Skip to content

Commit 7bb2748

Browse files
cpuguy83cpuguy83
authored
Run package spec tests for packaging targets (#397)
Before this the package tests were only run when building a container and not when just building a package. Signed-off-by: Brian Goff <[email protected]>
1 parent e9f1696 commit 7bb2748

File tree

6 files changed

+215
-163
lines changed

6 files changed

+215
-163
lines changed

‎frontend/azlinux/handle_container.go

+10-108
Original file line numberDiff line numberDiff line change
@@ -30,112 +30,18 @@ func handleContainer(w worker) gwclient.BuildFunc {
3030
return nil, nil, fmt.Errorf("error creating rpm: %w", err)
3131
}
3232

33-
rpms, err := readRPMs(ctx, client, rpmDir)
34-
if err != nil {
35-
return nil, nil, err
36-
}
37-
38-
st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg)
39-
if err != nil {
40-
return nil, nil, err
41-
}
42-
43-
def, err := st.Marshal(ctx, pg)
44-
if err != nil {
45-
return nil, nil, fmt.Errorf("error marshalling llb: %w", err)
46-
}
47-
48-
res, err := client.Solve(ctx, gwclient.SolveRequest{
49-
Definition: def.ToPB(),
50-
})
51-
if err != nil {
52-
return nil, nil, err
53-
}
54-
5533
img, err := resolveBaseConfig(ctx, w, client, platform, spec, targetKey)
5634
if err != nil {
5735
return nil, nil, errors.Wrap(err, "could not resolve base image config")
5836
}
5937

60-
ref, err := res.SingleRef()
61-
if err != nil {
62-
return nil, nil, err
63-
}
64-
65-
base, err := w.Base(sOpt, pg)
66-
if err != nil {
67-
return nil, nil, err
68-
}
69-
70-
withTestDeps := func(in llb.State) llb.State {
71-
deps := spec.GetTestDeps(targetKey)
72-
if len(deps) == 0 {
73-
return in
74-
}
75-
return base.Run(
76-
w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
77-
pg,
78-
dalec.ProgressGroup("Install test dependencies"),
79-
).AddMount("/tmp/rootfs", in)
80-
}
81-
82-
if err := frontend.RunTests(ctx, client, spec, ref, withTestDeps, targetKey); err != nil {
83-
return nil, nil, err
84-
}
85-
38+
ref, err := runTests(ctx, client, w, spec, sOpt, rpmDir, targetKey)
8639
return ref, img, err
8740
})
8841
}
8942
}
9043

91-
func readRPMs(ctx context.Context, client gwclient.Client, st llb.State) ([]string, error) {
92-
def, err := st.Marshal(ctx)
93-
if err != nil {
94-
return nil, err
95-
}
96-
97-
res, err := client.Solve(ctx, gwclient.SolveRequest{
98-
Definition: def.ToPB(),
99-
})
100-
if err != nil {
101-
return nil, err
102-
}
103-
104-
ref, err := res.SingleRef()
105-
if err != nil {
106-
return nil, err
107-
}
108-
109-
// Directory layout will have arch-specific sub-folders and/or `noarch`
110-
// RPMs will be in those subdirectories.
111-
arches, err := ref.ReadDir(ctx, gwclient.ReadDirRequest{
112-
Path: "/RPMS",
113-
})
114-
if err != nil {
115-
return nil, errors.Wrap(err, "error reading output state")
116-
}
117-
118-
var out []string
119-
120-
for _, arch := range arches {
121-
files, err := ref.ReadDir(ctx, gwclient.ReadDirRequest{
122-
Path: filepath.Join("/RPMS", arch.Path),
123-
IncludePattern: "*.rpm",
124-
})
125-
126-
if err != nil {
127-
return nil, errors.Wrap(err, "could not read arch specific output dir")
128-
}
129-
130-
for _, e := range files {
131-
out = append(out, filepath.Join(arch.Path, e.Path))
132-
}
133-
}
134-
135-
return out, nil
136-
}
137-
138-
func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb.State, files []string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
44+
func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb.State, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
13945
opts = append(opts, dalec.ProgressGroup("Install RPMs"))
14046
const workPath = "/tmp/rootfs"
14147

@@ -149,19 +55,15 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
14955
rootfs = llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
15056
}
15157

152-
if len(files) > 0 {
153-
rpmMountDir := "/tmp/rpms"
154-
updated := w.BasePackages()
155-
for _, f := range files {
156-
updated = append(updated, filepath.Join(rpmMountDir, f))
157-
}
58+
rpmMountDir := "/tmp/rpms"
59+
pkgs := w.BasePackages()
60+
pkgs = append(pkgs, filepath.Join(rpmMountDir, "**/*.rpm"))
15861

159-
rootfs = builderImg.Run(
160-
w.Install(updated, atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
161-
llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
162-
dalec.WithConstraints(opts...),
163-
).AddMount(workPath, rootfs)
164-
}
62+
rootfs = builderImg.Run(
63+
w.Install(pkgs, atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
64+
llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
65+
dalec.WithConstraints(opts...),
66+
).AddMount(workPath, rootfs)
16567

16668
if post := spec.GetImagePost(targetKey); post != nil && len(post.Symlinks) > 0 {
16769
rootfs = builderImg.

‎frontend/azlinux/handle_depsonly.go

+1-6
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
3131
).
3232
AddMount("/tmp/rpms", llb.Scratch())
3333

34-
files, err := readRPMs(ctx, client, rpmDir)
35-
if err != nil {
36-
return nil, nil, err
37-
}
38-
39-
st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg)
34+
st, err := specToContainerLLB(w, spec, targetKey, rpmDir, sOpt, pg)
4035
if err != nil {
4136
return nil, nil, err
4237
}

‎frontend/azlinux/handle_rpm.go

+60
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/moby/buildkit/client/llb"
1212
gwclient "github.com/moby/buildkit/frontend/gateway/client"
1313
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
14+
"github.com/pkg/errors"
1415
)
1516

1617
func handleRPM(w worker) gwclient.BuildFunc {
@@ -47,11 +48,70 @@ func handleRPM(w worker) gwclient.BuildFunc {
4748
if err != nil {
4849
return nil, nil, err
4950
}
51+
52+
if imgRef, err := runTests(ctx, client, w, spec, sOpt, st, targetKey, pg); err != nil {
53+
// return the container ref in case of error so it can be used to debug
54+
// the installed package state.
55+
cfg, _ := resolveBaseConfig(ctx, w, client, platform, spec, targetKey)
56+
return imgRef, cfg, err
57+
}
58+
5059
return ref, &dalec.DockerImageSpec{}, nil
5160
})
5261
}
5362
}
5463

64+
// runTests runs the package tests
65+
// The returned reference is the solved container state
66+
func runTests(ctx context.Context, client gwclient.Client, w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, rpmDir llb.State, targetKey string, opts ...llb.ConstraintsOpt) (gwclient.Reference, error) {
67+
withDeps, err := withTestDeps(w, spec, sOpt, targetKey)
68+
if err != nil {
69+
return nil, err
70+
}
71+
72+
imgSt, err := specToContainerLLB(w, spec, targetKey, rpmDir, sOpt, opts...)
73+
if err != nil {
74+
return nil, errors.Wrap(err, "error creating container image state")
75+
}
76+
77+
def, err := imgSt.Marshal(ctx, opts...)
78+
if err != nil {
79+
return nil, err
80+
}
81+
82+
res, err := client.Solve(ctx, gwclient.SolveRequest{Definition: def.ToPB()})
83+
if err != nil {
84+
return nil, errors.Wrap(err, "error solving container state")
85+
}
86+
87+
ref, err := res.SingleRef()
88+
if err != nil {
89+
return nil, err
90+
}
91+
92+
err = frontend.RunTests(ctx, client, spec, ref, withDeps, targetKey)
93+
return ref, errors.Wrap(err, "TESTS FAILED")
94+
}
95+
96+
func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
97+
base, err := w.Base(sOpt, opts...)
98+
if err != nil {
99+
return nil, err
100+
}
101+
return func(in llb.State) llb.State {
102+
deps := spec.GetTestDeps(targetKey)
103+
if len(deps) == 0 {
104+
return in
105+
}
106+
return base.Run(
107+
w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
108+
dalec.WithConstraints(opts...),
109+
dalec.ProgressGroup("Install test dependencies"),
110+
).AddMount("/tmp/rootfs", in)
111+
112+
}, nil
113+
}
114+
55115
// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
56116
func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc {
57117
// depsOnly is a simple dalec spec that only includes build dependencies and their constraints

‎frontend/jammy/handle_container.go

+1-48
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,10 @@ package jammy
33
import (
44
"context"
55
"encoding/json"
6-
"io/fs"
76
"strings"
87

98
"github.com/Azure/dalec"
109
"github.com/Azure/dalec/frontend"
11-
"github.com/Azure/dalec/frontend/pkg/bkfs"
1210
"github.com/moby/buildkit/client/llb"
1311
"github.com/moby/buildkit/client/llb/sourceresolver"
1412
gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -37,57 +35,12 @@ func handleContainer(ctx context.Context, client gwclient.Client) (*gwclient.Res
3735
return nil, nil, err
3836
}
3937

40-
worker, err := workerBase(sOpt, opt)
41-
if err != nil {
42-
return nil, nil, err
43-
}
44-
45-
var includeTestRepo bool
46-
47-
workerFS, err := bkfs.FromState(ctx, &worker, client)
48-
if err != nil {
49-
return nil, nil, err
50-
}
51-
52-
// Check if there there is a test repo in the worker image.
53-
// We'll mount that into the target container while installing packages.
54-
_, repoErr := fs.Stat(workerFS, testRepoPath[1:])
55-
_, listErr := fs.Stat(workerFS, testRepoSourceListPath[1:])
56-
if listErr == nil && repoErr == nil {
57-
// This is a test and we need to include the repo from the worker image
58-
// into target container.
59-
includeTestRepo = true
60-
frontend.Warn(ctx, client, worker, "Including test repo from worker image")
61-
}
62-
63-
st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opt)
64-
65-
def, err := st.Marshal(ctx)
66-
if err != nil {
67-
return nil, nil, err
68-
}
69-
7038
img, err := buildImageConfig(ctx, client, spec, platform, targetKey)
7139
if err != nil {
7240
return nil, nil, err
7341
}
7442

75-
res, err := client.Solve(ctx, gwclient.SolveRequest{
76-
Definition: def.ToPB(),
77-
})
78-
if err != nil {
79-
return nil, nil, err
80-
}
81-
82-
ref, err := res.SingleRef()
83-
if err != nil {
84-
return nil, nil, err
85-
}
86-
87-
if err := frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opt), targetKey); err != nil {
88-
return nil, nil, err
89-
}
90-
43+
ref, err := runTests(ctx, client, spec, sOpt, deb, targetKey, opt)
9144
return ref, img, err
9245
})
9346
}

‎frontend/jammy/handle_deb.go

+57-1
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ package jammy
33
import (
44
"context"
55
"fmt"
6+
"io/fs"
67
"strings"
78

89
"github.com/Azure/dalec"
910
"github.com/Azure/dalec/frontend"
1011
"github.com/Azure/dalec/frontend/deb"
12+
"github.com/Azure/dalec/frontend/pkg/bkfs"
1113
"github.com/containerd/platforms"
1214
"github.com/moby/buildkit/client/llb"
1315
gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -24,7 +26,8 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
2426
return nil, nil, err
2527
}
2628

27-
st, err := buildDeb(ctx, client, spec, sOpt, targetKey, dalec.ProgressGroup("Building Jammy deb package: "+spec.Name))
29+
opt := dalec.ProgressGroup("Building Jammy deb package: " + spec.Name)
30+
st, err := buildDeb(ctx, client, spec, sOpt, targetKey, opt)
2831
if err != nil {
2932
return nil, nil, err
3033
}
@@ -45,6 +48,12 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
4548
if err != nil {
4649
return nil, nil, err
4750
}
51+
52+
if ref, err := runTests(ctx, client, spec, sOpt, st, targetKey, opt); err != nil {
53+
cfg, _ := buildImageConfig(ctx, client, spec, platform, targetKey)
54+
return ref, cfg, err
55+
}
56+
4857
if platform == nil {
4958
p := platforms.DefaultSpec()
5059
platform = &p
@@ -53,6 +62,53 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
5362
})
5463
}
5564

65+
func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, opts ...llb.ConstraintsOpt) (gwclient.Reference, error) {
66+
worker, err := workerBase(sOpt, opts...)
67+
if err != nil {
68+
return nil, err
69+
}
70+
71+
var includeTestRepo bool
72+
73+
workerFS, err := bkfs.FromState(ctx, &worker, client)
74+
if err != nil {
75+
return nil, err
76+
}
77+
78+
// Check if there there is a test repo in the worker image.
79+
// We'll mount that into the target container while installing packages.
80+
_, repoErr := fs.Stat(workerFS, testRepoPath[1:])
81+
_, listErr := fs.Stat(workerFS, testRepoSourceListPath[1:])
82+
if listErr == nil && repoErr == nil {
83+
// This is a test and we need to include the repo from the worker image
84+
// into target container.
85+
includeTestRepo = true
86+
frontend.Warn(ctx, client, worker, "Including test repo from worker image")
87+
}
88+
89+
st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...)
90+
91+
def, err := st.Marshal(ctx, opts...)
92+
if err != nil {
93+
return nil, err
94+
}
95+
96+
res, err := client.Solve(ctx, gwclient.SolveRequest{
97+
Definition: def.ToPB(),
98+
})
99+
if err != nil {
100+
return nil, err
101+
}
102+
103+
ref, err := res.SingleRef()
104+
if err != nil {
105+
return nil, err
106+
}
107+
108+
err = frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opts...), targetKey)
109+
return ref, err
110+
}
111+
56112
func installPackages(ls ...string) llb.RunOption {
57113
return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
58114
// This only runs apt-get update if the pkgcache is older than 10 minutes.

0 commit comments

Comments
 (0)