diff --git a/commands/history/export.go b/commands/history/export.go index 5ccf25f3231a..af8149a7fb84 100644 --- a/commands/history/export.go +++ b/commands/history/export.go @@ -20,10 +20,11 @@ import ( ) type exportOptions struct { - builder string - refs []string - output string - all bool + builder string + refs []string + output string + all bool + finalize bool } func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error { @@ -62,6 +63,26 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e return errors.Errorf("no record found for ref %q", ref) } + if opts.finalize { + var finalized bool + for _, rec := range recs { + if rec.Trace == nil { + finalized = true + if err := finalizeRecord(ctx, rec.Ref, nodes); err != nil { + return err + } + } + } + if finalized { + recs, err = queryRecords(ctx, ref, nodes, &queryOptions{ + CompletedOnly: true, + }) + if err != nil { + return err + } + } + } + if ref == "" { slices.SortFunc(recs, func(a, b historyRecord) int { return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime()) @@ -154,7 +175,8 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command { flags := cmd.Flags() flags.StringVarP(&options.output, "output", "o", "", "Output file path") - flags.BoolVar(&options.all, "all", false, "Export all records for the builder") + flags.BoolVar(&options.all, "all", false, "Export all build records for the builder") + flags.BoolVar(&options.finalize, "finalize", false, "Ensure build records are finalized before exporting") return cmd } diff --git a/commands/history/trace.go b/commands/history/trace.go index 1e2b0f1ba775..b841ce357056 100644 --- a/commands/history/trace.go +++ b/commands/history/trace.go @@ -17,7 +17,6 @@ import ( "github.com/docker/buildx/util/otelutil" "github.com/docker/buildx/util/otelutil/jaeger" "github.com/docker/cli/cli/command" - controlapi "github.com/moby/buildkit/api/services/control" "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/browser" @@ -57,14 +56,7 @@ func loadTrace(ctx context.Context, ref string, nodes []builder.Node) (string, [ // build is complete but no trace yet. try to finalize the trace time.Sleep(1 * time.Second) // give some extra time for last parts of trace to be written - c, err := rec.node.Driver.Client(ctx) - if err != nil { - return "", nil, err - } - _, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{ - Ref: rec.Ref, - Finalize: true, - }) + err := finalizeRecord(ctx, rec.Ref, []builder.Node{*rec.node}) if err != nil { return "", nil, err } diff --git a/commands/history/utils.go b/commands/history/utils.go index 95c8702e15f4..fa070eaabeb8 100644 --- a/commands/history/utils.go +++ b/commands/history/utils.go @@ -248,6 +248,28 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q return out, nil } +func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error { + eg, ctx := errgroup.WithContext(ctx) + for _, node := range nodes { + node := node + eg.Go(func() error { + if node.Driver == nil { + return nil + } + c, err := node.Driver.Client(ctx) + if err != nil { + return err + } + _, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{ + Ref: ref, + Finalize: true, + }) + return err + }) + } + return eg.Wait() +} + func formatDuration(d time.Duration) string { if d < time.Minute { return fmt.Sprintf("%.1fs", d.Seconds()) diff --git a/docs/reference/buildx_history_export.md b/docs/reference/buildx_history_export.md index ae394077769d..edb99ee8f546 100644 --- a/docs/reference/buildx_history_export.md +++ b/docs/reference/buildx_history_export.md @@ -5,12 +5,13 @@ Export build records into Docker Desktop bundle ### Options -| Name | Type | Default | Description | -|:---------------------------------------|:---------|:--------|:-----------------------------------------| -| [`--all`](#all) | `bool` | | Export all records for the builder | -| [`--builder`](#builder) | `string` | | Override the configured builder instance | -| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging | -| [`-o`](#output), [`--output`](#output) | `string` | | Output file path | +| Name | Type | Default | Description | +|:---------------------------------------|:---------|:--------|:----------------------------------------------------| +| [`--all`](#all) | `bool` | | Export all build records for the builder | +| [`--builder`](#builder) | `string` | | Override the configured builder instance | +| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging | +| [`--finalize`](#finalize) | `bool` | | Ensure build records are finalized before exporting | +| [`-o`](#output), [`--output`](#output) | `string` | | Output file path | @@ -49,6 +50,16 @@ docker buildx history export --builder builder0 ^1 -o builder0-build.dockerbuild docker buildx history export --debug qu2gsuo8ejqrwdfii23xkkckt -o debug-build.dockerbuild ``` +### Ensure build records are finalized before exporting (--finalize) + +Clients can report their own traces concurrently, and not all traces may be +saved yet by the time of the export. Use the `--finalize` flag to ensure all +traces are finalized before exporting. + +```console +docker buildx history export --finalize qu2gsuo8ejqrwdfii23xkkckt -o finalized-build.dockerbuild +``` + ### Export a single build to a custom file (--output) ```console diff --git a/tests/history.go b/tests/history.go new file mode 100644 index 000000000000..6ced64cfedc4 --- /dev/null +++ b/tests/history.go @@ -0,0 +1,141 @@ +package tests + +import ( + "encoding/json" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/require" +) + +var historyTests = []func(t *testing.T, sb integration.Sandbox){ + testHistoryExport, + testHistoryExportFinalize, + testHistoryInspect, + testHistoryLs, + testHistoryRm, +} + +func testHistoryExport(t *testing.T, sb integration.Sandbox) { + ref := buildTestProject(t, sb) + require.NotEmpty(t, ref.Ref) + + outFile := path.Join(t.TempDir(), "export.dockerbuild") + cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile)) + out, err := cmd.Output() + require.NoError(t, err, string(out)) + require.FileExists(t, outFile) +} + +func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) { + ref := buildTestProject(t, sb) + require.NotEmpty(t, ref.Ref) + + outFile := path.Join(t.TempDir(), "export.dockerbuild") + cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile)) + out, err := cmd.Output() + require.NoError(t, err, string(out)) + require.FileExists(t, outFile) +} + +func testHistoryInspect(t *testing.T, sb integration.Sandbox) { + ref := buildTestProject(t, sb) + require.NotEmpty(t, ref.Ref) + + cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json")) + out, err := cmd.Output() + require.NoError(t, err, string(out)) + + type recT struct { + Name string + Ref string + Context string + Dockerfile string + StartedAt *time.Time + CompletedAt *time.Time + Duration time.Duration + Status string + NumCompletedSteps int32 + NumTotalSteps int32 + NumCachedSteps int32 + } + var rec recT + err = json.Unmarshal(out, &rec) + require.NoError(t, err) + require.Equal(t, ref.Ref, rec.Ref) + require.NotEmpty(t, rec.Name) +} + +func testHistoryLs(t *testing.T, sb integration.Sandbox) { + ref := buildTestProject(t, sb) + require.NotEmpty(t, ref.Ref) + + cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json")) + out, err := cmd.Output() + require.NoError(t, err, string(out)) + + type recT struct { + Ref string `json:"ref"` + Name string `json:"name"` + Status string `json:"status"` + CreatedAt *time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at"` + TotalSteps int32 `json:"total_steps"` + CompletedSteps int32 `json:"completed_steps"` + CachedSteps int32 `json:"cached_steps"` + } + var rec recT + err = json.Unmarshal(out, &rec) + require.NoError(t, err) + require.Equal(t, ref.String(), rec.Ref) + require.NotEmpty(t, rec.Name) +} + +func testHistoryRm(t *testing.T, sb integration.Sandbox) { + ref := buildTestProject(t, sb) + require.NotEmpty(t, ref.Ref) + + cmd := buildxCmd(sb, withArgs("history", "rm", ref.Ref)) + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +type buildRef struct { + Builder string + Node string + Ref string +} + +func (b buildRef) String() string { + return b.Builder + "/" + b.Node + "/" + b.Ref +} + +func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef { + dir := createTestProject(t) + out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir)) + require.NoError(t, err, string(out)) + + dt, err := os.ReadFile(filepath.Join(dir, "md.json")) + require.NoError(t, err) + + type mdT struct { + BuildRef string `json:"buildx.build.ref"` + } + var md mdT + err = json.Unmarshal(dt, &md) + require.NoError(t, err) + + refParts := strings.Split(md.BuildRef, "/") + require.Len(t, refParts, 3) + + return buildRef{ + Builder: refParts[0], + Node: refParts[1], + Ref: refParts[2], + } +} diff --git a/tests/integration_test.go b/tests/integration_test.go index 9cd76c7cd635..3884ca684430 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -24,6 +24,7 @@ func TestIntegration(t *testing.T) { tests = append(tests, commonTests...) tests = append(tests, buildTests...) tests = append(tests, bakeTests...) + tests = append(tests, historyTests...) tests = append(tests, inspectTests...) tests = append(tests, lsTests...) tests = append(tests, imagetoolsTests...)