From 257423c3231e1333ee8d8c14b81e11414a513caf Mon Sep 17 00:00:00 2001 From: Roberto Villarreal Date: Tue, 20 May 2025 13:09:13 -0600 Subject: [PATCH 1/4] Consider typed, value-less variables to have `null` value A variable with a type but no default value or override resulted in an empty string. This matches the legacy behavior of untyped variables, but does not make sense when using types (an empty string is itself a type violation for everything except `string`). All variables defined with a type but with no value are now a typed `null`. A variable explicitly typed `any` was previously treated as if the typing was omitted; with no defined value or override, that resulted in an empty string. The `any` type is now distinguished from an omitted type; these variables, with no default or override, are also `null`. In other respects, the behavior of `any` is unchanged and largely behaves as if the type was omitted. It's not clear whether it should be supported, let alone how it should behave, so these tests were removed. It's being treated as undefined behavior. Signed-off-by: Roberto Villarreal (cherry picked from commit 3c0f5c5c21155879ae68859a8771601a44e308ff) --- bake/hcl_test.go | 87 +++++++++++++++++++++++++------------ bake/hclparser/hclparser.go | 12 +++-- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/bake/hcl_test.go b/bake/hcl_test.go index 88972e8c686c..ee52352e6cae 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -423,6 +423,63 @@ func TestHCLNullVariables(t *testing.T) { require.Equal(t, ptrstr("bar"), c.Targets[0].Args["foo"]) } +func TestHCLTypedNullVariables(t *testing.T) { + types := []string{ + "any", + "string", "number", "bool", + "list(string)", "set(string)", "map(string)", + "tuple([string])", "object({val: string})", + } + for _, varType := range types { + tName := fmt.Sprintf("variable typed %q with null default remains null", varType) + t.Run(tName, func(t *testing.T) { + dt := fmt.Sprintf(` + variable "FOO" { + type = %s + default = null + } + + target "default" { + args = { + foo = equal(FOO, null) + } + }`, varType) + c, err := ParseFile([]byte(dt), "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) + }) + } +} + +func TestHCLTypedValuelessVariables(t *testing.T) { + types := []string{ + "any", + "string", "number", "bool", + "list(string)", "set(string)", "map(string)", + "tuple([string])", "object({val: string})", + } + for _, varType := range types { + tName := fmt.Sprintf("variable typed %q with no default is null", varType) + t.Run(tName, func(t *testing.T) { + dt := fmt.Sprintf(` + variable "FOO" { + type = %s + } + + target "default" { + args = { + foo = equal(FOO, null) + } + }`, varType) + c, err := ParseFile([]byte(dt), "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) + }) + } +} + func TestJSONNullVariables(t *testing.T) { dt := []byte(`{ "variable": { @@ -1877,19 +1934,6 @@ func TestTypedVarOverrides(t *testing.T) { override: `"hello"`, wantValue: `"hello"`, }, - { - name: "any", - varType: "any", - override: "[1,2]", - wantValue: "[1,2]", - }, - { - name: "any never convert to complex types", - varType: "any", - override: "[1,2]", - argValue: "length(FOO)", - wantErrorMsg: "collection must be a list", - }, { name: "proper CSV list of strings", varType: "list(string)", @@ -2090,19 +2134,6 @@ func TestTypedVarOverrides_JSON(t *testing.T) { override: `"hello"`, wantValue: "hello", }, - { - name: "any", - varType: "any", - override: "[1,2]", - wantValue: "[1,2]", - }, - { - name: "any never convert to complex types", - varType: "any", - override: "[1,2]", - argValue: "length(FOO)", - wantErrorMsg: "collection must be a list", - }, { name: "list of strings", varType: "list(string)", @@ -2313,6 +2344,7 @@ func TestJSONOverridePriority(t *testing.T) { dt := []byte(` variable "foo" { type = number + default = 101 } target "default" { @@ -2325,8 +2357,7 @@ func TestJSONOverridePriority(t *testing.T) { c, err := ParseFile(dt, "docker-bake.hcl") require.NoError(t, err) require.Equal(t, 1, len(c.Targets)) - // a variable with no value has always resulted in an empty string - require.Equal(t, "", *c.Targets[0].Args["bar"]) + require.Equal(t, "101", *c.Targets[0].Args["bar"]) t.Setenv("foo_JSON", "42") c, err = ParseFile(dt, "docker-bake.hcl") diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index d132e96618f9..d8f134590ebd 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -282,7 +282,7 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { } var diags hcl.Diagnostics - varType := cty.DynamicPseudoType + varType, typeSpecified := cty.DynamicPseudoType, false def, ok := p.attrs[name] if !ok { vr, ok := p.vars[name] @@ -295,12 +295,13 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { if diags.HasErrors() { return diags } + typeSpecified = !varType.Equals(cty.DynamicPseudoType) || hcl.ExprAsKeyword(vr.Type) == "any" } if def == nil { - // lack of specified value is considered to have an empty string value, - // but any overrides get type checked - if _, ok, _ := p.valueHasOverride(name, false); !ok { + // Lack of specified value, when untyped is considered to have an empty string value. + // A typed variable with no value will result in (typed) nil. + if _, ok, _ := p.valueHasOverride(name, false); !ok && !typeSpecified { vv := cty.StringVal("") v = &vv return @@ -322,9 +323,6 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { } } - // Not entirely true... this doesn't differentiate between a user that specified 'any' - // and a user that specified nothing. But the result is the same; both are treated as strings. - typeSpecified := !varType.Equals(cty.DynamicPseudoType) envv, hasEnv, jsonEnv := p.valueHasOverride(name, typeSpecified) _, isVar := p.vars[name] From d364dfeefcd1ba50ff83651a28e5034d914d419f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 21 May 2025 16:15:30 +0200 Subject: [PATCH 2/4] bake: TestEmptyVariable Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> (cherry picked from commit 291c3535757c58eb562beb2c1784f3e1a5eea539) --- bake/hcl_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bake/hcl_test.go b/bake/hcl_test.go index ee52352e6cae..db1ca9c6cf73 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -1622,6 +1622,20 @@ target "two" { require.Equal(t, map[string]*string{"b": ptrstr("pre-jkl")}, c.Targets[1].Args) } +func TestEmptyVariable(t *testing.T) { + dt := []byte(` + variable "FOO" {} + target "default" { + args = { + foo = equal(FOO, "") + } + }`) + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) +} + func TestEmptyVariableJSON(t *testing.T) { dt := []byte(`{ "variable": { From e66667d34ad8f98e510544ff5845ae42ef784fa4 Mon Sep 17 00:00:00 2001 From: Roberto Villarreal Date: Mon, 19 May 2025 00:22:04 -0600 Subject: [PATCH 3/4] Add variable typing to reference docs This documents the variable typing introduced in #3167. Signed-off-by: Roberto Villarreal (cherry picked from commit cfeca919a9e1142ecc106411fe38838348fd0101) --- docs/bake-reference.md | 202 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/docs/bake-reference.md b/docs/bake-reference.md index 92f39994ea55..5245f611dc38 100644 --- a/docs/bake-reference.md +++ b/docs/bake-reference.md @@ -1081,6 +1081,7 @@ or interpolate them in attribute values in your Bake file. ```hcl variable "TAG" { + type = string default = "latest" } @@ -1102,6 +1103,206 @@ overriding the default `latest` value shown in the previous example. $ TAG=dev docker buildx bake webapp-dev ``` +Variables can also be assigned an explicit type. +If provided, it will be used to validate the default value (if set), as well as any overrides. +This is particularly useful when using complex types which are intended to be overridden. +The previous example could be expanded to apply an arbitrary series of tags. +```hcl +variable "TAGS" { + default = ["latest"] + type = list(string) +} + +target "webapp-dev" { + dockerfile = "Dockerfile.webapp" + tags = [for tag in TAGS: "docker.io/username/webapp:${tag}"] +} +``` + +This example shows how to generate three tags without changing the file +or using custom functions/parsing: +```console +$ TAGS=dev,latest,2 docker buildx bake webapp-dev +``` + +### Variable typing + +The following primitive types are available: +* `string` +* `number` +* `bool` + +The type is expressed like a keyword; it must be expressed as a literal: +```hcl +variable "OK" { + type = string +} + +# cannot be an actual string +variable "BAD" { + type = "string" +} + +# cannot be the result of an expression +variable "ALSO_BAD" { + type = lower("string") +} +``` +Specifying primitive types can be valuable to show intent (especially when a default is not provided), +but bake will generally behave as expected without explicit typing. + +Complex types are expressed with "type constructors"; they are: +* `tuple([,...])` +* `list()` +* `set()` +* `map()` +* `object({=},...})` + +The following are examples of each of those, as well as how the (optional) default value would be expressed: +```hcl +# structured way to express "1.2.3-alpha" +variable "MY_VERSION" { + type = tuple([number, number, number, string]) + default = [1, 2, 3, "alpha"] +} + +# JDK versions used in a matrix build +variable "JDK_VERSIONS" { + type = list(number) + default = [11, 17, 21] +} + +# better way to express the previous example; this will also +# enforce set semantics and allow use of set-based functions +variable "JDK_VERSIONS" { + type = set(number) + default = [11, 17, 21] +} + +# with the help of lookup(), translate a 'feature' to a tag +variable "FEATURE_TO_NAME" { + type = map(string) + default = {featureA = "slim", featureB = "tiny"} +} + +# map a branch name to a registry location +variable "PUSH_DESTINATION" { + type = object({branch = string, registry = string}) + default = {branch = "main", registry = "prod-registry.invalid.com"} +} + +# make the previous example more useful with composition +variable "PUSH_DESTINATIONS" { + type = list(object({branch = string, registry = string})) + default = [ + {branch = "develop", registry = "test-registry.invalid.com"}, + {branch = "main", registry = "prod-registry.invalid.com"}, + ] +} +``` +Note that in each example, the default value would be valid even if typing was not present. +If typing was omitted, the first three would all be considered `tuple`; +you would be restricted to functions that operate on `tuple` and, for example, not be able to add elements. +Similarly, the third and fourth would both be considered `object`, with the limits and semantics of that type. +In short, in the absence of a type, any value delimited with `[]` is a `tuple` +and value delimited with `{}` is an `object`. +Explicit typing for complex types not only opens up the ability to use functions applicable to that specialized type, +but is also a precondition for providing overrides. + +> [!NOTE] +> See [HCL Type Expressions][typeexpr] page for more details. + +### Overriding variables + +As mentioned in the [intro to variables](#variable), primitive types (`string`, `number`, and `bool`) +can be overridden without typing and will generally behave as expected. +(When explicit typing is not provided, a variable is assumed to be primitive when the default value lacks `{}` or `[]` delimiters; +a variable with neither typing nor a default value is treated as `string`.) +Naturally, these same overrides can be used alongside explicit typing too; +they may help in edge cases where you want `VAR=true` to be a `string`, where without typing, +it may be a `string` or a `bool` depending on how/where it's used. +Overriding a variable with a complex type can only be done when the type is provided. +This is still done via environment variables, but the values can be provided via CSV or JSON. + +#### CSV overrides + +This is considered the canonical method and is well suited to interactive usage. +It is assumed that `list` and `set` will be the most common complex type, +as well as the most common complex type designed to be overridden. +Thus, there is full CSV support for `list` and `set` +(and `tuple`; despite being considered a structural type, it is more like a collection type in this regard). + + +There is limited support for `map` and `object` and no support for composite types; +for these advanced cases, an alternative mechanism [using JSON](#json-overrides) is available. + +#### JSON overrides + +Overrides can also be provided via JSON. +This is the only method available for providing some complex types and may be convenient if overrides are already JSON +(for example, if they come from a JSON API). +It can also be used when dealing with values are difficult or impossible to specify using CSV (e.g., values containing quotes or commas). +To use JSON, simply append `_JSON` to the variable name. +In this contrived example, CSV cannot handle the second value; despite being a supported CSV type, JSON must be used: +```hcl +variable "VALS" { + type = list(string) + default = ["some", "list"] +} +``` +```console +$ cat data.json +["hello","with,comma","with\"quote"] +$ VALS_JSON=$(< data.json) docker buildx bake + +# CSV equivalent, though the second value cannot be expressed at all +$ VALS='hello,"with""quote"' docker buildx bake +``` + +This example illustrates some precedence and usage rules: +```hcl +variable "FOO" { + type = string + default = "foo" +} + +variable "FOO_JSON" { + type = string + default = "foo" +} +``` + +The variable `FOO` can *only* be overridden using CSV because `FOO_JSON`, which would typically used for a JSON override, +is already a defined variable. +Since `FOO_JSON` is an actual variable, setting that environment variable would be expected to a CSV value. +A JSON override *is* possible for this variable, using environment variable `FOO_JSON_JSON`. + +```Console +# These three are all equivalent, setting variable FOO=bar +$ FOO=bar docker buildx bake <...> +$ FOO='bar' docker buildx bake <...> +$ FOO="bar" docker buildx bake <...> + +# Sets *only* variable FOO_JSON; FOO is untouched +$ FOO_JSON=bar docker buildx bake <...> + +# This also sets FOO_JSON, but will fail due to not being valid JSON +$ FOO_JSON_JSON=bar docker buildx bake <...> + +# These are all equivalent +$ cat data.json +"bar" +$ FOO_JSON_JSON=$(< data.json) docker buildx bake <...> +$ FOO_JSON_JSON='"bar"' docker buildx bake <...> +$ FOO_JSON=bar docker buildx bake <...> + +# This results in setting two different variables, both specified as CSV (FOO=bar and FOO_JSON="baz") +$ FOO=bar FOO_JSON='"baz"' docker buildx bake <...> + +# These refer to the same variable with FOO_JSON_JSON having precedence and read as JSON (FOO_JSON=baz) +$ FOO_JSON=bar FOO_JSON_JSON='"baz"' docker buildx bake <...> +``` + ### Built-in variables The following variables are built-ins that you can use with Bake without having @@ -1239,4 +1440,5 @@ target "webapp-dev" { [ssh]: https://docs.docker.com/reference/cli/docker/buildx/build/#ssh [tag]: https://docs.docker.com/reference/cli/docker/image/build/#tag [target]: https://docs.docker.com/reference/cli/docker/image/build/#target +[typeexpr]: https://github.com/hashicorp/hcl/tree/main/ext/typeexpr [userfunc]: https://github.com/hashicorp/hcl/tree/main/ext/userfunc From 3bb0ae451ca0cc6744112294d3129321945ebff1 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 21 May 2025 14:57:25 +0200 Subject: [PATCH 4/4] vendor: github.com/moby/buildkit v0.22.0 Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> (cherry picked from commit a11bb4985c333554d9a90b313c4b0dc3a5ef15e4) --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a796b6c9927d..06aa6815f16f 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/hcl/v2 v2.23.0 github.com/in-toto/in-toto-golang v0.5.0 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/moby/buildkit v0.22.0-rc2 + github.com/moby/buildkit v0.22.0 github.com/moby/go-archive v0.1.0 github.com/moby/sys/atomicwriter v0.1.0 github.com/moby/sys/mountinfo v0.7.2 diff --git a/go.sum b/go.sum index a156b17fb79e..2adf8018bbbd 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/buildkit v0.22.0-rc2 h1:B/1nFOqPO/ceJ8nzAbc+q3d0ckyGwlWtxVQXpeqXuwY= -github.com/moby/buildkit v0.22.0-rc2/go.mod h1:j4pP5hxiTWcz7xuTK2cyxQislHl/N2WWHzOy43DlLJw= +github.com/moby/buildkit v0.22.0 h1:aWN06w1YGSVN1XfeZbj2ZbgY+zi5xDAjEFI8Cy9fTjA= +github.com/moby/buildkit v0.22.0/go.mod h1:j4pP5hxiTWcz7xuTK2cyxQislHl/N2WWHzOy43DlLJw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index 7daf793ccf19..9492d6566ba0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -451,7 +451,7 @@ github.com/mitchellh/go-wordwrap # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.22.0-rc2 +# github.com/moby/buildkit v0.22.0 ## explicit; go 1.23.0 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types