From 0798b469d653a296bceadae3f0fd23b10517dd95 Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Fri, 28 Jan 2022 16:26:53 -0500 Subject: [PATCH 1/2] Support building from JAR files Prior to this PR, native-image builds would happen with the assumption that the contents of the `/workspace` directory was an exploded JAR file. The buildpack would configure options to build from that location. Recently, buildpacks have gained the capability to build and persist multiple files. This results in the `/workspace` directory not meeting the previous expectation on structure. This PR continues to support building from an exploded JAR file, but also supports building from a JAR file. This requires the user to specify a pattern (Glob match) or file name which points to the JAR file that should the target for the native-image build. When this is set, the buildpack will adjust the build arguments to build from the JAR file. The `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` argument is what controls this behavior. If the `/workspace` directory does not contain an exploded JAR structure, it must have `META-INF/MANIFEST.MF`, then we'll look for files that match the pattern set in `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` and attempt to build from the JAR file at that location. Signed-off-by: Daniel Mikusa --- README.md | 13 +- buildpack.toml | 10 ++ native/arguments.go | 160 ++++++++++++++++++++ native/arguments_test.go | 241 +++++++++++++++++++++++++++++++ native/build.go | 45 ++++-- native/build_test.go | 33 ++++- native/init_test.go | 1 + native/native_image.go | 85 ++++++----- native/native_image_test.go | 9 +- native/testdata/test-fixture.jar | Bin 0 -> 330 bytes 10 files changed, 536 insertions(+), 61 deletions(-) create mode 100644 native/arguments.go create mode 100644 native/arguments_test.go create mode 100644 native/testdata/test-fixture.jar diff --git a/README.md b/README.md index dc8f4bd..6429c2c 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,17 @@ The buildpack will do the following: * Requests that the Native Image builder be installed by requiring `native-image-builder` in the build plan. * If `$BP_BINARY_COMPRESSION_METHOD` is set to `upx`, requests that UPX be installed by requiring `upx` in the buildplan. -* Uses `native-image` a to build a GraalVM native image and removes existing bytecode. +* Uses `native-image` a to build a GraalVM native image and removes existing bytecode. Defaults to building the `/workspace` as an exploded JAR. If `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` is set, it will build from the specified JAR file. * Uses `$BP_BINARY_COMPRESSION_METHOD` if set to `upx` or `gzexe` to compress the native image. ## Configuration -| Environment Variable | Description | -| ---------------------------------- | --------------------------------------------------------------------------------------------- | -| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. | + +| Environment Variable | Description | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. | | `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` | Arguments to pass to directly to the `native-image` command. These arguments must be valid and correctly formed or the `native-image` command will fail. | -| `$BP_BINARY_COMPRESSION_METHOD` | Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe` | +| `$BP_BINARY_COMPRESSION_METHOD` | Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe` | +| `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` | Configure the built application artifact explicitly. This is required if building a native image from a JAR file | ### Compression Caveats @@ -32,6 +34,7 @@ The buildpack will do the following: 2. Using `upx` will create a compressed executable that fails to run on M1 Macs. There is at the time of writing a bug in the emulation layer used by Docker on M1 Macs that is triggered when you try to run amd64 executable that has been compressed using `upx`. This is a known issue and will hopefully be patched in a future release. ## License + This buildpack is released under version 2.0 of the [Apache License][a]. [a]: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/buildpack.toml b/buildpack.toml index 1ee2653..1bf51b3 100644 --- a/buildpack.toml +++ b/buildpack.toml @@ -46,6 +46,16 @@ name = "BP_NATIVE_IMAGE_BUILD_ARGUMENTS" description = "arguments to pass to the native-image command" build = true +[[metadata.configurations]] +name = "BP_BINARY_COMPRESSION_METHOD" +description = "Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe`" +build = true + +[[metadata.configurations]] +name = "BP_NATIVE_IMAGE_BUILT_ARTIFACT" +description = "the built application artifact explicitly, required if building from a JAR" +build = true + [metadata] pre-package = "scripts/build.sh" include-files = [ diff --git a/native/arguments.go b/native/arguments.go new file mode 100644 index 0000000..3536389 --- /dev/null +++ b/native/arguments.go @@ -0,0 +1,160 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package native + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/magiconair/properties" + "github.com/mattn/go-shellwords" + "github.com/paketo-buildpacks/libpak" +) + +type Arguments interface { + Configure(inputArgs []string) ([]string, string, error) +} + +// BaselineArguments provides a set of arguments that are always set +type BaselineArguments struct { + StackID string +} + +// Configure provides an initial set of arguments, it ignores any input arguments +func (b BaselineArguments) Configure(_ []string) ([]string, string, error) { + var newArguments []string + + if b.StackID == libpak.TinyStackID { + newArguments = append(newArguments, "-H:+StaticExecutableWithDynamicLibC") + } + + return newArguments, "", nil +} + +// UserArguments augments the existing arguments with those provided by the end user +type UserArguments struct { + Arguments string +} + +// Configure returns the inputArgs plus the additional arguments specified by the end user, preference given to user arguments +func (u UserArguments) Configure(inputArgs []string) ([]string, string, error) { + parsedArgs, err := shellwords.Parse(u.Arguments) + if err != nil { + return []string{}, "", fmt.Errorf("unable to parse arguments from %s\n%w", u.Arguments, err) + } + + var outputArgs []string + + for _, inputArg := range inputArgs { + if !containsArg(inputArg, parsedArgs) { + outputArgs = append(outputArgs, inputArg) + } + } + + outputArgs = append(outputArgs, parsedArgs...) + + return outputArgs, "", nil +} + +// containsArg checks if needle is found in haystack +// +// needle and haystack entries are processed as key=val strings where only the key must match +func containsArg(needle string, haystack []string) bool { + needleSplit := strings.SplitN(needle, "=", 2) + + for _, straw := range haystack { + targetSplit := strings.SplitN(straw, "=", 2) + if needleSplit[0] == targetSplit[0] { + return true + } + } + + return false +} + +// ExplodedJarArguments provides a set of arguments specific to building from an exploded jar directory +type ExplodedJarArguments struct { + ApplicationPath string + LayerPath string + Manifest *properties.Properties +} + +// NoStartOrMainClass is an error returned when a start or main class cannot be found +type NoStartOrMainClass struct{} + +func (e NoStartOrMainClass) Error() string { + return "unable to read Start-Class or Main-Class from MANIFEST.MF" +} + +// Configure appends arguments to inputArgs for building from an exploded JAR directory +func (e ExplodedJarArguments) Configure(inputArgs []string) ([]string, string, error) { + startClass, ok := e.Manifest.Get("Start-Class") + if !ok { + startClass, ok = e.Manifest.Get("Main-Class") + if !ok { + return []string{}, "", NoStartOrMainClass{} + } + } + + cp := os.Getenv("CLASSPATH") + if cp == "" { + // CLASSPATH should have been done by upstream buildpacks, but just in case + cp = e.ApplicationPath + if v, ok := e.Manifest.Get("Class-Path"); ok { + cp = strings.Join([]string{cp, v}, string(filepath.ListSeparator)) + } + } + + inputArgs = append(inputArgs, + fmt.Sprintf("-H:Name=%s", filepath.Join(e.LayerPath, startClass)), + "-cp", cp, + startClass, + ) + + return inputArgs, startClass, nil +} + +// JarArguments provides a set of arguments specific to building from a jar file +type JarArguments struct { + ApplicationPath string + JarFilePattern string +} + +func (j JarArguments) Configure(inputArgs []string) ([]string, string, error) { + file := filepath.Join(j.ApplicationPath, j.JarFilePattern) + fmt.Println("----", file) + candidates, err := filepath.Glob(file) + if err != nil { + return []string{}, "", fmt.Errorf("unable to find JAR with %s\n%w", j.JarFilePattern, err) + } + fmt.Println("----", candidates) + + if len(candidates) != 1 { + sort.Strings(candidates) + return []string{}, "", fmt.Errorf("unable to find single JAR in %s, candidates: %s", j.JarFilePattern, candidates) + } + + jarFileName := filepath.Base(candidates[0]) + startClass := strings.TrimSuffix(jarFileName, ".jar") + + inputArgs = append(inputArgs, "-jar", candidates[0]) + + return inputArgs, startClass, nil +} diff --git a/native/arguments_test.go b/native/arguments_test.go new file mode 100644 index 0000000..4426177 --- /dev/null +++ b/native/arguments_test.go @@ -0,0 +1,241 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package native_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/buildpacks/libcnb" + "github.com/magiconair/properties" + . "github.com/onsi/gomega" + "github.com/paketo-buildpacks/libpak" + "github.com/paketo-buildpacks/native-image/native" + "github.com/sclevine/spec" +) + +func testArguments(t *testing.T, context spec.G, it spec.S) { + var ( + Expect = NewWithT(t).Expect + + ctx libcnb.BuildContext + props *properties.Properties + ) + + it.Before(func() { + var err error + + ctx.Application.Path, err = ioutil.TempDir("", "native-image-application") + Expect(err).NotTo(HaveOccurred()) + + ctx.Layers.Path, err = ioutil.TempDir("", "native-image-layers") + Expect(err).NotTo(HaveOccurred()) + }) + + it.After(func() { + Expect(os.RemoveAll(ctx.Application.Path)).To(Succeed()) + Expect(os.RemoveAll(ctx.Layers.Path)).To(Succeed()) + }) + + context("baseline arguments", func() { + it("sets default arguments", func() { + args, startClass, err := native.BaselineArguments{}.Configure(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(0)) + }) + + it("ignores input arguments", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.BaselineArguments{}.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(0)) + }) + + it("sets defaults for tiny stack", func() { + args, startClass, err := native.BaselineArguments{StackID: libpak.TinyStackID}.Configure(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(1)) + Expect(args).To(Equal([]string{"-H:+StaticExecutableWithDynamicLibC"})) + }) + }) + + context("user arguments", func() { + it("has none", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.UserArguments{}.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(3)) + Expect(args).To(Equal([]string{"one", "two", "three"})) + }) + + it("has some and appends to end", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.UserArguments{ + Arguments: "more stuff", + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(5)) + Expect(args).To(Equal([]string{"one", "two", "three", "more", "stuff"})) + }) + + it("works with quotes", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.UserArguments{ + Arguments: `"more stuff"`, + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(4)) + Expect(args).To(Equal([]string{"one", "two", "three", "more stuff"})) + }) + + it("allows a user argument to override an input argument", func() { + inputArgs := []string{"one=input", "two", "three"} + args, startClass, err := native.UserArguments{ + Arguments: `one=output`, + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(3)) + Expect(args).To(Equal([]string{"two", "three", "one=output"})) + }) + }) + + context("exploded jar arguments", func() { + var layer libcnb.Layer + + it.Before(func() { + var err error + + layer, err = ctx.Layers.Layer("test-layer") + Expect(err).NotTo(HaveOccurred()) + + props = properties.NewProperties() + _, _, err = props.Set("Start-Class", "test-start-class") + Expect(err).NotTo(HaveOccurred()) + _, _, err = props.Set("Class-Path", "manifest-class-path") + Expect(err).NotTo(HaveOccurred()) + }) + + it("adds arguments, no CLASSPATH set", func() { + inputArgs := []string{"stuff"} + args, startClass, err := native.ExplodedJarArguments{ + ApplicationPath: ctx.Application.Path, + LayerPath: layer.Path, + Manifest: props, + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("test-start-class")) + Expect(args).To(HaveLen(5)) + Expect(args).To(Equal([]string{ + "stuff", + fmt.Sprintf("-H:Name=%s/test-start-class", layer.Path), + "-cp", + fmt.Sprintf("%s:%s", ctx.Application.Path, "manifest-class-path"), + "test-start-class"})) + }) + + it("fails to find start or main class", func() { + inputArgs := []string{"stuff"} + _, _, err := native.ExplodedJarArguments{ + ApplicationPath: ctx.Application.Path, + LayerPath: layer.Path, + Manifest: properties.NewProperties(), + }.Configure(inputArgs) + Expect(err).To(MatchError("unable to read Start-Class or Main-Class from MANIFEST.MF")) + }) + + context("CLASSPATH is set", func() { + it.Before(func() { + Expect(os.Setenv("CLASSPATH", "some-classpath")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("CLASSPATH")).To(Succeed()) + }) + + it("adds arguments", func() { + inputArgs := []string{"stuff"} + args, startClass, err := native.ExplodedJarArguments{ + ApplicationPath: ctx.Application.Path, + LayerPath: layer.Path, + Manifest: props, + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("test-start-class")) + Expect(args).To(HaveLen(5)) + Expect(args).To(Equal([]string{ + "stuff", + fmt.Sprintf("-H:Name=%s/test-start-class", layer.Path), + "-cp", + "some-classpath", + "test-start-class"})) + }) + }) + }) + + context("jar file", func() { + it.Before(func() { + Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "target"), 0755)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "found.jar"), []byte{}, 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "a.two"), []byte{}, 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "b.two"), []byte{}, 0644)).To(Succeed()) + }) + + it("adds arguments", func() { + inputArgs := []string{"stuff"} + args, startClass, err := native.JarArguments{ + ApplicationPath: ctx.Application.Path, + JarFilePattern: "target/*.jar", + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("found")) + Expect(args).To(HaveLen(3)) + Expect(args).To(Equal([]string{ + "stuff", + "-jar", + filepath.Join(ctx.Application.Path, "target", "found.jar"), + })) + }) + + it("pattern doesn't match", func() { + inputArgs := []string{"stuff"} + _, _, err := native.JarArguments{ + ApplicationPath: ctx.Application.Path, + JarFilePattern: "target/*.junk", + }.Configure(inputArgs) + Expect(err).To(MatchError("unable to find single JAR in target/*.junk, candidates: []")) + }) + + it("pattern matches multiple", func() { + inputArgs := []string{"stuff"} + _, _, err := native.JarArguments{ + ApplicationPath: ctx.Application.Path, + JarFilePattern: "target/*.two", + }.Configure(inputArgs) + Expect(err).To(MatchError(MatchRegexp(`unable to find single JAR in target/\*\.two, candidates: \[.*/target/a\.two .*/target/b\.two\]`))) + }) + }) +} diff --git a/native/build.go b/native/build.go index 2fc2242..e7f9d6e 100644 --- a/native/build.go +++ b/native/build.go @@ -17,6 +17,7 @@ package native import ( + "errors" "fmt" "path/filepath" @@ -57,8 +58,9 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { if err != nil { return libcnb.BuildResult{}, fmt.Errorf("unable to create configuration resolver\n%w", err) } + if _, ok := cr.Resolve(DeprecatedConfigNativeImage); ok { - b.warn(fmt.Sprintf("$%s has been deprecated. Please use $%s instead.", + warn(b.Logger, fmt.Sprintf("$%s has been deprecated. Please use $%s instead.", DeprecatedConfigNativeImage, ConfigNativeImage, )) @@ -67,31 +69,33 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { args, ok := cr.Resolve(ConfigNativeImageArgs) if !ok { if args, ok = cr.Resolve(DeprecatedConfigNativeImageArgs); ok { - b.warn(fmt.Sprintf("$%s has been deprecated. Please use $%s instead.", + warn(b.Logger, fmt.Sprintf("$%s has been deprecated. Please use $%s instead.", DeprecatedConfigNativeImageArgs, ConfigNativeImageArgs, )) } } + jarFilePattern, _ := cr.Resolve("BP_NATIVE_IMAGE_BUILT_ARTIFACT") + compressor, ok := cr.Resolve(BinaryCompressionMethod) if !ok { compressor = CompressorNone } else if ok { if compressor != CompressorUpx && compressor != CompressorGzexe && compressor != CompressorNone { - b.warn(fmt.Sprintf("Requested compression method [%s] is unknown, no compression will be performed", compressor)) + warn(b.Logger, fmt.Sprintf("Requested compression method [%s] is unknown, no compression will be performed", compressor)) compressor = CompressorNone } } - n, err := NewNativeImage(context.Application.Path, args, compressor, manifest, context.StackID) + n, err := NewNativeImage(context.Application.Path, args, compressor, jarFilePattern, manifest, context.StackID) if err != nil { return libcnb.BuildResult{}, fmt.Errorf("unable to create native image layer\n%w", err) } n.Logger = b.Logger result.Layers = append(result.Layers, n) - startClass, err := findStartOrMainClass(manifest) + startClass, err := findStartOrMainClass(manifest, context.Application.Path, jarFilePattern) if err != nil { return libcnb.BuildResult{}, fmt.Errorf("unable to find required manifest property\n%w", err) } @@ -114,21 +118,32 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { } // todo: move warn method to the logger -func (b Build) warn(msg string) { - b.Logger.Headerf( +func warn(l bard.Logger, msg string) { + l.Headerf( "\n%s %s\n\n", color.New(color.FgYellow, color.Bold).Sprintf("Warning:"), msg, ) } -func findStartOrMainClass(manifest *properties.Properties) (string, error) { - startClass, ok := manifest.Get("Start-Class") - if !ok { - startClass, ok = manifest.Get("Main-Class") - if !ok { - return "", fmt.Errorf("unable to read Start-Class or Main-Class from MANIFEST.MF") - } +func findStartOrMainClass(manifest *properties.Properties, appPath, jarFilePattern string) (string, error) { + _, startClass, err := ExplodedJarArguments{Manifest: manifest}.Configure(nil) + if err != nil && !errors.Is(err, NoStartOrMainClass{}) { + return "", fmt.Errorf("unable to find startClass\n%w", err) + } + + if startClass != "" { + return startClass, nil } - return startClass, nil + + _, startClass, err = JarArguments{JarFilePattern: jarFilePattern, ApplicationPath: appPath}.Configure(nil) + if err != nil { + return "", fmt.Errorf("unable to find startClass from JAR\n%w", err) + } + + if startClass != "" { + return startClass, nil + } + + return "", fmt.Errorf("unable to find a suitable startClass") } diff --git a/native/build_test.go b/native/build_test.go index 47472da..e9d2e1a 100644 --- a/native/build_test.go +++ b/native/build_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/paketo-buildpacks/libpak/sbom/mocks" + "github.com/paketo-buildpacks/libpak/sherpa" "github.com/buildpacks/libcnb" . "github.com/onsi/gomega" @@ -155,7 +156,7 @@ Start-Class: test-start-class result, err := build.Build(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(result.Layers[0].(native.NativeImage).Arguments).To(Equal([]string{"test-native-image-argument"})) + Expect(result.Layers[0].(native.NativeImage).Arguments).To(Equal("test-native-image-argument")) }) }) @@ -180,9 +181,37 @@ Start-Class: test-start-class result, err := build.Build(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(result.Layers[0].(native.NativeImage).Arguments).To(Equal([]string{"test-native-image-argument"})) + Expect(result.Layers[0].(native.NativeImage).Arguments).To(Equal("test-native-image-argument")) Expect(out.String()).To(ContainSubstring("$BP_BOOT_NATIVE_IMAGE_BUILD_ARGUMENTS has been deprecated. Please use $BP_NATIVE_IMAGE_BUILD_ARGUMENTS instead.")) }) }) + + context("BP_NATIVE_IMAGE_BUILT_ARTIFACT", func() { + it.Before(func() { + Expect(os.Setenv("BP_NATIVE_IMAGE_BUILT_ARTIFACT", "target/*.jar")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_NATIVE_IMAGE_BUILT_ARTIFACT")).To(Succeed()) + }) + + it("contributes native image layer to build against a JAR", func() { + Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "target"), 0755)).To(Succeed()) + + fp, err := os.Open("testdata/test-fixture.jar") + Expect(err).ToNot(HaveOccurred()) + Expect(sherpa.CopyFile(fp, filepath.Join(ctx.Application.Path, "target", "test-fixture.jar"))).To(Succeed()) + + result, err := build.Build(ctx) + Expect(err).NotTo(HaveOccurred()) + + Expect(result.Layers[0].(native.NativeImage).JarFilePattern).To(Equal("target/*.jar")) + Expect(result.Processes).To(ContainElements( + libcnb.Process{Type: "native-image", Command: filepath.Join(ctx.Application.Path, "test-fixture"), Direct: true}, + libcnb.Process{Type: "task", Command: filepath.Join(ctx.Application.Path, "test-fixture"), Direct: true}, + libcnb.Process{Type: "web", Command: filepath.Join(ctx.Application.Path, "test-fixture"), Direct: true, Default: true}, + )) + }) + }) } diff --git a/native/init_test.go b/native/init_test.go index 73e047f..01271c9 100644 --- a/native/init_test.go +++ b/native/init_test.go @@ -27,6 +27,7 @@ func TestUnit(t *testing.T) { suite := spec.New("native", spec.Report(report.Terminal{})) suite("Build", testBuild) suite("Detect", testDetect) + suite("Arguments", testArguments) suite("NativeImage", testNativeImage) suite.Run(t) } diff --git a/native/native_image.go b/native/native_image.go index 071bcfd..fdf5198 100644 --- a/native/native_image.go +++ b/native/native_image.go @@ -26,7 +26,6 @@ import ( "github.com/buildpacks/libcnb" "github.com/magiconair/properties" - "github.com/mattn/go-shellwords" "github.com/paketo-buildpacks/libpak" "github.com/paketo-buildpacks/libpak/bard" "github.com/paketo-buildpacks/libpak/effect" @@ -35,26 +34,21 @@ import ( type NativeImage struct { ApplicationPath string - Arguments []string + Arguments string Executor effect.Executor + JarFilePattern string Logger bard.Logger Manifest *properties.Properties StackID string Compressor string } -func NewNativeImage(applicationPath string, arguments string, compressor string, manifest *properties.Properties, stackID string) (NativeImage, error) { - var err error - - args, err := shellwords.Parse(arguments) - if err != nil { - return NativeImage{}, fmt.Errorf("unable to parse arguments from %s\n%w", arguments, err) - } - +func NewNativeImage(applicationPath string, arguments string, compressor string, jarFilePattern string, manifest *properties.Properties, stackID string) (NativeImage, error) { return NativeImage{ ApplicationPath: applicationPath, - Arguments: args, + Arguments: arguments, Executor: effect.NewExecutor(), + JarFilePattern: jarFilePattern, Manifest: manifest, StackID: stackID, Compressor: compressor, @@ -62,35 +56,14 @@ func NewNativeImage(applicationPath string, arguments string, compressor string, } func (n NativeImage) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { - startClass, err := findStartOrMainClass(n.Manifest) + files, err := sherpa.NewFileListing(n.ApplicationPath) if err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to find required manifest property\n%w", err) - } - - arguments := n.Arguments - - if n.StackID == libpak.TinyStackID { - arguments = append(arguments, "-H:+StaticExecutableWithDynamicLibC") - } - - cp := os.Getenv("CLASSPATH") - if cp == "" { - // CLASSPATH should have been done by upstream buildpacks, but just in case - cp = n.ApplicationPath - if v, ok := n.Manifest.Get("Class-Path"); ok { - cp = strings.Join([]string{cp, v}, string(filepath.ListSeparator)) - } + return libcnb.Layer{}, fmt.Errorf("unable to create file listing for %s\n%w", n.ApplicationPath, err) } - arguments = append(arguments, - fmt.Sprintf("-H:Name=%s", filepath.Join(layer.Path, startClass)), - "-cp", cp, - startClass, - ) - - files, err := sherpa.NewFileListing(n.ApplicationPath) + arguments, startClass, err := n.ProcessArguments(layer) if err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to create file listing for %s\n%w", n.ApplicationPath, err) + return libcnb.Layer{}, fmt.Errorf("unable to process arguments\n%w", err) } contributor := libpak.NewLayerContributor("Native Image", map[string]interface{}{ @@ -191,6 +164,46 @@ func (n NativeImage) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { return layer, nil } +func (n NativeImage) ProcessArguments(layer libcnb.Layer) ([]string, string, error) { + var arguments []string + var startClass string + var err error + + arguments, _, err = BaselineArguments{StackID: n.StackID}.Configure(nil) + if err != nil { + return []string{}, "", fmt.Errorf("unable to set baseline argument\n%w", err) + } + + arguments, _, err = UserArguments{Arguments: n.Arguments}.Configure(arguments) + if err != nil { + return []string{}, "", fmt.Errorf("unable to create baseline argument\n%w", err) + } + + _, err = os.Stat(filepath.Join(n.ApplicationPath, "META-INF", "MANIFEST.MF")) + if err != nil && !os.IsNotExist(err) { + return []string{}, "", fmt.Errorf("unable to check for manifest\n%w", err) + } else if err != nil && os.IsNotExist(err) { + arguments, startClass, err = JarArguments{ + ApplicationPath: n.ApplicationPath, + JarFilePattern: n.JarFilePattern, + }.Configure(arguments) + if err != nil { + return []string{}, "", fmt.Errorf("unable to append jar arguments\n%w", err) + } + } else { + arguments, startClass, err = ExplodedJarArguments{ + ApplicationPath: n.ApplicationPath, + LayerPath: layer.Path, + Manifest: n.Manifest, + }.Configure(arguments) + if err != nil { + return []string{}, "", fmt.Errorf("unable to append exploded-jar directory arguments\n%w", err) + } + } + + return arguments, startClass, err +} + func (NativeImage) Name() string { return "native-image" } diff --git a/native/native_image_test.go b/native/native_image_test.go index 4f8cad1..bfc70b2 100644 --- a/native/native_image_test.go +++ b/native/native_image_test.go @@ -69,8 +69,10 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "fixture-marker"), []byte{}, 0644)).To(Succeed()) Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "BOOT-INF"), 0755)).To(Succeed()) + Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "META-INF"), 0755)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "META-INF", "MANIFEST.MF"), []byte{}, 0644)).To(Succeed()) - nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", "none", props, ctx.StackID) + nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", "none", "", props, ctx.StackID) nativeImage.Logger = bard.NewLogger(io.Discard) Expect(err).NotTo(HaveOccurred()) nativeImage.Executor = executor @@ -84,7 +86,8 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { }).Return(nil) executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { - return e.Command == "native-image" && e.Args[0] == "test-argument-1" + return e.Command == "native-image" && + (e.Args[0] == "test-argument-1" || (e.Args[0] == "-H:+StaticExecutableWithDynamicLibC" && e.Args[1] == "test-argument-1")) })).Run(func(args mock.Arguments) { exec := args.Get(0).(effect.Execution) lastArg := exec.Args[len(exec.Args)-1] @@ -253,9 +256,9 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { execution := executor.Calls[1].Arguments[0].(effect.Execution) Expect(execution.Command).To(Equal("native-image")) Expect(execution.Args).To(Equal([]string{ + "-H:+StaticExecutableWithDynamicLibC", "test-argument-1", "test-argument-2", - "-H:+StaticExecutableWithDynamicLibC", fmt.Sprintf("-H:Name=%s", filepath.Join(layer.Path, "test-start-class")), "-cp", strings.Join([]string{ diff --git a/native/testdata/test-fixture.jar b/native/testdata/test-fixture.jar new file mode 100644 index 0000000000000000000000000000000000000000..289ee912f9470b52d1d0749fb86fe1f5edc0f312 GIT binary patch literal 330 zcmWIWW@Zs#;Nak3NU5<2VL$?$3@i-3t|5-Po_=on|4uP5Ff#;rvvYt{FhP|C;M6Pv zQ~}rQ>*(j{<{BKL=j-;__snS@Z(Y5MyxzK6=gyqp9At3C_`%a6JuhD!Pv48Bt5~=g zT)*)6ikQp^DlTgJeA&ySa%aU(u+3F>wG}OZ=2`-AfHxzP2m`9aVXgu>92LM_0`dmB hR%ByA&PD)RAQP??$rAzItZX1vOh8x*q_=}O3;?&zJh=b> literal 0 HcmV?d00001 From e7c0c928124abad740ee7c28a148f8a1f5e47423 Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Sat, 29 Jan 2022 15:44:59 -0500 Subject: [PATCH 2/2] Adds support for an arguments file Previously, you could set arguments through the `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` env variable, but this is a static set of arguments.. You may now set `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS_FILE` and point to a file that contains arguments, which is handy if your build system generates a file containing arguments to pass to `native-image`. Arguments are now applied in this order: 1. Baseline arguments set by the buildpack 2. If set, arguments from the file `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS_FILE` 3. If any, arguments from `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` 4. Either arguments required to build from a JAR file or arguments required to build from an exploded JAR directory Arguments from later steps override arguments from lower steps. Signed-off-by: Daniel Mikusa --- README.md | 13 ++++---- buildpack.toml | 5 +++ native/arguments.go | 52 ++++++++++++++++++++++++++++-- native/arguments_test.go | 64 +++++++++++++++++++++++++++++++++++++ native/build.go | 3 +- native/native_image.go | 15 +++++++-- native/native_image_test.go | 29 ++++++++++++++++- 7 files changed, 168 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6429c2c..7b47ea6 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,13 @@ The buildpack will do the following: ## Configuration -| Environment Variable | Description | -| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. | -| `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` | Arguments to pass to directly to the `native-image` command. These arguments must be valid and correctly formed or the `native-image` command will fail. | -| `$BP_BINARY_COMPRESSION_METHOD` | Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe` | -| `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` | Configure the built application artifact explicitly. This is required if building a native image from a JAR file | +| Environment Variable | Description | +| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. | +| `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` | Arguments to pass to directly to the `native-image` command. These arguments must be valid and correctly formed or the `native-image` command will fail. | +| `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS_FILE` | A file containing arguments to pass to directly to the `native-image` command. The file must exist and the contents must contain a single line of arguments which must be valid and correctly formed or the `native-image` command will fail. | +| `$BP_BINARY_COMPRESSION_METHOD` | Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe` | +| `$BP_NATIVE_IMAGE_BUILT_ARTIFACT` | Configure the built application artifact explicitly. This is required if building a native image from a JAR file | ### Compression Caveats diff --git a/buildpack.toml b/buildpack.toml index 1bf51b3..3506c52 100644 --- a/buildpack.toml +++ b/buildpack.toml @@ -46,6 +46,11 @@ name = "BP_NATIVE_IMAGE_BUILD_ARGUMENTS" description = "arguments to pass to the native-image command" build = true +[[metadata.configurations]] +name = "BP_NATIVE_IMAGE_BUILD_ARGUMENTS_FILE" +description = "a file with arguments to pass to the native-image command" +build = true + [[metadata.configurations]] name = "BP_BINARY_COMPRESSION_METHOD" description = "Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe`" diff --git a/native/arguments.go b/native/arguments.go index 3536389..dda2836 100644 --- a/native/arguments.go +++ b/native/arguments.go @@ -18,6 +18,7 @@ package native import ( "fmt" + "io/ioutil" "os" "path/filepath" "sort" @@ -73,6 +74,36 @@ func (u UserArguments) Configure(inputArgs []string) ([]string, string, error) { return outputArgs, "", nil } +// UserFileArguments augments the existing arguments with those provided by the end user through a file +type UserFileArguments struct { + ArgumentsFile string +} + +// Configure returns the inputArgs plus the additional arguments specified by the end user through the file, preference given to user arguments +func (u UserFileArguments) Configure(inputArgs []string) ([]string, string, error) { + rawArgs, err := ioutil.ReadFile(u.ArgumentsFile) + if err != nil { + return []string{}, "", fmt.Errorf("read arguments from %s\n%w", u.ArgumentsFile, err) + } + + parsedArgs, err := shellwords.Parse(string(rawArgs)) + if err != nil { + return []string{}, "", fmt.Errorf("unable to parse arguments from %s\n%w", string(rawArgs), err) + } + + var outputArgs []string + + for _, inputArg := range inputArgs { + if !containsArg(inputArg, parsedArgs) { + outputArgs = append(outputArgs, inputArg) + } + } + + outputArgs = append(outputArgs, parsedArgs...) + + return outputArgs, "", nil +} + // containsArg checks if needle is found in haystack // // needle and haystack entries are processed as key=val strings where only the key must match @@ -139,12 +170,10 @@ type JarArguments struct { func (j JarArguments) Configure(inputArgs []string) ([]string, string, error) { file := filepath.Join(j.ApplicationPath, j.JarFilePattern) - fmt.Println("----", file) candidates, err := filepath.Glob(file) if err != nil { return []string{}, "", fmt.Errorf("unable to find JAR with %s\n%w", j.JarFilePattern, err) } - fmt.Println("----", candidates) if len(candidates) != 1 { sort.Strings(candidates) @@ -154,6 +183,25 @@ func (j JarArguments) Configure(inputArgs []string) ([]string, string, error) { jarFileName := filepath.Base(candidates[0]) startClass := strings.TrimSuffix(jarFileName, ".jar") + if containsArg("-jar", inputArgs) { + var tmpArgs []string + var skip bool + for _, inputArg := range inputArgs { + if skip { + skip = false + break + } + + if inputArg == "-jar" { + skip = true + break + } + + tmpArgs = append(tmpArgs, inputArg) + } + inputArgs = tmpArgs + } + inputArgs = append(inputArgs, "-jar", candidates[0]) return inputArgs, startClass, nil diff --git a/native/arguments_test.go b/native/arguments_test.go index 4426177..7596957 100644 --- a/native/arguments_test.go +++ b/native/arguments_test.go @@ -123,6 +123,54 @@ func testArguments(t *testing.T, context spec.G, it spec.S) { }) }) + context("user arguments from file", func() { + it.Before(func() { + Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "target"), 0755)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "more-stuff.txt"), []byte("more stuff"), 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "more-stuff-quotes.txt"), []byte(`"more stuff"`), 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "target", "override.txt"), []byte(`one=output`), 0644)).To(Succeed()) + }) + + it("has none", func() { + inputArgs := []string{"one", "two", "three"} + _, _, err := native.UserFileArguments{}.Configure(inputArgs) + Expect(err).To(MatchError(os.ErrNotExist)) + }) + + it("has some and appends to end", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.UserFileArguments{ + ArgumentsFile: filepath.Join(ctx.Application.Path, "target/more-stuff.txt"), + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(5)) + Expect(args).To(Equal([]string{"one", "two", "three", "more", "stuff"})) + }) + + it("works with quotes", func() { + inputArgs := []string{"one", "two", "three"} + args, startClass, err := native.UserFileArguments{ + ArgumentsFile: filepath.Join(ctx.Application.Path, "target/more-stuff-quotes.txt"), + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(4)) + Expect(args).To(Equal([]string{"one", "two", "three", "more stuff"})) + }) + + it("allows a user argument to override an input argument", func() { + inputArgs := []string{"one=input", "two", "three"} + args, startClass, err := native.UserFileArguments{ + ArgumentsFile: filepath.Join(ctx.Application.Path, "target/override.txt"), + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("")) + Expect(args).To(HaveLen(3)) + Expect(args).To(Equal([]string{"two", "three", "one=output"})) + }) + }) + context("exploded jar arguments", func() { var layer libcnb.Layer @@ -220,6 +268,22 @@ func testArguments(t *testing.T, context spec.G, it spec.S) { })) }) + it("overrides -jar arguments", func() { + inputArgs := []string{"stuff", "-jar", "no-where"} + args, startClass, err := native.JarArguments{ + ApplicationPath: ctx.Application.Path, + JarFilePattern: "target/*.jar", + }.Configure(inputArgs) + Expect(err).ToNot(HaveOccurred()) + Expect(startClass).To(Equal("found")) + Expect(args).To(HaveLen(3)) + Expect(args).To(Equal([]string{ + "stuff", + "-jar", + filepath.Join(ctx.Application.Path, "target", "found.jar"), + })) + }) + it("pattern doesn't match", func() { inputArgs := []string{"stuff"} _, _, err := native.JarArguments{ diff --git a/native/build.go b/native/build.go index e7f9d6e..14eb93f 100644 --- a/native/build.go +++ b/native/build.go @@ -77,6 +77,7 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { } jarFilePattern, _ := cr.Resolve("BP_NATIVE_IMAGE_BUILT_ARTIFACT") + argsFile, _ := cr.Resolve("BP_NATIVE_IMAGE_BUILD_ARGUMENTS_FILE") compressor, ok := cr.Resolve(BinaryCompressionMethod) if !ok { @@ -88,7 +89,7 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { } } - n, err := NewNativeImage(context.Application.Path, args, compressor, jarFilePattern, manifest, context.StackID) + n, err := NewNativeImage(context.Application.Path, args, argsFile, compressor, jarFilePattern, manifest, context.StackID) if err != nil { return libcnb.BuildResult{}, fmt.Errorf("unable to create native image layer\n%w", err) } diff --git a/native/native_image.go b/native/native_image.go index fdf5198..faeee4c 100644 --- a/native/native_image.go +++ b/native/native_image.go @@ -35,6 +35,7 @@ import ( type NativeImage struct { ApplicationPath string Arguments string + ArgumentsFile string Executor effect.Executor JarFilePattern string Logger bard.Logger @@ -43,10 +44,11 @@ type NativeImage struct { Compressor string } -func NewNativeImage(applicationPath string, arguments string, compressor string, jarFilePattern string, manifest *properties.Properties, stackID string) (NativeImage, error) { +func NewNativeImage(applicationPath string, arguments string, argumentsFile string, compressor string, jarFilePattern string, manifest *properties.Properties, stackID string) (NativeImage, error) { return NativeImage{ ApplicationPath: applicationPath, Arguments: arguments, + ArgumentsFile: argumentsFile, Executor: effect.NewExecutor(), JarFilePattern: jarFilePattern, Manifest: manifest, @@ -171,12 +173,19 @@ func (n NativeImage) ProcessArguments(layer libcnb.Layer) ([]string, string, err arguments, _, err = BaselineArguments{StackID: n.StackID}.Configure(nil) if err != nil { - return []string{}, "", fmt.Errorf("unable to set baseline argument\n%w", err) + return []string{}, "", fmt.Errorf("unable to set baseline arguments\n%w", err) + } + + if n.ArgumentsFile != "" { + arguments, _, err = UserFileArguments{ArgumentsFile: n.ArgumentsFile}.Configure(arguments) + if err != nil { + return []string{}, "", fmt.Errorf("unable to create user file arguments\n%w", err) + } } arguments, _, err = UserArguments{Arguments: n.Arguments}.Configure(arguments) if err != nil { - return []string{}, "", fmt.Errorf("unable to create baseline argument\n%w", err) + return []string{}, "", fmt.Errorf("unable to create user arguments\n%w", err) } _, err = os.Stat(filepath.Join(n.ApplicationPath, "META-INF", "MANIFEST.MF")) diff --git a/native/native_image_test.go b/native/native_image_test.go index bfc70b2..12b5e66 100644 --- a/native/native_image_test.go +++ b/native/native_image_test.go @@ -72,7 +72,7 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "META-INF"), 0755)).To(Succeed()) Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "META-INF", "MANIFEST.MF"), []byte{}, 0644)).To(Succeed()) - nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", "none", "", props, ctx.StackID) + nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", "", "none", "", props, ctx.StackID) nativeImage.Logger = bard.NewLogger(io.Discard) Expect(err).NotTo(HaveOccurred()) nativeImage.Executor = executor @@ -145,6 +145,33 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { "test-start-class", })) }) + + it("contributes native image with Class-Path from manifest and args from a file", func() { + argsFile := filepath.Join(ctx.Application.Path, "target", "args.txt") + Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "target"), 0755)).To(Succeed()) + Expect(ioutil.WriteFile(argsFile, []byte(`test-argument-1 test-argument-2`), 0644)).To(Succeed()) + + nativeImage, err := native.NewNativeImage(ctx.Application.Path, "", argsFile, "none", "", props, ctx.StackID) + nativeImage.Logger = bard.NewLogger(io.Discard) + Expect(err).NotTo(HaveOccurred()) + nativeImage.Executor = executor + + _, err = nativeImage.Contribute(layer) + Expect(err).NotTo(HaveOccurred()) + + execution := executor.Calls[1].Arguments[0].(effect.Execution) + Expect(execution.Args).To(Equal([]string{ + "test-argument-1", + "test-argument-2", + fmt.Sprintf("-H:Name=%s", filepath.Join(layer.Path, "test-start-class")), + "-cp", + strings.Join([]string{ + ctx.Application.Path, + "manifest-class-path", + }, ":"), + "test-start-class", + })) + }) }) context("Not a Spring Boot app", func() {