From 5b4096c6021030d469bf4119ba2e1e907927a590 Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Fri, 9 Jul 2021 10:13:39 -0400 Subject: [PATCH] Implements Paketo Utilities RFC 0001 to add compression support for native image executables --- README.md | 19 ++- native/build.go | 15 ++- native/detect.go | 35 ++++-- native/detect_test.go | 240 ++++++++++++++++++++++++++++++++++++ native/native_image.go | 37 +++++- native/native_image_test.go | 64 +++++++++- 6 files changed, 390 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 55b50c8..40b1174 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # `gcr.io/paketo-buildpacks/native-image` + The Paketo Native Image Buildpack is a Cloud Native Buildpack that uses the [GraalVM Native Image builder][native-image] (`native-image`) to compile a standalone executable from an executable JAR. Most users should not use this component buildpack directly and should instead use the [Paketo Java Native Image][bp/java-native-image], which provides the full set of buildpacks required to build a native image application. ## Behavior + This buildpack will participate if one the following conditions are met: * `$BP_NATIVE_IMAGE` is set. @@ -12,13 +14,22 @@ This buildpack will participate if one the following conditions are met: The buildpack will do the following: * Requests that the Native Image builder be installed by requiring `native-image-builder` in the build plan. +* If `$BP_BINARY_COMPRESSION_METHOD` is set to `upx`, requests that UPX be installed by requiring `upx` in the buildplan. * Uses `native-image` a to build a GraalVM native image and removes existing bytecode. +* Uses `$BP_BINARY_COMPRESSION_METHOD` if set to `upx` or `gzexe` to compress the native image. ## Configuration -| Environment Variable | Description -| -------------------- | ----------- -| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. -| `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` | Arguments to pass to the `native-image` command. +| Environment Variable | Description | +| ---------------------------------- | --------------------------------------------------------------------------------------------- | +| `$BP_NATIVE_IMAGE` | Whether to build a native image from the application. Defaults to false. | +| `$BP_NATIVE_IMAGE_BUILD_ARGUMENTS` | Arguments to pass to the `native-image` command. | +| `$BP_BINARY_COMPRESSION_METHOD` | Compression mechanism used to reduce binary size. Options: `none` (default), `upx` or `gzexe` | + +### Compression Caveats + +1. Using `gzexe` if you intend to run your application on the Paketo Tiny image is not currently supported. The `gzexe` utility will compress your executable into what is a shell script, which executes and extracts the actual binary to a temp location. This process requires `/bin/sh` and that is not in the Tiny image. If you try using `gzexe` with the Tiny stack, it'll build OK but fail to run saying a file is missing. + +2. Using `upx` will create a compressed executable that fails to run on M1 Macs. There is at the time of writing a bug in the emulation layer used by Docker on M1 Macs that is triggered when you try to run amd64 executable that has been compressed using `upx`. This is a known issue and will hopefully be patched in a future release. ## License This buildpack is released under version 2.0 of the [Apache License][a]. diff --git a/native/build.go b/native/build.go index 77c0a96..8f87378 100644 --- a/native/build.go +++ b/native/build.go @@ -30,6 +30,9 @@ import ( const ( ConfigNativeImageArgs = "BP_NATIVE_IMAGE_BUILD_ARGUMENTS" DeprecatedConfigNativeImageArgs = "BP_BOOT_NATIVE_IMAGE_BUILD_ARGUMENTS" + CompressorUpx = "upx" + CompressorGzexe = "gzexe" + CompressorNone = "none" ) type Build struct { @@ -66,7 +69,17 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) { } } - n, err := NewNativeImage(context.Application.Path, args, manifest, context.StackID) + compressor, ok := cr.Resolve(BinaryCompressionMethod) + if !ok { + compressor = CompressorNone + } else if ok { + if compressor != CompressorUpx && compressor != CompressorGzexe && compressor != CompressorNone { + b.warn(fmt.Sprintf("Requested compression method [%s] is unknown, no compression will be performed", compressor)) + compressor = CompressorNone + } + } + + n, err := NewNativeImage(context.Application.Path, args, compressor, manifest, context.StackID) if err != nil { return libcnb.BuildResult{}, fmt.Errorf("unable to create native image layer\n%w", err) } diff --git a/native/detect.go b/native/detect.go index 747897b..dbf1aeb 100644 --- a/native/detect.go +++ b/native/detect.go @@ -27,11 +27,13 @@ import ( const ( ConfigNativeImage = "BP_NATIVE_IMAGE" DeprecatedConfigNativeImage = "BP_BOOT_NATIVE_IMAGE" + BinaryCompressionMethod = "BP_BINARY_COMPRESSION_METHOD" - PlatEntryNativeImage = "native-image-application" + PlanEntryNativeImage = "native-image-application" PlanEntryNativeImageBuilder = "native-image-builder" PlanEntryJVMApplication = "jvm-application" PlanEntrySpringBoot = "spring-boot" + PlanEntryUpx = "upx" ) type Detect struct{} @@ -48,7 +50,7 @@ func (d Detect) Detect(context libcnb.DetectContext) (libcnb.DetectResult, error { Provides: []libcnb.BuildPlanProvide{ { - Name: PlatEntryNativeImage, + Name: PlanEntryNativeImage, }, }, Requires: []libcnb.BuildPlanRequire{ @@ -68,7 +70,7 @@ func (d Detect) Detect(context libcnb.DetectContext) (libcnb.DetectResult, error { Provides: []libcnb.BuildPlanProvide{ { - Name: PlatEntryNativeImage, + Name: PlanEntryNativeImage, }, }, Requires: []libcnb.BuildPlanRequire{ @@ -86,20 +88,33 @@ func (d Detect) Detect(context libcnb.DetectContext) (libcnb.DetectResult, error if ok, err := d.nativeImageEnabled(cr); err != nil { return libcnb.DetectResult{}, err - } else if !ok { - // still participates if a downstream buildpack requires native-image-applications - return result, nil + } else if ok { + for i := range result.Plans { + result.Plans[i].Requires = append(result.Plans[i].Requires, libcnb.BuildPlanRequire{ + Name: PlanEntryNativeImage, + }) + } } - for i := range result.Plans { - result.Plans[i].Requires = append(result.Plans[i].Requires, libcnb.BuildPlanRequire{ - Name: PlatEntryNativeImage, - }) + if d.upxCompressionEnabled(cr) { + for i := range result.Plans { + result.Plans[i].Requires = append(result.Plans[i].Requires, libcnb.BuildPlanRequire{ + Name: PlanEntryUpx, + }) + } } + // still participates if a downstream buildpack requires native-image-applications or upx return result, nil } +func (d Detect) upxCompressionEnabled(cr libpak.ConfigurationResolver) bool { + if val, ok := cr.Resolve(BinaryCompressionMethod); ok { + return val == CompressorUpx + } + return false +} + func (d Detect) nativeImageEnabled(cr libpak.ConfigurationResolver) (bool, error) { if val, ok := cr.Resolve(ConfigNativeImage); ok { enable, err := strconv.ParseBool(val) diff --git a/native/detect_test.go b/native/detect_test.go index 667d8bd..b3f8af1 100644 --- a/native/detect_test.go +++ b/native/detect_test.go @@ -200,6 +200,246 @@ func testDetect(t *testing.T, context spec.G, it spec.S) { }) }) + context("$BP_BINARY_COMPRESSION_METHOD", func() { + it.Before(func() { + Expect(os.Setenv("BP_NATIVE_IMAGE", "true")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_NATIVE_IMAGE")).To(Succeed()) + }) + + context("upx", func() { + it.Before(func() { + Expect(os.Setenv("BP_BINARY_COMPRESSION_METHOD", "upx")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_BINARY_COMPRESSION_METHOD")).To(Succeed()) + }) + + it("requires upx", func() { + Expect(detect.Detect(ctx)).To(Equal(libcnb.DetectResult{ + Pass: true, + Plans: []libcnb.BuildPlan{ + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "spring-boot", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + { + Name: "upx", + }, + }, + }, + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + { + Name: "upx", + }, + }, + }, + }, + })) + }) + }) + + context("gzexe", func() { + it.Before(func() { + Expect(os.Setenv("BP_BINARY_COMPRESSION_METHOD", "gzexe")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_BINARY_COMPRESSION_METHOD")).To(Succeed()) + }) + + it("no additional provides or requires", func() { + Expect(detect.Detect(ctx)).To(Equal(libcnb.DetectResult{ + Pass: true, + Plans: []libcnb.BuildPlan{ + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "spring-boot", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + }, + })) + }) + }) + + context("none", func() { + it.Before(func() { + Expect(os.Setenv("BP_BINARY_COMPRESSION_METHOD", "none")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_BINARY_COMPRESSION_METHOD")).To(Succeed()) + }) + + it("no additional provides or requires", func() { + Expect(detect.Detect(ctx)).To(Equal(libcnb.DetectResult{ + Pass: true, + Plans: []libcnb.BuildPlan{ + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "spring-boot", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + }, + })) + }) + }) + + context("not a supported method", func() { + it.Before(func() { + Expect(os.Setenv("BP_BINARY_COMPRESSION_METHOD", "foo")).To(Succeed()) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_BINARY_COMPRESSION_METHOD")).To(Succeed()) + }) + + it("ignore and no additional provides or requires", func() { + Expect(detect.Detect(ctx)).To(Equal(libcnb.DetectResult{ + Pass: true, + Plans: []libcnb.BuildPlan{ + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "spring-boot", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + { + Provides: []libcnb.BuildPlanProvide{ + {Name: "native-image-application"}, + }, + Requires: []libcnb.BuildPlanRequire{ + { + Name: "native-image-builder", + }, + { + Name: "jvm-application", + Metadata: map[string]interface{}{"native-image": true}, + }, + { + Name: "native-image-application", + }, + }, + }, + }, + })) + }) + }) + }) + context("$BP_BOOT_NATIVE_IMAGE", func() { it.Before(func() { Expect(os.Setenv("BP_BOOT_NATIVE_IMAGE", "true")).To(Succeed()) diff --git a/native/native_image.go b/native/native_image.go index fbbcd40..d192540 100644 --- a/native/native_image.go +++ b/native/native_image.go @@ -40,9 +40,10 @@ type NativeImage struct { Logger bard.Logger Manifest *properties.Properties StackID string + Compressor string } -func NewNativeImage(applicationPath string, arguments string, manifest *properties.Properties, stackID string) (NativeImage, error) { +func NewNativeImage(applicationPath string, arguments string, compressor string, manifest *properties.Properties, stackID string) (NativeImage, error) { var err error args, err := shellwords.Parse(arguments) @@ -56,6 +57,7 @@ func NewNativeImage(applicationPath string, arguments string, manifest *properti Executor: effect.NewExecutor(), Manifest: manifest, StackID: stackID, + Compressor: compressor, }, nil } @@ -92,8 +94,9 @@ func (n NativeImage) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { } contributor := libpak.NewLayerContributor("Native Image", map[string]interface{}{ - "files": files, - "arguments": arguments, + "files": files, + "arguments": arguments, + "compression": n.Compressor, }, libcnb.LayerTypes{ Cache: true, }) @@ -121,6 +124,34 @@ func (n NativeImage) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { return libcnb.Layer{}, fmt.Errorf("error running build\n%w", err) } + if n.Compressor == CompressorUpx { + n.Logger.Bodyf("Executing %s to compress native image", n.Compressor) + if err := n.Executor.Execute(effect.Execution{ + Command: "upx", + Args: []string{"-q", "-9", filepath.Join(layer.Path, startClass)}, + Dir: layer.Path, + Stdout: n.Logger.InfoWriter(), + Stderr: n.Logger.InfoWriter(), + }); err != nil { + return libcnb.Layer{}, fmt.Errorf("error compressing\n%w", err) + } + } else if n.Compressor == CompressorGzexe { + n.Logger.Bodyf("Executing %s to compress native image", n.Compressor) + if err := n.Executor.Execute(effect.Execution{ + Command: "gzexe", + Args: []string{filepath.Join(layer.Path, startClass)}, + Dir: layer.Path, + Stdout: n.Logger.InfoWriter(), + Stderr: n.Logger.InfoWriter(), + }); err != nil { + return libcnb.Layer{}, fmt.Errorf("error compressing\n%w", err) + } + + if err := os.Remove(filepath.Join(layer.Path, fmt.Sprintf("%s~", startClass))); err != nil { + return libcnb.Layer{}, fmt.Errorf("error removing\n%w", err) + } + } + return layer, nil }) if err != nil { diff --git a/native/native_image_test.go b/native/native_image_test.go index 2c32ef8..8841e3a 100644 --- a/native/native_image_test.go +++ b/native/native_image_test.go @@ -68,11 +68,13 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { Expect(ioutil.WriteFile(filepath.Join(ctx.Application.Path, "fixture-marker"), []byte{}, 0644)).To(Succeed()) Expect(os.MkdirAll(filepath.Join(ctx.Application.Path, "BOOT-INF"), 0755)).To(Succeed()) - nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", props, ctx.StackID) + nativeImage, err = native.NewNativeImage(ctx.Application.Path, "test-argument-1 test-argument-2", "none", props, ctx.StackID) Expect(err).NotTo(HaveOccurred()) nativeImage.Executor = executor - executor.On("Execute", mock.Anything).Run(func(args mock.Arguments) { + executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { + return e.Command == "native-image" + })).Run(func(args mock.Arguments) { Expect(ioutil.WriteFile(filepath.Join(layer.Path, "test-start-class"), []byte{}, 0644)).To(Succeed()) }).Return(nil) @@ -129,6 +131,64 @@ func testNativeImage(t *testing.T, context spec.G, it spec.S) { }) }) + context("upx compression is used", func() { + it("contributes native image and runs compression", func() { + nativeImage.Compressor = "upx" + + executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { + return e.Command == "upx" + })).Run(func(args mock.Arguments) { + Expect(ioutil.WriteFile(filepath.Join(layer.Path, "test-start-class"), []byte("upx-compressed"), 0644)).To(Succeed()) + }).Return(nil) + + _, err := nativeImage.Contribute(layer) + Expect(err).NotTo(HaveOccurred()) + + execution := executor.Calls[1].Arguments[0].(effect.Execution) + Expect(execution.Command).To(Equal("native-image")) + + execution = executor.Calls[2].Arguments[0].(effect.Execution) + Expect(execution.Command).To(Equal("upx")) + + bin := filepath.Join(layer.Path, "test-start-class") + Expect(bin).To(BeARegularFile()) + + data, err := ioutil.ReadFile(bin) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(ContainSubstring("upx-compressed")) + }) + }) + + context("gzexe compression is used", func() { + it("contributes native image and runs compression", func() { + nativeImage.Compressor = "gzexe" + + executor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool { + return e.Command == "gzexe" + })).Run(func(args mock.Arguments) { + Expect(ioutil.WriteFile(filepath.Join(layer.Path, "test-start-class"), []byte("gzexe-compressed"), 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(layer.Path, "test-start-class~"), []byte("original"), 0644)).To(Succeed()) + }).Return(nil) + + _, err := nativeImage.Contribute(layer) + Expect(err).NotTo(HaveOccurred()) + + execution := executor.Calls[1].Arguments[0].(effect.Execution) + Expect(execution.Command).To(Equal("native-image")) + + execution = executor.Calls[2].Arguments[0].(effect.Execution) + Expect(execution.Command).To(Equal("gzexe")) + + bin := filepath.Join(layer.Path, "test-start-class") + Expect(bin).To(BeARegularFile()) + + data, err := ioutil.ReadFile(bin) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(ContainSubstring("gzexe-compressed")) + Expect(filepath.Join(layer.Path, "test-start-class~")).ToNot(BeAnExistingFile()) + }) + }) + context("tiny stack", func() { it.Before(func() { nativeImage.StackID = libpak.TinyStackID