chore: integrate binary build in windows dockerfile

Makefile

@@ -45,6 +45,9 @@ ARCH ?= amd64
 WINDOWS_OSVERSION ?= 1809
 # The output type for `docker buildx build` could either be docker (local), or registry.
 OUTPUT_TYPE ?= docker
+# LOCAL_WINDOWS_BUILD, when set to true, allows picking a local windows binary built locally.
+# This is beneficial when the windows binary requires additional security protection like binary signing.
+LOCAL_WINDOWS_BUILD ?=false
 
 BASE.windows := mcr.microsoft.com/windows/nanoserver
 
@@ -168,24 +171,26 @@ build-node-image-linux: buildx-setup ## Build node-manager image.
 		--sbom=false
 
 .PHONY: build-node-image-windows
-build-node-image-windows: buildx-setup $(BIN_DIR)/azure-cloud-node-manager.exe ## Build node-manager image for Windows.
+build-node-image-windows: buildx-setup ## Build node-manager image for Windows.
 	$(DOCKER_BUILDX) build --pull \
 		--output=type=$(OUTPUT_TYPE) \
 		--platform windows/$(ARCH) \
 		-t $(NODE_MANAGER_WINDOWS_FULL_IMAGE_PREFIX)-$(WINDOWS_OSVERSION)-$(ARCH) \
 		--build-arg OSVERSION=$(WINDOWS_OSVERSION) \
 		--build-arg ARCH=$(ARCH) \
+		--build-arg LOCAL_BUILD=$(LOCAL_WINDOWS_BUILD) \
 		-f cloud-node-manager-windows.Dockerfile . \
 		--provenance=false \
 		--sbom=false
 
 .PHONY: build-node-image-windows-hpc
-build-node-image-windows-hpc: buildx-setup $(BIN_DIR)/azure-cloud-node-manager.exe ## Build node-manager image for Windows.
+build-node-image-windows-hpc: buildx-setup ## Build node-manager image for Windows.
 	$(DOCKER_BUILDX) build --pull \
 		--output=type=$(OUTPUT_TYPE) \
 		--platform windows/$(ARCH) \
 		-t $(NODE_MANAGER_WINDOWS_FULL_IMAGE_PREFIX)-hpc-$(ARCH) \
 		--build-arg ARCH=$(ARCH) \
+		--build-arg LOCAL_BUILD=$(LOCAL_WINDOWS_BUILD) \
 		-f cloud-node-manager-windows-hpc.Dockerfile . \
 		--provenance=false \
 		--sbom=false

cloud-node-manager-windows-hpc.Dockerfile

@@ -13,8 +13,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# check cloud-node-manager-windows.Dockerfile for the context of conditional builder image
 ARG ARCH=amd64
+ARG LOCAL_BUILD=false
+# build windows cloud node manager binary
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-false
+ARG ENABLE_GIT_COMMAND=true
+ARG ARCH
+WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
+COPY . .
+RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH}
+
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-true
+WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
+COPY . .
+COPY bin/azure-cloud-node-manager-*.exe bin/
+
+FROM local-build-${LOCAL_BUILD} AS builder
+
 FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0
 ARG ARCH
-COPY bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe
+ARG LOCAL_BUILD
+COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe
 ENTRYPOINT ["/cloud-node-manager.exe"]

cloud-node-manager-windows.Dockerfile

@@ -13,21 +13,45 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 ARG OSVERSION=1809
 ARG ARCH=amd64
+ARG LOCAL_BUILD=false
+
+# NOTE(mainred): BuildKit-based builder will skip the unused stage determined by  the value LOCAL_BUILD.
+
+# Build windows cloud node manager binary from golang build stage
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-false
+ARG ENABLE_GIT_COMMAND=true
+ARG ARCH
+WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
+COPY . .
+# Build the Go app
+RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH}
+
+# COPY the binary built locally to the builder container to normalize the following COPY behavior
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-true
+WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
+COPY . .
+COPY bin/azure-cloud-node-manager-*.exe bin/
+
+# mutli-stage dependency are determined before the build starts, so it's invalid to use command like
+# `COPY --from=local-build-{LOCAL_BUILD}` to copy the binary from a dynamic source, so we create a
+# normalized builder base as a workaround.
+# Example of the error:
+# ERROR: failed to solve: failed to parse stage name "local-build-{LOCAL_BUILD}": invalid reference format: repository name (library/local-build-{LOCAL_BUILD}) must be lowercase
+FROM local-build-${LOCAL_BUILD} AS builder
+
 
 # NOTE(claudiub): Instead of pulling the servercore image, which is ~2GB in side, we
 # can instead pull the windows-servercore-cache image, which is only a few MBs in size.
 # The image contains the netapi32.dll we need.
-FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-${ARCH}-$OSVERSION as servercore-helper
+FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-${ARCH}-$OSVERSION AS servercore-helper
 
 FROM mcr.microsoft.com/windows/nanoserver:$OSVERSION
 
-ARG OSVERSION
 ARG ARCH
 
 COPY --from=servercore-helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
-COPY bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe
+COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe
 USER ContainerUser
 ENTRYPOINT ["/cloud-node-manager.exe"]

cloud-node-manager.Dockerfile

@@ -27,8 +27,6 @@ RUN if [ "$ARCH" = "arm64" ] ; then \
 
 WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
 COPY . .
-
-# Build the Go app
 RUN make bin/azure-cloud-node-manager ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH}
 
 # Use distroless base image for a lean production container.