Merge pull request #3940 from tzneal/build-pause-image-with-k8s

build the pause image from upstream source and use as the default
bottlerocket-os · Jun 6, 2024 · 9b04b6e · 9b04b6e
2 parents 892e31f + 27f758b
commit 9b04b6e
Show file tree

Hide file tree

Showing 77 changed files with 614 additions and 117 deletions.
diff --git a/Release.toml b/Release.toml
@@ -315,4 +315,7 @@ version = "1.21.0"
 "(1.20.0, 1.20.1)" = []
 "(1.20.1, 1.21.0)" = [
     "migrate_v1.21.0_pluto-remove-generators-v0-1-0.lz4",
+    "migrate_v1.21.0_pod-infra-container-image-remove-settings-generator.lz4",
+    "migrate_v1.21.0_pod-infra-container-image-affected-services.lz4",
+    "migrate_v1.21.0_pod-infra-container-image-services.lz4",
 ]
diff --git a/packages/containerd/containerd-config-toml_k8s_containerd_sock b/packages/containerd/containerd-config-toml_k8s_containerd_sock
@@ -21,7 +21,7 @@ address = "/run/containerd/containerd.sock"
 [plugins."io.containerd.grpc.v1.cri"]
 enable_selinux = true
 # Pause container image is specified here, shares the same image as kubelet's pod-infra-container-image
-sandbox_image = "{{settings.kubernetes.pod-infra-container-image}}"
+sandbox_image = "localhost/kubernetes/pause:0.1.0"
 {{#if settings.container-runtime.max-container-log-line-size}}
 max_container_log_line_size = {{settings.container-runtime.max-container-log-line-size}}
 {{/if}}

diff --git a/packages/containerd/containerd-config-toml_k8s_nvidia_containerd_sock b/packages/containerd/containerd-config-toml_k8s_nvidia_containerd_sock
@@ -21,7 +21,7 @@ address = "/run/containerd/containerd.sock"
 [plugins."io.containerd.grpc.v1.cri"]
 enable_selinux = true
 # Pause container image is specified here, shares the same image as kubelet's pod-infra-container-image
-sandbox_image = "{{settings.kubernetes.pod-infra-container-image}}"
+sandbox_image = "localhost/kubernetes/pause:0.1.0"
 {{#if settings.container-runtime.max-container-log-line-size}}
 max_container_log_line_size = {{settings.container-runtime.max-container-log-line-size}}
 {{/if}}

diff --git a/packages/kubernetes-1.23/kubelet-env b/packages/kubernetes-1.23/kubelet-env
@@ -5,4 +5,3 @@ std = { version = "v1", helpers = ["join_map"] }
 NODE_IP={{settings.kubernetes.node-ip}}
 NODE_LABELS={{join_map "=" "," "no-fail-if-missing" settings.kubernetes.node-labels}}
 NODE_TAINTS={{join_node_taints settings.kubernetes.node-taints}}
-POD_INFRA_CONTAINER_IMAGE={{settings.kubernetes.pod-infra-container-image}}
diff --git a/packages/kubernetes-1.23/kubelet-exec-start-conf b/packages/kubernetes-1.23/kubelet-exec-start-conf
@@ -38,5 +38,5 @@ ExecStart=/usr/bin/kubelet \
 {{#if settings.kubernetes.log-level includeZero=true}}
     -v {{settings.kubernetes.log-level}} \
 {{/if}}
-    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE} \
+    --pod-infra-container-image localhost/kubernetes/pause:0.1.0 \
     --runtime-cgroups=/runtime.slice/containerd.service
diff --git a/packages/kubernetes-1.23/kubernetes-1.23.spec b/packages/kubernetes-1.23/kubernetes-1.23.spec
@@ -7,6 +7,11 @@
 
 %global _dwz_low_mem_die_limit 0
 
+# Construct reproducible tar archives
+# See https://reproducible-builds.org/docs/archives/
+%global source_date_epoch 1234567890
+%global tar_cf tar --sort=name --mtime="@%{source_date_epoch}" --owner=0 --group=0 --numeric-owner -cf
+
 # The kubernetes build process expects the cross-compiler to be specified via `KUBE_*_CC`
 # Here we generate that variable to use bottlerocket-specific compile aliases
 # Examples of the generated variable:
@@ -42,10 +47,15 @@ Source14: credential-provider-config-yaml
 Source15: logdog.kubelet.conf
 
 # ExecStartPre drop-ins
-Source20: prestart-pull-pause-ctr.conf
+Source20: prestart-load-pause-ctr.conf
 Source21: dockershim-symlink.conf
 Source22: make-kubelet-dirs.conf
 
+# pause image components
+Source100: pause-config.json
+Source101: pause-manifest.json
+Source102: pod-infra-container-image
+
 Source1000: clarify.toml
 
 # Additional patches on top of last 1.23 point release
@@ -121,6 +131,23 @@ export KUBE_OUTPUT_SUBPATH="_fips_output/local"
 export GOEXPERIMENT="boringcrypto"
 make WHAT="cmd/kubelet"
 
+# build the pause container
+cd build/pause/linux/
+
+# Build static pause executable for container image.
+mkdir -p rootfs/usr/bin
+%{_cross_triple}-musl-gcc %{_cross_cflags} %{_cross_ldflags} -static-pie pause.c -o rootfs/pause
+
+# Construct container image.
+mkdir -p image/rootfs
+%tar_cf image/rootfs/layer.tar -C rootfs .
+DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//')
+install -m 0644 %{S:100} image/config.json
+sed -i "s/~~digest~~/${DIGEST}/" image/config.json
+install -m 0644 %{S:101} image/manifest.json
+
+%tar_cf ../../../_output/local/bin/linux/%{_cross_go_arch}/kubernetes-pause.tar -C image .
+
 %install
 output="./_output/local/bin/linux/%{_cross_go_arch}"
 install -d %{buildroot}%{_cross_bindir}
@@ -163,6 +190,10 @@ ln -rs \
 install -d %{buildroot}%{_cross_datadir}/logdog.d
 install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 
+install -d %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 ${output}/kubernetes-pause.tar %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 %{S:102} %{buildroot}%{_cross_templatedir}/pod-infra-container-image
+
 %files -n %{_cross_os}kubelet-1.23
 %license LICENSE LICENSE.gonum.graph LICENSE.shell2junit LICENSE.golang PATENTS.golang
 %{_cross_attribution_file}
@@ -171,7 +202,7 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_unitdir}/prepare-var-lib-kubelet.service
 %{_cross_unitdir}/etc-kubernetes-pki-private.mount
 %dir %{_cross_unitdir}/kubelet.service.d
-%{_cross_unitdir}/kubelet.service.d/prestart-pull-pause-ctr.conf
+%{_cross_unitdir}/kubelet.service.d/prestart-load-pause-ctr.conf
 %{_cross_unitdir}/kubelet.service.d/make-kubelet-dirs.conf
 %{_cross_unitdir}/kubelet.service.d/dockershim-symlink.conf
 %dir %{_cross_templatedir}
@@ -188,6 +219,8 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes
 %{_cross_libexecdir}/kubernetes/kubelet-plugins
+%{_cross_libexecdir}/kubernetes/kubernetes-pause.tar
+%{_cross_templatedir}/pod-infra-container-image
 %{_cross_datadir}/logdog.d/logdog.kubelet.conf
 
 %files -n %{_cross_os}kubelet-1.23-bin

diff --git a/packages/kubernetes-1.23/pause-config.json b/packages/kubernetes-1.23/pause-config.json
@@ -0,0 +1 @@
+{"author":"Bottlerocket","config":{"Cmd":["/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Bottlerocket","created_by":"Bottlerocket","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
diff --git a/packages/kubernetes-1.23/pause-manifest.json b/packages/kubernetes-1.23/pause-manifest.json
@@ -0,0 +1 @@
+[{"Config":"config.json","RepoTags":["localhost/kubernetes/pause:0.1.0"],"Layers":["rootfs/layer.tar"]}]
diff --git a/packages/kubernetes-1.23/pod-infra-container-image b/packages/kubernetes-1.23/pod-infra-container-image
@@ -0,0 +1,6 @@
+[required-extensions]
+kubernetes = { version = "v1", optional = true }
++++
+{{~#if settings.kubernetes.pod-infra-container-image~}}
+DEPRECATED_SETTING=settings.kubernetes.pod-infra-container-image
+{{~/if~}}
diff --git a/packages/kubernetes-1.23/prestart-load-pause-ctr.conf b/packages/kubernetes-1.23/prestart-load-pause-ctr.conf
@@ -0,0 +1,14 @@
+[Service]
+# load the built-in pause image
+ExecStartPre=/usr/bin/ctr \
+    --namespace=k8s.io \
+    image import \
+    --all-platforms \
+    /usr/libexec/kubernetes/kubernetes-pause.tar
+
+# label it to prevent it from being removed
+ExecStartPre=/usr/bin/ctr \
+    --namespace=k8s.io \
+    image label \
+    localhost/kubernetes/pause:0.1.0 \
+    io.cri-containerd.pinned=pinned
diff --git a/packages/kubernetes-1.23/prestart-pull-pause-ctr.conf b/packages/kubernetes-1.23/prestart-pull-pause-ctr.conf
diff --git a/packages/kubernetes-1.24/kubelet-env b/packages/kubernetes-1.24/kubelet-env
@@ -5,4 +5,3 @@ std = { version = "v1", helpers = ["join_map"] }
 NODE_IP={{settings.kubernetes.node-ip}}
 NODE_LABELS={{join_map "=" "," "no-fail-if-missing" settings.kubernetes.node-labels}}
 NODE_TAINTS={{join_node_taints settings.kubernetes.node-taints}}
-POD_INFRA_CONTAINER_IMAGE={{settings.kubernetes.pod-infra-container-image}}
diff --git a/packages/kubernetes-1.24/kubelet-exec-start-conf b/packages/kubernetes-1.24/kubelet-exec-start-conf
@@ -37,5 +37,5 @@ ExecStart=/usr/bin/kubelet \
 {{#if settings.kubernetes.log-level includeZero=true}}
     -v {{settings.kubernetes.log-level}} \
 {{/if}}
-    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE} \
+    --pod-infra-container-image localhost/kubernetes/pause:0.1.0 \
     --runtime-cgroups=/runtime.slice/containerd.service
diff --git a/packages/kubernetes-1.24/kubernetes-1.24.spec b/packages/kubernetes-1.24/kubernetes-1.24.spec
@@ -15,6 +15,11 @@
 
 %global _dwz_low_mem_die_limit 0
 
+# Construct reproducible tar archives
+# See https://reproducible-builds.org/docs/archives/
+%global source_date_epoch 1234567890
+%global tar_cf tar --sort=name --mtime="@%{source_date_epoch}" --owner=0 --group=0 --numeric-owner -cf
+
 # The kubernetes build process expects the cross-compiler to be specified via `KUBE_*_CC`
 # Here we generate that variable to use bottlerocket-specific compile aliases
 # Examples of the generated variable:
@@ -50,10 +55,15 @@ Source14: credential-provider-config-yaml
 Source15: logdog.kubelet.conf
 
 # ExecStartPre drop-ins
-Source20: prestart-pull-pause-ctr.conf
+Source20: prestart-load-pause-ctr.conf
 Source21: dockershim-symlink.conf
 Source22: make-kubelet-dirs.conf
 
+# pause image components
+Source100: pause-config.json
+Source101: pause-manifest.json
+Source102: pod-infra-container-image
+
 Source1000: clarify.toml
 
 BuildRequires: git
@@ -122,6 +132,23 @@ export KUBE_OUTPUT_SUBPATH="_fips_output/local"
 export GOEXPERIMENT="boringcrypto"
 make WHAT="cmd/kubelet"
 
+# build the pause container
+cd build/pause/linux/
+
+# Build static pause executable for container image.
+mkdir -p rootfs/usr/bin
+%{_cross_triple}-musl-gcc %{_cross_cflags} %{_cross_ldflags} -static-pie pause.c -o rootfs/pause
+
+# Construct container image.
+mkdir -p image/rootfs
+%tar_cf image/rootfs/layer.tar -C rootfs .
+DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//')
+install -m 0644 %{S:100} image/config.json
+sed -i "s/~~digest~~/${DIGEST}/" image/config.json
+install -m 0644 %{S:101} image/manifest.json
+
+%tar_cf ../../../_output/local/bin/linux/%{_cross_go_arch}/kubernetes-pause.tar -C image .
+
 %install
 output="./_output/local/bin/linux/%{_cross_go_arch}"
 install -d %{buildroot}%{_cross_bindir}
@@ -164,6 +191,10 @@ ln -rs \
 install -d %{buildroot}%{_cross_datadir}/logdog.d
 install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 
+install -d %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 ${output}/kubernetes-pause.tar %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 %{S:102} %{buildroot}%{_cross_templatedir}/pod-infra-container-image
+
 %files -n %{_cross_os}kubelet-1.24
 %license LICENSE LICENSE.gonum.graph LICENSE.shell2junit LICENSE.golang PATENTS.golang
 %{_cross_attribution_file}
@@ -172,7 +203,7 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_unitdir}/prepare-var-lib-kubelet.service
 %{_cross_unitdir}/etc-kubernetes-pki-private.mount
 %dir %{_cross_unitdir}/kubelet.service.d
-%{_cross_unitdir}/kubelet.service.d/prestart-pull-pause-ctr.conf
+%{_cross_unitdir}/kubelet.service.d/prestart-load-pause-ctr.conf
 %{_cross_unitdir}/kubelet.service.d/make-kubelet-dirs.conf
 %{_cross_unitdir}/kubelet.service.d/dockershim-symlink.conf
 %dir %{_cross_templatedir}
@@ -189,6 +220,8 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes
 %{_cross_libexecdir}/kubernetes/kubelet-plugins
+%{_cross_libexecdir}/kubernetes/kubernetes-pause.tar
+%{_cross_templatedir}/pod-infra-container-image
 %{_cross_datadir}/logdog.d/logdog.kubelet.conf
 
 %files -n %{_cross_os}kubelet-1.24-bin

diff --git a/packages/kubernetes-1.24/pause-config.json b/packages/kubernetes-1.24/pause-config.json
@@ -0,0 +1 @@
+{"author":"Bottlerocket","config":{"Cmd":["/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Bottlerocket","created_by":"Bottlerocket","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
diff --git a/packages/kubernetes-1.24/pause-manifest.json b/packages/kubernetes-1.24/pause-manifest.json
@@ -0,0 +1 @@
+[{"Config":"config.json","RepoTags":["localhost/kubernetes/pause:0.1.0"],"Layers":["rootfs/layer.tar"]}]
diff --git a/packages/kubernetes-1.24/pod-infra-container-image b/packages/kubernetes-1.24/pod-infra-container-image
@@ -0,0 +1,6 @@
+[required-extensions]
+kubernetes = { version = "v1", optional = true }
++++
+{{~#if settings.kubernetes.pod-infra-container-image~}}
+DEPRECATED_SETTING=settings.kubernetes.pod-infra-container-image
+{{~/if~}}
diff --git a/packages/kubernetes-1.24/prestart-load-pause-ctr.conf b/packages/kubernetes-1.24/prestart-load-pause-ctr.conf
@@ -0,0 +1,14 @@
+[Service]
+# load the built-in pause image
+ExecStartPre=/usr/bin/ctr \
+    --namespace=k8s.io \
+    image import \
+    --all-platforms \
+    /usr/libexec/kubernetes/kubernetes-pause.tar
+
+# label it to prevent it from being removed
+ExecStartPre=/usr/bin/ctr \
+    --namespace=k8s.io \
+    image label \
+    localhost/kubernetes/pause:0.1.0 \
+    io.cri-containerd.pinned=pinned
diff --git a/packages/kubernetes-1.24/prestart-pull-pause-ctr.conf b/packages/kubernetes-1.24/prestart-pull-pause-ctr.conf
diff --git a/packages/kubernetes-1.25/kubelet-env b/packages/kubernetes-1.25/kubelet-env
@@ -5,4 +5,3 @@ std = { version = "v1", helpers = ["join_map"] }
 NODE_IP={{settings.kubernetes.node-ip}}
 NODE_LABELS={{join_map "=" "," "no-fail-if-missing" settings.kubernetes.node-labels}}
 NODE_TAINTS={{join_node_taints settings.kubernetes.node-taints}}
-POD_INFRA_CONTAINER_IMAGE={{settings.kubernetes.pod-infra-container-image}}
diff --git a/packages/kubernetes-1.25/kubelet-exec-start-conf b/packages/kubernetes-1.25/kubelet-exec-start-conf
@@ -37,5 +37,5 @@ ExecStart=/usr/bin/kubelet \
 {{#if settings.kubernetes.log-level includeZero=true}}
     -v {{settings.kubernetes.log-level}} \
 {{/if}}
-    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE} \
+    --pod-infra-container-image localhost/kubernetes/pause:0.1.0 \
     --runtime-cgroups=/runtime.slice/containerd.service
diff --git a/packages/kubernetes-1.25/kubernetes-1.25.spec b/packages/kubernetes-1.25/kubernetes-1.25.spec
@@ -15,6 +15,11 @@
 
 %global _dwz_low_mem_die_limit 0
 
+# Construct reproducible tar archives
+# See https://reproducible-builds.org/docs/archives/
+%global source_date_epoch 1234567890
+%global tar_cf tar --sort=name --mtime="@%{source_date_epoch}" --owner=0 --group=0 --numeric-owner -cf
+
 # The kubernetes build process expects the cross-compiler to be specified via `KUBE_*_CC`
 # Here we generate that variable to use bottlerocket-specific compile aliases
 # Examples of the generated variable:
@@ -50,10 +55,15 @@ Source14: credential-provider-config-yaml
 Source15: logdog.kubelet.conf
 
 # ExecStartPre drop-ins
-Source20: prestart-pull-pause-ctr.conf
+Source20: prestart-load-pause-ctr.conf
 Source21: dockershim-symlink.conf
 Source22: make-kubelet-dirs.conf
 
+# pause image components
+Source100: pause-config.json
+Source101: pause-manifest.json
+Source102: pod-infra-container-image
+
 Source1000: clarify.toml
 
 BuildRequires: git
@@ -122,6 +132,23 @@ export KUBE_OUTPUT_SUBPATH="_fips_output/local"
 export GOEXPERIMENT="boringcrypto"
 make WHAT="cmd/kubelet"
 
+# build the pause container
+cd build/pause/linux/
+
+# Build static pause executable for container image.
+mkdir -p rootfs/usr/bin
+%{_cross_triple}-musl-gcc %{_cross_cflags} %{_cross_ldflags} -static-pie pause.c -o rootfs/pause
+
+# Construct container image.
+mkdir -p image/rootfs
+%tar_cf image/rootfs/layer.tar -C rootfs .
+DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//')
+install -m 0644 %{S:100} image/config.json
+sed -i "s/~~digest~~/${DIGEST}/" image/config.json
+install -m 0644 %{S:101} image/manifest.json
+
+%tar_cf ../../../_output/local/bin/linux/%{_cross_go_arch}/kubernetes-pause.tar -C image .
+
 %install
 output="./_output/local/bin/linux/%{_cross_go_arch}"
 install -d %{buildroot}%{_cross_bindir}
@@ -164,6 +191,10 @@ ln -rs \
 install -d %{buildroot}%{_cross_datadir}/logdog.d
 install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 
+install -d %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 ${output}/kubernetes-pause.tar %{buildroot}%{_cross_libexecdir}/kubernetes
+install -p -m 0644 %{S:102} %{buildroot}%{_cross_templatedir}/pod-infra-container-image
+
 %files -n %{_cross_os}kubelet-1.25
 %license LICENSE LICENSE.gonum.graph LICENSE.shell2junit LICENSE.golang PATENTS.golang
 %{_cross_attribution_file}
@@ -172,7 +203,7 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_unitdir}/prepare-var-lib-kubelet.service
 %{_cross_unitdir}/etc-kubernetes-pki-private.mount
 %dir %{_cross_unitdir}/kubelet.service.d
-%{_cross_unitdir}/kubelet.service.d/prestart-pull-pause-ctr.conf
+%{_cross_unitdir}/kubelet.service.d/prestart-load-pause-ctr.conf
 %{_cross_unitdir}/kubelet.service.d/make-kubelet-dirs.conf
 %{_cross_unitdir}/kubelet.service.d/dockershim-symlink.conf
 %dir %{_cross_templatedir}
@@ -189,6 +220,8 @@ install -p -m 0644 %{S:15} %{buildroot}%{_cross_datadir}/logdog.d
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes
 %{_cross_libexecdir}/kubernetes/kubelet-plugins
+%{_cross_libexecdir}/kubernetes/kubernetes-pause.tar
+%{_cross_templatedir}/pod-infra-container-image
 %{_cross_datadir}/logdog.d/logdog.kubelet.conf
 
 %files -n %{_cross_os}kubelet-1.25-bin

diff --git a/packages/kubernetes-1.25/pause-config.json b/packages/kubernetes-1.25/pause-config.json
@@ -0,0 +1 @@
+{"author":"Bottlerocket","config":{"Cmd":["/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Bottlerocket","created_by":"Bottlerocket","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
diff --git a/packages/kubernetes-1.25/pause-manifest.json b/packages/kubernetes-1.25/pause-manifest.json
@@ -0,0 +1 @@
+[{"Config":"config.json","RepoTags":["localhost/kubernetes/pause:0.1.0"],"Layers":["rootfs/layer.tar"]}]
diff --git a/packages/kubernetes-1.25/pod-infra-container-image b/packages/kubernetes-1.25/pod-infra-container-image
@@ -0,0 +1,6 @@
+[required-extensions]
+kubernetes = { version = "v1", optional = true }
++++
+{{~#if settings.kubernetes.pod-infra-container-image~}}
+DEPRECATED_SETTING=settings.kubernetes.pod-infra-container-image
+{{~/if~}}