From 6a151e6158195abda8cd451d7d41309cc70e0782 Mon Sep 17 00:00:00 2001 From: YuryLysov Date: Wed, 13 Aug 2025 18:24:31 +0400 Subject: [PATCH 1/2] chore(module): add pre-created mount points to images Images with pre-created mount points: - cdi-apiserver - cdi-cloner - cdi-controller - cdi-importer - cdi-operator - dvcr - dvcr-importer - dvcr-uploader - kube-api-rewriter - virt-api - virt-controller - virt-handler - virt-launcher - virt-operator - virtualization-api - virtualization-audit - virtualization-controller - hp pods Some notes: - Create /var/run subdirectories in /run, as /var/run is a symlink to ../run. - Add /var, /run and symlink /var/run -> ../run in 'distroless' base image. - Pre-create /var, /run and symlink /var/run -> ../run in kube-api-rewriter image. - Remove unused extraheaders settings in dvcr-importer and dvcr-uploader. Signed-off-by: YuryLysov Co-authored-by: Ivan Mikheykin Co-authored-by: Nikita Korolev Signed-off-by: Nikita Korolev Signed-off-by: Ivan Mikheykin --- .prettierignore | 1 + .werf/defines/image-mountpoints.tmpl | 32 +++++++++++++ images/cdi-apiserver/mount-points.yaml | 7 +++ images/cdi-apiserver/werf.inc.yaml | 2 + images/cdi-cloner/mount-points.yaml | 7 +++ images/cdi-cloner/werf.inc.yaml | 2 + images/cdi-controller/mount-points.yaml | 13 +++++ images/cdi-controller/werf.inc.yaml | 2 + images/cdi-importer/mount-points.yaml | 17 +++++++ images/cdi-importer/werf.inc.yaml | 2 + images/cdi-operator/mount-points.yaml | 4 ++ images/cdi-operator/werf.inc.yaml | 2 + images/distroless/werf.inc.yaml | 10 +++- images/dvcr-importer/mount-points.yaml | 7 +++ images/dvcr-importer/werf.inc.yaml | 2 + images/dvcr-uploader/mount-points.yaml | 4 ++ images/dvcr-uploader/werf.inc.yaml | 2 + images/dvcr/mount-points.yaml | 7 +++ images/dvcr/werf.inc.yaml | 2 + images/kube-api-rewriter/mount-points.yaml | 7 +++ images/kube-api-rewriter/werf.inc.yaml | 11 ++++- images/virt-api/mount-points.yaml | 10 ++++ images/virt-api/werf.inc.yaml | 2 + images/virt-controller/mount-points.yaml | 7 +++ images/virt-controller/werf.inc.yaml | 2 + images/virt-handler/mount-points.yaml | 21 ++++++++ images/virt-handler/werf.inc.yaml | 2 + images/virt-launcher/mount-points.yaml | 48 +++++++++++++++++++ images/virt-launcher/werf.inc.yaml | 2 + images/virt-operator/mount-points.yaml | 6 +++ images/virt-operator/werf.inc.yaml | 2 + images/virtualization-api/mount-points.yaml | 6 +++ images/virtualization-api/werf.inc.yaml | 2 + .../pkg/common/consts.go | 6 --- .../pkg/controller/importer/importer_pod.go | 20 -------- .../pkg/controller/importer/settings.go | 1 - .../pkg/controller/uploader/settings.go | 1 - .../pkg/controller/uploader/uploader_pod.go | 20 -------- images/virtualization-audit/mount-points.yaml | 4 ++ images/virtualization-audit/werf.inc.yaml | 2 + .../mount-points.yaml | 5 ++ .../virtualization-controller/werf.inc.yaml | 2 + templates/virtualization-api/deployment.yaml | 12 ++--- tools/mounts/README.md | 3 ++ tools/mounts/mountdir/.gitkeep | 0 tools/mounts/mountfile | 0 46 files changed, 272 insertions(+), 57 deletions(-) create mode 100644 .werf/defines/image-mountpoints.tmpl create mode 100644 images/cdi-apiserver/mount-points.yaml create mode 100644 images/cdi-cloner/mount-points.yaml create mode 100644 images/cdi-controller/mount-points.yaml create mode 100644 images/cdi-importer/mount-points.yaml create mode 100644 images/cdi-operator/mount-points.yaml create mode 100644 images/dvcr-importer/mount-points.yaml create mode 100644 images/dvcr-uploader/mount-points.yaml create mode 100644 images/dvcr/mount-points.yaml create mode 100644 images/kube-api-rewriter/mount-points.yaml create mode 100644 images/virt-api/mount-points.yaml create mode 100644 images/virt-controller/mount-points.yaml create mode 100644 images/virt-handler/mount-points.yaml create mode 100644 images/virt-launcher/mount-points.yaml create mode 100644 images/virt-operator/mount-points.yaml create mode 100644 images/virtualization-api/mount-points.yaml create mode 100644 images/virtualization-audit/mount-points.yaml create mode 100644 images/virtualization-controller/mount-points.yaml create mode 100644 tools/mounts/README.md create mode 100644 tools/mounts/mountdir/.gitkeep create mode 100644 tools/mounts/mountfile diff --git a/.prettierignore b/.prettierignore index e1b45c5caa..40ab026e2e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,6 +1,7 @@ **/*.git **/.svn **/.hg +images/**/mount-points.yaml **/werf*.yaml **/werf*.yml .werf/** diff --git a/.werf/defines/image-mountpoints.tmpl b/.werf/defines/image-mountpoints.tmpl new file mode 100644 index 0000000000..9c76a3f917 --- /dev/null +++ b/.werf/defines/image-mountpoints.tmpl @@ -0,0 +1,32 @@ +{{/* + +Template to bake mount points in the image. These static mount points +are required so containerd can start a container with image integrity check. + +Problem: each directory specified in volumeMounts items should exist +in image, containerd is unable to create mount point for us when +integrity check is enabled. + +Solution: define all possible mount points in mount-points.yaml file and +include this template in git section of the werf.inc.yaml. + +*/}} +{{/* NOTE: Keep in sync with version in Deckhouse CSE */}} +{{- define "image mount points" }} +{{- $mountPoints := ($.Files.Get (printf "images/%s/mount-points.yaml" $.ImageName) | fromYaml) }} +{{- $context := . }} +{{- range $v := $mountPoints.dirs }} +- add: /tools/mounts/mountdir + to: {{ $v | trimSuffix "/" }} + stageDependencies: + install: + - "**/*" +{{- end }} +{{- range $v := $mountPoints.files }} +- add: /tools/mounts/mountfile + to: {{ $v }} + stageDependencies: + install: + - "**/*" +{{- end }} +{{- end }} diff --git a/images/cdi-apiserver/mount-points.yaml b/images/cdi-apiserver/mount-points.yaml new file mode 100644 index 0000000000..7f9f0c920b --- /dev/null +++ b/images/cdi-apiserver/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + # Create dirs in /run, as /var/run is a symlink to /run. + - /run/certs/cdi-apiserver-signer-bundle + - /run/certs/cdi-apiserver-server-cert + - /kubeconfig.local diff --git a/images/cdi-apiserver/werf.inc.yaml b/images/cdi-apiserver/werf.inc.yaml index fe1a3e7539..a005cef951 100644 --- a/images/cdi-apiserver/werf.inc.yaml +++ b/images/cdi-apiserver/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}cdi-artifact add: /cdi-binaries diff --git a/images/cdi-cloner/mount-points.yaml b/images/cdi-cloner/mount-points.yaml new file mode 100644 index 0000000000..4b3de32b9b --- /dev/null +++ b/images/cdi-cloner/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. +# +# See https://github.com/deckhouse/3p-containerized-data-importer/blob/80d763d788e06b3decaf22e4762076cec64582b3/pkg/controller/clone-controller.go#L699 + +dirs: + # Create dirs in /run, as /var/run is a symlink to /run. + - /run/cdi/clone/source diff --git a/images/cdi-cloner/werf.inc.yaml b/images/cdi-cloner/werf.inc.yaml index 3f4976946f..f08ea278ed 100644 --- a/images/cdi-cloner/werf.inc.yaml +++ b/images/cdi-cloner/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-bins add: /relocate diff --git a/images/cdi-controller/mount-points.yaml b/images/cdi-controller/mount-points.yaml new file mode 100644 index 0000000000..d68ce54296 --- /dev/null +++ b/images/cdi-controller/mount-points.yaml @@ -0,0 +1,13 @@ +# A list of pre-created mount points for containerd strict mode. +# +# Some volume mounts are ignored: +# - /tmp - already in the 'distroless' base image. + +dirs: + # Create dirs in /run, as /var/run is a symlink to /run. + - /run/cdi/token/keys + - /run/certs/cdi-uploadserver-signer + - /run/certs/cdi-uploadserver-client-signer + - /run/ca-bundle/cdi-uploadserver-signer-bundle + - /run/ca-bundle/cdi-uploadserver-client-signer-bundle + - /kubeconfig.local diff --git a/images/cdi-controller/werf.inc.yaml b/images/cdi-controller/werf.inc.yaml index a01afca53e..814dde77fa 100644 --- a/images/cdi-controller/werf.inc.yaml +++ b/images/cdi-controller/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-bins add: /relocate diff --git a/images/cdi-importer/mount-points.yaml b/images/cdi-importer/mount-points.yaml new file mode 100644 index 0000000000..f926961f28 --- /dev/null +++ b/images/cdi-importer/mount-points.yaml @@ -0,0 +1,17 @@ +# A list of pre-created mount points for containerd strict mode. +# +# See https://github.com/deckhouse/3p-containerized-data-importer/blob/d5fa5124b8a645521843814fffecdf385b74b379/pkg/controller/import-controller.go#L962 +# +# Some volume mounts are ignored: +# - /extraheaders - Etra headers not implemented in virtualization-controller. +# - /google - No support for GCS data source in VirtualImage. +# - /tmp - already in the 'distroless' base image. + +dirs: + - /certs + - /data + - /opt + - /proxycerts + - /scratch + - /shared + diff --git a/images/cdi-importer/werf.inc.yaml b/images/cdi-importer/werf.inc.yaml index 8da94bd533..8b24b87039 100644 --- a/images/cdi-importer/werf.inc.yaml +++ b/images/cdi-importer/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-bins add: /relocate diff --git a/images/cdi-operator/mount-points.yaml b/images/cdi-operator/mount-points.yaml new file mode 100644 index 0000000000..624df72961 --- /dev/null +++ b/images/cdi-operator/mount-points.yaml @@ -0,0 +1,4 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /kubeconfig.local diff --git a/images/cdi-operator/werf.inc.yaml b/images/cdi-operator/werf.inc.yaml index 5b6030cd58..c720c33d50 100644 --- a/images/cdi-operator/werf.inc.yaml +++ b/images/cdi-operator/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}cdi-artifact add: /cdi-binaries diff --git a/images/distroless/werf.inc.yaml b/images/distroless/werf.inc.yaml index ac7d5ce6ac..da9e12b236 100644 --- a/images/distroless/werf.inc.yaml +++ b/images/distroless/werf.inc.yaml @@ -27,13 +27,18 @@ shell: install: - | mkdir -p /relocate/etc/{pki,ssl} /relocate/usr/{bin,sbin,share,lib,lib64} - + cd /relocate for dir in {bin,sbin,lib,lib64};do ln -s usr/$dir $dir done + # /var/run -> ../run symlink to prevent making /var/run a directory during the build. + # It is needed for better compatibility with containerd default top layer. + mkdir -p run + mkdir -p var + ln -s var/run ../run cd / - + cp -pr /tmp /relocate cp -pr /etc/passwd /etc/group /etc/hostname /etc/hosts /etc/shadow /etc/protocols /etc/services /etc/nsswitch.conf /relocate/etc cp -pr /usr/share/ca-certificates /relocate/usr/share @@ -41,6 +46,7 @@ shell: cp -pr /etc/pki/tls/cert.pem /relocate/etc/ssl cp -pr /etc/pki/tls/certs /relocate/etc/ssl cp -pr /etc/pki/ca-trust/ /relocate/etc/ + # Create 'deckhouse' user to run without root. echo "deckhouse:x:64535:64535:deckhouse:/:/sbin/nologin" >> /relocate/etc/passwd echo "deckhouse:x:64535:" >> /relocate/etc/group echo "deckhouse:!::0:::::" >> /relocate/etc/shadow diff --git a/images/dvcr-importer/mount-points.yaml b/images/dvcr-importer/mount-points.yaml new file mode 100644 index 0000000000..1795c5aae4 --- /dev/null +++ b/images/dvcr-importer/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /dvcr-src-auth + - /dvcr-auth + - /certs + - /proxycerts diff --git a/images/dvcr-importer/werf.inc.yaml b/images/dvcr-importer/werf.inc.yaml index 331c26202e..6afb9ec24e 100644 --- a/images/dvcr-importer/werf.inc.yaml +++ b/images/dvcr-importer/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}dvcr-artifact-bins add: /relocate diff --git a/images/dvcr-uploader/mount-points.yaml b/images/dvcr-uploader/mount-points.yaml new file mode 100644 index 0000000000..14d3dcb3d0 --- /dev/null +++ b/images/dvcr-uploader/mount-points.yaml @@ -0,0 +1,4 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /dvcr-auth diff --git a/images/dvcr-uploader/werf.inc.yaml b/images/dvcr-uploader/werf.inc.yaml index 0eedc4ca25..fcd1090632 100644 --- a/images/dvcr-uploader/werf.inc.yaml +++ b/images/dvcr-uploader/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}dvcr-artifact-bins add: /relocate diff --git a/images/dvcr/mount-points.yaml b/images/dvcr/mount-points.yaml new file mode 100644 index 0000000000..b844c9dc7c --- /dev/null +++ b/images/dvcr/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/docker/registry + - /etc/ssl/docker + - /var/lib/registry + - /auth diff --git a/images/dvcr/werf.inc.yaml b/images/dvcr/werf.inc.yaml index 2d6a1672fc..b1a24c19a6 100644 --- a/images/dvcr/werf.inc.yaml +++ b/images/dvcr/werf.inc.yaml @@ -19,6 +19,8 @@ shell: --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder add: /container-registry-binary diff --git a/images/kube-api-rewriter/mount-points.yaml b/images/kube-api-rewriter/mount-points.yaml new file mode 100644 index 0000000000..fa5ef6daed --- /dev/null +++ b/images/kube-api-rewriter/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/virt-operator/certificates + - /etc/virt-api/certificates + # Create dirs in /run, as /var/run is a symlink to /run. + - /run/certs/cdi-apiserver-server-cert diff --git a/images/kube-api-rewriter/werf.inc.yaml b/images/kube-api-rewriter/werf.inc.yaml index b698b1fe31..0b4f559c24 100644 --- a/images/kube-api-rewriter/werf.inc.yaml +++ b/images/kube-api-rewriter/werf.inc.yaml @@ -35,13 +35,22 @@ shell: image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: builder/scratch +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder add: /src/kube-api-rewriter/kube-api-rewriter to: /app/kube-api-rewriter after: install + # Make containerd compatible directories structure. + - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder + add: /var + to: /var + includePaths: + - run + after: install imageSpec: config: - user: "65532:65532" + user: "64535:64535" workingDir: "/app" entrypoint: ["/app/kube-api-rewriter"] diff --git a/images/virt-api/mount-points.yaml b/images/virt-api/mount-points.yaml new file mode 100644 index 0000000000..eb2d220cf6 --- /dev/null +++ b/images/virt-api/mount-points.yaml @@ -0,0 +1,10 @@ +# A list of pre-created mount points for containerd strict mode. +# +# Some volume mounts are ignored: +# - /tmp - already in the 'distroless' base image. + +dirs: + - /etc/virt-api/certificates + - /etc/virt-handler/clientcertificates + - /profile-data + - /kubeconfig.local diff --git a/images/virt-api/werf.inc.yaml b/images/virt-api/werf.inc.yaml index 47432f599f..bb6bd3757a 100644 --- a/images/virt-api/werf.inc.yaml +++ b/images/virt-api/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virt-artifact add: /kubevirt-binaries/ diff --git a/images/virt-controller/mount-points.yaml b/images/virt-controller/mount-points.yaml new file mode 100644 index 0000000000..183768973f --- /dev/null +++ b/images/virt-controller/mount-points.yaml @@ -0,0 +1,7 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/virt-controller/certificates + - /etc/virt-controller/exportca + - /profile-data + - /kubeconfig.local diff --git a/images/virt-controller/werf.inc.yaml b/images/virt-controller/werf.inc.yaml index 3ad212b26c..ede2c542d7 100644 --- a/images/virt-controller/werf.inc.yaml +++ b/images/virt-controller/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virt-artifact add: /kubevirt-binaries/ diff --git a/images/virt-handler/mount-points.yaml b/images/virt-handler/mount-points.yaml new file mode 100644 index 0000000000..680aedd7f9 --- /dev/null +++ b/images/virt-handler/mount-points.yaml @@ -0,0 +1,21 @@ +# A list of pre-created mount points for containerd strict mode. +# +# Some volume mounts are ignored: +# - /tmp - already in the 'distroless' base image. + +dirs: + - /etc/virt-handler/clientcertificates + - /etc/virt-handler/servercertificates + - /kubeconfig.local + - /profile-data + - /etc/podinfo + - /pods + - /var/lib/kubevirt + - /var/lib/kubelet/device-plugins + - /var/lib/kubelet/pods + - /var/lib/kubevirt-node-labeller + # Create dirs in /run, as /var/run is a symlink to /run. + - /run/kubevirt + - /run/kubevirt-libvirt-runtimes + - /run/kubevirt-private + diff --git a/images/virt-handler/werf.inc.yaml b/images/virt-handler/werf.inc.yaml index cd44dff356..fb1b970762 100644 --- a/images/virt-handler/werf.inc.yaml +++ b/images/virt-handler/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-bins add: /relocate diff --git a/images/virt-launcher/mount-points.yaml b/images/virt-launcher/mount-points.yaml new file mode 100644 index 0000000000..643cfdb1fe --- /dev/null +++ b/images/virt-launcher/mount-points.yaml @@ -0,0 +1,48 @@ +# A list of pre-created mount points for containerd strict mode. +# +# See https://github.com/deckhouse/3p-kubevirt/blob/8aed630/pkg/virt-controller/services/rendervolumes.go +# +# Some volume mounts are ignored: +# - /tmp - already in the 'distroless' base image. +# - /var/run - already in the 'distroless' base image. +# No need to pre-create a plethora of /var/run descendants, +# as deckhouse/3p-kubevirt is patched to mount /var/run as emptyDir: +# - /var/run/libvirt +# - /var/run/kubevirt-ephemeral-disks +# - /var/run/kubevirt-hooks +# - /var/run/kubevirt-private +# - /var/run/kubevirt-private/sysprep/ +# - /var/run/kubevirt-private/secret/cloudinit/userdata +# - /var/run/kubevirt-private/secret/cloudinit/userData +# - /var/run/kubevirt-private/secret/cloudinit/networkdata +# - /var/run/kubevirt-private/secret/cloudinit/networkData +# - /var/run/kubevirt-private/config-map +# - /var/run/kubevirt-private/downwardapi +# - /var/run/kubevirt-private/downwardapi-disks +# - /var/run/kubevirt-private/vmi-disks +# - /var/run/kubevirt-private/libvirt +# - /var/run/kubevirt-private/libvirt/qemu +# - /var/run/kubevirt-private/libvirt/qemu/nvram +# - /var/run/kubevirt-private/libvirt/qemu/swtpm +# - /var/run/kubevirt-private/var/lib/swtpm-localca +# - There are more dirs in /var/run/kubevirt-private/ +# - /var/run/kubevirt +# - /var/run/kubevirt/container-disks +# - /var/run/kubevirt/sockets +# - /var/run/kubevirt/hotplug-disks +# - /var/run/kubevirt/virtiofs-containers +# /var/log is mounted as emptyDir too: +# - /var/log/libvirt + +dirs: + - /etc/libvirt + - /etc/podinfo + - /var/cache/libvirt + - /var/lib/libvirt + - /var/lib/libvirt/swtpm + - /var/lib/libvirt/qemu/nvram + - /var/lib/kubevirt-node-labeller + - /var/lib/swtpm-localca + - /var/log + - /path # For hot-plugged disks, used in "hp Pods". + - /init/usr/bin # For attaching images as "container disks". diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index ac7c0903a8..e653963e36 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -2,6 +2,8 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }} final: true fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-binaries add: /relocate diff --git a/images/virt-operator/mount-points.yaml b/images/virt-operator/mount-points.yaml new file mode 100644 index 0000000000..3c674da58c --- /dev/null +++ b/images/virt-operator/mount-points.yaml @@ -0,0 +1,6 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/virt-operator/certificates + - /profile-data + - /kubeconfig.local diff --git a/images/virt-operator/werf.inc.yaml b/images/virt-operator/werf.inc.yaml index 022ad77e2a..dda81277a1 100644 --- a/images/virt-operator/werf.inc.yaml +++ b/images/virt-operator/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virt-artifact add: /kubevirt-binaries/ diff --git a/images/virtualization-api/mount-points.yaml b/images/virtualization-api/mount-points.yaml new file mode 100644 index 0000000000..cab24f0ee2 --- /dev/null +++ b/images/virtualization-api/mount-points.yaml @@ -0,0 +1,6 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/virtualization-api/certificates + - /etc/virtualization-api-proxy/certificates + - /etc/virt-api/certificates diff --git a/images/virtualization-api/werf.inc.yaml b/images/virtualization-api/werf.inc.yaml index 108b3a98e4..a9d75809c3 100644 --- a/images/virtualization-api/werf.inc.yaml +++ b/images/virtualization-api/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virtualization-artifact add: /out/virtualization-api diff --git a/images/virtualization-artifact/pkg/common/consts.go b/images/virtualization-artifact/pkg/common/consts.go index fb16ef545b..8aedc71927 100644 --- a/images/virtualization-artifact/pkg/common/consts.go +++ b/images/virtualization-artifact/pkg/common/consts.go @@ -79,10 +79,6 @@ const ( ImportProxyNoProxy = "no_proxy" // ImporterProxyCertDirVar provides a constant to capture our env variable "IMPORTER_PROXY_CERT_DIR" ImporterProxyCertDirVar = "IMPORTER_PROXY_CERT_DIR" - // ImporterExtraHeader provides a constant to include extra HTTP headers, as the prefix to a format string - ImporterExtraHeader = "IMPORTER_EXTRA_HEADER_" - // ImporterSecretExtraHeadersDir is where the secrets containing extra HTTP headers will be mounted - ImporterSecretExtraHeadersDir = "/extraheaders" // ImporterDestinationAuthConfigDir is a mount directory for auth Secret. ImporterDestinationAuthConfigDir = "/dvcr-auth" @@ -101,10 +97,8 @@ const ( UploaderDestinationEndpoint = "UPLOADER_DESTINATION_ENDPOINT" UploaderDestinationAuthConfigVar = "UPLOADER_DESTINATION_AUTH_CONFIG" - UploaderExtraHeader = "UPLOADER_EXTRA_HEADER_" UploaderDestinationAuthConfigDir = "/dvcr-auth" UploaderDestinationAuthConfigFile = "/dvcr-auth/.dockerconfigjson" - UploaderSecretExtraHeadersDir = "/extraheaders" DockerRegistrySchemePrefix = "docker://" diff --git a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go index 5aae08181d..b3aa834760 100644 --- a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go +++ b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go @@ -18,8 +18,6 @@ package importer import ( "context" - "fmt" - "path" "strconv" corev1 "k8s.io/api/core/v1" @@ -52,9 +50,6 @@ const ( // ProxyCertVolName is the name of the volumecontaining certs proxyCertVolName = "cdi-proxy-cert-vol" - // secretExtraHeadersVolumeName is the format string that specifies where extra HTTP header secrets will be mounted - secretExtraHeadersVolumeName = "import-extra-headers-vol-%d" - // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" @@ -389,21 +384,6 @@ func (imp *Importer) addVolumes(pod *corev1.Pod, container *corev1.Container) { }, ) } - - // Mount extra headers Secrets. - for index, header := range imp.EnvSettings.SecretExtraHeaders { - volName := fmt.Sprintf(secretExtraHeadersVolumeName, index) - mountPath := path.Join(common.ImporterSecretExtraHeadersDir, fmt.Sprint(index)) - envName := fmt.Sprintf("%s%d", common.ImporterExtraHeader, index) - podutil.AddVolume(pod, container, - podutil.CreateSecretVolume(volName, header), - podutil.CreateVolumeMount(volName, mountPath), - corev1.EnvVar{ - Name: envName, - Value: header, - }, - ) - } } type PodNamer interface { diff --git a/images/virtualization-artifact/pkg/controller/importer/settings.go b/images/virtualization-artifact/pkg/controller/importer/settings.go index 25e93e3917..91888b14f1 100644 --- a/images/virtualization-artifact/pkg/controller/importer/settings.go +++ b/images/virtualization-artifact/pkg/controller/importer/settings.go @@ -61,7 +61,6 @@ type Settings struct { NoProxy string CertConfigMapProxy string ExtraHeaders []string - SecretExtraHeaders []string DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string diff --git a/images/virtualization-artifact/pkg/controller/uploader/settings.go b/images/virtualization-artifact/pkg/controller/uploader/settings.go index 33f86407f5..f32e1ac425 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/settings.go +++ b/images/virtualization-artifact/pkg/controller/uploader/settings.go @@ -25,7 +25,6 @@ import ( // Fields from this struct are passed via environment variables. type Settings struct { Verbose string - SecretExtraHeaders []string DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string diff --git a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go index 9bc98f11ab..0472ee3839 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go +++ b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go @@ -18,8 +18,6 @@ package uploader import ( "context" - "fmt" - "path" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -36,9 +34,6 @@ import ( ) const ( - // secretExtraHeadersVolumeName is the format string that specifies where extra HTTP header secrets will be mounted - secretExtraHeadersVolumeName = "import-extra-headers-vol-%d" - // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" ) @@ -197,21 +192,6 @@ func (p *Pod) addVolumes(pod *corev1.Pod, container *corev1.Container) { }, ) } - - // Mount extra headers Secrets. - for index, header := range p.Settings.SecretExtraHeaders { - volName := fmt.Sprintf(secretExtraHeadersVolumeName, index) - mountPath := path.Join(common.UploaderSecretExtraHeadersDir, fmt.Sprint(index)) - envName := fmt.Sprintf("%s%d", common.UploaderExtraHeader, index) - podutil.AddVolume(pod, container, - podutil.CreateSecretVolume(volName, header), - podutil.CreateVolumeMount(volName, mountPath), - corev1.EnvVar{ - Name: envName, - Value: header, - }, - ) - } } type PodNamer interface { diff --git a/images/virtualization-audit/mount-points.yaml b/images/virtualization-audit/mount-points.yaml new file mode 100644 index 0000000000..393d1fda58 --- /dev/null +++ b/images/virtualization-audit/mount-points.yaml @@ -0,0 +1,4 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /etc/virtualization-audit/certificates diff --git a/images/virtualization-audit/werf.inc.yaml b/images/virtualization-audit/werf.inc.yaml index 80491c16b2..1d3d3d9974 100644 --- a/images/virtualization-audit/werf.inc.yaml +++ b/images/virtualization-audit/werf.inc.yaml @@ -2,6 +2,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virtualization-artifact add: /out/virtualization-audit diff --git a/images/virtualization-controller/mount-points.yaml b/images/virtualization-controller/mount-points.yaml new file mode 100644 index 0000000000..80ba2a6cc0 --- /dev/null +++ b/images/virtualization-controller/mount-points.yaml @@ -0,0 +1,5 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /tmp/k8s-webhook-server/serving-certs + - /kubeconfig.local diff --git a/images/virtualization-controller/werf.inc.yaml b/images/virtualization-controller/werf.inc.yaml index f23d868b55..73e6f0de5e 100644 --- a/images/virtualization-controller/werf.inc.yaml +++ b/images/virtualization-controller/werf.inc.yaml @@ -1,6 +1,8 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }} fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} import: - image: {{ .ModuleNamePrefix }}virtualization-artifact add: /out/virtualization-controller diff --git a/templates/virtualization-api/deployment.yaml b/templates/virtualization-api/deployment.yaml index 178810e23f..722a01b7b1 100644 --- a/templates/virtualization-api/deployment.yaml +++ b/templates/virtualization-api/deployment.yaml @@ -85,19 +85,19 @@ spec: {{- else }} - --v=3 {{- end }} - - --tls-cert-file=/etc/virtualziation-api/certificates/tls.crt - - --tls-private-key-file=/etc/virtualziation-api/certificates/tls.key - - --proxy-client-cert-file=/etc/virtualziation-api-proxy/certificates/tls.crt - - --proxy-client-key-file=/etc/virtualziation-api-proxy/certificates/tls.key + - --tls-cert-file=/etc/virtualization-api/certificates/tls.crt + - --tls-private-key-file=/etc/virtualization-api/certificates/tls.key + - --proxy-client-cert-file=/etc/virtualization-api-proxy/certificates/tls.crt + - --proxy-client-key-file=/etc/virtualization-api-proxy/certificates/tls.key - --service-account-name=virtualization-api - --service-account-namespace=d8-{{ .Chart.Name }} image: {{ include "helm_lib_module_image" (list . "virtualizationApi") }} imagePullPolicy: IfNotPresent volumeMounts: - - mountPath: /etc/virtualziation-api/certificates + - mountPath: /etc/virtualization-api/certificates name: virtualization-api-tls readOnly: true - - mountPath: /etc/virtualziation-api-proxy/certificates + - mountPath: /etc/virtualization-api-proxy/certificates name: virtualization-api-proxy-tls readOnly: true - mountPath: /etc/virt-api/certificates diff --git a/tools/mounts/README.md b/tools/mounts/README.md new file mode 100644 index 0000000000..728514dae9 --- /dev/null +++ b/tools/mounts/README.md @@ -0,0 +1,3 @@ +# Mount primitives + +This dir contains empty dir and empty file to use as mountpoints in the images. diff --git a/tools/mounts/mountdir/.gitkeep b/tools/mounts/mountdir/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/mounts/mountfile b/tools/mounts/mountfile new file mode 100644 index 0000000000..e69de29bb2 From 8d9d3d1e2152a59f19fbe1bad5d4a6f1fa0deca0 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Thu, 4 Sep 2025 19:39:56 +0300 Subject: [PATCH 2/2] chore(module): rename containers to support integrity checks 1. Rename containers of kubevirt Pods, cdi Pods and dvcr Pods created in non-system namespaces (namespaces without d8- prefix). 2. Mount container-disk binary into /var/run in container with user uploaded image (support attaching cvi, vi to vm). Related PRs: https://github.com/deckhouse/3p-kubevirt/pull/19 https://github.com/deckhouse/3p-containerized-data-importer/pull/17 Signed-off-by: Ivan Mikheykin --- .werf/defines/images.tmpl | 2 +- build/components/versions.yml | 4 ++-- .../pkg/audit/events/vm/vm_control.go | 3 ++- .../pkg/audit/events/vm/vm_control_test.go | 4 ++-- images/virtualization-artifact/pkg/common/consts.go | 6 +++--- images/virtualization-artifact/pkg/common/vm/vm.go | 10 ++++++++++ .../pkg/controller/powerstate/shutdown_reason.go | 10 ++++------ .../pkg/controller/vm/internal/statistic.go | 2 +- .../pkg/controller/vm/internal/statistic_test.go | 2 +- 9 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.werf/defines/images.tmpl b/.werf/defines/images.tmpl index 86f19cf68b..51152c5e52 100644 --- a/.werf/defines/images.tmpl +++ b/.werf/defines/images.tmpl @@ -46,4 +46,4 @@ Result: {{- end }} {{- end -}} {{- end }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/build/components/versions.yml b/build/components/versions.yml index e26ae5392c..887ed72f3d 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,8 +3,8 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.3.1-v12n.11 - 3p-containerized-data-importer: v1.60.3-v12n.9 + 3p-kubevirt: v1.3.1-v12n.12 + 3p-containerized-data-importer: v1.60.3-v12n.10 distribution: 2.8.3 package: acl: v2.3.1 diff --git a/images/virtualization-artifact/pkg/audit/events/vm/vm_control.go b/images/virtualization-artifact/pkg/audit/events/vm/vm_control.go index a58c3fe818..4f5ef8abe6 100644 --- a/images/virtualization-artifact/pkg/audit/events/vm/vm_control.go +++ b/images/virtualization-artifact/pkg/audit/events/vm/vm_control.go @@ -25,6 +25,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/virtualization-controller/pkg/audit/events" "github.com/deckhouse/virtualization-controller/pkg/audit/util" + vmutil "github.com/deckhouse/virtualization-controller/pkg/common/vm" ) func NewVMControl(options events.EventLoggerOptions) *VMControl { @@ -73,7 +74,7 @@ func (m *VMControl) Fill() error { var terminatedStatuses string for _, status := range pod.Status.ContainerStatuses { - if status.Name == "compute" && status.State.Terminated != nil { + if vmutil.IsComputeContainer(status.Name) && status.State.Terminated != nil { terminatedStatuses = status.State.Terminated.Message } } diff --git a/images/virtualization-artifact/pkg/audit/events/vm/vm_control_test.go b/images/virtualization-artifact/pkg/audit/events/vm/vm_control_test.go index 87e81d9744..8c41e18abf 100644 --- a/images/virtualization-artifact/pkg/audit/events/vm/vm_control_test.go +++ b/images/virtualization-artifact/pkg/audit/events/vm/vm_control_test.go @@ -86,7 +86,7 @@ var _ = Describe("VMOP Events", func() { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "compute", + Name: "d8v-compute", Image: "test-image", }, }, @@ -95,7 +95,7 @@ var _ = Describe("VMOP Events", func() { Status: corev1.PodStatus{ ContainerStatuses: []corev1.ContainerStatus{ { - Name: "compute", + Name: "d8v-compute", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: "guest-shutdown"}}, }, }, diff --git a/images/virtualization-artifact/pkg/common/consts.go b/images/virtualization-artifact/pkg/common/consts.go index 8aedc71927..1a78a19bf3 100644 --- a/images/virtualization-artifact/pkg/common/consts.go +++ b/images/virtualization-artifact/pkg/common/consts.go @@ -23,11 +23,11 @@ const ( OwnerUID = "OWNER_UID" // BounderContainerName provides a constant to use as a name for bounder Container - BounderContainerName = "bounder" + BounderContainerName = "d8v-dvcr-bounder" // ImporterContainerName provides a constant to use as a name for importer Container - ImporterContainerName = "importer" + ImporterContainerName = "d8v-dvcr-importer" // UploaderContainerName provides a constant to use as a name for uploader Container - UploaderContainerName = "uploader" + UploaderContainerName = "d8v-dvcr-uploader" // UploaderPortName provides a constant to use as a port name for uploader Service UploaderPortName = "uploader" // UploaderPort provides a constant to use as a port for uploader Service diff --git a/images/virtualization-artifact/pkg/common/vm/vm.go b/images/virtualization-artifact/pkg/common/vm/vm.go index 4eeca7b6ce..d246617be7 100644 --- a/images/virtualization-artifact/pkg/common/vm/vm.go +++ b/images/virtualization-artifact/pkg/common/vm/vm.go @@ -17,9 +17,15 @@ limitations under the License. package vm import ( + "strings" + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" ) +// VMContainerNameSuffix - a name suffix for container with virt-launcher, libvirt and qemu processes. +// Container name is "d8v-compute", but previous versions may have "compute" container. +const VMContainerNameSuffix = "compute" + // CalculateCoresAndSockets calculates the number of sockets and cores per socket needed to achieve // the desired total number of CPU cores. // The function tries to minimize the number of sockets while ensuring the desired core count. @@ -59,3 +65,7 @@ func ApprovalMode(vm *virtv2.VirtualMachine) virtv2.RestartApprovalMode { } return vm.Spec.Disruptions.RestartApprovalMode } + +func IsComputeContainer(name string) bool { + return strings.HasSuffix(name, VMContainerNameSuffix) +} diff --git a/images/virtualization-artifact/pkg/controller/powerstate/shutdown_reason.go b/images/virtualization-artifact/pkg/controller/powerstate/shutdown_reason.go index a06fec4d32..cbc49ae3bc 100644 --- a/images/virtualization-artifact/pkg/controller/powerstate/shutdown_reason.go +++ b/images/virtualization-artifact/pkg/controller/powerstate/shutdown_reason.go @@ -22,14 +22,13 @@ import ( corev1 "k8s.io/api/core/v1" kvv1 "kubevirt.io/api/core/v1" + + vmutil "github.com/deckhouse/virtualization-controller/pkg/common/vm" ) type GuestSignalReason string const ( - // DefaultVMContainerName - a container name with virt-launcher, libvirt and qemu processes. - DefaultVMContainerName = "compute" - // GuestResetReason - a reboot command was issued from inside the VM. GuestResetReason GuestSignalReason = "guest-reset" @@ -65,10 +64,9 @@ func ShutdownReason(kvvmi *kvv1.VirtualMachineInstance, kvPods *corev1.PodList) return ShutdownInfo{} } - // Extract termination mesage from the "compute" container. + // Extract termination message from the container with VM. for _, contStatus := range recentPod.Status.ContainerStatuses { - // "compute" is a default container name for VM Pod. - if contStatus.Name != DefaultVMContainerName { + if !vmutil.IsComputeContainer(contStatus.Name) { continue } msg := "" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go b/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go index 4c4a4e672b..15d0e293ef 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go @@ -125,7 +125,7 @@ func (h *StatisticHandler) syncResources(changed *virtv2.VirtualMachine, } var ctr corev1.Container for _, container := range pod.Spec.Containers { - if container.Name == "compute" { + if vm.IsComputeContainer(container.Name) { ctr = container } } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go index 05a0340518..a68885634a 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go @@ -89,7 +89,7 @@ var _ = Describe("TestStatisticHandler", func() { NodeName: nodeName, Containers: []corev1.Container{ { - Name: "compute", + Name: "d8v-compute", Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(requestCPU),