From 9f25c3e989229445eb1f4089519052b06138c95f Mon Sep 17 00:00:00 2001 From: vimiix Date: Mon, 6 Jul 2026 17:55:21 +0800 Subject: [PATCH 1/9] feat(kubernetes): add v0.5.0 Helm chart for CubeSandbox Add a Helm chart under deploy/kubernetes/ so CubeSandbox v0.5.0 can be delivered on Kubernetes/TKE from an internal registry. deploy/kubernetes/chart/ - Helm chart pinned to appVersion 0.5.0. - Control-plane workloads: cube-master, cube-api, cubemastercli, cube-proxy-node, cube-webui, plus optional built-in mysql and redis StatefulSets. - Compute-side workloads: cube-node DaemonSet with cube-egress sidecar, cube-egress-net, cube-node-init, cube-pvm-host-bootstrap init containers, and node-local cube-dns. - Preflight validation (templates/validate.yaml) and Helm test hooks (templates/tests/) covering node-health and RBAC. - All Cube-owned images.*.repository default to ccr.ccs.tencentyun.com/cubesandbox-chart/*, tag v0.5.0. deploy/kubernetes/images/ - Role-specific Dockerfiles for cube-node, cube-node-init, cube-pvm-host-bootstrap, cube-webui, cube-egress-net, and cubemastercli. - build-cube-images.sh builds and optionally pushes all ten Cube images. cube-master, cube-api, cube-proxy-node, and cube-egress (plus its openresty tproxy base) are built from the in-tree CubeMaster/CubeAPI/CubeProxy/CubeEgress source. cube-node, cube-master, and cubemastercli additionally consume binaries extracted from the v0.5.0 one-click release bundle on SourceForge. - Downloads honor the v0.5.0 arch-suffixed layout via a new ONE_CLICK_ARCH knob (default amd64), which drives both the tarball URL and the extracted top-level directory name. - The curl invocation gates --retry-all-errors on runtime feature detection so the script also works on distributions shipping curl < 7.71. Default registry / tag: - REGISTRY=ccr.ccs.tencentyun.com/cubesandbox-chart - IMAGE_TAG=v0.5.0 Verified: - helm lint passes. - helm template with representative overrides renders every Cube-owned image as ccr.ccs.tencentyun.com/cubesandbox-chart/:v0.5.0. - All ten images built and pushed successfully with the default REGISTRY and IMAGE_TAG. --- deploy/kubernetes/chart/Chart.yaml | 14 + deploy/kubernetes/chart/README.md | 496 ++++++++++++++ deploy/kubernetes/chart/docs/ARCHITECTURE.md | 488 +++++++++++++ .../chart/files/cube-master/conf.yaml | 105 +++ deploy/kubernetes/chart/templates/NOTES.txt | 64 ++ .../kubernetes/chart/templates/_helpers.tpl | 240 +++++++ deploy/kubernetes/chart/templates/api.yaml | 140 ++++ .../chart/templates/cubemastercli.yaml | 86 +++ .../chart/templates/diagnostics.yaml | 61 ++ deploy/kubernetes/chart/templates/dns.yaml | 219 ++++++ .../kubernetes/chart/templates/egress-ca.yaml | 58 ++ .../chart/templates/master-config-secret.yaml | 14 + .../chart/templates/master-pvc.yaml | 18 + deploy/kubernetes/chart/templates/master.yaml | 161 +++++ deploy/kubernetes/chart/templates/mysql.yaml | 131 ++++ .../chart/templates/node-daemonset.yaml | 481 +++++++++++++ .../chart/templates/proxy-node.yaml | 310 +++++++++ deploy/kubernetes/chart/templates/rbac.yaml | 30 + deploy/kubernetes/chart/templates/redis.yaml | 121 ++++ deploy/kubernetes/chart/templates/secret.yaml | 13 + .../chart/templates/serviceaccount.yaml | 11 + .../chart/templates/storageclass.yaml | 15 + .../chart/templates/tests/node-health.yaml | 405 +++++++++++ .../chart/templates/tests/rbac.yaml | 40 ++ .../kubernetes/chart/templates/validate.yaml | 119 ++++ deploy/kubernetes/chart/templates/webui.yaml | 155 +++++ deploy/kubernetes/chart/values.yaml | 646 ++++++++++++++++++ deploy/kubernetes/images/.gitignore | 1 + deploy/kubernetes/images/README.md | 72 ++ deploy/kubernetes/images/build-cube-images.sh | 434 ++++++++++++ .../images/cube-egress-net/Dockerfile | 22 + .../images/cube-node-init/Dockerfile | 22 + deploy/kubernetes/images/cube-node/Dockerfile | 56 ++ .../images/cube-pvm-host-bootstrap/Dockerfile | 27 + .../kubernetes/images/cube-webui/Dockerfile | 8 + .../images/cubemastercli/Dockerfile | 52 ++ .../scripts/cube-egress-net-entrypoint.sh | 49 ++ .../images/scripts/cube-node-entrypoint.sh | 247 +++++++ .../images/scripts/cube-node-init.sh | 326 +++++++++ .../images/scripts/pvm-host-bootstrap.sh | 205 ++++++ 40 files changed, 6162 insertions(+) create mode 100644 deploy/kubernetes/chart/Chart.yaml create mode 100644 deploy/kubernetes/chart/README.md create mode 100644 deploy/kubernetes/chart/docs/ARCHITECTURE.md create mode 100644 deploy/kubernetes/chart/files/cube-master/conf.yaml create mode 100644 deploy/kubernetes/chart/templates/NOTES.txt create mode 100644 deploy/kubernetes/chart/templates/_helpers.tpl create mode 100644 deploy/kubernetes/chart/templates/api.yaml create mode 100644 deploy/kubernetes/chart/templates/cubemastercli.yaml create mode 100644 deploy/kubernetes/chart/templates/diagnostics.yaml create mode 100644 deploy/kubernetes/chart/templates/dns.yaml create mode 100644 deploy/kubernetes/chart/templates/egress-ca.yaml create mode 100644 deploy/kubernetes/chart/templates/master-config-secret.yaml create mode 100644 deploy/kubernetes/chart/templates/master-pvc.yaml create mode 100644 deploy/kubernetes/chart/templates/master.yaml create mode 100644 deploy/kubernetes/chart/templates/mysql.yaml create mode 100644 deploy/kubernetes/chart/templates/node-daemonset.yaml create mode 100644 deploy/kubernetes/chart/templates/proxy-node.yaml create mode 100644 deploy/kubernetes/chart/templates/rbac.yaml create mode 100644 deploy/kubernetes/chart/templates/redis.yaml create mode 100644 deploy/kubernetes/chart/templates/secret.yaml create mode 100644 deploy/kubernetes/chart/templates/serviceaccount.yaml create mode 100644 deploy/kubernetes/chart/templates/storageclass.yaml create mode 100644 deploy/kubernetes/chart/templates/tests/node-health.yaml create mode 100644 deploy/kubernetes/chart/templates/tests/rbac.yaml create mode 100644 deploy/kubernetes/chart/templates/validate.yaml create mode 100644 deploy/kubernetes/chart/templates/webui.yaml create mode 100644 deploy/kubernetes/chart/values.yaml create mode 100644 deploy/kubernetes/images/.gitignore create mode 100644 deploy/kubernetes/images/README.md create mode 100755 deploy/kubernetes/images/build-cube-images.sh create mode 100644 deploy/kubernetes/images/cube-egress-net/Dockerfile create mode 100644 deploy/kubernetes/images/cube-node-init/Dockerfile create mode 100644 deploy/kubernetes/images/cube-node/Dockerfile create mode 100644 deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile create mode 100644 deploy/kubernetes/images/cube-webui/Dockerfile create mode 100644 deploy/kubernetes/images/cubemastercli/Dockerfile create mode 100755 deploy/kubernetes/images/scripts/cube-egress-net-entrypoint.sh create mode 100755 deploy/kubernetes/images/scripts/cube-node-entrypoint.sh create mode 100644 deploy/kubernetes/images/scripts/cube-node-init.sh create mode 100644 deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh diff --git a/deploy/kubernetes/chart/Chart.yaml b/deploy/kubernetes/chart/Chart.yaml new file mode 100644 index 000000000..bdd38f622 --- /dev/null +++ b/deploy/kubernetes/chart/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: cube +description: CubeSandbox on Kubernetes/TKE Helm chart with separate control-plane, node, proxy, and host bootstrap images. +type: application +version: 0.5.0 +appVersion: "0.5.0" +home: https://github.com/TencentCloud/CubeSandbox +keywords: + - cubesandbox + - pvm + - sandbox + - tke +maintainers: + - name: CubeSandbox maintainers diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md new file mode 100644 index 000000000..693dc5fbd --- /dev/null +++ b/deploy/kubernetes/chart/README.md @@ -0,0 +1,496 @@ +# CubeSandbox Kubernetes/TKE Helm Chart + +This chart delivers CubeSandbox on Kubernetes/TKE as chart-managed resources. +It follows the final Big Pod delivery design: + +- bootstrap images are separate from runtime images; +- `cube-node` is a DaemonSet Big Pod; +- control-plane and compute/data-plane scheduling are separated through `placement.controlPlane` and `placement.compute`; +- PVM host kernel installation and host reboot are handled by a dedicated Init Container; +- Cube Node host preparation is handled by a second Init Container; +- MySQL schema migration is handled by CubeMaster itself using embedded migrations; +- Cube Master, Cube API, cubemastercli, Cube Proxy Node, WebUI, Template Builder, and Cube Node use separate images. + +## Directory + +```text +deploy/kubernetes/chart/ + Chart.yaml + values.yaml + docs/ + ARCHITECTURE.md + templates/ +``` + +## Architecture + +整体组件关系、安装流程、节点启动流程、DNS/Proxy/Egress 数据流和 +external control plane / compute-only 模式见： + +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) + +## Image responsibilities + +| Image | Role | +|---|---| +| `cube-pvm-host-bootstrap` | Init Container only. Installs/configures PVM host kernel and may reboot the node. | +| `cube-node-init` | Init Container only. Loads KVM module, prepares host paths, validates `/dev/kvm` and XFS. | +| `cube-node` | Runtime container only. Runs `cubelet` and `network-agent`. | +| `cube-master` | Control-plane master only. Built from `CubeMaster/docker/Dockerfile`; runs `cubemaster`; schema migrations are embedded in the binary. | +| `cube-api` | HTTP API only. Runs `cube-api`. | +| `cubemastercli` | Operational CLI only. Packages the real `CubeMaster/bin/cubemastercli` binary for exec-based operations. | +| `cube-proxy-node` | Data-plane proxy. Reuses `CubeProxy/Dockerfile` and runs as a chart-managed control-plane Deployment when `cubeProxy.enabled=true`. | +| `cube-egress` | CubeEgress transparent outbound proxy. Reuses `CubeEgress/Dockerfile` and runs as a Cube Node sidecar when `cubeEgress.enabled=true`. | +| `cube-egress-net` | Host network rule helper for CubeEgress TPROXY/ip-rule/sysctl setup. | +| `cube-webui` | One-click WebUI static assets and OpenResty runtime. | +| template builder sidecar | Optional template builder with `dockerd`/BuildKit. Uses `docker:27-dind` by default. | + +## Node selection + +The chart separates placement into dedicated control-plane nodes and compute +nodes. Control-plane Deployments, StatefulSets, and `cube-proxy-node` use +`placement.controlPlane`; `cube-node` and node-local `cube-dns` use +`placement.compute`. + +The chart refuses to render host-mutating compute components without +`placement.compute.nodeSelector`. This prevents PVM bootstrap and Cube runtime +setup from running on ordinary nodes. The default compute selector includes +`cube.tencent.com/allow-pvm-bootstrap=true` because the default profile +initializes the PVM host kernel and may reboot selected compute nodes. + +Default placement values: + +```yaml +global: + timezone: Asia/Shanghai + +storageClass: + create: true + name: cube-cbs-wffc + provisioner: com.tencent.cloud.csi.cbs + volumeBindingMode: WaitForFirstConsumer + +placement: + controlPlane: + nodeSelector: + cube.tencent.com/role: control + cube.tencent.com/cube-control: "true" + tolerations: + - key: cube.tencent.com/control + operator: Equal + value: "true" + effect: NoSchedule + + compute: + nodeSelector: + cube.tencent.com/role: compute + cube.tencent.com/cube-node: "true" + cube.tencent.com/allow-pvm-bootstrap: "true" + tolerations: + - key: cube.tencent.com/compute + operator: Equal + value: "true" + effect: NoSchedule +``` + +Recommended labels: + +```bash +kubectl label node cube.tencent.com/role=control cube.tencent.com/cube-control=true --overwrite + +kubectl taint node cube.tencent.com/control=true:NoSchedule --overwrite + +kubectl label node cube.tencent.com/role=compute cube.tencent.com/cube-node=true cube.tencent.com/allow-pvm-bootstrap=true --overwrite + +kubectl taint node cube.tencent.com/compute=true:NoSchedule --overwrite +``` + +The chart does not label or taint nodes. The platform operator must prepare node labels and taints before installation. +All chart-managed Cube containers and init containers receive `TZ` from +`global.timezone`. + +## Cubelet data path + +`bootstrap.nodeInit.dataCubelet.loopback.enabled` defaults to `true` so the +chart can create and mount a loopback XFS image for `/data/cubelet` during +bootstrap. Production environments that pre-provision `/data/cubelet` as XFS +can set it to `false`. + +## Cube Node one-click parity + +`cube-node` mirrors the one-click runtime layout: + +- runtime tools are available through `/usr/local/bin/containerd-shim-cube-rs`, `/usr/local/bin/cube-runtime`, `/usr/local/bin/cubecli`, and `/usr/local/bin/cubevsmapdump`; +- `cubeNode.pvmGuestKernel.enabled` defaults to `true` and controls the one-click `CUBE_PVM_ENABLE` behavior, selecting `cube-kernel-scf/vmlinux -> vmlinux-pvm` or `vmlinux-bm`; +- `cubeNode.network.autoDetectEthName=true` auto-detects the primary host NIC and patches Cubelet `eth_name`; +- `cubeNode.network.cidr` can patch Cubelet cubevs CIDR when the packaged default conflicts with the host network. + +`bootstrap.pvmHostKernel.enabled` also defaults to `true`, so the PVM host +kernel bootstrap Init Container can install/configure the host kernel and +perform the configured coordinated reboot. The default +`bootstrap.pvmHostKernel.bootArgs` is `nopti pti=off` because the current +`kvm_pvm` module does not support host KPTI. `cube-node-init` performs the same +fail-fast style checks as one-click for memory, glibc, cgroup v2 cpu +controller, cubecow dependencies, KVM, XFS, and PVM consistency. It fails when +a host has `kvm_pvm` loaded but `cubeNode.pvmGuestKernel.enabled=false`, or +when `cubeNode.pvmGuestKernel.enabled=true` but the host has not booted a PVM +kernel with `kvm_pvm` loaded. + +## Build and push images + +```bash +PUSH=1 REGISTRY=ccr.ccs.tencentyun.com/cubesandbox-chart IMAGE_TAG=v0.5.0 ./deploy/kubernetes/images/build-cube-images.sh +``` + +Cube-owned images default to `imagePullPolicy: Always` because this chart uses the release tag directly and environments are expected to pull the pushed image from the registry during deployment. + +If the target registry requires authentication, create a Kubernetes +`kubernetes.io/dockerconfigjson` Secret in the release namespace and pass it to +the chart: + +```yaml +imagePullSecrets: + - name: +``` + +## Install + +```bash +helm upgrade --install cube ./deploy/kubernetes/chart -n cube-system --create-namespace -f --wait --timeout 90m +``` + +> Do not store SSH passwords or node login credentials in this chart. Host mutation is performed through Kubernetes privileged Pods, not through SSH. + +## Use third-party MySQL or Redis + +The chart installs `cube-mysql` StatefulSet only when `mysql.enabled=true` and `mysql.host` is empty. +Set `mysql.host` to use an existing MySQL service; the chart will not install `cube-mysql`. + +The chart installs `cube-redis` StatefulSet only when `redis.enabled=true` and `redis.host` is empty. +Set `redis.host` to use an existing Redis service; the chart will not install `cube-redis`. + +## CubeMaster configuration + +The `cube-master` image uses `CubeMaster/docker/Dockerfile` directly and does not carry a Kubernetes-specific entrypoint or bundled `conf.yaml`. +The chart stores the One-click `CubeMaster/conf.yaml` at `deploy/kubernetes/chart/files/cube-master/conf.yaml`, renders MySQL/Redis values into it, creates a release-scoped Secret named `-master-config`, and mounts it to `/usr/local/services/cubemaster/conf.yaml`; `CUBE_MASTER_CONFIG_PATH` points CubeMaster to that mounted file. + +CubeMaster artifact storage maps to `/data/CubeMaster/storage`, matching one-click. +The chart uses PVC-backed persistence by default so state can survive +rescheduling across dedicated control nodes: + +```yaml +controlPlane: + master: + persistence: + enabled: true + hostPath: "" + storageClassName: cube-cbs-wffc +mysql: + persistence: + enabled: true + hostPath: "" + storageClassName: cube-cbs-wffc +redis: + persistence: + enabled: true + hostPath: "" + storageClassName: cube-cbs-wffc +``` + +Set `storageClassName` / `size` to tune dynamic PVCs, or `existingClaim` to +bind pre-created volumes. Use `hostPath` only for single-node throwaway +environments; multi-control-node deployments must use PVCs or external +MySQL/Redis. + +Built-in MySQL and Redis use StatefulSet `volumeClaimTemplates` by default. +For a release named `cube`, the generated claims are owned by the StatefulSet +Pod, such as `mysql-data-cube-mysql-0` and `redis-data-cube-redis-0`. + +The default `cube-cbs-wffc` StorageClass uses `WaitForFirstConsumer`, which is +important on TKE multi-zone clusters: CBS disks are provisioned in the same zone +as the selected control node instead of being created in a random zone before +the Pod is scheduled. + +## Database migration + +The chart does not deliver a separate DB migration Job or image. CubeMaster owns MySQL schema migration and runs its embedded `CubeMaster/pkg/base/dao/migrate/migrations/mysql` migrations during startup. + +- CubeMaster uses the configured MySQL endpoint, user, password, and database. +- The chart does not package or maintain SQL files under `files/`; do not add migration SQL copies to the chart. +- CubeMaster records applied versions in `goose_db_version` and serializes concurrent migration attempts through the migration lock implemented by CubeMaster. +- There is no chart-managed SQL data seed, and the one-click single-node seed file `sql/002_seed_single_node.sql` is intentionally not rendered by the chart. Node registration must come from real Cube Node Pods selected by `placement.compute.nodeSelector`. +- When using third-party MySQL, set `mysql.host` and ensure the configured MySQL user can create/alter tables in `mysql.database`. + +## cubemastercli operational CLI + +`cubemastercli.enabled=true` installs a chart-managed +`-cubemastercli` Deployment. The image contains the real +`CubeMaster/bin/cubemastercli` binary only; it does not provide a wrapper or +fake `ctl` command. + +The chart injects `CUBEMASTERCLI_ADDRESS` and `CUBEMASTERCLI_PORT` from the +current CubeMaster endpoint. Because upstream `cubemastercli` does not read +environment variables as flag defaults, commands should pass those values to +the real binary: + +```bash +kubectl exec -n cube-system deploy/cube-cubemastercli -- cubemastercli --help +kubectl exec -n cube-system deploy/cube-cubemastercli -- \ + sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" node list' +kubectl exec -n cube-system deploy/cube-cubemastercli -- \ + sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" template list' +``` + +The `cubemastercli` image is intentionally independent from `cube-master` and +`cube-node`. It contains CLI/operator tooling only; the runtime images do not +carry this operational entry point. + +## Cube Proxy Node + +`cube-proxy-node` is a Cube data-plane component. It is enabled by default to match one-click behavior and is installed, upgraded, and uninstalled with the Cube release as a control-plane Deployment. + +The default TLS mode is `selfSigned`, matching the one-click mkcert-style test experience. Production environments should provide a real TLS certificate and reserve node host ports 80/443 on selected nodes. The image reuses `CubeProxy/Dockerfile`; the chart does not override nginx with a Kubernetes-only configuration. + +`cube-proxy-node` also starts the built-in `cube-proxy-sidecar`. The chart wires the sidecar to the chart-managed or third-party Redis endpoint and to the CubeMaster Kubernetes Service. Do not run a separate unmanaged CubeProxy sidecar. + +### Production TLS Secret + +```yaml +cubeProxy: + enabled: true + domain: sandbox.example.com + tls: + mode: existingSecret + existingSecret: cube-proxy-certs + certSecretKey: tls.crt + keySecretKey: tls.key +``` + +The Secret keys are mounted to the file names required by the `CubeProxy` image: + +- `cube.app+3.pem` +- `cube.app+3-key.pem` + +The certificate SAN should cover the sandbox domain used by CubeAPI, typically: + +```text +sandbox.example.com +*.sandbox.example.com +``` + +Keep `controlPlane.api.sandboxDomain` and `cubeProxy.domain` consistent. Configure DNS so the domain and wildcard subdomains resolve to the CubeProxy entrypoint. + +### cert-manager TLS + +When cert-manager is installed in the cluster, let the chart create a `Certificate`: + +```yaml +controlPlane: + api: + sandboxDomain: sandbox.example.com +cubeProxy: + enabled: true + domain: sandbox.example.com + tls: + mode: certManager + certManager: + issuerRef: + kind: ClusterIssuer + name: letsencrypt-prod + dnsNames: + - sandbox.example.com + - "*.sandbox.example.com" +``` + +Wildcard public certificates usually require a DNS-01 issuer. + +### Self-signed TLS for test only + +For offline test environments, explicitly opt in to a chart-generated self-signed certificate: + +```yaml +cubeProxy: + enabled: true + domain: cube.app + tls: + mode: selfSigned + selfSigned: + dnsNames: + - cube.app + - "*.cube.app" + - localhost + ipAddresses: + - 127.0.0.1 +``` + +This mode creates a release-scoped Secret with `tls.crt`, `tls.key`, and `ca.crt`. Import `ca.crt` into clients if browser or SDK trust is required. Do not use this mode for production. + +`cube-proxy-node` uses `placement.controlPlane`, matching the one-click control-node placement for CubeProxy. The chart does not create node labels. + +`cubeProxy.hostNetwork=true` is also the default. This is required for one-click +parity: CubeProxy must terminate `cube.app` / wildcard traffic on a node-local +host-network endpoint. When the sandbox owner is on a compute node, CubeProxy +uses Redis routing metadata to connect to the owner `HostIP:hostPort`. The +chart patches the image's default nginx listeners to +the configured `cubeProxy.ports.*.containerPort` values, which default to `80` +and `443`. + +The chart does not create a `cube-proxy-node` ClusterIP Service. A normal +Kubernetes Service load-balances requests across proxy Pods, which does not +match the one-click model where callers reach an explicit CubeProxy host +endpoint. For sandbox data-plane traffic, point wildcard DNS at a specific +control-node CubeProxy IP or at an external load balancer that preserves the intended +CubeProxy topology. + +CubeProxy admin health remains loopback-only inside each Pod, matching the image's nginx admin listener. The chart validates it through Pod readiness/liveness probes rather than exposing it through a Service. + +CubeProxy reads sandbox routing metadata from Redis in nginx Lua. Because nginx +does not automatically inherit Kubernetes DNS resolution for Lua cosocket +connections, the chart renders an nginx `resolver` into +`/usr/local/openresty/nginx/conf/global/global.conf`. By default the proxy +discovers resolver addresses from the Pod `/etc/resolv.conf`, which resolves the +chart-managed `cube-redis..svc.cluster.local` Service name and +third-party Redis DNS names. Override only when the cluster requires explicit +DNS servers: + +```yaml +cubeProxy: + resolver: + addresses: + - 172.18.0.10 + valid: 30s + timeout: 5s + ipv6: false +``` + +## Cube DNS + +`cubeDns.enabled=true` delivers CoreDNS for one-click style sandbox domain +resolution. The default `cubeDns.mode=nodeLocal` runs `cube-dns` as a +hostNetwork DaemonSet on Cube compute nodes selected by `placement.compute` +and listens on `127.0.0.54:53`. +`cube-node` Pods use `dnsPolicy: None` and explicitly set +`dnsConfig.nameservers: [127.0.0.54]`, so `cube.app` and wildcard domains are +resolved inside the Big Pod without modifying host-wide DNS. + +Sandbox guest DNS is configured separately from the `cube-node` Pod DNS. The +guest cannot use `127.0.0.54` because that address is the guest loopback inside +the sandbox. By default `cubeDns.sandboxGateway.enabled=true` also binds +node-local `cube-dns` on the compute node HostIP, and +`cubeNode.dns.sandbox.useCubeDns=true` injects that HostIP into Cubelet +`default_dns_servers`. + +CubeVS eBPF egress does not traverse host kube-proxy ClusterIP DNAT, so guests +should not rely on Kubernetes Service ClusterIPs as if they were ordinary +external addresses. The chart no longer renders sandbox service proxy resources +or DNS overrides for them; expose required in-cluster services through the +platform networking layer, AgentWay provider configuration, or an +operator-managed proxy outside this chart. + +Set `cubeProxy.advertiseIP` or `cubeDns.answerIP` to return the control-node +CubeProxy entrypoint. If both are empty in node-local mode, `cube-dns` falls +back to the current compute HostIP for compatibility with older local-proxy +topologies; that fallback is not suitable when CubeProxy runs only on control +nodes. Optional `cubeDns.mode=service` keeps the older ClusterIP DNS model, +where callers must explicitly point their DNS policy or upstream DNS to the +`cube-dns` Service. In service mode, set `cubeDns.answerIP` or +`cubeProxy.advertiseIP` to an explicit CubeProxy entrypoint. + +The chart does not silently rewrite Kubernetes nodes' host DNS settings. +For external clients, browsers, SDKs, or any Pod that is not explicitly using this `dnsConfig`, configure DNS/LB/Ingress outside the chart so `cubeProxy.domain` and wildcard subdomains resolve to an explicit control-node CubeProxy endpoint or external load balancer. + +## WebUI + +`webui.enabled=true` delivers the one-click WebUI by default: + +- `cube-webui` image packages one-click `webui/dist` static assets; +- a chart-rendered nginx config proxies `/cubeapi/` to the CubeAPI Service; +- the Service listens on port `12088`, matching one-click `WEB_UI_HOST_PORT`. + +Expose the WebUI externally by changing `webui.service.type` or by adding your platform's ingress/load balancer configuration. + +## Diagnostics + +One-click delivers `cube-diag` scripts on the host. The Kubernetes chart delivers the equivalent operational entry point as a ConfigMap when `diagnostics.enabled=true`: + +```bash +kubectl get configmap -n cube-system cube-diagnostics -o jsonpath='{.data.cube-diag-k8s\.sh}' > /tmp/cube-diag-k8s.sh +sh /tmp/cube-diag-k8s.sh cube-system cube +``` + +The script collects Pods, DaemonSets, Deployments, Services, Endpoints, Events, Helm values/manifests, Pod descriptions, and recent logs for Cube components into a timestamped directory. + +## CubeEgress + +`cubeEgress.enabled=true` runs CubeEgress inside the Cube Node Big Pod: + +- `cube-egress` mounts `/etc/cube/ca` and exposes the loopback admin API on `127.0.0.1:9090`; +- `cube-egress-net` waits for the `cube-dev` interface, applies the upstream `CubeEgress/scripts/cube-proxy-iptables-init.sh` rules, periodically reapplies them, and removes them on Pod termination; +- CubeMaster and CubeAPI both mount the same CA Secret at `/etc/cube/ca` so template CA bake and AgentHub/OpenClaw CA injection use the same trust root. + +Default CA mode is `selfSigned`; the chart creates and reuses a release-scoped Secret named `-egress-ca` with: + +```text +cube-root-ca.crt +cube-root-ca.key +placeholder.crt +placeholder.key +``` + +For production CA lifecycle control, pre-create a Secret and use: + +```yaml +cubeEgress: + enabled: true + ca: + mode: existingSecret + existingSecret: cube-egress-ca +``` + +Do not rotate the CubeEgress CA casually: templates baked with the old CA and sandboxes trusting the old CA must be considered during rotation. + +## Render and lint + +```bash +helm lint ./deploy/kubernetes/chart +helm template cube ./deploy/kubernetes/chart -n cube-system > /tmp/cube-rendered.yaml +``` + +## Verify + +```bash +kubectl get pods -n cube-system -o wide +kubectl get ds -n cube-system cube-node +kubectl get deploy -n cube-system cube-proxy-node +kubectl get sts -n cube-system cube-mysql cube-redis +kubectl logs -n cube-system -l app.kubernetes.io/component=cube-node -c pvm-host-bootstrap --tail=100 +kubectl logs -n cube-system -l app.kubernetes.io/component=cube-node -c cube-node-init --tail=100 +kubectl logs -n cube-system -l app.kubernetes.io/component=cube-node -c cube-node --tail=100 +kubectl logs -n cube-system deploy/cube-master -c cube-master --tail=100 +kubectl exec -n cube-system deploy/cube-cubemastercli -- \ + sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" node list' +helm test cube -n cube-system --timeout 20m +``` + +## Upgrade policy + +`cubeNode.updateStrategy.type` defaults to `RollingUpdate`. For production +maintenance windows, set it to `OnDelete` if you need to upgrade one compute +node at a time after draining or cleaning Cube sandboxes on that node. + +## Rollback warning + +Helm rollback only rolls back Kubernetes resources. It does not undo host kernel, GRUB, udev, fstab, or XFS changes made by the bootstrap Init Containers. +Prepare a separate host-kernel rollback runbook for production. + +## Uninstall cleanup + +`helm uninstall cube -n cube-system` removes chart-managed Kubernetes resources, including CubeProxy, CubeDNS, WebUI, CubeEgress, MySQL/Redis when they are chart-managed, and diagnostic ConfigMaps. It intentionally does not remove: + +- operator-provided node labels/taints; +- external MySQL/Redis resources; +- hostPath data such as `/data/CubeMaster/storage`, `/data/cubelet`, `/data/cube-shim`, `/data/snapshot_pack`, and logs; +- host kernel, GRUB, udev, fstab, or XFS changes made by bootstrap containers; +- external DNS or load balancer records. + +Clean those items using the platform runbook for the target environment. diff --git a/deploy/kubernetes/chart/docs/ARCHITECTURE.md b/deploy/kubernetes/chart/docs/ARCHITECTURE.md new file mode 100644 index 000000000..95d0b9a22 --- /dev/null +++ b/deploy/kubernetes/chart/docs/ARCHITECTURE.md @@ -0,0 +1,488 @@ +# CubeSandbox Chart 组件关系与运行流程 + +本文说明 `deploy/kubernetes/chart` 的整体架构、组件关系、安装流程和运行期关键链路，帮助交付、运维和后续开发快速理解 Chart 如何还原 One Click 包能力。 + +## 1. 总体分层 + +CubeSandbox Chart 按职责分为 7 层： + +| 层级 | 组件 | Kubernetes 形态 | 主要职责 | +| --- | --- | --- | --- | +| 控制面 | CubeMaster | Deployment + Service + Secret + PVC/hostPath | 节点注册、模板/rootfs artifact 管理、内置 DB migration、核心调度/元数据能力 | +| 控制面 API | CubeAPI | Deployment + Service | 对外 HTTP API，读写 MySQL，访问 CubeMaster | +| 管理入口 | WebUI | Deployment + Service + ConfigMap | 静态控制台，反向代理 `/cubeapi/` 到 CubeAPI | +| 运维入口 | cubemastercli | Deployment | 面向 `kubectl exec` 的 CLI Pod，交付真实 `cubemastercli` 并注入本 Release 的 CubeMaster endpoint | +| 依赖存储 | MySQL / Redis | 内置 StatefulSet + Headless Service + volumeClaimTemplates/hostPath，或第三方服务 | MySQL 存储业务数据；Redis 存储 CubeProxy/Sidecar 状态 | +| 计算面 | Cube Node Big Pod | DaemonSet | 节点初始化、运行 cubelet/network-agent、透明 egress sidecar | +| 数据面入口 | CubeProxy / CubeDNS | CubeProxy Deployment；CubeDNS DaemonSet 或 Deployment | HTTP/HTTPS sandbox 入口；sandbox 域名解析 | + +默认完整部署形态： + +```mermaid +flowchart TB + subgraph CP["Control Plane selected by placement.controlPlane"] + CM["cube-master Deployment\ncubemaster:8089"] + API["cube-api Deployment\nHTTP API:3000"] + WEB["cube-webui Deployment\nWebUI:12088"] + CLI["cubemastercli Deployment\nreal cubemastercli binary"] + MYSQL[("cube-mysql\nMySQL 8.0 + PVC")] + REDIS[("cube-redis\nRedis 7-alpine + PVC")] + PROXY["cube-proxy-node Deployment\nHTTP/HTTPS 80/443"] + CMS["cube-master-config Secret\nconf.yaml"] + CA["cube-egress-ca Secret"] + CERT["cube-proxy-certs Secret"] + end + + subgraph COMPUTE["Compute Nodes selected by placement.compute"] + DNS["cube-dns DaemonSet\nCoreDNS 127.0.0.54:53"] + subgraph NODE["cube-node DaemonSet Big Pod"] + WAITDNS["init: wait-cube-dns"] + PVM["init: pvm-host-bootstrap\noptional"] + INIT["init: cube-node-init"] + CUBELET["container: cube-node\ncubelet + network-agent"] + EG["sidecar: cube-egress\nOpenResty transparent proxy"] + EGNET["sidecar: cube-egress-net\nTPROXY/ip-rule/sysctl"] + end + end + + WEB --> API + CLI --> CM + API --> CM + API --> MYSQL + CM --> MYSQL + CM --> REDIS + CM --> CMS + CM --> CA + API --> CA + PROXY --> REDIS + PROXY --> CM + PROXY --> CERT + DNS --> PROXY + WAITDNS --> DNS + INIT --> CM + CUBELET --> CM + EG --> CA + EG --> CUBELET + EGNET --> EG +``` + +## 2. 资源与镜像职责 + +### 2.1 控制面 + +| 资源 | 模板 | 镜像 / 数据 | 说明 | +| --- | --- | --- | --- | +| `cube-master` | `templates/master.yaml` | `images.master` | 复用 `CubeMaster/docker/Dockerfile`；启动时挂载 Chart 渲染的 `conf.yaml`；数据库迁移由 CubeMaster 内置逻辑完成 | +| `cube-master-config` | `templates/master-config-secret.yaml` | `deploy/kubernetes/chart/files/cube-master/conf.yaml` 渲染结果 | 注入 MySQL / Redis / CA 等运行配置 | +| `cube-master-storage` | `templates/master.yaml` / `templates/master-pvc.yaml` | PVC / existingClaim / hostPath / emptyDir | 对应 One Click `/data/CubeMaster/storage` artifact 目录；默认使用 PVC，避免多 control 节点场景下 hostPath 数据绑定单节点 | +| `cube-api` | `templates/api.yaml` | `images.api` | 暴露 HTTP API，连接 CubeMaster 和 MySQL | +| `cubemastercli` | `templates/cubemastercli.yaml` | `images.cubemastercli` | 运维 CLI Pod；只内置真实 `CubeMaster/bin/cubemastercli`，Chart 注入本 Release 的 CubeMaster endpoint | +| `cube-webui` | `templates/webui.yaml` | `images.webui` + nginx ConfigMap | 提供控制台入口，`/cubeapi/` 代理到 CubeAPI | +| `cube-secret` | `templates/secret.yaml` | MySQL / Redis / Proxy 密码 | Chart 管理内置依赖和组件间连接密码 | + +### 2.2 内置或第三方 MySQL / Redis + +| 模式 | 行为 | +| --- | --- | +| 内置 MySQL | `mysql.host=""` 时安装 `cube-mysql` StatefulSet / Headless Service / volumeClaimTemplates，显式配置 `mysql.persistence.hostPath` 时才使用 hostPath | +| 第三方 MySQL | `mysql.host` 非空时不安装内置 MySQL，CubeMaster / CubeAPI 使用外部地址 | +| 内置 Redis | `redis.host=""` 且控制面或 CubeProxy 需要 Redis 时安装 `cube-redis` StatefulSet / Headless Service / volumeClaimTemplates | +| 第三方 Redis | `redis.host` 非空时不安装内置 Redis，CubeProxy / CubeMaster 使用外部地址 | + +### 2.3 计算面 Big Pod + +`cube-node` DaemonSet 是计算节点上的 Big Pod。它只调度到 +`placement.compute.nodeSelector` 命中的节点，Chart 不负责给节点打 label。 + +| 容器 | 类型 | 镜像 | 职责 | +| --- | --- | --- | --- | +| `wait-cube-dns` | Init Container | `cubeNode.dns.checkImage` | 等待 node-local `cube-dns` 可用，并验证 `cube.app`、wildcard、Kubernetes Service 域名解析 | +| `pvm-host-bootstrap` | Init Container，可选 | `images.pvmHostBootstrap` | 安装/配置 PVM host kernel，必要时协调节点重启 | +| `cube-node-init` | Init Container | `images.nodeInit` | 节点预检和准备：KVM、XFS、内存、glibc、cgroup、cubecow 依赖、CIDR 冲突、CubeMaster 连通性 | +| `cube-node` | 主容器 | `images.node` | 运行 cubelet 和 network-agent；选择 guest kernel；向 CubeMaster 注册节点 | +| `cube-egress` | Sidecar | `images.cubeEgress` | 透明出站代理，提供 loopback admin health | +| `cube-egress-net` | Sidecar | `images.cubeEgressNet` | 管理 host network namespace 中的 TPROXY、ip rule、sysctl 规则 | + +关键 hostPath： + +| hostPath | 用途 | +| --- | --- | +| `/data/cubelet` | cubelet 数据和 `cubelet.sock` | +| `/tmp/cube` | network-agent gRPC socket | +| `/data/cube-shim` | cube runtime/shim 运行数据 | +| `/data/snapshot_pack` | snapshot pack 数据 | +| `/data/log` | Cube Node / shim / egress 日志 | +| `/dev`、`/sys`、`/lib/modules` | KVM、内核模块、网络和系统能力 | + +### 2.4 数据面入口 + +| 资源 | 模板 | 职责 | +| --- | --- | --- | +| `cube-proxy-node` Deployment | `templates/proxy-node.yaml` | 提供 sandbox HTTP/HTTPS 数据面入口，使用 `placement.controlPlane` 与 one-click control 节点语义对齐，并使用 `hostNetwork` 监听节点 `80/443` | +| `cube-proxy-certs` Secret / Certificate | `templates/proxy-node.yaml` | TLS 证书，支持 selfSigned、inline、existingSecret、certManager | +| `cube-dns` | `templates/dns.yaml` | 提供 sandbox 域名解析；默认 node-local | + +CubeProxy 默认保持 One Click 的 host-network 语义： + +- `cubeProxy.hostNetwork=true`，Pod IP 等于所在节点 HostIP。 +- `cube-proxy-node` 复用 `placement.controlPlane`，与 one-click control 节点上的 CubeProxy 对齐。 +- `cube-dns` 复用 `placement.compute`，与 `cube-node` 保持同一组计算节点。 +- nginx 监听节点 `80/443`，Chart 启动脚本会把镜像默认 `8081/8080` patch 为 values 中配置的端口。 +- node-local `cube-dns` 应通过 `cubeProxy.advertiseIP` 或 `cubeDns.answerIP` 返回 control 节点 CubeProxy 入口。 +- CubeProxy 通过 Redis 中的 owner `HostIP:hostPort` 元数据转发到目标 compute 节点 sandbox。 +- nginx `global.conf` 中写入 `resolver`，Lua Redis 客户端可以解析内置或第三方 Redis DNS 名称。 +- Chart 不修改 CubeProxy Lua 后端解析语义；跨节点访问仍遵循社区 CubeProxy 使用 Redis `HostIP:hostPort` 的原始路径。 + +## 3. 默认 DNS 架构 + +默认 `cubeDns.mode=nodeLocal`： + +```mermaid +sequenceDiagram + participant CN as cube-node Pod + participant DNS as cube-dns on same node + participant KDNS as Kubernetes DNS + participant PX as cube-proxy-node on control node + + CN->>DNS: resolve cube.app / *.cube.app via 127.0.0.54 + DNS-->>CN: A record = cubeProxy.advertiseIP / cubeDns.answerIP + CN->>DNS: resolve kubernetes.default.svc.cluster.local + DNS->>KDNS: forward cluster/service DNS query + KDNS-->>DNS: ClusterIP answer + DNS-->>CN: ClusterIP answer + CN->>PX: sandbox HTTP/HTTPS traffic +``` + +关键点： + +- `cube-dns` 以 DaemonSet + `hostNetwork` 运行在计算节点。 +- `cube-dns` 监听 `127.0.0.54:53`；启用 `cubeDns.sandboxGateway.enabled` + 时也监听当前 compute 节点 HostIP，供 sandbox guest 作为 nameserver。 +- `cube-proxy-node` 以 Deployment + `hostNetwork` 运行在 control 节点，监听节点 `80/443`。 +- `cube-node` 使用 `dnsPolicy: None`，显式配置 `nameserver 127.0.0.54`。 +- `cube.app` / `*.cube.app` 优先解析为 `cubeDns.answerIP`，其次为 `cubeProxy.advertiseIP`；两者为空时才回退到当前 compute HostIP。 +- 其他域名转发到 `cubeDns.forward.upstreams`，为空时使用 `/etc/resolv.conf`。 +- Chart 不修改宿主机全局 DNS，不影响非 Cube Pod。 +- Cube sandbox guest DNS 由 `cubeNode.dns.sandbox` 单独写入 Cubelet dynamicconf。 +- 默认写入当前 compute 节点 HostIP，让 guest 使用 node-local `cube-dns`。 +- sandbox 访问 Kubernetes Service ClusterIP 可能绕过宿主机 kube-proxy DNAT；Chart 不再渲染 sandbox service proxy 资源或对应 DNS override。需要暴露给 sandbox 的 in-cluster Service 应由平台网络层、AgentWay provider 或 operator 管理的外部代理处理。 +- 外部客户端、浏览器、SDK 或未显式使用该 `dnsConfig` 的 Pod，需要由使用方配置 DNS / 负载均衡 / Ingress，把 `cubeProxy.domain` 与 wildcard 子域名指向 CubeProxy 入口。 + +可选 `cubeDns.mode=service`： + +- `cube-dns` 以 Deployment + ClusterIP Service 运行。 +- `cube.app` / wildcard 必须通过 `cubeDns.answerIP` 或 `cubeProxy.advertiseIP` 返回明确的 CubeProxy 入口。 +- 使用方需要自行把客户端 DNS 或上游 DNS 指向该 Service。 + +## 4. 安装与启动流程 + +### 4.1 Helm 渲染与校验 + +```mermaid +flowchart TD + A["helm upgrade/install"] --> B["templates/validate.yaml"] + B --> C{"values 组合是否合法?"} + C -- 否 --> X["fail render"] + C -- 是 --> D["渲染 Secret / ConfigMap / 持久化卷"] + D --> E["渲染 MySQL / Redis 或使用第三方服务"] + E --> F["渲染控制面 Deployment"] + F --> G["渲染 cube-dns / cube-proxy-node"] + G --> H["渲染 cube-node DaemonSet"] + H --> I["等待 --wait / rollout / helm test"] +``` + +主要 validate 规则： + +- `controlPlane.enabled=true` 时必须配置 `placement.controlPlane.nodeSelector`。 +- `cubeNode.enabled=true` 时必须配置 `placement.compute.nodeSelector`。 +- `cubeProxy.enabled=true` 时必须配置 `placement.controlPlane.nodeSelector`。 +- `cubeDns.enabled=true` 时必须配置 `placement.compute.nodeSelector`。 +- compute-only 模式必须显式配置 `externalControlPlane.masterEndpoint`。 +- `cubeNode.dns.useCubeDns=true` 时要求 `cubeDns.enabled=true`、`cubeDns.mode=nodeLocal`、`security.hostNetwork=true`。 +- `cubeDns.mode` 只能为 `nodeLocal` 或 `service`。 +- PVM host kernel bootstrap 只能在明确命中 selector 的节点上执行。 + +### 4.1.1 调度与时区 + +- CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 使用 `placement.controlPlane`。 +- `cube-node`、`cube-dns` 使用 `placement.compute`。 +- 所有 Chart 管理的 Cube 容器、sidecar 和 initContainer 都通过 `global.timezone` 注入 `TZ`，默认 `Asia/Shanghai`。 + +### 4.2 控制面启动 + +```mermaid +sequenceDiagram + participant H as Helm + participant DB as MySQL + participant R as Redis + participant CM as CubeMaster + participant API as CubeAPI + participant WEB as WebUI + participant CLI as cubemastercli + + H->>DB: create/use MySQL + H->>R: create/use Redis + H->>CM: mount conf.yaml + storage + CA + CM->>DB: run embedded schema migration + CM-->>H: /notify/health ready + H->>API: start with CUBE_MASTER_ENDPOINT + MySQL config + API->>CM: call CubeMaster + API->>DB: read/write business data + API-->>H: /health ready + H->>WEB: render nginx upstream to CubeAPI + WEB->>API: proxy /cubeapi/ + H->>CLI: inject CUBEMASTERCLI_ADDRESS / CUBEMASTERCLI_PORT + CLI->>CM: cubemastercli --address ... --port ... node list +``` + +说明： + +- Chart 不交付独立 `cube-db-migrate` Job。 +- `cubemastercli` 只通过独立 `cubemastercli` 运维镜像交付；不混入 `cube-master` 或 `cube-node` 运行镜像，不提供 `ctl` wrapper。 +- CubeMaster 内置 migration SQL，启动时自行迁移。 +- CubeMaster artifact storage 默认使用 PVC，可切换到 existingClaim；hostPath 仅适合单节点临时环境。 + +### 4.3 计算节点启动 + +```mermaid +sequenceDiagram + participant DS as cube-node DaemonSet + participant DNS as wait-cube-dns + participant PVM as pvm-host-bootstrap + participant INIT as cube-node-init + participant CN as cube-node + participant EG as cube-egress + participant EN as cube-egress-net + participant CM as CubeMaster + + DS->>DNS: wait 127.0.0.54 and validate DNS records + DNS-->>DS: success + opt bootstrap.pvmHostKernel.enabled=true + DS->>PVM: install/check PVM host kernel + PVM-->>DS: success or reboot-required failure signal + end + DS->>INIT: host preflight and preparation + INIT->>INIT: check kvm_pvm / CUBE_PVM_ENABLE consistency + INIT->>CM: /notify/health connectivity check + INIT-->>DS: success + DS->>CN: start cubelet + network-agent + CN->>CN: select guest kernel vmlinux-bm/vmlinux-pvm + CN->>CM: register node and heartbeat + DS->>EG: start egress worker + DS->>EN: wait cube-dev and apply TPROXY rules +``` + +`cube-node` 使用 startupProbe、readinessProbe、livenessProbe： + +- startupProbe：等待 cubelet 9999 启动，避免慢启动期间被 liveness 提前杀死。 +- readiness/liveness：检查 cubelet 9999。 +- `cube-egress`：检查 `127.0.0.1:9090/admin/v1/health`。 +- `cube-egress-net`：检查 `cube-dev`、ip rule、table 100 local route、mangle `TRANSPROXY` 80/443 规则。 + +### 4.4 节点注册与健康链路 + +```mermaid +flowchart LR + CN["cube-node\ncubelet"] -->|register / heartbeat| CM["CubeMaster"] + API["CubeAPI"] -->|list nodes / sandbox API| CM + WEB["WebUI"] -->|/cubeapi/*| API + HT["helm test cube-health-test"] --> API + HT --> CM + HT --> K8S["Kubernetes API\nDaemonSet ready / Pod containers"] +``` + +验收关注点： + +- CubeMaster `/notify/health` 成功。 +- CubeAPI `/health` 成功。 +- CubeAPI 能查询到 healthy node。 +- `cube-node` DaemonSet ready 数等于命中 selector 的节点数。 +- `cube-egress` / `cube-egress-net` sidecar 存在且 Ready。 + +## 5. 运行期关键数据流 + +### 5.1 WebUI / API / Master / DB + +```mermaid +flowchart LR + U["Browser / Operator"] --> WEB["cube-webui Service"] + WEB -->|/cubeapi/*| API["cube-api Service"] + API --> CM["cube-master Service"] + API --> MYSQL[("MySQL")] + CM --> MYSQL + CM --> REDIS[("Redis")] +``` + +用途： + +- 控制台操作、模板管理、sandbox 查询等通过 CubeAPI。 +- CubeAPI 读写业务数据到 MySQL。 +- CubeMaster 维护节点和任务元数据，使用内置迁移保证 schema。 + +### 5.2 Sandbox 入口流量 + +```mermaid +flowchart LR + CLIENT["Client / Sandbox domain"] --> DNS["DNS: cube.app / wildcard"] + DNS --> PROXY["cube-proxy-node"] + PROXY --> REDIS[("Redis state")] + PROXY --> CM["CubeMaster"] + PROXY --> NODE["Target Cube Node / Sandbox"] +``` + +说明： + +- `cube-proxy-node` 默认启用，随 Chart 一起安装和卸载。 +- `cube-proxy-node` 默认使用 `hostNetwork`，确保 control 节点能够接入 `80/443` 流量。 +- Chart 不创建 `cube-proxy-node` ClusterIP Service，避免 Kubernetes 随机分发到非目标节点后偏离 One Click 的显式 CubeProxy host 入口模型。 +- 需要多节点统一入口时，应由外部 DNS/LB 明确指向预期 control 节点 CubeProxy 或保证社区 CubeProxy 的 `HostIP:hostPort` 跨节点路径可用，而不是依赖默认 ClusterIP 随机分流。 +- TLS 支持 selfSigned、existingSecret、inline、certManager。 +- 生产环境应提供正式证书，并把 sandbox domain / wildcard DNS 指向明确的 CubeProxy 入口。 +- Chart 自带 `cube-dns` 默认只服务 `cube-node` Big Pod；面向用户访问的外部 DNS、LB 或 Ingress 需要由使用方显式配置。 + +### 5.3 Sandbox 出站 egress + +```mermaid +flowchart LR + SB["Sandbox traffic"] --> DEV["cube-dev interface"] + DEV --> EGNET["cube-egress-net\nTPROXY/ip-rule"] + EGNET --> EG["cube-egress\nOpenResty"] + EG --> EXT["External endpoint"] + EG --> CA["cube-egress-ca"] +``` + +说明： + +- `cube-egress-net` 只负责 host network 规则。 +- `cube-egress` 负责透明代理和证书能力。 +- CubeMaster / CubeAPI / Cube Node 共享 `cube-egress-ca` Secret，保证模板构建、AgentHub/OpenClaw 注入和运行期信任一致。 + +### 5.4 模板构建 + +```mermaid +flowchart LR + API["CubeAPI"] --> CM["CubeMaster"] + CM --> TB["template-builder sidecar\noptional docker:dind"] + CM --> ST[("CubeMaster storage hostPath/PVC")] + TB --> ST +``` + +说明： + +- `controlPlane.templateBuilder.enabled=true` 时，CubeMaster Pod 内增加 `template-builder` sidecar。 +- sidecar 默认使用 `docker:27-dind`，给模板构建提供 Docker/BuildKit 能力。 +- 构建产物写入 CubeMaster artifact storage。 + +## 6. external control plane / compute-only 模式 + +compute-only 模式对齐 One Click 的计算节点单独交付场景。 + +```mermaid +flowchart TB + subgraph EXT["External Control Plane"] + ECM["External CubeMaster"] + EAPI["External CubeAPI optional"] + EDB[("External MySQL / Redis")] + end + + subgraph NS["Compute Namespace"] + DNS["cube-dns"] + NODE["cube-node DaemonSet"] + end + + NODE --> ECM + DNS --> NODE + EAPI --> ECM + ECM --> EDB +``` + +关键 values： + +```yaml +controlPlane: + enabled: false +externalControlPlane: + enabled: true + masterEndpoint: :8089 + apiEndpoint: http://:3000 # optional, for helm test +``` + +行为： + +- 不安装 Chart 内置 Master / API / MySQL / Redis / WebUI。 +- `cube-node` 使用 `externalControlPlane.masterEndpoint` 注册外部 CubeMaster。 +- 如果配置 `externalControlPlane.apiEndpoint`，Helm test 会校验外部 API 和节点注册。 +- 默认不安装 `cube-proxy-node`，避免 compute-only release 留下与外部控制面不一致的数据面资源。 + +## 7. 关键 values 开关 + +| values 路径 | 默认 | 影响 | +| --- | --- | --- | +| `global.timezone` | `Asia/Shanghai` | 注入所有 Chart 管理的 Cube 容器、sidecar 和 initContainer 的 `TZ` | +| `storageClass.create` | `true` | 是否创建 Chart 默认的状态组件 StorageClass | +| `storageClass.name` | `cube-cbs-wffc` | CubeMaster storage、内置 MySQL、内置 Redis 默认使用的 StorageClass | +| `storageClass.volumeBindingMode` | `WaitForFirstConsumer` | 多可用区 TKE 集群中等待 Pod 选中 control 节点后再创建 CBS 盘，避免 PV zone 与 control 节点不匹配 | +| `controlPlane.enabled` | `true` | 是否部署内置控制面 | +| `externalControlPlane.enabled` | `false` | 是否使用外部 CubeMaster | +| `placement.controlPlane.nodeSelector` | `cube.tencent.com/role=control` | 控制 CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 调度范围 | +| `placement.compute.nodeSelector` | 含 `allow-pvm-bootstrap=true` | 控制 `cube-node`、`cube-dns` 调度范围，并要求节点显式允许 PVM bootstrap | +| `cubeDns.enabled` | `true` | 是否交付 CubeDNS | +| `cubeDns.mode` | `nodeLocal` | node-local DNS 或 ClusterIP DNS | +| `cubeDns.sandboxGateway.enabled` | `true` | node-local DNS 是否同时监听 compute HostIP，供 sandbox guest 使用 | +| `cubeNode.dns.useCubeDns` | `true` | `cube-node` 是否显式使用 `127.0.0.54` | +| `cubeNode.dns.sandbox.useCubeDns` | `true` | 是否把 sandbox guest `/etc/resolv.conf` 指到 cube-dns；默认写当前 compute HostIP，让 guest 使用 node-local `cube-dns` | +| `cubeNode.dns.sandbox.nameservers` | `[]` | 覆盖写入 sandbox guest `/etc/resolv.conf` 的 DNS server | +| `cubeNode.pvmGuestKernel.enabled` | `true` | 是否选择 PVM guest kernel；`cube-node-init` 校验该值与 `kvm_pvm` 状态一致 | +| `bootstrap.pvmHostKernel.enabled` | `true` | 是否执行 host kernel bootstrap；默认可能安装 host kernel 并按租约重启计算节点 | +| `bootstrap.pvmHostKernel.bootArgs` | `nopti pti=off` | PVM host kernel 启动参数；当前 `kvm_pvm` 不支持 host KPTI，默认关闭 PTI | +| `bootstrap.nodeInit.*` | 多项 | 控制节点预检、XFS、KVM、CIDR 检测 | +| `mysql.host` | `""` | 非空时使用第三方 MySQL | +| `redis.host` | `""` | 非空时使用第三方 Redis | +| `cubeProxy.enabled` | `true` | 是否部署 control 节点 CubeProxy 数据面入口 | +| `cubeProxy.advertiseIP` | `""` | `cubeDns.answerIP` 为空时返回给 `cube.app` / wildcard 的 control 节点 CubeProxy 入口 IP | +| `cubeEgress.enabled` | `true` | 是否在 Big Pod 中启用 egress sidecar | +| `webui.enabled` | `true` | 是否部署 WebUI | +| `controlPlane.templateBuilder.enabled` | `false` | 是否启用模板构建 sidecar | + +## 8. Helm test 覆盖 + +`templates/tests/` 提供 Chart 内置验收： + +| Test Pod | 覆盖内容 | +| --- | --- | +| `-health-test` | CubeMaster、CubeAPI、节点注册、WebUI、CubeProxy、DaemonSet/Deployment/StatefulSet ready、Egress sidecar 存在性 | +| `-mysql-test` | 内置 MySQL `mysqladmin ping` | +| `-redis-test` | 内置 Redis `PING` | +| `-dns-test` | `cube.app`、wildcard、Kubernetes Service 域名解析 | +| `-node-image-test` | `cube-node` 镜像内 runtime 工具和必需 asset | +| `-node-runtime-test` | 计算节点 host runtime：`/dev/kvm`、cubelet socket、network-agent socket、node-local DNS | + +执行： + +```bash +helm test -n --timeout 20m --logs +``` + +## 9. 资源所有权与卸载边界 + +Chart 管理并随 release 卸载： + +- 控制面 Deployment / Service； +- 内置 MySQL / Redis； +- CubeDNS； +- CubeProxy； +- CubeNode DaemonSet； +- CA / TLS / config Secret； +- Helm test RBAC； +- diagnostics ConfigMap。 + +Chart 不管理： + +- 使用方给节点打的 label / taint； +- 第三方 MySQL / Redis； +- 外部 DNS / 负载均衡； +- hostPath 数据目录； +- host kernel、GRUB、udev、fstab 或 XFS 等节点级持久修改。 +- One Click 单节点 seed SQL / demo 数据；Chart 依赖真实 `cube-node` Pod 注册节点。 + +因此卸载后，节点级数据和外部接入应按平台 runbook 清理。 diff --git a/deploy/kubernetes/chart/files/cube-master/conf.yaml b/deploy/kubernetes/chart/files/cube-master/conf.yaml new file mode 100644 index 000000000..18f9934b5 --- /dev/null +++ b/deploy/kubernetes/chart/files/cube-master/conf.yaml @@ -0,0 +1,105 @@ +common: + http_port: {{ .Values.controlPlane.master.service.port }} + http_readtimeout: 120 + http_writetimeout: 360 + http_idletimeout: 360 + sync_meta_data_interval: 30s + sync_metric_data_interval: 1s + collect_metric_interval: 1s + default_headless_service_nodes_num: 1 + enable_check_com_net_id_param: false + +log: + module: "cubemaster" + path: "/data/log/CubeMaster" + file_size: 100 + file_num: 10 + level: "info" + +cubelet_conf: + grpc: + grpc_port: 9999 + common_timeout_insec: 30 + create_image_timeout_insec: 300 + create_concurrent_limit: 100 + destroy_concurent_limit: 100 + enable_exposed_port: true + exposed_port_list: + - "80" + disable_redis_proxy_port: true + +auth: + enable: false + +req_template_conf: + whitelist_req_tag: + WorkingDir: true + RLimit: true + DnsConfig: true + HostAliases: true + Poststop: true + Prestop: true + cube_box_req_template: >- + {"volumes":[{"name":"tmp","volume_source":{"empty_dir":{"medium":0}}}],"containers":[{"name":"cubebox-default","envs":[{"key":"TZ","value":"Asia/Shanghai"},{"key":"TERM","value":"xterm"}],"volume_mounts":[{"name":"tmp","container_path":"/"}],"security_context":{"privileged":true,"readonly_rootfs":false,"no_new_privs":false}}],"network_type":"tap","cubevs_context":{"allowInternetAccess":true,"denyOut":["10.0.0.0/8","100.64.0.0/10","172.16.0.0/12","192.168.0.0/18"]}} + +ossdb_config: + addr: {{ printf "%s:%v" (include "cube.mysqlHost" .) .Values.mysql.port | quote }} + user: {{ .Values.mysql.user | quote }} + pwd: {{ .Values.mysql.password | quote }} + db_name: {{ .Values.mysql.database | quote }} + conn_timeout: 5 + read_timeout: 5 + write_timeout: 5 + max_idle_conns: 5 + max_open_conns: 20 + max_conn_life_time_seconds: 300 + +instance_db_config: + addr: {{ printf "%s:%v" (include "cube.mysqlHost" .) .Values.mysql.port | quote }} + user: {{ .Values.mysql.user | quote }} + pwd: {{ .Values.mysql.password | quote }} + db_name: {{ .Values.mysql.database | quote }} + conn_timeout: 5 + read_timeout: 5 + write_timeout: 5 + max_idle_conns: 5 + max_open_conns: 20 + max_conn_life_time_seconds: 300 + +redis: + nodes: {{ printf "%s:%v" (include "cube.redisHost" .) .Values.redis.port | quote }} + password: {{ .Values.redis.password | quote }} + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + +redis_read: + nodes: {{ printf "%s:%v" (include "cube.redisHost" .) .Values.redis.port | quote }} + password: {{ .Values.redis.password | quote }} + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + +redis_write: + nodes: {{ printf "%s:%v" (include "cube.redisHost" .) .Values.redis.port | quote }} + password: {{ .Values.redis.password | quote }} + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + +scheduler: + priority_select_num: 1 + metric_update_timeout: 300s + local_metric_update_timeout: 300s + filter: + enable_filters: + - "cpu" + - "mem" + - "template_locality" + - "realtime_create_num" diff --git a/deploy/kubernetes/chart/templates/NOTES.txt b/deploy/kubernetes/chart/templates/NOTES.txt new file mode 100644 index 000000000..e4a164a2b --- /dev/null +++ b/deploy/kubernetes/chart/templates/NOTES.txt @@ -0,0 +1,64 @@ +CubeSandbox chart installed. + +Release: {{ .Release.Name }} +Namespace: {{ .Release.Namespace }} + +{{- if .Values.controlPlane.enabled }} +Control plane: + CubeMaster Service: {{ include "cube.masterName" . }}:{{ .Values.controlPlane.master.service.port }} + CubeAPI Service: {{ include "cube.apiName" . }}:{{ .Values.controlPlane.api.service.port }} +{{- if .Values.webui.enabled }} + WebUI Service: {{ include "cube.webuiName" . }}:{{ .Values.webui.service.port }} +{{- end }} +{{- if eq (include "cube.cubemastercliEnabled" .) "true" }} + cubemastercli Deployment: {{ include "cube.cubemastercliName" . }} +{{- end }} +{{- else if .Values.externalControlPlane.enabled }} +External control plane: + CubeMaster Endpoint: {{ .Values.externalControlPlane.masterEndpoint }} +{{- if .Values.externalControlPlane.apiEndpoint }} + CubeAPI Endpoint: {{ .Values.externalControlPlane.apiEndpoint }} +{{- end }} +{{- if eq (include "cube.cubemastercliEnabled" .) "true" }} + cubemastercli Deployment: {{ include "cube.cubemastercliName" . }} +{{- end }} +{{- end }} +{{- if .Values.cubeDns.enabled }} +{{- if eq .Values.cubeDns.mode "nodeLocal" }} + CubeDNS DaemonSet: {{ include "cube.fullname" . }}-dns (node-local {{ .Values.cubeDns.bindAddress }}:53) +{{- else }} + CubeDNS Service: {{ include "cube.fullname" . }}-dns:{{ .Values.cubeDns.service.port }} +{{- end }} +{{- end }} + +Cube Node DaemonSet: + {{ include "cube.nodeName" . }} +{{- if eq (include "cube.proxyEnabled" .) "true" }} + +Cube Proxy Node Deployment: + {{ include "cube.proxyName" . }} + HTTP/HTTPS control-node endpoint: :{{ .Values.cubeProxy.ports.http.containerPort }}/{{ .Values.cubeProxy.ports.https.containerPort }} + No ClusterIP Service is created; point wildcard DNS at the control node or external load balancer. +{{- end }} + +Useful commands: + kubectl get pods -n {{ .Release.Namespace }} -o wide + kubectl get ds -n {{ .Release.Namespace }} {{ include "cube.nodeName" . }} +{{- if eq (include "cube.proxyEnabled" .) "true" }} + kubectl get deploy -n {{ .Release.Namespace }} {{ include "cube.proxyName" . }} +{{- end }} + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=cube-node -c pvm-host-bootstrap --tail=100 + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=cube-node -c cube-node-init --tail=100 + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=cube-node -c cube-node --tail=100 +{{- if eq (include "cube.cubemastercliEnabled" .) "true" }} + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "cube.cubemastercliName" . }} -- cubemastercli --help + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "cube.cubemastercliName" . }} -- sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" node list' +{{- end }} + helm test {{ .Release.Name }} -n {{ .Release.Namespace }} --timeout 20m +{{- if .Values.diagnostics.enabled }} + kubectl get configmap -n {{ .Release.Namespace }} {{ include "cube.fullname" . }}-diagnostics -o jsonpath='{.data.cube-diag-k8s\.sh}' > /tmp/cube-diag-k8s.sh + sh /tmp/cube-diag-k8s.sh {{ .Release.Namespace }} {{ .Release.Name }} +{{- end }} + +WARNING: If bootstrap.pvmHostKernel.enabled=true, this release may install a host kernel and reboot selected compute nodes. +Helm rollback does not roll back host kernel/GRUB changes. diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl new file mode 100644 index 000000000..d37c8f51b --- /dev/null +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -0,0 +1,240 @@ +{{/* Common template helpers for CubeSandbox chart. */}} +{{- define "cube.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "cube.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "cube.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }} +app.kubernetes.io/name: {{ include "cube.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "cube.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cube.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "cube.image" -}} +{{- printf "%s:%s" .repository .tag -}} +{{- end -}} + +{{- define "cube.timezoneEnv" -}} +{{- with .Values.global.timezone }} +- name: TZ + value: {{ . | quote }} +{{- end }} +{{- end -}} + +{{- define "cube.controlPlanePlacement" -}} +{{- with .Values.placement.controlPlane.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.placement.controlPlane.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + +{{- define "cube.computePlacement" -}} +{{- with .Values.placement.compute.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.placement.compute.affinity }} +affinity: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.placement.compute.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + +{{- define "cube.nodeServiceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- printf "%s-node" (include "cube.fullname" .) -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{- define "cube.masterName" -}} +{{- printf "%s-master" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.apiName" -}} +{{- printf "%s-api" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.cubemastercliName" -}} +{{- printf "%s-cubemastercli" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.webuiName" -}} +{{- printf "%s-webui" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.nodeName" -}} +{{- printf "%s-node" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.proxyName" -}} +{{- printf "%s-proxy-node" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.proxyEnabled" -}} +{{- if and .Values.cubeProxy.enabled (or .Values.controlPlane.enabled (not .Values.externalControlPlane.enabled)) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "cube.cubemastercliEnabled" -}} +{{- $cubemastercli := default dict .Values.cubemastercli -}} +{{- if and (dig "enabled" true $cubemastercli) (or .Values.controlPlane.enabled .Values.externalControlPlane.enabled) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "cube.mysqlName" -}} +{{- printf "%s-mysql" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.redisName" -}} +{{- printf "%s-redis" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.secretName" -}} +{{- printf "%s-secret" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.masterConfigSecretName" -}} +{{- printf "%s-master-config" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.masterStoragePVCName" -}} +{{- if .Values.controlPlane.master.persistence.existingClaim -}} +{{- .Values.controlPlane.master.persistence.existingClaim -}} +{{- else -}} +{{- printf "%s-master-storage" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{- define "cube.mysqlPVCName" -}} +{{- if .Values.mysql.persistence.existingClaim -}} +{{- .Values.mysql.persistence.existingClaim -}} +{{- else -}} +{{- printf "%s-mysql-data" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{- define "cube.redisPVCName" -}} +{{- if .Values.redis.persistence.existingClaim -}} +{{- .Values.redis.persistence.existingClaim -}} +{{- else -}} +{{- printf "%s-redis-data" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{- define "cube.proxyCertSecretName" -}} +{{- if and (eq .Values.cubeProxy.tls.mode "existingSecret") .Values.cubeProxy.tls.existingSecret -}} +{{- .Values.cubeProxy.tls.existingSecret -}} +{{- else if .Values.cubeProxy.tls.secretName -}} +{{- .Values.cubeProxy.tls.secretName -}} +{{- else -}} +{{- printf "%s-proxy-certs" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{- define "cube.egressCASecretName" -}} +{{- if and (eq .Values.cubeEgress.ca.mode "existingSecret") .Values.cubeEgress.ca.existingSecret -}} +{{- .Values.cubeEgress.ca.existingSecret -}} +{{- else if .Values.cubeEgress.ca.secretName -}} +{{- .Values.cubeEgress.ca.secretName -}} +{{- else -}} +{{- printf "%s-egress-ca" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{- define "cube.masterEndpoint" -}} +{{- if .Values.externalControlPlane.enabled -}} +{{- .Values.externalControlPlane.masterEndpoint -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local:%v" (include "cube.masterName" .) .Release.Namespace .Values.controlPlane.master.service.port -}} +{{- end -}} +{{- end -}} + +{{- define "cube.cubemastercliMasterEndpoint" -}} +{{- if .Values.externalControlPlane.enabled -}} +{{- .Values.externalControlPlane.masterEndpoint -}} +{{- else if and .Values.controlPlane.enabled .Values.controlPlane.master.enabled -}} +{{- include "cube.masterEndpoint" . -}} +{{- end -}} +{{- end -}} + +{{- define "cube.cubemastercliMasterAddress" -}} +{{- $endpoint := include "cube.cubemastercliMasterEndpoint" . -}} +{{- $withoutHTTP := trimPrefix "http://" (trimPrefix "https://" $endpoint) -}} +{{- $hostPort := first (splitList "/" $withoutHTTP) -}} +{{- regexReplaceAll ":[0-9]+$" $hostPort "" -}} +{{- end -}} + +{{- define "cube.cubemastercliMasterPort" -}} +{{- $endpoint := include "cube.cubemastercliMasterEndpoint" . -}} +{{- $withoutHTTP := trimPrefix "http://" (trimPrefix "https://" $endpoint) -}} +{{- $hostPort := first (splitList "/" $withoutHTTP) -}} +{{- $port := regexFind "[0-9]+$" $hostPort -}} +{{- default "8089" $port -}} +{{- end -}} + +{{- define "cube.apiEndpoint" -}} +{{- if .Values.externalControlPlane.enabled -}} +{{- .Values.externalControlPlane.apiEndpoint -}} +{{- else -}} +{{- printf "http://%s.%s.svc.cluster.local:%v" (include "cube.apiName" .) .Release.Namespace .Values.controlPlane.api.service.port -}} +{{- end -}} +{{- end -}} + +{{- define "cube.mysqlHost" -}} +{{- if .Values.mysql.host -}}{{ .Values.mysql.host }}{{- else -}}{{ include "cube.mysqlName" . }}.{{ .Release.Namespace }}.svc.cluster.local{{- end -}} +{{- end -}} + +{{- define "cube.mysqlBuiltinEnabled" -}} +{{- if and .Values.controlPlane.enabled .Values.mysql.enabled (not .Values.mysql.host) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "cube.redisHost" -}} +{{- if .Values.redis.host -}}{{ .Values.redis.host }}{{- else -}}{{ include "cube.redisName" . }}.{{ .Release.Namespace }}.svc.cluster.local{{- end -}} +{{- end -}} + +{{- define "cube.redisBuiltinEnabled" -}} +{{- if and (or .Values.controlPlane.enabled (eq (include "cube.proxyEnabled" .) "true")) .Values.redis.enabled (not .Values.redis.host) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "cube.egressNetProbeCommand" -}} +set -e +iface="${CUBE_INGRESS_IFACE:-cube-dev}" +table="${CUBE_EGRESS_NET_ROUTE_TABLE:-100}" +chain="${CUBE_EGRESS_NET_CHAIN:-TRANSPROXY}" +ip link show "${iface}" >/dev/null +ip rule show | grep -q "iif ${iface} ipproto tcp dport 80 lookup ${table}" +ip rule show | grep -q "iif ${iface} ipproto tcp dport 443 lookup ${table}" +ip route show table "${table}" | grep -Eq "local (default|0\\.0\\.0\\.0/0) dev lo" +iptables -t mangle -S "${chain}" | grep -q -- "--dport 80" +iptables -t mangle -S "${chain}" | grep -q -- "--dport 443" +{{- end -}} + +{{- define "cube.secretEnabled" -}} +{{- if or (and .Values.controlPlane.enabled (or .Values.controlPlane.master.enabled .Values.controlPlane.api.enabled)) (eq (include "cube.proxyEnabled" .) "true") (eq (include "cube.mysqlBuiltinEnabled" .) "true") (eq (include "cube.redisBuiltinEnabled" .) "true") -}}true{{- else -}}false{{- end -}} +{{- end -}} diff --git a/deploy/kubernetes/chart/templates/api.yaml b/deploy/kubernetes/chart/templates/api.yaml new file mode 100644 index 000000000..157070e82 --- /dev/null +++ b/deploy/kubernetes/chart/templates/api.yaml @@ -0,0 +1,140 @@ +{{- if and .Values.controlPlane.enabled .Values.controlPlane.api.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.apiName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + replicas: {{ .Values.controlPlane.api.replicas }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: api + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: api + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: cube-api + image: {{ include "cube.image" .Values.images.api | quote }} + imagePullPolicy: {{ .Values.images.api.pullPolicy }} + {{- with .Values.controlPlane.api.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.controlPlane.api.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_MASTER_ENDPOINT + value: {{ include "cube.masterEndpoint" . | quote }} + - name: CUBE_MASTER_ADDR + value: {{ printf "http://%s" (include "cube.masterEndpoint" .) | quote }} + - name: CUBE_API_BIND + value: {{ .Values.controlPlane.api.bind | quote }} + - name: CUBE_SANDBOX_MYSQL_HOST + value: {{ include "cube.mysqlHost" . | quote }} + - name: CUBE_SANDBOX_MYSQL_PORT + value: {{ .Values.mysql.port | quote }} + - name: CUBE_SANDBOX_MYSQL_USER + value: {{ .Values.mysql.user | quote }} + - name: CUBE_SANDBOX_MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: mysql-password + - name: CUBE_SANDBOX_MYSQL_DB + value: {{ .Values.mysql.database | quote }} + - name: CUBE_API_SANDBOX_DOMAIN + value: {{ .Values.controlPlane.api.sandboxDomain | quote }} + {{- with .Values.controlPlane.api.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-api + containerPort: 3000 + {{- if .Values.controlPlane.api.probes.readiness.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.controlPlane.api.probes.readiness.path | quote }} + port: http-api + initialDelaySeconds: {{ .Values.controlPlane.api.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.controlPlane.api.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.controlPlane.api.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.controlPlane.api.probes.liveness.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.controlPlane.api.probes.liveness.path | quote }} + port: http-api + initialDelaySeconds: {{ .Values.controlPlane.api.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.controlPlane.api.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.controlPlane.api.probes.liveness.failureThreshold }} + {{- end }} + {{- if .Values.cubeEgress.enabled }} + volumeMounts: + - name: cube-egress-ca + mountPath: /etc/cube/ca + readOnly: true + {{- end }} + {{- with .Values.controlPlane.api.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.cubeEgress.enabled }} + volumes: + - name: cube-egress-ca + secret: + secretName: {{ include "cube.egressCASecretName" . }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.apiName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + type: {{ .Values.controlPlane.api.service.type }} + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: api + ports: + - name: http-api + port: {{ .Values.controlPlane.api.service.port }} + targetPort: http-api +{{- if .Values.controlPlane.api.internalLoadBalancer.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.apiName" . }}-internal + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: api + annotations: + {{- toYaml .Values.controlPlane.api.internalLoadBalancer.annotations | nindent 4 }} +spec: + type: LoadBalancer + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: api + ports: + - name: http-api + protocol: TCP + port: {{ .Values.controlPlane.api.internalLoadBalancer.port }} + targetPort: http-api +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/cubemastercli.yaml b/deploy/kubernetes/chart/templates/cubemastercli.yaml new file mode 100644 index 000000000..44ac05643 --- /dev/null +++ b/deploy/kubernetes/chart/templates/cubemastercli.yaml @@ -0,0 +1,86 @@ +{{- if eq (include "cube.cubemastercliEnabled" .) "true" }} +{{- $cubemastercli := default dict .Values.cubemastercli -}} +{{- $images := default dict .Values.images -}} +{{- $image := default (dict "repository" "ccr.ccs.tencentyun.com/cubesandbox-chart/cubemastercli" "tag" "v0.5.0" "pullPolicy" "Always") (get $images "cubemastercli") -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.cubemastercliName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cubemastercli +spec: + replicas: {{ dig "replicas" 1 $cubemastercli }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cubemastercli + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cubemastercli + {{- with (dig "podAnnotations" dict $cubemastercli) }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + terminationGracePeriodSeconds: 5 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: cubemastercli + image: {{ include "cube.image" $image | quote }} + imagePullPolicy: {{ dig "pullPolicy" "Always" $image }} + {{- with (dig "command" list $cubemastercli) }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with (dig "args" list $cubemastercli) }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_RELEASE + value: {{ .Release.Name | quote }} + - name: CUBE_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: CUBE_MASTER_ENDPOINT + value: {{ include "cube.cubemastercliMasterEndpoint" . | quote }} + - name: CUBEMASTERCLI_ADDRESS + value: {{ include "cube.cubemastercliMasterAddress" . | quote }} + - name: CUBEMASTERCLI_PORT + value: {{ include "cube.cubemastercliMasterPort" . | quote }} + {{- with (dig "env" list $cubemastercli) }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if dig "probes" "readiness" "enabled" true $cubemastercli }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - cubemastercli --address "${CUBEMASTERCLI_ADDRESS}" --port "${CUBEMASTERCLI_PORT}" node list >/tmp/cubemastercli-node-list.out + initialDelaySeconds: {{ dig "probes" "readiness" "initialDelaySeconds" 5 $cubemastercli }} + periodSeconds: {{ dig "probes" "readiness" "periodSeconds" 10 $cubemastercli }} + failureThreshold: {{ dig "probes" "readiness" "failureThreshold" 12 $cubemastercli }} + {{- end }} + {{- if dig "probes" "liveness" "enabled" true $cubemastercli }} + livenessProbe: + exec: + command: + - /usr/local/bin/cubemastercli + - --help + initialDelaySeconds: {{ dig "probes" "liveness" "initialDelaySeconds" 30 $cubemastercli }} + periodSeconds: {{ dig "probes" "liveness" "periodSeconds" 20 $cubemastercli }} + failureThreshold: {{ dig "probes" "liveness" "failureThreshold" 6 $cubemastercli }} + {{- end }} + {{- with (dig "resources" dict $cubemastercli) }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/diagnostics.yaml b/deploy/kubernetes/chart/templates/diagnostics.yaml new file mode 100644 index 000000000..a6880835c --- /dev/null +++ b/deploy/kubernetes/chart/templates/diagnostics.yaml @@ -0,0 +1,61 @@ +{{- if .Values.diagnostics.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cube.fullname" . }}-diagnostics + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: diagnostics +data: + cube-diag-k8s.sh: |- + #!/bin/sh + set -eu + + NS="${1:-{{ .Release.Namespace }}}" + RELEASE="${2:-{{ .Release.Name }}}" + OUT="${3:-cube-diag-${RELEASE}-$(date +%Y%m%d%H%M%S)}" + + mkdir -p "${OUT}" + + run() { + name="$1" + shift + echo "==> ${name}" + { + echo "# ${name}" + echo "# $*" + "$@" + } > "${OUT}/${name}.txt" 2>&1 || true + } + + run pods kubectl get pods -n "${NS}" -o wide + run daemonsets kubectl get daemonsets -n "${NS}" -o wide + run deployments kubectl get deployments -n "${NS}" -o wide + run statefulsets kubectl get statefulsets -n "${NS}" -o wide + run services kubectl get services -n "${NS}" -o wide + run endpoints kubectl get endpoints -n "${NS}" -o wide + run events kubectl get events -n "${NS}" --sort-by=.lastTimestamp + run helm-values helm get values "${RELEASE}" -n "${NS}" -o yaml + run helm-manifest helm get manifest "${RELEASE}" -n "${NS}" + + for component in master api cube-node cube-proxy-node webui dns mysql redis; do + run "pods-${component}" kubectl get pods -n "${NS}" -l "app.kubernetes.io/component=${component}" -o wide + kubectl get pods -n "${NS}" -l "app.kubernetes.io/component=${component}" -o name 2>/dev/null \ + | while read -r pod; do + pod_name="${pod#pod/}" + run "describe-${pod_name}" kubectl describe -n "${NS}" "${pod}" + for container in $(kubectl get -n "${NS}" "${pod}" -o jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}' 2>/dev/null); do + run "logs-${pod_name}-${container}" kubectl logs -n "${NS}" "${pod}" -c "${container}" --tail=300 + done + done + done + + for ds in cube-node; do + run "rollout-${ds}" kubectl rollout status "daemonset/${RELEASE}-${ds#cube-}" -n "${NS}" --timeout=30s + done + run rollout-proxy-node kubectl rollout status "deployment/${RELEASE}-proxy-node" -n "${NS}" --timeout=30s + run rollout-mysql kubectl rollout status "statefulset/${RELEASE}-mysql" -n "${NS}" --timeout=30s + run rollout-redis kubectl rollout status "statefulset/${RELEASE}-redis" -n "${NS}" --timeout=30s + + echo "diagnostics written to ${OUT}" +{{- end }} diff --git a/deploy/kubernetes/chart/templates/dns.yaml b/deploy/kubernetes/chart/templates/dns.yaml new file mode 100644 index 000000000..fcf824800 --- /dev/null +++ b/deploy/kubernetes/chart/templates/dns.yaml @@ -0,0 +1,219 @@ +{{- if .Values.cubeDns.enabled }} +{{- $domain := default .Values.cubeProxy.domain .Values.cubeDns.domain -}} +{{- $domainRegex := replace "." "\\." $domain -}} +{{- $forward := "/etc/resolv.conf" -}} +{{- $answerIP := default .Values.cubeProxy.advertiseIP .Values.cubeDns.answerIP -}} +{{- if .Values.cubeDns.forward.upstreams -}} +{{- $forward = join " " .Values.cubeDns.forward.upstreams -}} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cube.fullname" . }}-dns-config + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: dns +data: + Corefile: |- + .:53 { + {{- if eq .Values.cubeDns.mode "nodeLocal" }} + bind {{ .Values.cubeDns.bindAddress }}{{- if .Values.cubeDns.sandboxGateway.enabled }} {$NODE_IP}{{- end }} + {{- end }} + {{- if $answerIP }} + template IN A {{ $domain }} { + answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {{ $answerIP }}" + } + template IN A (.*)\.{{ $domainRegex }} { + answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {{ $answerIP }}" + } + {{- else if eq .Values.cubeDns.mode "nodeLocal" }} + template IN A {{ $domain }} { + answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {$NODE_IP}" + } + template IN A (.*)\.{{ $domainRegex }} { + answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {$NODE_IP}" + } + {{- end }} + template IN AAAA {{ $domain }} { + rcode NOERROR + } + template IN AAAA (.*)\.{{ $domainRegex }} { + rcode NOERROR + } + log + cache 300 + forward . {{ $forward }} + } +{{- if eq .Values.cubeDns.mode "nodeLocal" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "cube.fullname" . }}-dns + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: dns +spec: + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: dns + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: dns + annotations: + checksum/corefile: {{ printf "%s|%s|%s|%s|%s|%t|node-local-v3" $domain $answerIP .Values.cubeDns.bindAddress .Values.cubeDns.mode $forward .Values.cubeDns.sandboxGateway.enabled | sha256sum | quote }} + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.computePlacement" . | nindent 6 }} + containers: + - name: coredns + image: {{ include "cube.image" .Values.images.coredns | quote }} + imagePullPolicy: {{ .Values.images.coredns.pullPolicy }} + args: + - -conf + - /etc/coredns/Corefile + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + ports: + - name: dns-udp + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP + readinessProbe: + tcpSocket: + host: {{ .Values.cubeDns.bindAddress | quote }} + port: dns-tcp + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + tcpSocket: + host: {{ .Values.cubeDns.bindAddress | quote }} + port: dns-tcp + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + volumeMounts: + - name: coredns-config + mountPath: /etc/coredns + readOnly: true + {{- with .Values.cubeDns.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: coredns-config + configMap: + name: {{ include "cube.fullname" . }}-dns-config +{{- else }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.fullname" . }}-dns + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: dns +spec: + replicas: {{ .Values.cubeDns.replicas }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: dns + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: dns + annotations: + checksum/corefile: {{ printf "%s|%s|%s|service-v1" $domain .Values.cubeDns.answerIP $forward | sha256sum | quote }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.computePlacement" . | nindent 6 }} + containers: + - name: coredns + image: {{ include "cube.image" .Values.images.coredns | quote }} + imagePullPolicy: {{ .Values.images.coredns.pullPolicy }} + args: + - -conf + - /etc/coredns/Corefile + {{- if .Values.global.timezone }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + {{- end }} + ports: + - name: dns-udp + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP + readinessProbe: + tcpSocket: + port: dns-tcp + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + tcpSocket: + port: dns-tcp + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + volumeMounts: + - name: coredns-config + mountPath: /etc/coredns + readOnly: true + {{- with .Values.cubeDns.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: coredns-config + configMap: + name: {{ include "cube.fullname" . }}-dns-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.fullname" . }}-dns + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: dns + {{- with .Values.cubeDns.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.cubeDns.service.type }} + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: dns + ports: + - name: dns-udp + protocol: UDP + port: {{ .Values.cubeDns.service.port }} + targetPort: dns-udp + - name: dns-tcp + protocol: TCP + port: {{ .Values.cubeDns.service.port }} + targetPort: dns-tcp +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/egress-ca.yaml b/deploy/kubernetes/chart/templates/egress-ca.yaml new file mode 100644 index 000000000..6f1640304 --- /dev/null +++ b/deploy/kubernetes/chart/templates/egress-ca.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.cubeEgress.enabled (ne .Values.cubeEgress.ca.mode "existingSecret") }} +{{- $secretName := include "cube.egressCASecretName" . -}} +{{- if eq .Values.cubeEgress.ca.mode "inline" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-egress + app.kubernetes.io/part-of: cube-egress +type: Opaque +stringData: + {{ .Values.cubeEgress.ca.rootCertKey | quote }}: |- +{{ .Values.cubeEgress.ca.inline.rootCert | nindent 4 }} + {{ .Values.cubeEgress.ca.rootKeyKey | quote }}: |- +{{ .Values.cubeEgress.ca.inline.rootKey | nindent 4 }} + {{ .Values.cubeEgress.ca.placeholderCertKey | quote }}: |- +{{ .Values.cubeEgress.ca.inline.placeholderCert | nindent 4 }} + {{ .Values.cubeEgress.ca.placeholderKeyKey | quote }}: |- +{{ .Values.cubeEgress.ca.inline.placeholderKey | nindent 4 }} +{{- else if eq .Values.cubeEgress.ca.mode "selfSigned" }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName -}} +{{- $rootCertKey := .Values.cubeEgress.ca.rootCertKey -}} +{{- $rootKeyKey := .Values.cubeEgress.ca.rootKeyKey -}} +{{- $placeholderCertKey := .Values.cubeEgress.ca.placeholderCertKey -}} +{{- $placeholderKeyKey := .Values.cubeEgress.ca.placeholderKeyKey -}} +{{- $hasExisting := and $existing (hasKey $existing.data $rootCertKey) (hasKey $existing.data $rootKeyKey) (hasKey $existing.data $placeholderCertKey) (hasKey $existing.data $placeholderKeyKey) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-egress + app.kubernetes.io/part-of: cube-egress +type: Opaque +{{ if $hasExisting }} +data: + {{ $rootCertKey | quote }}: {{ index $existing.data $rootCertKey | quote }} + {{ $rootKeyKey | quote }}: {{ index $existing.data $rootKeyKey | quote }} + {{ $placeholderCertKey | quote }}: {{ index $existing.data $placeholderCertKey | quote }} + {{ $placeholderKeyKey | quote }}: {{ index $existing.data $placeholderKeyKey | quote }} +{{- else }} +{{- $ca := genCA .Values.cubeEgress.ca.selfSigned.caCommonName (int .Values.cubeEgress.ca.selfSigned.caValidityDays) -}} +{{- $placeholder := genSelfSignedCert .Values.cubeEgress.ca.selfSigned.placeholderCommonName nil nil (int .Values.cubeEgress.ca.selfSigned.placeholderValidityDays) -}} +stringData: + {{ $rootCertKey | quote }}: |- +{{ $ca.Cert | nindent 4 }} + {{ $rootKeyKey | quote }}: |- +{{ $ca.Key | nindent 4 }} + {{ $placeholderCertKey | quote }}: |- +{{ $placeholder.Cert | nindent 4 }} + {{ $placeholderKeyKey | quote }}: |- +{{ $placeholder.Key | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/master-config-secret.yaml b/deploy/kubernetes/chart/templates/master-config-secret.yaml new file mode 100644 index 000000000..7b0d438be --- /dev/null +++ b/deploy/kubernetes/chart/templates/master-config-secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.controlPlane.enabled .Values.controlPlane.master.enabled }} +{{- $conf := tpl (.Files.Get "files/cube-master/conf.yaml") . }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cube.masterConfigSecretName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: master +type: Opaque +stringData: + conf.yaml: |- +{{ $conf | nindent 4 }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/master-pvc.yaml b/deploy/kubernetes/chart/templates/master-pvc.yaml new file mode 100644 index 000000000..48058ab32 --- /dev/null +++ b/deploy/kubernetes/chart/templates/master-pvc.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.controlPlane.enabled .Values.controlPlane.master.enabled .Values.controlPlane.master.persistence.enabled (not .Values.controlPlane.master.persistence.hostPath) (not .Values.controlPlane.master.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cube.masterStoragePVCName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: master +spec: + accessModes: + {{- toYaml .Values.controlPlane.master.persistence.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.controlPlane.master.persistence.size | quote }} + {{- if .Values.controlPlane.master.persistence.storageClassName }} + storageClassName: {{ .Values.controlPlane.master.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/master.yaml b/deploy/kubernetes/chart/templates/master.yaml new file mode 100644 index 000000000..8e1f7cbf2 --- /dev/null +++ b/deploy/kubernetes/chart/templates/master.yaml @@ -0,0 +1,161 @@ +{{- if and .Values.controlPlane.enabled .Values.controlPlane.master.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.masterName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: master +spec: + replicas: {{ .Values.controlPlane.master.replicas }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: master + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: master + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: cube-master + image: {{ include "cube.image" .Values.images.master | quote }} + imagePullPolicy: {{ .Values.images.master.pullPolicy }} + {{- with .Values.controlPlane.master.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.controlPlane.master.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_MASTER_CONFIG_PATH + value: /usr/local/services/cubemaster/conf.yaml + - name: CUBEMASTER_ROOTFS_ARTIFACT_STORE_DIR + value: /data/CubeMaster/storage + - name: CUBE_NODE_IPS + value: {{ join "," .Values.controlPlane.master.nodeIPs | quote }} + {{- with .Values.controlPlane.master.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: cubemaster + containerPort: {{ .Values.controlPlane.master.service.port }} + {{- if .Values.controlPlane.master.probes.readiness.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.controlPlane.master.probes.readiness.path | quote }} + port: cubemaster + initialDelaySeconds: {{ .Values.controlPlane.master.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.controlPlane.master.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.controlPlane.master.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.controlPlane.master.probes.liveness.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.controlPlane.master.probes.liveness.path | quote }} + port: cubemaster + initialDelaySeconds: {{ .Values.controlPlane.master.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.controlPlane.master.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.controlPlane.master.probes.liveness.failureThreshold }} + {{- end }} + volumeMounts: + - name: cube-master-config + mountPath: /usr/local/services/cubemaster/conf.yaml + subPath: conf.yaml + readOnly: true + {{- if .Values.cubeEgress.enabled }} + - name: cube-egress-ca + mountPath: /etc/cube/ca + readOnly: true + {{- end }} + - name: cube-master-log + mountPath: /data/log/CubeMaster + - name: cube-master-storage + mountPath: /data/CubeMaster/storage + {{- if .Values.controlPlane.templateBuilder.enabled }} + - name: docker-run + mountPath: /var/run + {{- end }} + {{- with .Values.controlPlane.master.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.controlPlane.templateBuilder.enabled }} + - name: template-builder + image: {{ include "cube.image" .Values.images.templateBuilder | quote }} + imagePullPolicy: {{ .Values.images.templateBuilder.pullPolicy }} + securityContext: + privileged: {{ .Values.controlPlane.templateBuilder.privileged }} + {{- if .Values.global.timezone }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + {{- end }} + {{- with .Values.controlPlane.templateBuilder.dockerdArgs }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: docker-run + mountPath: /var/run + - name: docker-graph + mountPath: /var/lib/docker + {{- with .Values.controlPlane.templateBuilder.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + volumes: + - name: cube-master-config + secret: + secretName: {{ include "cube.masterConfigSecretName" . }} + {{- if .Values.cubeEgress.enabled }} + - name: cube-egress-ca + secret: + secretName: {{ include "cube.egressCASecretName" . }} + {{- end }} + - name: cube-master-log + emptyDir: {} + - name: cube-master-storage + {{- if and .Values.controlPlane.master.persistence.enabled .Values.controlPlane.master.persistence.hostPath }} + hostPath: + path: {{ .Values.controlPlane.master.persistence.hostPath | quote }} + type: DirectoryOrCreate + {{- else if .Values.controlPlane.master.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "cube.masterStoragePVCName" . }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.controlPlane.templateBuilder.enabled }} + - name: docker-run + emptyDir: {} + - name: docker-graph + emptyDir: {} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.masterName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: master +spec: + type: {{ .Values.controlPlane.master.service.type }} + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: master + ports: + - name: cubemaster + port: {{ .Values.controlPlane.master.service.port }} + targetPort: cubemaster +{{- end }} diff --git a/deploy/kubernetes/chart/templates/mysql.yaml b/deploy/kubernetes/chart/templates/mysql.yaml new file mode 100644 index 000000000..a2e8e1929 --- /dev/null +++ b/deploy/kubernetes/chart/templates/mysql.yaml @@ -0,0 +1,131 @@ +{{- if eq (include "cube.mysqlBuiltinEnabled" .) "true" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.mysqlName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: mysql +spec: + clusterIP: None + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: mysql + ports: + - name: mysql + port: {{ .Values.mysql.port }} + targetPort: mysql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "cube.mysqlName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: mysql +spec: + serviceName: {{ include "cube.mysqlName" . }} + replicas: 1 + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: mysql + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: mysql + spec: + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: mysql + image: {{ include "cube.image" .Values.mysql.image | quote }} + imagePullPolicy: {{ .Values.mysql.image.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: mysql-root-password + - name: MYSQL_DATABASE + value: {{ .Values.mysql.database | quote }} + - name: MYSQL_USER + value: {{ .Values.mysql.user | quote }} + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: mysql-password + args: + - --default-authentication-plugin=mysql_native_password + - --skip-name-resolve + ports: + - name: mysql + containerPort: 3306 + startupProbe: + exec: + command: + - /bin/sh + - -ec + - mysqladmin ping -h 127.0.0.1 -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --silent + initialDelaySeconds: 10 + periodSeconds: 3 + failureThreshold: 40 + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - mysqladmin ping -h 127.0.0.1 -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --silent + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - mysqladmin ping -h 127.0.0.1 -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --silent + initialDelaySeconds: 60 + periodSeconds: 20 + failureThreshold: 6 + {{- if .Values.mysql.persistence.enabled }} + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + {{- end }} + {{- with .Values.mysql.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if and .Values.mysql.persistence.enabled (or .Values.mysql.persistence.hostPath .Values.mysql.persistence.existingClaim) }} + volumes: + - name: mysql-data + {{- if .Values.mysql.persistence.hostPath }} + hostPath: + path: {{ .Values.mysql.persistence.hostPath | quote }} + type: DirectoryOrCreate + {{- else }} + persistentVolumeClaim: + claimName: {{ include "cube.mysqlPVCName" . }} + {{- end }} + {{- end }} + {{- if and .Values.mysql.persistence.enabled (not .Values.mysql.persistence.hostPath) (not .Values.mysql.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: mysql-data + labels: + {{- include "cube.labels" . | nindent 10 }} + app.kubernetes.io/component: mysql + spec: + accessModes: + {{- toYaml .Values.mysql.persistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ .Values.mysql.persistence.size | quote }} + {{- if .Values.mysql.persistence.storageClassName }} + storageClassName: {{ .Values.mysql.persistence.storageClassName | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml new file mode 100644 index 000000000..f4975c707 --- /dev/null +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -0,0 +1,481 @@ +{{- if .Values.cubeNode.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "cube.nodeName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-node +spec: + updateStrategy: + {{- toYaml .Values.cubeNode.updateStrategy | nindent 4 }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cube-node + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cube-node + annotations: + {{- with .Values.cubeNode.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "cube.nodeServiceAccountName" . }} + automountServiceAccountToken: true + hostNetwork: {{ .Values.security.hostNetwork }} + hostPID: {{ .Values.security.hostPID }} + {{- if .Values.cubeNode.dns.useCubeDns }} + dnsPolicy: None + dnsConfig: + nameservers: + - {{ .Values.cubeNode.dns.nameserver | quote }} + searches: + - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} + - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} + - {{ .Values.cubeNode.dns.clusterDomain | quote }} + options: + - name: ndots + value: "5" + {{- else }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.computePlacement" . | nindent 6 }} + initContainers: + {{- if .Values.cubeNode.dns.useCubeDns }} + - name: wait-cube-dns + image: {{ include "cube.image" .Values.cubeNode.dns.checkImage | quote }} + imagePullPolicy: {{ .Values.cubeNode.dns.checkImage.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_DNS_NAMESERVER + value: {{ .Values.cubeNode.dns.nameserver | quote }} + - name: CUBE_DNS_DOMAIN + value: {{ (default .Values.cubeProxy.domain .Values.cubeDns.domain) | quote }} + - name: CUBE_DNS_TIMEOUT + value: {{ .Values.cubeNode.dns.waitTimeoutSeconds | quote }} + command: + - /bin/sh + - -ec + - | + deadline=$(( $(date +%s) + ${CUBE_DNS_TIMEOUT} )) + while [ "$(date +%s)" -le "${deadline}" ]; do + if nslookup "${CUBE_DNS_DOMAIN}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1 \ + && nslookup "wildcard-check.${CUBE_DNS_DOMAIN}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1 \ + && nslookup "kubernetes.default.svc.{{ .Values.cubeNode.dns.clusterDomain }}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1; then + echo "cube dns is ready at ${CUBE_DNS_NAMESERVER}" + exit 0 + fi + echo "waiting for cube dns at ${CUBE_DNS_NAMESERVER}" + sleep 2 + done + echo "cube dns is not ready at ${CUBE_DNS_NAMESERVER}" >&2 + exit 1 + {{- end }} + {{- if .Values.bootstrap.pvmHostKernel.enabled }} + - name: pvm-host-bootstrap + image: {{ include "cube.image" .Values.images.pvmHostBootstrap | quote }} + imagePullPolicy: {{ .Values.images.pvmHostBootstrap.pullPolicy }} + securityContext: + privileged: {{ .Values.security.privileged }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HOST_ROOT + value: /host + - name: DESIRED_KERNEL_PATTERN + value: {{ .Values.bootstrap.pvmHostKernel.desiredKernelPattern | quote }} + - name: PVM_KERNEL_BOOT_ARGS + value: {{ .Values.bootstrap.pvmHostKernel.bootArgs | quote }} + - name: PVM_KERNEL_RPM_PATH + value: {{ .Values.bootstrap.pvmHostKernel.package.rpmPath | quote }} + - name: PVM_KERNEL_DEB_PATH + value: {{ .Values.bootstrap.pvmHostKernel.package.debPath | quote }} + - name: PVM_KERNEL_RPM_URL + value: {{ .Values.bootstrap.pvmHostKernel.package.rpmUrl | quote }} + - name: PVM_KERNEL_DEB_URL + value: {{ .Values.bootstrap.pvmHostKernel.package.debUrl | quote }} + - name: PVM_KERNEL_PACKAGE_SOURCE + value: {{ .Values.bootstrap.pvmHostKernel.package.source | quote }} + - name: REBOOT_ENABLED + value: {{ .Values.bootstrap.pvmHostKernel.reboot.enabled | quote }} + - name: REBOOT_COORDINATED + value: {{ .Values.bootstrap.pvmHostKernel.reboot.coordinated | quote }} + - name: LEASE_NAME + value: {{ .Values.bootstrap.pvmHostKernel.reboot.leaseName | quote }} + - name: LEASE_TTL_SECONDS + value: {{ .Values.bootstrap.pvmHostKernel.reboot.leaseTTLSeconds | quote }} + - name: REBOOT_MAX_COUNT + value: {{ .Values.bootstrap.pvmHostKernel.reboot.maxCount | quote }} + volumeMounts: + - name: host-root + mountPath: /host + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + {{- end }} + {{- if .Values.bootstrap.nodeInit.enabled }} + - name: cube-node-init + image: {{ include "cube.image" .Values.images.nodeInit | quote }} + imagePullPolicy: {{ .Values.images.nodeInit.pullPolicy }} + securityContext: + privileged: {{ .Values.security.privileged }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: HOST_ROOT + value: /host + - name: DATA_CUBELET + value: {{ .Values.hostPaths.dataCubelet | quote }} + - name: REQUIRE_KVM + value: {{ .Values.bootstrap.nodeInit.requireKVM | quote }} + - name: REQUIRE_XFS + value: {{ .Values.bootstrap.nodeInit.requireXFS | quote }} + - name: CHMOD_KVM + value: {{ .Values.bootstrap.nodeInit.chmodKVM | quote }} + - name: LOAD_KVM_MODULE + value: {{ .Values.bootstrap.nodeInit.loadKVMModule | quote }} + - name: WRITE_UDEV_RULE + value: {{ .Values.bootstrap.nodeInit.writeUdevRule | quote }} + - name: CREATE_HOST_DIRS + value: {{ .Values.bootstrap.nodeInit.createHostDirs | quote }} + - name: CHECK_MASTER_CONNECTIVITY + value: {{ .Values.bootstrap.nodeInit.checkMasterConnectivity | quote }} + - name: CUBE_MASTER_ENDPOINT + value: {{ include "cube.masterEndpoint" . | quote }} + - name: LOOPBACK_ENABLED + value: {{ .Values.bootstrap.nodeInit.dataCubelet.loopback.enabled | quote }} + - name: LOOPBACK_IMAGE_PATH + value: {{ .Values.bootstrap.nodeInit.dataCubelet.loopback.imagePath | quote }} + - name: LOOPBACK_SIZE + value: {{ .Values.bootstrap.nodeInit.dataCubelet.loopback.size | quote }} + - name: CHECK_MEMORY + value: {{ .Values.bootstrap.nodeInit.checkMemory | quote }} + - name: MIN_MEMORY_KB + value: {{ .Values.bootstrap.nodeInit.minMemoryKB | quote }} + - name: CHECK_CGROUP_CPU + value: {{ .Values.bootstrap.nodeInit.checkCgroupCPU | quote }} + - name: CHECK_GLIBC + value: {{ .Values.bootstrap.nodeInit.checkGlibc | quote }} + - name: CHECK_CIDR + value: {{ .Values.bootstrap.nodeInit.checkCIDR | quote }} + - name: CHECK_CUBECOW_DEPS + value: {{ .Values.bootstrap.nodeInit.checkCubecowDeps | quote }} + - name: CUBE_PVM_ENABLE + value: {{ ternary "1" "0" .Values.cubeNode.pvmGuestKernel.enabled | quote }} + - name: CUBE_SANDBOX_NETWORK_CIDR + value: {{ .Values.cubeNode.network.cidr | quote }} + - name: CUBE_SANDBOX_NETWORK_CIDR_SKIP_CONFLICT_CHECK + value: {{ ternary "1" "0" .Values.cubeNode.network.cidrSkipConflictCheck | quote }} + volumeMounts: + - name: host-root + mountPath: /host + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: data-cubelet + mountPath: {{ .Values.hostPaths.dataCubelet }} + mountPropagation: Bidirectional + - name: data-log + mountPath: {{ .Values.hostPaths.dataLog }} + - name: data-cube-shim + mountPath: {{ .Values.hostPaths.dataCubeShim }} + mountPropagation: Bidirectional + - name: data-snapshot-pack + mountPath: {{ .Values.hostPaths.dataSnapshotPack }} + - name: tmp-cube + mountPath: {{ .Values.hostPaths.tmpCube }} + mountPropagation: Bidirectional + {{- end }} + containers: + - name: cube-node + image: {{ include "cube.image" .Values.images.node | quote }} + imagePullPolicy: {{ .Values.images.node.pullPolicy }} + securityContext: + privileged: {{ .Values.security.privileged }} + capabilities: + add: + - SYS_ADMIN + - NET_ADMIN + - SYS_MODULE + - SYS_RESOURCE + - IPC_LOCK + - SYS_PTRACE + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_ROLE + value: node + - name: CUBE_MASTER_ENDPOINT + value: {{ include "cube.masterEndpoint" . | quote }} + - name: CUBE_SANDBOX_NODE_IP + {{- if eq .Values.cubeNode.ipSource "statusHostIP" }} + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- else }} + value: {{ .Values.cubeNode.staticNodeIP | quote }} + {{- end }} + - name: CUBE_PVM_ENABLE + value: {{ ternary "1" "0" .Values.cubeNode.pvmGuestKernel.enabled | quote }} + - name: CUBE_SANDBOX_AUTO_DETECT_ETH + value: {{ .Values.cubeNode.network.autoDetectEthName | quote }} + - name: CUBE_SANDBOX_ETH_NAME + value: {{ .Values.cubeNode.network.ethName | quote }} + - name: CUBE_SANDBOX_NETWORK_CIDR + value: {{ .Values.cubeNode.network.cidr | quote }} + - name: CUBE_SANDBOX_DNS_SERVERS + {{- if .Values.cubeNode.dns.sandbox.nameservers }} + value: {{ join "," . | quote }} + {{- else if and .Values.cubeNode.dns.sandbox.useCubeDns .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") .Values.cubeDns.sandboxGateway.enabled }} + value: "$(CUBE_SANDBOX_NODE_IP)" + {{- else }} + value: "" + {{- end }} + {{- with .Values.cubeNode.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: cubelet-grpc + containerPort: 9999 + - name: cubelet-http + containerPort: 9998 + - name: cubelet-debug + containerPort: 9966 + - name: network-agent + containerPort: 19090 + {{- if .Values.cubeNode.probes.startup.enabled }} + startupProbe: + tcpSocket: + port: {{ .Values.cubeNode.probes.startup.tcpSocketPort }} + periodSeconds: {{ .Values.cubeNode.probes.startup.periodSeconds }} + failureThreshold: {{ .Values.cubeNode.probes.startup.failureThreshold }} + {{- end }} + {{- if .Values.cubeNode.probes.readiness.enabled }} + readinessProbe: + tcpSocket: + port: {{ .Values.cubeNode.probes.readiness.tcpSocketPort }} + initialDelaySeconds: {{ .Values.cubeNode.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeNode.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.cubeNode.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.cubeNode.probes.liveness.enabled }} + livenessProbe: + tcpSocket: + port: {{ .Values.cubeNode.probes.liveness.tcpSocketPort }} + initialDelaySeconds: {{ .Values.cubeNode.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeNode.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.cubeNode.probes.liveness.failureThreshold }} + {{- end }} + volumeMounts: + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: data-cubelet + mountPath: {{ .Values.hostPaths.dataCubelet }} + mountPropagation: Bidirectional + - name: data-log + mountPath: {{ .Values.hostPaths.dataLog }} + - name: data-cube-shim + mountPath: {{ .Values.hostPaths.dataCubeShim }} + mountPropagation: Bidirectional + - name: data-snapshot-pack + mountPath: {{ .Values.hostPaths.dataSnapshotPack }} + - name: tmp-cube + mountPath: {{ .Values.hostPaths.tmpCube }} + mountPropagation: Bidirectional + {{- with .Values.cubeNode.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.cubeEgress.enabled }} + - name: cube-egress + image: {{ include "cube.image" .Values.images.cubeEgress | quote }} + imagePullPolicy: {{ .Values.images.cubeEgress.pullPolicy }} + command: + - /bin/bash + - -ec + - | + mkdir -p /data/log/cube-egress + exec /usr/local/openresty/nginx/sbin/start.sh + securityContext: + privileged: {{ .Values.security.privileged }} + capabilities: + add: + - NET_ADMIN + - NET_RAW + - NET_BIND_SERVICE + - CHOWN + - SETUID + - SETGID + - DAC_READ_SEARCH + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_EGRESS_BOOTSTRAP_URL + value: {{ .Values.cubeEgress.bootstrapURL | quote }} + - name: CUBE_EGRESS_DEBUG_DUMP + value: {{ ternary "1" "0" .Values.cubeEgress.debugDump | quote }} + ports: + - name: egress-admin + containerPort: 9090 + - name: egress-http + containerPort: 8080 + - name: egress-https + containerPort: 8443 + {{- if .Values.cubeEgress.probes.readiness.enabled }} + readinessProbe: + exec: + command: + - /bin/bash + - -ec + - {{ printf "curl -fsS http://127.0.0.1:%v%s >/dev/null" .Values.cubeEgress.probes.readiness.port .Values.cubeEgress.probes.readiness.path | quote }} + initialDelaySeconds: {{ .Values.cubeEgress.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeEgress.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.cubeEgress.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.cubeEgress.probes.liveness.enabled }} + livenessProbe: + exec: + command: + - /bin/bash + - -ec + - {{ printf "curl -fsS http://127.0.0.1:%v%s >/dev/null" .Values.cubeEgress.probes.liveness.port .Values.cubeEgress.probes.liveness.path | quote }} + initialDelaySeconds: {{ .Values.cubeEgress.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeEgress.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.cubeEgress.probes.liveness.failureThreshold }} + {{- end }} + volumeMounts: + - name: cube-egress-ca + mountPath: /etc/cube/ca + readOnly: true + - name: data-log + mountPath: {{ .Values.hostPaths.dataLog }} + - name: cube-egress-run + mountPath: /var/run/openresty + {{- with .Values.cubeEgress.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.cubeEgress.network.enabled }} + - name: cube-egress-net + image: {{ include "cube.image" .Values.images.cubeEgressNet | quote }} + imagePullPolicy: {{ .Values.images.cubeEgressNet.pullPolicy }} + securityContext: + privileged: {{ .Values.security.privileged }} + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_MODULE + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_INGRESS_IFACE + value: {{ .Values.cubeEgress.network.ingressInterface | quote }} + - name: CUBE_TPROXY_ON_IP + value: {{ .Values.cubeEgress.network.tproxyOnIP | quote }} + - name: CUBE_EGRESS_NET_INITIAL_WAIT_SECONDS + value: {{ .Values.cubeEgress.network.initialWaitSeconds | quote }} + - name: CUBE_EGRESS_NET_REAPPLY_INTERVAL_SECONDS + value: {{ .Values.cubeEgress.network.reapplyIntervalSeconds | quote }} + {{- if .Values.cubeEgress.network.probes.readiness.enabled }} + readinessProbe: + exec: + command: + - /bin/bash + - -ec + - {{ include "cube.egressNetProbeCommand" . | quote }} + initialDelaySeconds: {{ .Values.cubeEgress.network.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeEgress.network.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.cubeEgress.network.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.cubeEgress.network.probes.liveness.enabled }} + livenessProbe: + exec: + command: + - /bin/bash + - -ec + - {{ include "cube.egressNetProbeCommand" . | quote }} + initialDelaySeconds: {{ .Values.cubeEgress.network.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeEgress.network.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.cubeEgress.network.probes.liveness.failureThreshold }} + {{- end }} + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + {{- with .Values.cubeEgress.netResources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + volumes: + - name: host-root + hostPath: + path: {{ .Values.hostPaths.root }} + type: Directory + - name: dev + hostPath: + path: {{ .Values.hostPaths.dev }} + type: Directory + - name: sys + hostPath: + path: {{ .Values.hostPaths.sys }} + type: Directory + - name: lib-modules + hostPath: + path: {{ .Values.hostPaths.libModules }} + type: Directory + - name: data-cubelet + hostPath: + path: {{ .Values.hostPaths.dataCubelet }} + type: DirectoryOrCreate + - name: data-log + hostPath: + path: {{ .Values.hostPaths.dataLog }} + type: DirectoryOrCreate + - name: data-cube-shim + hostPath: + path: {{ .Values.hostPaths.dataCubeShim }} + type: DirectoryOrCreate + - name: data-snapshot-pack + hostPath: + path: {{ .Values.hostPaths.dataSnapshotPack }} + type: DirectoryOrCreate + - name: tmp-cube + hostPath: + path: {{ .Values.hostPaths.tmpCube }} + type: DirectoryOrCreate + {{- if .Values.cubeEgress.enabled }} + - name: cube-egress-ca + secret: + secretName: {{ include "cube.egressCASecretName" . }} + - name: cube-egress-run + emptyDir: + medium: Memory + sizeLimit: 64Mi + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/proxy-node.yaml b/deploy/kubernetes/chart/templates/proxy-node.yaml new file mode 100644 index 000000000..872dff29b --- /dev/null +++ b/deploy/kubernetes/chart/templates/proxy-node.yaml @@ -0,0 +1,310 @@ +{{- if eq (include "cube.proxyEnabled" .) "true" }} +{{- $redisHost := default (include "cube.redisHost" .) .Values.cubeProxy.redis.host -}} +{{- $redisPort := default .Values.redis.port .Values.cubeProxy.redis.port -}} +{{- $proxyCertSecretName := include "cube.proxyCertSecretName" . -}} +{{- if eq .Values.cubeProxy.tls.mode "inline" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $proxyCertSecretName }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node +type: Opaque +stringData: + {{ .Values.cubeProxy.tls.certSecretKey | quote }}: |- +{{ .Values.cubeProxy.tls.cert | nindent 4 }} + {{ .Values.cubeProxy.tls.keySecretKey | quote }}: |- +{{ .Values.cubeProxy.tls.key | nindent 4 }} +--- +{{- else if eq .Values.cubeProxy.tls.mode "selfSigned" }} +{{- $commonName := default .Values.cubeProxy.domain .Values.cubeProxy.tls.selfSigned.commonName -}} +{{- $dnsNames := list .Values.cubeProxy.domain -}} +{{- range .Values.cubeProxy.tls.selfSigned.dnsNames }} +{{- $dnsNames = append $dnsNames . -}} +{{- end }} +{{- $dnsNames = uniq $dnsNames -}} +{{- $ipAddresses := list -}} +{{- range .Values.cubeProxy.tls.selfSigned.ipAddresses }} +{{- $ipAddresses = append $ipAddresses . -}} +{{- end }} +{{- $certKey := .Values.cubeProxy.tls.certSecretKey -}} +{{- $keyKey := .Values.cubeProxy.tls.keySecretKey -}} +{{- $caKey := .Values.cubeProxy.tls.caSecretKey -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $proxyCertSecretName -}} +{{- $hasExisting := and $existing (hasKey $existing.data $certKey) (hasKey $existing.data $keyKey) (hasKey $existing.data $caKey) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $proxyCertSecretName }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node +type: Opaque +{{ if $hasExisting }} +data: + {{ $certKey | quote }}: {{ index $existing.data $certKey | quote }} + {{ $keyKey | quote }}: {{ index $existing.data $keyKey | quote }} + {{ $caKey | quote }}: {{ index $existing.data $caKey | quote }} +{{- else }} +{{- $ca := genCA (printf "%s-proxy-ca" (include "cube.fullname" .)) (int .Values.cubeProxy.tls.selfSigned.caValidityDays) -}} +{{- $cert := genSignedCert $commonName $ipAddresses $dnsNames (int .Values.cubeProxy.tls.selfSigned.certValidityDays) $ca -}} +stringData: + {{ .Values.cubeProxy.tls.certSecretKey | quote }}: |- +{{ $cert.Cert | nindent 4 }} + {{ .Values.cubeProxy.tls.keySecretKey | quote }}: |- +{{ $cert.Key | nindent 4 }} + {{ .Values.cubeProxy.tls.caSecretKey | quote }}: |- +{{ $ca.Cert | nindent 4 }} +{{- end }} +--- +{{- else if eq .Values.cubeProxy.tls.mode "certManager" }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $proxyCertSecretName }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node +spec: + secretName: {{ $proxyCertSecretName }} + {{- if .Values.cubeProxy.tls.certManager.commonName }} + commonName: {{ .Values.cubeProxy.tls.certManager.commonName | quote }} + {{- else }} + commonName: {{ .Values.cubeProxy.domain | quote }} + {{- end }} + dnsNames: + {{- if .Values.cubeProxy.tls.certManager.dnsNames }} + {{- toYaml .Values.cubeProxy.tls.certManager.dnsNames | nindent 4 }} + {{- else }} + - {{ .Values.cubeProxy.domain | quote }} + {{- end }} + {{- with .Values.cubeProxy.tls.certManager.ipAddresses }} + ipAddresses: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.cubeProxy.tls.certManager.duration }} + duration: {{ . | quote }} + {{- end }} + {{- with .Values.cubeProxy.tls.certManager.renewBefore }} + renewBefore: {{ . | quote }} + {{- end }} + usages: + {{- toYaml .Values.cubeProxy.tls.certManager.usages | nindent 4 }} + issuerRef: + group: {{ .Values.cubeProxy.tls.certManager.issuerRef.group | quote }} + kind: {{ .Values.cubeProxy.tls.certManager.issuerRef.kind | quote }} + name: {{ .Values.cubeProxy.tls.certManager.issuerRef.name | quote }} +--- +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.proxyName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node +spec: + replicas: {{ .Values.cubeProxy.replicas }} + strategy: + {{- toYaml .Values.cubeProxy.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cube-proxy-node + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cube-proxy-node + {{- with .Values.cubeProxy.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: false + hostNetwork: {{ .Values.cubeProxy.hostNetwork }} + dnsPolicy: {{ ternary "ClusterFirstWithHostNet" "ClusterFirst" .Values.cubeProxy.hostNetwork }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: cube-proxy-node + image: {{ include "cube.image" .Values.images.proxyNode | quote }} + imagePullPolicy: {{ .Values.images.proxyNode.pullPolicy }} + command: + - /bin/sh + - -ec + - | + set -u + mkdir -p /usr/local/openresty/nginx/conf/global /data /data/log/cube-proxy /cache + + case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in + *[!0-9:]*|:*|*:) + echo "invalid CubeProxy listen ports: http=${CUBE_PROXY_HTTP_LISTEN_PORT} https=${CUBE_PROXY_HTTPS_LISTEN_PORT}" >&2 + exit 1 + ;; + esac + sed -i \ + -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ + -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ + -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ + -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ + /usr/local/openresty/nginx/conf/nginx.conf + + escape_nginx_value() { + printf '%s' "$1" | sed 's/[\\"]/\\&/g' + } + + resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" + if [ -z "${resolver_addrs}" ]; then + resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" + fi + [ -n "${resolver_addrs}" ] || { + echo "unable to determine nginx DNS resolver for CubeProxy Redis lookups" >&2 + exit 1 + } + case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT}${CUBE_PROXY_RESOLVER_IPV6}" in + *[\;\{\}\$\`]*) + echo "invalid CubeProxy resolver configuration" >&2 + exit 1 + ;; + esac + + cat > /usr/local/openresty/nginx/conf/global/global.conf </dev/null" .Values.cubeProxy.probes.readiness.port .Values.cubeProxy.probes.readiness.path | quote }} + initialDelaySeconds: {{ .Values.cubeProxy.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeProxy.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.cubeProxy.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.cubeProxy.probes.liveness.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - {{ printf "curl -fsS http://127.0.0.1:%v%s >/dev/null" .Values.cubeProxy.probes.liveness.port .Values.cubeProxy.probes.liveness.path | quote }} + initialDelaySeconds: {{ .Values.cubeProxy.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.cubeProxy.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.cubeProxy.probes.liveness.failureThreshold }} + {{- end }} + volumeMounts: + - name: proxy-certs + mountPath: /usr/local/openresty/nginx/certs + readOnly: true + - name: proxy-logs + mountPath: /data/log/cube-proxy + {{- with .Values.cubeProxy.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: proxy-certs + secret: + secretName: {{ $proxyCertSecretName }} + items: + - key: {{ .Values.cubeProxy.tls.certSecretKey | quote }} + path: cube.app+3.pem + - key: {{ .Values.cubeProxy.tls.keySecretKey | quote }} + path: cube.app+3-key.pem + - name: proxy-logs + hostPath: + path: {{ .Values.cubeProxy.logs.hostPath | quote }} + type: DirectoryOrCreate +{{- end }} diff --git a/deploy/kubernetes/chart/templates/rbac.yaml b/deploy/kubernetes/chart/templates/rbac.yaml new file mode 100644 index 000000000..a79d963cc --- /dev/null +++ b/deploy/kubernetes/chart/templates/rbac.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.rbac.create .Values.cubeNode.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "cube.nodeServiceAccountName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "cube.nodeServiceAccountName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "cube.nodeServiceAccountName" . }} +subjects: + - kind: ServiceAccount + name: {{ include "cube.nodeServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/redis.yaml b/deploy/kubernetes/chart/templates/redis.yaml new file mode 100644 index 000000000..8e18da7d3 --- /dev/null +++ b/deploy/kubernetes/chart/templates/redis.yaml @@ -0,0 +1,121 @@ +{{- if eq (include "cube.redisBuiltinEnabled" .) "true" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.redisName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + clusterIP: None + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: redis + ports: + - name: redis + port: {{ .Values.redis.port }} + targetPort: redis +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "cube.redisName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + serviceName: {{ include "cube.redisName" . }} + replicas: 1 + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + spec: + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: redis + image: {{ include "cube.image" .Values.redis.image | quote }} + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + command: ["redis-server"] + args: ["--requirepass", "$(REDIS_PASSWORD)"] + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: redis-password + ports: + - name: redis + containerPort: 6379 + startupProbe: + exec: + command: + - /bin/sh + - -ec + - redis-cli -a "${REDIS_PASSWORD}" ping | grep -x PONG + initialDelaySeconds: 5 + periodSeconds: 3 + failureThreshold: 40 + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - redis-cli -a "${REDIS_PASSWORD}" ping | grep -x PONG + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - redis-cli -a "${REDIS_PASSWORD}" ping | grep -x PONG + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + {{- if .Values.redis.persistence.enabled }} + volumeMounts: + - name: redis-data + mountPath: /data + {{- end }} + {{- with .Values.redis.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if and .Values.redis.persistence.enabled (or .Values.redis.persistence.hostPath .Values.redis.persistence.existingClaim) }} + volumes: + - name: redis-data + {{- if .Values.redis.persistence.hostPath }} + hostPath: + path: {{ .Values.redis.persistence.hostPath | quote }} + type: DirectoryOrCreate + {{- else }} + persistentVolumeClaim: + claimName: {{ include "cube.redisPVCName" . }} + {{- end }} + {{- end }} + {{- if and .Values.redis.persistence.enabled (not .Values.redis.persistence.hostPath) (not .Values.redis.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: redis-data + labels: + {{- include "cube.labels" . | nindent 10 }} + app.kubernetes.io/component: redis + spec: + accessModes: + {{- toYaml .Values.redis.persistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ .Values.redis.persistence.size | quote }} + {{- if .Values.redis.persistence.storageClassName }} + storageClassName: {{ .Values.redis.persistence.storageClassName | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/secret.yaml b/deploy/kubernetes/chart/templates/secret.yaml new file mode 100644 index 000000000..2f3366c0d --- /dev/null +++ b/deploy/kubernetes/chart/templates/secret.yaml @@ -0,0 +1,13 @@ +{{- if eq (include "cube.secretEnabled" .) "true" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cube.secretName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} +type: Opaque +stringData: + mysql-root-password: {{ .Values.mysql.rootPassword | quote }} + mysql-password: {{ .Values.mysql.password | quote }} + redis-password: {{ .Values.redis.password | quote }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/serviceaccount.yaml b/deploy/kubernetes/chart/templates/serviceaccount.yaml new file mode 100644 index 000000000..34c6675f3 --- /dev/null +++ b/deploy/kubernetes/chart/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.cubeNode.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cube.nodeServiceAccountName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-node + annotations: + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/storageclass.yaml b/deploy/kubernetes/chart/templates/storageclass.yaml new file mode 100644 index 000000000..76b06b1b3 --- /dev/null +++ b/deploy/kubernetes/chart/templates/storageclass.yaml @@ -0,0 +1,15 @@ +{{- if .Values.storageClass.create }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name | quote }} + labels: + {{- include "cube.labels" . | nindent 4 }} +provisioner: {{ .Values.storageClass.provisioner | quote }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +volumeBindingMode: {{ .Values.storageClass.volumeBindingMode }} +{{- with .Values.storageClass.parameters }} +parameters: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml new file mode 100644 index 000000000..6b5607247 --- /dev/null +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -0,0 +1,405 @@ +{{- if .Values.helmTest.enabled }} +{{- $domain := default .Values.cubeProxy.domain .Values.cubeDns.domain -}} +{{- $internalMaster := and .Values.controlPlane.enabled .Values.controlPlane.master.enabled -}} +{{- $internalAPI := and .Values.controlPlane.enabled .Values.controlPlane.api.enabled -}} +{{- $externalAPIEndpoint := default "" .Values.externalControlPlane.apiEndpoint -}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-health-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + serviceAccountName: {{ include "cube.fullname" . }}-test + containers: + - name: curl + image: {{ include "cube.image" .Values.helmTest.image | quote }} + imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} + command: + - sh + - -ec + - | + token_path=/var/run/secrets/kubernetes.io/serviceaccount/token + ca_path=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + ns={{ .Release.Namespace | quote }} + kube_api="https://kubernetes.default.svc" + kget() { + curl --connect-timeout 5 --max-time 20 --cacert "${ca_path}" \ + -H "Authorization: Bearer $(cat "${token_path}")" \ + -fsS "${kube_api}$1" + } + + {{- if or $internalMaster .Values.externalControlPlane.enabled }} + echo '[health-test] check CubeMaster' + curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.masterEndpoint" . }}/notify/health + {{- else }} + echo '[health-test] skip CubeMaster: no internal or external control plane' + {{- end }} + + {{- if or $internalAPI $externalAPIEndpoint }} + echo '[health-test] check CubeAPI' + api_endpoint={{ include "cube.apiEndpoint" . | quote }} + case "${api_endpoint}" in + http://*|https://*) api_base="${api_endpoint}" ;; + *) api_base="http://${api_endpoint}" ;; + esac + curl --connect-timeout 5 --max-time 15 -fsS "${api_base%/}/health" + + {{- if .Values.cubeNode.enabled }} + echo '[health-test] check node registration' + nodes_json="$(curl --connect-timeout 5 --max-time 15 -fsS "${api_base%/}/cubeapi/v1/nodes")" + printf '%s\n' "${nodes_json}" + healthy_nodes="$(printf '%s\n' "${nodes_json}" | grep -o '"healthy"[[:space:]]*:[[:space:]]*true' | wc -l | tr -d ' ')" + node_ids="$(printf '%s\n' "${nodes_json}" | grep -o '"nodeID"[[:space:]]*:' | wc -l | tr -d ' ')" + test "${healthy_nodes}" -ge 1 + test "${node_ids}" -ge 1 + {{- else }} + echo '[health-test] skip node registration: cubeNode.enabled=false' + {{- end }} + {{- else }} + echo '[health-test] skip CubeAPI: no internal API and externalControlPlane.apiEndpoint is empty' + {{- end }} + + {{- if and .Values.webui.enabled $internalAPI }} + echo '[health-test] check WebUI' + curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.webuiName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.webui.service.port }}/ >/dev/null + curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.webuiName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.webui.service.port }}/cubeapi/v1/health + {{- end }} + + {{- if eq (include "cube.proxyEnabled" .) "true" }} + echo '[health-test] check CubeProxy through control-node test pod' + {{- end }} + + {{- if .Values.cubeNode.enabled }} + echo '[health-test] check CubeNode DaemonSet readiness' + node_ds="$(kget "/apis/apps/v1/namespaces/${ns}/daemonsets/{{ include "cube.nodeName" . }}")" + printf '%s\n' "${node_ds}" | grep -Eq '"numberReady"[[:space:]]*:[[:space:]]*[1-9]' + printf '%s\n' "${node_ds}" | grep -Eq '"desiredNumberScheduled"[[:space:]]*:[[:space:]]*[1-9]' + {{- end }} + + {{- if .Values.cubeEgress.enabled }} + echo '[health-test] check CubeEgress containers are present in CubeNode pods' + pods_json="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io/component%3Dcube-node")" + printf '%s\n' "${pods_json}" | grep -qF 'cube-egress' + printf '%s\n' "${pods_json}" | grep -qF 'cube-egress-net' + {{- end }} +{{ if eq (include "cube.cubemastercliEnabled" .) "true" }} +{{- $images := default dict .Values.images -}} +{{- $cubemastercliImage := default (dict "repository" "ccr.ccs.tencentyun.com/cubesandbox-chart/cubemastercli" "tag" "v0.5.0" "pullPolicy" "Always") (get $images "cubemastercli") -}} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-cubemastercli-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: cubemastercli + image: {{ include "cube.image" $cubemastercliImage | quote }} + imagePullPolicy: {{ dig "pullPolicy" "Always" $cubemastercliImage }} + env: + {{- include "cube.timezoneEnv" . | nindent 8 }} + - name: CUBE_RELEASE + value: {{ .Release.Name | quote }} + - name: CUBE_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: CUBE_MASTER_ENDPOINT + value: {{ include "cube.cubemastercliMasterEndpoint" . | quote }} + - name: CUBEMASTERCLI_ADDRESS + value: {{ include "cube.cubemastercliMasterAddress" . | quote }} + - name: CUBEMASTERCLI_PORT + value: {{ include "cube.cubemastercliMasterPort" . | quote }} + command: + - /bin/sh + - -ec + - cubemastercli --address "${CUBEMASTERCLI_ADDRESS}" --port "${CUBEMASTERCLI_PORT}" node list +{{- end }} +{{ if eq (include "cube.mysqlBuiltinEnabled" .) "true" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-mysql-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: mysql + image: {{ include "cube.image" .Values.mysql.image | quote }} + imagePullPolicy: {{ .Values.mysql.image.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 8 }} + - name: MYSQL_PWD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: mysql-password + command: + - sh + - -ec + - | + echo '[health-test] check MySQL' + mysqladmin ping -h {{ include "cube.mysqlHost" . | quote }} -P {{ .Values.mysql.port }} -u{{ .Values.mysql.user | quote }} --silent +{{- end }} +{{ if eq (include "cube.redisBuiltinEnabled" .) "true" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-redis-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: redis + image: {{ include "cube.image" .Values.redis.image | quote }} + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 8 }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: redis-password + command: + - sh + - -ec + - | + echo '[health-test] check Redis' + redis-cli -h {{ include "cube.redisHost" . | quote }} -p {{ .Values.redis.port }} -a "${REDIS_PASSWORD}" ping | grep -x PONG +{{- end }} +{{ if eq (include "cube.proxyEnabled" .) "true" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-proxy-control-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 2 }} + containers: + - name: proxy + image: {{ include "cube.image" .Values.helmTest.image | quote }} + imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 8 }} + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + command: + - sh + - -ec + - | + echo '[health-test] check control-node CubeProxy' + curl --connect-timeout 5 --max-time 15 -fsS -o /dev/null http://127.0.0.1:{{ .Values.cubeProxy.probes.readiness.port }}{{ .Values.cubeProxy.probes.readiness.path }} +{{- end }} +{{ if .Values.cubeDns.enabled }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-dns-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if eq .Values.cubeDns.mode "nodeLocal" }} + hostNetwork: true + dnsPolicy: None + dnsConfig: + nameservers: + - {{ .Values.cubeNode.dns.nameserver | quote }} + searches: + - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} + - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} + - {{ .Values.cubeNode.dns.clusterDomain | quote }} + options: + - name: ndots + value: "5" + {{- include "cube.computePlacement" . | nindent 2 }} + {{- end }} + containers: + - name: dns + image: {{ include "cube.image" .Values.helmTest.dnsImage | quote }} + imagePullPolicy: {{ .Values.helmTest.dnsImage.pullPolicy }} + command: + - sh + - -ec + - | + echo '[health-test] check CubeDNS' + {{- if eq .Values.cubeDns.mode "nodeLocal" }} + nslookup {{ $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' + nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' + nslookup {{ printf "kubernetes.default.svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' + {{- else }} + nslookup {{ $domain | quote }} {{ printf "%s-dns.%s.svc.cluster.local" (include "cube.fullname" .) .Release.Namespace | quote }} | grep -Eq 'canonical name|Address' + nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ printf "%s-dns.%s.svc.cluster.local" (include "cube.fullname" .) .Release.Namespace | quote }} | grep -Eq 'canonical name|Address' + {{- end }} +{{- end }} +{{ if .Values.cubeNode.enabled }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-node-image-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- include "cube.computePlacement" . | nindent 2 }} + containers: + - name: node-image-assets + image: {{ include "cube.image" .Values.images.node | quote }} + imagePullPolicy: {{ .Values.images.node.pullPolicy }} + command: + - sh + - -ec + - | + TOOLBOX_ROOT="${TOOLBOX_ROOT:-/usr/local/services/cubetoolbox}" + echo '[health-test] check cube-node image runtime assets' + test -x /usr/local/bin/cube-runtime + test -x /usr/local/bin/containerd-shim-cube-rs + test -x /usr/local/bin/cubecli + test -x /usr/local/bin/cubevsmapdump + test -f "${TOOLBOX_ROOT}/Cubelet/config/config.toml" + test -f "${TOOLBOX_ROOT}/Cubelet/dynamicconf/conf.yaml" + test -f "${TOOLBOX_ROOT}/cube-kernel-scf/vmlinux-bm" + test -f "${TOOLBOX_ROOT}/cube-kernel-scf/vmlinux-pvm" + test -f "${TOOLBOX_ROOT}/cube-image/cube-guest-image-cpu.img" +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "cube.fullname" . }}-node-runtime-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") }} + hostNetwork: true + dnsPolicy: None + dnsConfig: + nameservers: + - {{ .Values.cubeNode.dns.nameserver | quote }} + searches: + - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} + - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} + - {{ .Values.cubeNode.dns.clusterDomain | quote }} + options: + - name: ndots + value: "5" + {{- end }} + {{- include "cube.computePlacement" . | nindent 2 }} + containers: + - name: node-runtime + image: {{ include "cube.image" .Values.helmTest.dnsImage | quote }} + imagePullPolicy: {{ .Values.helmTest.dnsImage.pullPolicy }} + command: + - sh + - -ec + - | + echo '[health-test] check cube-node host runtime assets' + test -e /dev/kvm + test -d {{ .Values.hostPaths.dataCubelet | quote }} + test -S {{ printf "%s/cubelet.sock" .Values.hostPaths.dataCubelet | quote }} + test -S {{ printf "%s/network-agent-grpc.sock" .Values.hostPaths.tmpCube | quote }} + {{- if and .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") }} + nslookup {{ $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} >/dev/null + {{- end }} + volumeMounts: + - name: dev + mountPath: /dev + readOnly: true + - name: data-cubelet + mountPath: {{ .Values.hostPaths.dataCubelet }} + - name: tmp-cube + mountPath: {{ .Values.hostPaths.tmpCube }} + volumes: + - name: dev + hostPath: + path: {{ .Values.hostPaths.dev }} + type: Directory + - name: data-cubelet + hostPath: + path: {{ .Values.hostPaths.dataCubelet }} + type: DirectoryOrCreate + - name: tmp-cube + hostPath: + path: {{ .Values.hostPaths.tmpCube }} + type: DirectoryOrCreate +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/tests/rbac.yaml b/deploy/kubernetes/chart/templates/tests/rbac.yaml new file mode 100644 index 000000000..424e0a70d --- /dev/null +++ b/deploy/kubernetes/chart/templates/tests/rbac.yaml @@ -0,0 +1,40 @@ +{{- if .Values.helmTest.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cube.fullname" . }}-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "cube.fullname" . }}-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test +rules: + - apiGroups: [""] + resources: ["pods", "services", "endpoints"] + verbs: ["get", "list"] + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets", "statefulsets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "cube.fullname" . }}-test + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "cube.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "cube.fullname" . }}-test + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/validate.yaml b/deploy/kubernetes/chart/templates/validate.yaml new file mode 100644 index 000000000..487257336 --- /dev/null +++ b/deploy/kubernetes/chart/templates/validate.yaml @@ -0,0 +1,119 @@ +{{- $computeSelector := .Values.placement.compute.nodeSelector -}} +{{- $controlPlaneSelector := .Values.placement.controlPlane.nodeSelector -}} +{{- if and .Values.controlPlane.enabled (not $controlPlaneSelector) }} +{{- fail "controlPlane.enabled=true requires placement.controlPlane.nodeSelector" }} +{{- end }} +{{- if .Values.cubeNode.enabled }} +{{- if not $computeSelector }} +{{- fail "cube-node requires placement.compute.nodeSelector, otherwise host-mutating bootstrap may run on all nodes" }} +{{- end }} +{{- if and .Values.bootstrap.pvmHostKernel.enabled (not (hasKey $computeSelector "cube.tencent.com/allow-pvm-bootstrap")) }} +{{- fail "bootstrap.pvmHostKernel.enabled=true requires placement.compute.nodeSelector to include cube.tencent.com/allow-pvm-bootstrap=true" }} +{{- end }} +{{- if and (eq .Values.cubeNode.ipSource "static") (not .Values.cubeNode.staticNodeIP) }} +{{- fail "cubeNode.ipSource=static requires cubeNode.staticNodeIP" }} +{{- end }} +{{- if and .Values.bootstrap.pvmHostKernel.enabled (not .Values.cubeNode.pvmGuestKernel.enabled) }} +{{- fail "bootstrap.pvmHostKernel.enabled=true requires cubeNode.pvmGuestKernel.enabled=true to match one-click CUBE_PVM_ENABLE consistency" }} +{{- end }} +{{- if and .Values.cubeNode.dns.useCubeDns (not .Values.cubeDns.enabled) }} +{{- fail "cubeNode.dns.useCubeDns=true requires cubeDns.enabled=true" }} +{{- end }} +{{- if and .Values.cubeNode.dns.useCubeDns (ne .Values.cubeDns.mode "nodeLocal") }} +{{- fail "cubeNode.dns.useCubeDns=true requires cubeDns.mode=nodeLocal" }} +{{- end }} +{{- if and .Values.cubeNode.dns.useCubeDns (not .Values.security.hostNetwork) }} +{{- fail "cubeNode.dns.useCubeDns=true requires security.hostNetwork=true because node-local DNS is bound on host loopback" }} +{{- end }} +{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (not .Values.cubeDns.sandboxGateway.enabled) }} +{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.sandboxGateway.enabled=true" }} +{{- end }} +{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (not .Values.cubeDns.enabled) }} +{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.enabled=true" }} +{{- end }} +{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (ne .Values.cubeDns.mode "nodeLocal") }} +{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.mode=nodeLocal" }} +{{- end }} +{{- end }} +{{- if .Values.externalControlPlane.enabled }} +{{- if not .Values.externalControlPlane.masterEndpoint }} +{{- fail "externalControlPlane.enabled=true requires externalControlPlane.masterEndpoint" }} +{{- end }} +{{- end }} +{{- if and .Values.cubeNode.enabled (not .Values.controlPlane.enabled) (not .Values.externalControlPlane.enabled) }} +{{- fail "cubeNode.enabled=true with controlPlane.enabled=false requires externalControlPlane.enabled=true" }} +{{- end }} +{{- $controlPlaneNeedsMysql := and .Values.controlPlane.enabled (or .Values.controlPlane.master.enabled .Values.controlPlane.api.enabled) -}} +{{- if and $controlPlaneNeedsMysql (not (eq (include "cube.mysqlBuiltinEnabled" .) "true")) (not .Values.mysql.host) }} +{{- fail "control plane requires mysql.host when mysql.enabled=false" }} +{{- end }} +{{- $controlPlaneNeedsRedis := and .Values.controlPlane.enabled .Values.controlPlane.master.enabled -}} +{{- if and $controlPlaneNeedsRedis (not (eq (include "cube.redisBuiltinEnabled" .) "true")) (not .Values.redis.host) }} +{{- fail "control plane master requires redis.host when redis.enabled=false" }} +{{- end }} +{{- if eq (include "cube.proxyEnabled" .) "true" }} +{{- if not $controlPlaneSelector }} +{{- fail "cubeProxy.enabled=true requires placement.controlPlane.nodeSelector because CubeProxy runs on control nodes" }} +{{- end }} +{{- if and (not (eq (include "cube.redisBuiltinEnabled" .) "true")) (not .Values.redis.host) (not .Values.cubeProxy.redis.host) }} +{{- fail "cubeProxy.enabled=true requires redis.host or cubeProxy.redis.host when redis.enabled=false" }} +{{- end }} +{{- if not (has .Values.cubeProxy.tls.mode (list "existingSecret" "certManager" "selfSigned" "inline")) }} +{{- fail "cubeProxy.tls.mode must be one of: existingSecret, certManager, selfSigned, inline" }} +{{- end }} +{{- if and (eq .Values.cubeProxy.tls.mode "existingSecret") (not .Values.cubeProxy.tls.existingSecret) }} +{{- fail "cubeProxy.tls.mode=existingSecret requires cubeProxy.tls.existingSecret" }} +{{- end }} +{{- if and (eq .Values.cubeProxy.tls.mode "inline") (or (not .Values.cubeProxy.tls.cert) (not .Values.cubeProxy.tls.key)) }} +{{- fail "cubeProxy.tls.mode=inline requires both cubeProxy.tls.cert and cubeProxy.tls.key" }} +{{- end }} +{{- if and (eq .Values.cubeProxy.tls.mode "certManager") (not .Values.cubeProxy.tls.certManager.issuerRef.name) }} +{{- fail "cubeProxy.tls.mode=certManager requires cubeProxy.tls.certManager.issuerRef.name" }} +{{- end }} +{{- if .Values.cubeProxy.sidecar.adminToken }} +{{- fail "cubeProxy.sidecar.adminToken is not supported until CubeProxy nginx admin server token injection is chart-managed; leave it empty" }} +{{- end }} +{{- if not .Values.cubeProxy.hostNetwork }} +{{- fail "cubeProxy.hostNetwork=true is required to keep CubeProxy aligned with one-click control-node ingress semantics" }} +{{- end }} +{{- if and .Values.cubeProxy.hostNetwork .Values.cubeProxy.ports.http.hostPort (ne (int .Values.cubeProxy.ports.http.hostPort) (int .Values.cubeProxy.ports.http.containerPort)) }} +{{- fail "cubeProxy.hostNetwork=true requires cubeProxy.ports.http.hostPort to equal cubeProxy.ports.http.containerPort or be empty" }} +{{- end }} +{{- if and .Values.cubeProxy.hostNetwork .Values.cubeProxy.ports.https.hostPort (ne (int .Values.cubeProxy.ports.https.hostPort) (int .Values.cubeProxy.ports.https.containerPort)) }} +{{- fail "cubeProxy.hostNetwork=true requires cubeProxy.ports.https.hostPort to equal cubeProxy.ports.https.containerPort or be empty" }} +{{- end }} +{{- end }} +{{- if .Values.cubeEgress.enabled }} +{{- if not (has .Values.cubeEgress.ca.mode (list "existingSecret" "selfSigned" "inline")) }} +{{- fail "cubeEgress.ca.mode must be one of: existingSecret, selfSigned, inline" }} +{{- end }} +{{- if and (eq .Values.cubeEgress.ca.mode "existingSecret") (not .Values.cubeEgress.ca.existingSecret) }} +{{- fail "cubeEgress.ca.mode=existingSecret requires cubeEgress.ca.existingSecret" }} +{{- end }} +{{- if and (eq .Values.cubeEgress.ca.mode "inline") (or (not .Values.cubeEgress.ca.inline.rootCert) (not .Values.cubeEgress.ca.inline.rootKey) (not .Values.cubeEgress.ca.inline.placeholderCert) (not .Values.cubeEgress.ca.inline.placeholderKey)) }} +{{- fail "cubeEgress.ca.mode=inline requires rootCert, rootKey, placeholderCert, and placeholderKey" }} +{{- end }} +{{- if and .Values.cubeEgress.network.enabled (not .Values.cubeEgress.network.ingressInterface) }} +{{- fail "cubeEgress.network.enabled=true requires cubeEgress.network.ingressInterface" }} +{{- end }} +{{- end }} +{{- if .Values.cubeDns.enabled }} +{{- if not $computeSelector }} +{{- fail "cubeDns.enabled=true requires placement.compute.nodeSelector because cube-dns runs on compute nodes" }} +{{- end }} +{{- if not (has .Values.cubeDns.mode (list "nodeLocal" "service")) }} +{{- fail "cubeDns.mode must be one of: nodeLocal, service" }} +{{- end }} +{{- if and (eq .Values.cubeDns.mode "nodeLocal") (not .Values.cubeDns.bindAddress) }} +{{- fail "cubeDns.mode=nodeLocal requires cubeDns.bindAddress" }} +{{- end }} +{{- if and .Values.cubeDns.sandboxGateway.enabled (ne .Values.cubeDns.mode "nodeLocal") }} +{{- fail "cubeDns.sandboxGateway.enabled=true requires cubeDns.mode=nodeLocal" }} +{{- end }} +{{- if and (eq .Values.cubeDns.mode "service") (not (default .Values.cubeProxy.advertiseIP .Values.cubeDns.answerIP)) }} +{{- fail "cubeDns.mode=service requires cubeDns.answerIP or cubeProxy.advertiseIP because CubeProxy is exposed on control-node hostNetwork" }} +{{- end }} +{{- if not (default .Values.cubeProxy.domain .Values.cubeDns.domain) }} +{{- fail "cubeDns.enabled=true requires cubeDns.domain or cubeProxy.domain" }} +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/webui.yaml b/deploy/kubernetes/chart/templates/webui.yaml new file mode 100644 index 000000000..25722d140 --- /dev/null +++ b/deploy/kubernetes/chart/templates/webui.yaml @@ -0,0 +1,155 @@ +{{- if and .Values.webui.enabled .Values.controlPlane.enabled .Values.controlPlane.api.enabled }} +{{- $apiUpstream := default (printf "http://%s.%s.svc.cluster.local:%v" (include "cube.apiName" .) .Release.Namespace .Values.controlPlane.api.service.port) .Values.webui.upstream | trimSuffix "/" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cube.webuiName" . }}-config + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: webui +data: + nginx.conf: |- + worker_processes auto; + + events { + worker_connections 1024; + } + + http { + include mime.types; + default_type application/octet-stream; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + gzip on; + gzip_types text/plain text/css application/javascript application/json application/xml; + + server { + listen 80; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + location = /cubeapi { + return 308 /cubeapi/; + } + + location /cubeapi/ { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + proxy_pass {{ $apiUpstream }}/cubeapi/; + } + + location /assets/ { + try_files $uri =404; + expires 1y; + add_header Cache-Control "public, immutable"; + } + + location / { + try_files $uri $uri/ /index.html; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.webuiName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: webui +spec: + replicas: {{ .Values.webui.replicas }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: webui + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: webui + annotations: + checksum/nginx-conf: {{ $apiUpstream | sha256sum | quote }} + {{- with .Values.webui.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: webui + image: {{ include "cube.image" .Values.images.webui | quote }} + imagePullPolicy: {{ .Values.images.webui.pullPolicy }} + {{- if .Values.global.timezone }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 80 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + volumeMounts: + - name: webui-config + mountPath: /usr/local/openresty/nginx/conf/nginx.conf + subPath: nginx.conf + readOnly: true + {{- with .Values.webui.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: webui-config + configMap: + name: {{ include "cube.webuiName" . }}-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.webuiName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: webui + {{- with .Values.webui.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.webui.service.type }} + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: webui + ports: + - name: http + port: {{ .Values.webui.service.port }} + targetPort: {{ .Values.webui.service.targetPort }} +{{- end }} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml new file mode 100644 index 000000000..ba5e0ae37 --- /dev/null +++ b/deploy/kubernetes/chart/values.yaml @@ -0,0 +1,646 @@ +# Default values for CubeSandbox on Kubernetes/TKE. +# This chart intentionally separates bootstrap images from runtime images. + +nameOverride: "" +fullnameOverride: "" + +global: + timezone: Asia/Shanghai + +images: + # Init container 1: installs/configures PVM host kernel and may reboot the host. + pvmHostBootstrap: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-pvm-host-bootstrap + tag: v0.5.0 + pullPolicy: Always + + # Init container 2: validates KVM/XFS, prepares host directories, checks CubeMaster reachability. + nodeInit: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-node-init + tag: v0.5.0 + pullPolicy: Always + + # Runtime container for Cube compute nodes. It should contain node-only components. + node: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-node + tag: v0.5.0 + pullPolicy: Always + + master: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-master + tag: v0.5.0 + pullPolicy: Always + + api: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-api + tag: v0.5.0 + pullPolicy: Always + + cubemastercli: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cubemastercli + tag: v0.5.0 + pullPolicy: Always + + proxyNode: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-proxy-node + tag: v0.5.0 + pullPolicy: Always + + cubeEgress: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-egress + tag: v0.5.0 + pullPolicy: Always + + cubeEgressNet: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-egress-net + tag: v0.5.0 + pullPolicy: Always + + webui: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-webui + tag: v0.5.0 + pullPolicy: Always + + coredns: + repository: ccr.ccs.tencentyun.com/tkeimages/coredns + tag: v1.11.1-tke.1 + pullPolicy: IfNotPresent + + templateBuilder: + repository: docker + tag: "27-dind" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +storageClass: + create: true + name: cube-cbs-wffc + provisioner: com.tencent.cloud.csi.cbs + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer + parameters: + type: cbs + +placement: + # Control-plane components run only on dedicated control nodes. + controlPlane: + nodeSelector: + cube.tencent.com/role: control + cube.tencent.com/cube-control: "true" + tolerations: + - key: cube.tencent.com/control + operator: Equal + value: "true" + effect: NoSchedule + + # Compute placement is used by cube-node and node-local cube-dns. + compute: + nodeSelector: + cube.tencent.com/role: compute + cube.tencent.com/cube-node: "true" + cube.tencent.com/allow-pvm-bootstrap: "true" + affinity: {} + tolerations: + - key: cube.tencent.com/compute + operator: Equal + value: "true" + effect: NoSchedule + +externalControlPlane: + # Compute-only mode equivalent to one-click ONE_CLICK_DEPLOY_ROLE=compute. + # When enabled, Cube Node and CubeAPI use the supplied CubeMaster endpoint + # instead of the chart-managed CubeMaster Service. + enabled: false + masterEndpoint: "" + apiEndpoint: "" + +controlPlane: + enabled: true + namespaceScoped: true + + master: + enabled: true + replicas: 1 + command: [] + args: [] + env: [] + nodeIPs: [] + service: + type: ClusterIP + port: 8089 + persistence: + # One-click persists CubeMaster artifacts under /data/CubeMaster/storage. + # Kubernetes multi-control-node deployments use PVC by default so data + # follows the Pod when it is rescheduled across control nodes. + enabled: true + hostPath: "" + existingClaim: "" + storageClassName: cube-cbs-wffc + size: 20Gi + accessModes: + - ReadWriteOnce + probes: + readiness: + enabled: true + path: /notify/health + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + liveness: + enabled: true + path: /notify/health + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: {} + + api: + enabled: true + replicas: 1 + command: [] + args: [] + env: [] + bind: "0.0.0.0:3000" + sandboxDomain: "cube.app" + service: + type: ClusterIP + port: 3000 + internalLoadBalancer: + enabled: false + annotations: {} + port: 3000 + probes: + readiness: + enabled: true + path: /health + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + liveness: + enabled: true + path: /health + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: {} + + templateBuilder: + # Optional sidecar in cube-master Pod. Enable only when template build is required. + enabled: false + privileged: true + dockerdArgs: [] + resources: {} + +cubemastercli: + # Operational CLI Pod delivered with the chart. It contains the real + # CubeMaster/cmd/cubemastercli binary only. The chart injects + # CUBEMASTERCLI_ADDRESS and CUBEMASTERCLI_PORT for convenience; commands + # still execute the upstream cubemastercli binary with explicit flags. + enabled: true + replicas: 1 + podAnnotations: {} + command: [] + args: [] + env: [] + probes: + readiness: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + liveness: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: {} + +webui: + # One-click enables WebUI by default on host port 12088. In Kubernetes the + # chart delivers it as a Service on port 12088 and proxies /cubeapi to CubeAPI. + enabled: true + replicas: 1 + podAnnotations: {} + service: + type: ClusterIP + port: 12088 + targetPort: 80 + annotations: {} + # Empty upstream uses the chart-managed CubeAPI Service. + upstream: "" + resources: {} + +cubeDns: + # One-click deploys CoreDNS for cube.app / wildcard resolution and lets the + # compute runtime resolve through a local DNS address. The default Kubernetes + # mode runs CoreDNS on Cube compute nodes and lets cube-node Pods use + # 127.0.0.54 explicitly via dnsConfig, without modifying host /etc/resolv.conf. + enabled: true + # nodeLocal: DaemonSet on Cube nodes, binds bindAddress. Set answerIP or + # cubeProxy.advertiseIP when CubeProxy runs only on control nodes. + # service: Deployment + ClusterIP Service for DNS itself. Requires + # answerIP or cubeProxy.advertiseIP for cube.app records because + # CubeProxy is exposed by control-node hostNetwork. + mode: nodeLocal + replicas: 1 + bindAddress: 127.0.0.54 + sandboxGateway: + # Also bind node-local cube-dns on the compute node HostIP so Cube sandbox + # guests can use it as their nameserver. The regular cube-node Pod DNS + # keeps using bindAddress. + enabled: true + domain: cube.app + # Empty answerIP uses cubeProxy.advertiseIP when set. If both are empty in + # nodeLocal mode, cube-dns returns the local compute HostIP for older local + # proxy topologies; control-node CubeProxy deployments should set one of them. + # service mode requires answerIP or cubeProxy.advertiseIP because CubeProxy is + # not exposed by Service. + answerIP: "" + forward: + # Empty upstreams forwards to /etc/resolv.conf. Set explicit upstreams to + # avoid forwarding loops in custom clusters. + upstreams: [] + service: + type: ClusterIP + port: 53 + annotations: {} + resources: {} + +diagnostics: + # One-click ships cube-diag scripts. In Kubernetes the chart exposes an + # equivalent kubectl-based diagnostic script through a ConfigMap so operators + # can collect release state without maintaining a second script copy. + enabled: true + +mysql: + enabled: true + image: + repository: mysql + tag: "8.0" + pullPolicy: IfNotPresent + # Empty host installs and uses the chart-managed cube-mysql service. + # Non-empty host uses the configured third-party MySQL service and skips cube-mysql. + host: "" + port: 3306 + database: cube_mvp + user: cube + password: cube_pass + rootPassword: cube_root + persistence: + enabled: true + existingClaim: "" + hostPath: "" + size: 20Gi + storageClassName: cube-cbs-wffc + accessModes: + - ReadWriteOnce + resources: {} + +redis: + enabled: true + image: + repository: redis + tag: "7-alpine" + pullPolicy: IfNotPresent + # Empty host installs and uses the chart-managed cube-redis service. + # Non-empty host uses the configured third-party Redis service and skips cube-redis. + host: "" + port: 6379 + password: ceuhvu123 + persistence: + enabled: true + existingClaim: "" + hostPath: "" + size: 10Gi + storageClassName: cube-cbs-wffc + accessModes: + - ReadWriteOnce + resources: {} + +cubeNode: + enabled: true + podAnnotations: {} + updateStrategy: + type: RollingUpdate + + # How CUBE_SANDBOX_NODE_IP is populated. Currently statusHostIP or static are supported. + ipSource: statusHostIP + staticNodeIP: "" + + # Default to the PVM profile so a labelled compute node can be initialized by + # this chart without an extra values file. cube-node-init fails fast if this + # value does not match host kvm_pvm state. + pvmGuestKernel: + enabled: true + + network: + # One-click auto-detects the primary NIC and patches Cubelet eth_name. + autoDetectEthName: true + # Leave empty to keep auto-detection. Set to pin Cubelet eth_name. + ethName: "" + # Leave empty to use packaged Cubelet config.toml CIDR. Set to patch cidr. + cidr: "" + cidrSkipConflictCheck: false + + dns: + useCubeDns: true + nameserver: 127.0.0.54 + clusterDomain: cluster.local + sandbox: + # DNS injected into Cube sandbox guest /etc/resolv.conf. When useCubeDns + # is true and nameservers is empty, the chart injects the current compute + # node HostIP so guests use the node-local cube-dns sandboxGateway. + useCubeDns: true + nameservers: [] + waitTimeoutSeconds: 120 + checkImage: + repository: busybox + tag: "1.36" + pullPolicy: IfNotPresent + + env: [] + resources: {} + + probes: + startup: + enabled: true + tcpSocketPort: 9999 + periodSeconds: 10 + failureThreshold: 30 + readiness: + enabled: true + tcpSocketPort: 9999 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + liveness: + enabled: true + tcpSocketPort: 9999 + initialDelaySeconds: 60 + periodSeconds: 20 + failureThreshold: 6 + +cubeProxy: + # Cube data-plane proxy. When enabled, it is delivered and removed by this + # chart together with the rest of CubeSandbox. + # + # One-click enables CubeProxy by default. This chart follows that behavior + # and creates a self-signed test certificate unless production TLS is provided. + enabled: true + # Public sandbox domain returned by CubeAPI and terminated by CubeProxy. + # For production, configure DNS for this domain and wildcard subdomains to + # the CubeProxy entrypoint, for example *.sandbox.example.com. + domain: cube.app + podAnnotations: {} + replicas: 1 + strategy: + type: RollingUpdate + # One-click CubeProxy is part of the control-node stack. This chart follows + # that model and schedules CubeProxy through placement.controlPlane. + hostNetwork: true + # IP returned by cube-dns for cube.app / *.cube.app when cubeDns.answerIP is + # empty. Set this to the selected control node IP, or set cubeDns.answerIP + # directly when using an external load balancer. + advertiseIP: "" + ports: + http: + containerPort: 80 + hostPort: "" + https: + containerPort: 443 + hostPort: "" + redis: + # Empty host uses redis.host or chart-managed cube-redis. + host: "" + port: "" + db: 0 + timeout: + min: 500 + max: 700 + tls: + # CubeProxy nginx.conf expects these file names inside the container: + # /usr/local/openresty/nginx/certs/cube.app+3.pem + # /usr/local/openresty/nginx/certs/cube.app+3-key.pem + # + # Supported modes: + # existingSecret - production. The operator provides a TLS Secret. + # certManager - chart creates a cert-manager.io/v1 Certificate. + # selfSigned - chart creates a self-signed certificate Secret; test only. + # inline - chart creates a Secret from tls.cert/tls.key values. + mode: selfSigned + # Optional for certManager/selfSigned/inline. Defaults to -proxy-certs. + secretName: "" + existingSecret: "" + certSecretKey: tls.crt + keySecretKey: tls.key + caSecretKey: ca.crt + certManager: + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: "" + commonName: "" + dnsNames: [] + ipAddresses: [] + duration: "" + renewBefore: "" + usages: + - digital signature + - key encipherment + - server auth + selfSigned: + commonName: "" + dnsNames: + - cube.app + - "*.cube.app" + - localhost + ipAddresses: + - 127.0.0.1 + caValidityDays: 3650 + certValidityDays: 3650 + # Used only when tls.mode=inline. + cert: "" + key: "" + sidecar: + # CubeProxy image starts cube-proxy-sidecar from CubeProxy/Dockerfile. + # These values wire it to Kubernetes Services instead of the one-click + # loopback defaults. + listenAddr: 127.0.0.1:8083 + proxyAdminURLs: + - http://127.0.0.1:8082 + defaultIdleTimeout: 5m + lastActivePoll: 5s + idleSweepInterval: 5s + bootstrapWarmup: 30s + stateLockTTL: 60s + # Leave empty until CubeProxy nginx admin server also has token injection. + adminToken: "" + resolver: + # CubeProxy nginx Lua Redis client needs an nginx resolver when Redis is + # configured by DNS name (the default Kubernetes Service DNS name). + # Empty addresses auto-discovers nameservers from the Pod /etc/resolv.conf. + addresses: [] + valid: 30s + timeout: 5s + ipv6: false + probes: + readiness: + enabled: true + path: /admin/healthz + port: 8082 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + liveness: + enabled: true + path: /admin/healthz + port: 8082 + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + logs: + hostPath: /data/log/cube-proxy + resources: {} + +cubeEgress: + # Transparent outbound egress/MITM data-plane component. When enabled, the + # Cube Node Big Pod gets a cube-egress sidecar and a cube-egress-net helper + # sidecar that owns host TPROXY iptables/ip-rule state. + enabled: true + bootstrapURL: http://127.0.0.1:19090/v1/policies/dump + debugDump: false + ca: + # Supported modes: + # selfSigned - chart creates/reuses a release-scoped CA Secret. + # existingSecret - operator provides cube-root-ca.* and placeholder.*. + # inline - chart creates a Secret from the inline PEM values. + mode: selfSigned + secretName: "" + existingSecret: "" + rootCertKey: cube-root-ca.crt + rootKeyKey: cube-root-ca.key + placeholderCertKey: placeholder.crt + placeholderKeyKey: placeholder.key + selfSigned: + caCommonName: CubeSandbox Egress MITM CA + placeholderCommonName: cube-proxy-placeholder + caValidityDays: 3650 + placeholderValidityDays: 36500 + inline: + rootCert: "" + rootKey: "" + placeholderCert: "" + placeholderKey: "" + network: + enabled: true + ingressInterface: cube-dev + tproxyOnIP: 192.168.0.1 + initialWaitSeconds: 300 + reapplyIntervalSeconds: 30 + probes: + readiness: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 30 + liveness: + enabled: true + initialDelaySeconds: 330 + periodSeconds: 20 + failureThreshold: 6 + probes: + readiness: + enabled: true + path: /admin/v1/health + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + liveness: + enabled: true + path: /admin/v1/health + port: 9090 + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 6 + resources: {} + netResources: {} + +bootstrap: + pvmHostKernel: + enabled: true + desiredKernelPattern: pvm.host + bootArgs: "nopti pti=off" + package: + # image: rpm/deb is baked into cube-pvm-host-bootstrap image under /artifacts. + # production delivery should use image and normal registry pulls. + source: image + rpmPath: /artifacts/kernel-pvm-host.rpm + debPath: /artifacts/linux-image-pvm-host.deb + rpmUrl: "" + debUrl: "" + sha256: "" + reboot: + enabled: true + coordinated: true + leaseName: cube-node-pvm-bootstrap + leaseTTLSeconds: 900 + maxConcurrent: 1 + maxCount: 1 + + nodeInit: + enabled: true + requireKVM: true + requireXFS: true + chmodKVM: true + loadKVMModule: true + writeUdevRule: true + createHostDirs: true + checkMasterConnectivity: true + checkMemory: true + minMemoryKB: "7500000" + checkCgroupCPU: true + checkGlibc: true + checkCIDR: true + checkCubecowDeps: true + dataCubelet: + path: /data/cubelet + loopback: + enabled: true + imagePath: /data/cubelet-xfs.img + size: 25G + +security: + hostNetwork: true + hostPID: true + privileged: true + +hostPaths: + root: / + dev: /dev + sys: /sys + libModules: /lib/modules + dataCubelet: /data/cubelet + dataLog: /data/log + dataCubeShim: /data/cube-shim + dataSnapshotPack: /data/snapshot_pack + tmpCube: /tmp/cube + +serviceAccount: + create: true + annotations: {} + +rbac: + create: true + +helmTest: + enabled: true + image: + repository: curlimages/curl + tag: "8.10.1" + pullPolicy: IfNotPresent + dnsImage: + repository: busybox + tag: "1.36" + pullPolicy: IfNotPresent diff --git a/deploy/kubernetes/images/.gitignore b/deploy/kubernetes/images/.gitignore new file mode 100644 index 000000000..3c1b3b23b --- /dev/null +++ b/deploy/kubernetes/images/.gitignore @@ -0,0 +1 @@ +.work/ diff --git a/deploy/kubernetes/images/README.md b/deploy/kubernetes/images/README.md new file mode 100644 index 000000000..b3deae9c2 --- /dev/null +++ b/deploy/kubernetes/images/README.md @@ -0,0 +1,72 @@ +# CubeSandbox delivery images + +This directory contains image build definitions used by the Kubernetes/TKE chart. + +## Build entrypoint + +```bash +PUSH=1 REGISTRY=ccr.ccs.tencentyun.com/cubesandbox-chart IMAGE_TAG=v0.5.0 ./deploy/kubernetes/images/build-cube-images.sh +``` + +Use `NO_CACHE=1` when every Docker image layer must be rebuilt instead of +using Docker's build cache: + +```bash +NO_CACHE=1 PUSH=1 REGISTRY=ccr.ccs.tencentyun.com/cubesandbox-chart IMAGE_TAG=v0.5.0 ./deploy/kubernetes/images/build-cube-images.sh +``` + +The script defaults its temporary `BUILD_ROOT` to +`/tmp/cube-kubernetes-images-` so large image contexts and downloads do +not land in the Git worktree. Override `BUILD_ROOT` only when you intentionally +want a different cache location. + +The script reuses valid artifacts already present under `${BUILD_ROOT}/downloads` +and does not require a `.complete` marker. + +When the release package is older than the verified Kubernetes node runtime, +build `cube-node` by rebasing a known-good node image and copying the current +entrypoint into it: + +```bash +CUBE_NODE_BASE_IMAGE=ccr.ccs.tencentyun.com/pavleli/cube-node:v0.4.0-cubevsfix-20260627 \ + PUSH=1 REGISTRY=ccr.ccs.tencentyun.com/cubesandbox-chart IMAGE_TAG=v0.5.0 \ + ./deploy/kubernetes/images/build-cube-images.sh +``` + +This keeps the CubeVS/network-agent runtime fix while preserving the chart-side +entrypoint behavior. + +## Image source policy + +- `cube-node` continues to use `deploy/kubernetes/images/cube-node/Dockerfile`. + It is a Kubernetes delivery image that bundles the node-side runtime components required by the Cube Node Big Pod, including `Cubelet`, `network-agent`, `cube-shim`, `cube-kernel-scf`, `cube-image`, `cube-vs`, and `cube-snapshot`. `cube-egress` is intentionally not bundled in this image because it is delivered as a separate sidecar image. + If `CUBE_NODE_BASE_IMAGE` is set, the build script rebases that image instead + and only replaces `/usr/local/bin/cube-node-entrypoint.sh`. +- `cube-node-init` and `cube-pvm-host-bootstrap` are Kubernetes Init Container images and stay in this delivery image directory. +- `cube-master` is built directly from `CubeMaster/docker/Dockerfile`. The build script prepares a temporary Docker context with the release-package `cubemaster` binary and the `CubeMaster/docker/tools` directory expected by that Dockerfile. +- `cube-api` is built from `CubeAPI/Dockerfile`; no duplicate Dockerfile is kept here. +- `cubemastercli` is an operational CLI image. It packages only the + release-package `CubeMaster/bin/cubemastercli` binary and minimal runtime + dependencies. It is separate from `cube-master` and `cube-node` so runtime + image responsibilities remain clean. +- `cube-proxy-node` is built from `CubeProxy/Dockerfile`; no duplicate Dockerfile is kept here. +- `cube-egress` is built from `CubeEgress/Dockerfile`; no duplicate Dockerfile is kept here. Its `cube-egress/openresty:1.29.2.5-tproxy` base image is built first from `CubeEgress/openresty/Dockerfile`, because that patched OpenResty base is part of the upstream CubeEgress build chain rather than a public pull-only dependency. + The build script also tags that local base as + `cube-sandbox-cn.tencentcloudcr.com/cube-sandbox/openresty-tproxy`, matching + the upstream `CubeEgress/Dockerfile` `FROM` line, so the final egress image + uses the just-built base instead of pulling a drifting external image. +- `cube-egress-net` is a Kubernetes helper image that owns the host TPROXY + iptables/ip-rule setup for CubeEgress. It packages the upstream + `CubeEgress/scripts/cube-proxy-iptables-init.sh` plus a small idempotent + entrypoint that waits for `cube-dev`, applies rules, and removes them on + termination. +- `cube-webui` packages the one-click `webui/dist` static assets and reuses + the one-click WebUI nginx layout. The chart mounts a rendered nginx config so + `/cubeapi/` proxies to the chart-managed CubeAPI Service. +- The template builder sidecar uses a dind image by default; no duplicate Dockerfile is kept here. + +The Helm chart stays under `deploy/kubernetes/chart`; image build logic stays here to avoid coupling chart templates with image construction. + +`build-cube-images.sh` copies only the scripts required by each image into that image's build context. Do not add generic helper scripts here unless they are referenced by a Dockerfile or explicitly copied by the build script. + +CubeMaster runtime configuration is delivered by the Helm chart from `deploy/kubernetes/chart/files/cube-master/conf.yaml` as a Secret mounted at `/usr/local/services/cubemaster/conf.yaml`. CubeMaster schema migrations are embedded in the `cubemaster` binary at compile time from `CubeMaster/pkg/base/dao/migrate/migrations/mysql`; this image build does not package a second SQL copy. diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh new file mode 100755 index 000000000..3e9418c10 --- /dev/null +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# Build and optionally push CubeSandbox images for the Kubernetes/TKE chart. +# +# This script builds role-specific images directly from the CubeSandbox release +# package (sandbox-package). cube-node intentionally uses +# deploy/kubernetes/images/cube-node/Dockerfile because it is a Kubernetes delivery image +# that bundles node-side runtime components. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +VERSION="${VERSION:-v0.5.0}" +IMAGE_TAG="${IMAGE_TAG:-${VERSION}}" +REGISTRY="${REGISTRY:-ccr.ccs.tencentyun.com/cubesandbox-chart}" +PUSH="${PUSH:-0}" +NO_CACHE="${NO_CACHE:-0}" +BUILD_ROOT="${BUILD_ROOT:-/tmp/cube-kubernetes-images-${VERSION}}" +CUBE_NODE_BASE_IMAGE="${CUBE_NODE_BASE_IMAGE:-}" +CUBE_EGRESS_OPENRESTY_BASE_IMAGE="${CUBE_EGRESS_OPENRESTY_BASE_IMAGE:-cube-sandbox-cn.tencentcloudcr.com/cube-sandbox/openresty-tproxy}" + +ONE_CLICK_ARCH="${ONE_CLICK_ARCH:-amd64}" +ONE_CLICK_URL="${ONE_CLICK_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/cube-sandbox-one-click-${VERSION}-${ONE_CLICK_ARCH}.tar.gz}" +PVM_KERNEL_RPM_URL="${PVM_KERNEL_RPM_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/kernel-6.6.69_opencloudos9.cubesandbox.pvm.host_gb85200d80fa2-1.x86_64.rpm}" +PVM_KERNEL_DEB_URL="${PVM_KERNEL_DEB_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/linux-image-6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2_6.6.69-gb85200d80fa2-1_amd64.deb}" + +# Bake kernel packages into cube-pvm-host-bootstrap by default so delivery only +# depends on normal image pulls from the target registry. +INCLUDE_PVM_KERNEL_RPM="${INCLUDE_PVM_KERNEL_RPM:-1}" +INCLUDE_PVM_KERNEL_DEB="${INCLUDE_PVM_KERNEL_DEB:-1}" +DOWNLOAD_RETRIES="${DOWNLOAD_RETRIES:-5}" +DOWNLOAD_CONNECT_TIMEOUT="${DOWNLOAD_CONNECT_TIMEOUT:-20}" + +log() { printf '[build-cube-images] %s\n' "$*"; } +fail() { printf '[build-cube-images] ERROR: %s\n' "$*" >&2; exit 1; } + +need() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +validate_download() { + local out="$1" + local validator="${2:-file}" + case "${validator}" in + tar.gz) + tar -tzf "${out}" >/dev/null + ;; + file) + [[ -s "${out}" ]] + ;; + *) + fail "unknown download validator: ${validator}" + ;; + esac +} + +download_file() { + local url="$1" + local out="$2" + local validator="${3:-file}" + local attempt + + if [[ -f "${out}" ]]; then + if validate_download "${out}" "${validator}"; then + log "reusing existing download: ${out}" + return 0 + fi + log "existing download is invalid, redownloading: ${out}" + rm -f "${out}" + fi + + mkdir -p "$(dirname "${out}")" + for attempt in $(seq 1 "${DOWNLOAD_RETRIES}"); do + log "downloading $(basename "${out}") attempt ${attempt}/${DOWNLOAD_RETRIES}: ${url}" + local curl_extra_args=() + if curl --help all 2>/dev/null | grep -q -- '--retry-all-errors'; then + curl_extra_args+=(--retry-all-errors) + fi + if curl \ + --fail \ + --location \ + --continue-at - \ + --retry 3 \ + "${curl_extra_args[@]}" \ + --retry-delay 5 \ + --connect-timeout "${DOWNLOAD_CONNECT_TIMEOUT}" \ + --show-error \ + --progress-bar \ + -o "${out}" \ + "${url}"; then + if validate_download "${out}" "${validator}"; then + return 0 + fi + log "downloaded file failed ${validator} validation: ${out}" + fi + if [[ "${attempt}" != "${DOWNLOAD_RETRIES}" ]]; then + log "retrying download after 5 seconds" + sleep 5 + fi + done + fail "failed to download valid file after ${DOWNLOAD_RETRIES} attempts: ${url}" +} + +need docker +need tar +need curl +need go + +DOWNLOAD_DIR="${BUILD_ROOT}/downloads" +EXTRACT_DIR="${BUILD_ROOT}/extract" +CONTEXT_DIR="${BUILD_ROOT}/contexts" +ONE_CLICK_DIRNAME="cube-sandbox-one-click-${VERSION}-${ONE_CLICK_ARCH}" +ONE_CLICK_TAR="${DOWNLOAD_DIR}/${ONE_CLICK_DIRNAME}.tar.gz" +PVM_KERNEL_RPM="${DOWNLOAD_DIR}/$(basename "${PVM_KERNEL_RPM_URL}")" +PVM_KERNEL_DEB="${DOWNLOAD_DIR}/$(basename "${PVM_KERNEL_DEB_URL}")" +SANDBOX_PACKAGE_TAR="${EXTRACT_DIR}/${ONE_CLICK_DIRNAME}/assets/package/sandbox-package.tar.gz" +PACKAGE_DIR="${BUILD_ROOT}/sandbox-package" + +mkdir -p "${DOWNLOAD_DIR}" "${EXTRACT_DIR}" "${CONTEXT_DIR}" + +if [[ -n "${PACKAGE_DIR_OVERRIDE:-}" ]]; then + PACKAGE_DIR="${PACKAGE_DIR_OVERRIDE}" + [[ -d "${PACKAGE_DIR}" ]] || fail "PACKAGE_DIR_OVERRIDE does not exist: ${PACKAGE_DIR}" +else + if [[ ! -f "${ONE_CLICK_TAR}" ]] || ! tar -tzf "${ONE_CLICK_TAR}" >/dev/null 2>&1; then + log "downloading one-click release package: ${ONE_CLICK_URL}" + download_file "${ONE_CLICK_URL}" "${ONE_CLICK_TAR}" tar.gz + fi + if [[ ! -f "${SANDBOX_PACKAGE_TAR}" ]]; then + log "extracting one-click release package" + rm -rf "${EXTRACT_DIR}/${ONE_CLICK_DIRNAME}" + tar -C "${EXTRACT_DIR}" -xzf "${ONE_CLICK_TAR}" + fi + if [[ ! -d "${PACKAGE_DIR}" ]]; then + log "extracting sandbox-package" + rm -rf "${PACKAGE_DIR}" + mkdir -p "${BUILD_ROOT}" + tar -C "${BUILD_ROOT}" -xzf "${SANDBOX_PACKAGE_TAR}" + fi +fi + +[[ -d "${PACKAGE_DIR}/CubeMaster" ]] || fail "invalid package dir: missing CubeMaster" +[[ -d "${PACKAGE_DIR}/Cubelet" ]] || fail "invalid package dir: missing Cubelet" +[[ -d "${PACKAGE_DIR}/CubeAPI" ]] || fail "invalid package dir: missing CubeAPI" + +copy_scripts() { + local ctx="$1" + shift + mkdir -p "${ctx}/scripts" + for script in "$@"; do + cp "${SCRIPT_DIR}/scripts/${script}" "${ctx}/scripts/${script}" + chmod +x "${ctx}/scripts/${script}" + done +} + +prepare_context() { + local name="$1" + local ctx="${CONTEXT_DIR}/${name}" + rm -rf "${ctx}" + mkdir -p "${ctx}/package" "${ctx}/scripts" "${ctx}/artifacts" + printf '%s\n' "${ctx}" +} + +copy_if_exists() { + local src="$1" + local dst="$2" + if [[ -e "${src}" ]]; then + mkdir -p "$(dirname "${dst}")" + cp -a "${src}" "${dst}" + fi +} + +copy_cube_master_component_context() { + local ctx="$1" + local src="${PACKAGE_DIR}/CubeMaster" + local bin="${src}/bin/cubemaster" + + [[ -x "${bin}" ]] || fail "invalid CubeMaster package: missing executable ${bin}" + [[ -f "${REPO_ROOT}/CubeMaster/docker/tools/gracestop.sh" ]] || fail "missing CubeMaster docker tools/gracestop.sh" + + cp "${bin}" "${ctx}/cubemaster" + chmod +x "${ctx}/cubemaster" + cp -a "${REPO_ROOT}/CubeMaster/docker/tools" "${ctx}/tools" +} + +copy_cubemastercli_context() { + local ctx="$1" + local src="${PACKAGE_DIR}/CubeMaster" + local bin="${src}/bin/cubemastercli" + + [[ -x "${bin}" ]] || fail "invalid CubeMaster package: missing executable ${bin}" + + cp "${bin}" "${ctx}/cubemastercli" + chmod +x "${ctx}/cubemastercli" +} + +copy_cube_proxy_component_context() { + local ctx="$1" + local src="${REPO_ROOT}/CubeProxy" + local sidecar_src="${src}/sidecar" + local sidecar_out="${ctx}/bin/cube-proxy-sidecar" + + [[ -f "${src}/nginx.conf" ]] || fail "missing CubeProxy nginx.conf" + [[ -d "${src}/conf/includes" ]] || fail "missing CubeProxy conf/includes" + [[ -f "${sidecar_src}/go.mod" ]] || fail "missing CubeProxy sidecar source" + + cp -a "${src}/lua" "${ctx}/lua" + mkdir -p "${ctx}/conf" + cp -a "${src}/conf/includes" "${ctx}/conf/includes" + cp "${src}/nginx.conf" "${ctx}/nginx.conf" + cp "${src}/rotate_nginx_log.sh" "${ctx}/rotate_nginx_log.sh" + cp "${src}/root" "${ctx}/root" + cp "${src}/start.sh" "${ctx}/start.sh" + + mkdir -p "${ctx}/bin" + log "building CubeProxy sidecar for cube-proxy-node image context" + ( + cd "${sidecar_src}" + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -trimpath -tags 'netgo osusergo' -ldflags '-s -w' \ + -o "${sidecar_out}" ./cmd/sidecar + ) + chmod +x "${sidecar_out}" +} + +build_image() { + local name="$1" + local ctx="$2" + local dockerfile="${SCRIPT_DIR}/${name}/Dockerfile" + local image="${REGISTRY}/${name}:${IMAGE_TAG}" + local docker_args=(-f "${dockerfile}" -t "${image}") + if [[ "${NO_CACHE}" == "1" ]]; then + docker_args=(--no-cache --pull "${docker_args[@]}") + fi + log "building ${image}" + docker build "${docker_args[@]}" "${ctx}" + if [[ "${PUSH}" == "1" ]]; then + log "pushing ${image}" + docker push "${image}" + fi +} + +build_component_image() { + local name="$1" + local dockerfile="$2" + local ctx="$3" + local image="${REGISTRY}/${name}:${IMAGE_TAG}" + local docker_args=(-f "${dockerfile}" -t "${image}") + if [[ "${NO_CACHE}" == "1" ]]; then + docker_args=(--no-cache --pull "${docker_args[@]}") + fi + log "building ${image} from ${dockerfile}" + docker build "${docker_args[@]}" "${ctx}" + if [[ "${PUSH}" == "1" ]]; then + log "pushing ${image}" + docker push "${image}" + fi +} + +build_cube_api_image() { + local dockerfile="${CONTEXT_DIR}/cube-api.Dockerfile" + + # CubeAPI/Dockerfile first compiles a dummy main to cache dependencies. Docker + # preserves source mtimes on COPY, so Cargo can incorrectly keep that dummy + # binary if the real src/main.rs is older than the cached artifact. Keep the + # upstream Dockerfile unchanged and inject one cache-busting cleanup layer for + # Kubernetes image builds. + awk ' + { + print + if ($0 == "COPY src/ src/") { + print "RUN rust_target=\"$(cat /etc/rust-target)\" \\" + print " && rm -f \"target/${rust_target}/release/cube-api\" target/${rust_target}/release/deps/cube_api-*" + } + } + ' "${REPO_ROOT}/CubeAPI/Dockerfile" > "${dockerfile}" + + build_component_image cube-api "${dockerfile}" "${REPO_ROOT}/CubeAPI" +} + +build_cube_egress_openresty_base_image() { + local image="cube-egress/openresty:1.29.2.5-tproxy" + local docker_args=( + -f "${REPO_ROOT}/CubeEgress/openresty/Dockerfile" + -t "${image}" + -t "${CUBE_EGRESS_OPENRESTY_BASE_IMAGE}" + ) + if [[ "${NO_CACHE}" == "1" ]]; then + docker_args=(--no-cache --pull "${docker_args[@]}") + fi + log "building ${image} from ${REPO_ROOT}/CubeEgress/openresty/Dockerfile" + log "tagging ${image} as ${CUBE_EGRESS_OPENRESTY_BASE_IMAGE} for CubeEgress/Dockerfile" + docker build "${docker_args[@]}" "${REPO_ROOT}/CubeEgress/openresty" +} + +build_cube_egress_image() { + local image="${REGISTRY}/cube-egress:${IMAGE_TAG}" + local docker_args=( + -f "${REPO_ROOT}/CubeEgress/Dockerfile" + -t "${image}" + --build-arg "CUBE_EGRESS_VERSION=${VERSION}" + --build-arg "CUBE_EGRESS_COMMIT=$(git -C "${REPO_ROOT}" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" + --build-arg "CUBE_EGRESS_BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + ) + if [[ "${NO_CACHE}" == "1" ]]; then + # Do not add --pull here: CubeEgress/Dockerfile uses a fixed FROM name. + # build_cube_egress_openresty_base_image tags the locally built base with + # that exact name so the egress image remains reproducible. + docker_args=(--no-cache "${docker_args[@]}") + fi + log "building ${image} from ${REPO_ROOT}/CubeEgress/Dockerfile" + docker build "${docker_args[@]}" "${REPO_ROOT}/CubeEgress" + if [[ "${PUSH}" == "1" ]]; then + log "pushing ${image}" + docker push "${image}" + fi +} + +build_cube_node_from_base_image() { + local ctx + local dockerfile + + [[ -n "${CUBE_NODE_BASE_IMAGE}" ]] || fail "CUBE_NODE_BASE_IMAGE is required" + ctx="$(prepare_context cube-node)" + copy_scripts "${ctx}" cube-node-entrypoint.sh + dockerfile="${ctx}/Dockerfile.rebase" + + cat > "${dockerfile}" < + images.*.tag: ${IMAGE_TAG} + +Template builder is not built by this script. The chart uses a dind image by +default and can be overridden through images.templateBuilder.* when needed. +EOF diff --git a/deploy/kubernetes/images/cube-egress-net/Dockerfile b/deploy/kubernetes/images/cube-egress-net/Dockerfile new file mode 100644 index 000000000..5cc528acb --- /dev/null +++ b/deploy/kubernetes/images/cube-egress-net/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iptables \ + kmod \ + procps \ + tini \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/cube-proxy-iptables-init.sh /usr/local/bin/cube-proxy-iptables-init.sh +COPY scripts/cube-egress-net-entrypoint.sh /usr/local/bin/cube-egress-net-entrypoint.sh + +RUN chmod +x \ + /usr/local/bin/cube-proxy-iptables-init.sh \ + /usr/local/bin/cube-egress-net-entrypoint.sh + +ENTRYPOINT ["/usr/bin/tini", "-s", "--", "/usr/local/bin/cube-egress-net-entrypoint.sh"] diff --git a/deploy/kubernetes/images/cube-node-init/Dockerfile b/deploy/kubernetes/images/cube-node-init/Dockerfile new file mode 100644 index 000000000..43fbe2211 --- /dev/null +++ b/deploy/kubernetes/images/cube-node-init/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + e2fsprogs \ + iproute2 \ + kmod \ + mount \ + tini \ + util-linux \ + xfsprogs \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/cube-node-init.sh /scripts/cube-node-init.sh + +RUN chmod +x /scripts/cube-node-init.sh + +ENTRYPOINT ["/usr/bin/tini", "-s", "--", "/scripts/cube-node-init.sh"] diff --git a/deploy/kubernetes/images/cube-node/Dockerfile b/deploy/kubernetes/images/cube-node/Dockerfile new file mode 100644 index 000000000..eef753da5 --- /dev/null +++ b/deploy/kubernetes/images/cube-node/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TOOLBOX_ROOT=/usr/local/services/cubetoolbox + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + iproute2 \ + iptables \ + iputils-ping \ + kmod \ + lsof \ + net-tools \ + procps \ + psmisc \ + e2fsprogs \ + tini \ + util-linux \ + xfsprogs \ + && rm -rf /var/lib/apt/lists/* + +COPY package/Cubelet ${TOOLBOX_ROOT}/Cubelet +COPY package/network-agent ${TOOLBOX_ROOT}/network-agent +COPY package/cube-shim ${TOOLBOX_ROOT}/cube-shim +COPY package/cube-kernel-scf ${TOOLBOX_ROOT}/cube-kernel-scf +COPY package/cube-image ${TOOLBOX_ROOT}/cube-image +COPY package/cube-vs ${TOOLBOX_ROOT}/cube-vs +COPY package/cube-snapshot ${TOOLBOX_ROOT}/cube-snapshot +COPY package/scripts/common ${TOOLBOX_ROOT}/scripts/common +COPY scripts/cube-node-entrypoint.sh /usr/local/bin/cube-node-entrypoint.sh + +RUN chmod +x \ + ${TOOLBOX_ROOT}/Cubelet/bin/cubelet \ + ${TOOLBOX_ROOT}/Cubelet/bin/cubecli \ + ${TOOLBOX_ROOT}/network-agent/bin/network-agent \ + ${TOOLBOX_ROOT}/network-agent/bin/cubevsmapdump \ + ${TOOLBOX_ROOT}/cube-shim/bin/cube-runtime \ + ${TOOLBOX_ROOT}/cube-shim/bin/containerd-shim-cube-rs \ + /usr/local/bin/cube-node-entrypoint.sh \ + && ln -sfn ${TOOLBOX_ROOT}/cube-shim/bin/containerd-shim-cube-rs /usr/local/bin/containerd-shim-cube-rs \ + && ln -sfn ${TOOLBOX_ROOT}/cube-shim/bin/cube-runtime /usr/local/bin/cube-runtime \ + && ln -sfn ${TOOLBOX_ROOT}/Cubelet/bin/cubecli /usr/local/bin/cubecli \ + && ln -sfn ${TOOLBOX_ROOT}/network-agent/bin/cubevsmapdump /usr/local/bin/cubevsmapdump \ + && mkdir -p \ + /data/cubelet \ + /data/log/Cubelet \ + /data/log/CubeShim \ + /data/log/CubeVmm \ + /data/cube-shim/disks \ + /data/snapshot_pack/disks \ + /tmp/cube + +EXPOSE 9999 9998 9966 19090 +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/cube-node-entrypoint.sh"] diff --git a/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile new file mode 100644 index 000000000..b18faab18 --- /dev/null +++ b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile @@ -0,0 +1,27 @@ +FROM bitnami/kubectl:1.30 AS kubectl + +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + kmod \ + rpm \ + tini \ + util-linux \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=kubectl /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl +COPY scripts/pvm-host-bootstrap.sh /scripts/pvm-host-bootstrap.sh + +# Optional kernel packages. Put files under build context artifacts/ to build a fully self-contained image. +COPY artifacts/ /artifacts/ + +RUN chmod +x \ + /usr/local/bin/kubectl \ + /scripts/pvm-host-bootstrap.sh + +ENTRYPOINT ["/usr/bin/tini", "-s", "--", "/scripts/pvm-host-bootstrap.sh"] diff --git a/deploy/kubernetes/images/cube-webui/Dockerfile b/deploy/kubernetes/images/cube-webui/Dockerfile new file mode 100644 index 000000000..d32e324f1 --- /dev/null +++ b/deploy/kubernetes/images/cube-webui/Dockerfile @@ -0,0 +1,8 @@ +FROM cube-sandbox-image.tencentcloudcr.com/opensource/openresty:1.21.4.1-6-alpine-fat + +COPY package/webui/dist /usr/share/nginx/html +COPY package/webui/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf + +EXPOSE 80 + +CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"] diff --git a/deploy/kubernetes/images/cubemastercli/Dockerfile b/deploy/kubernetes/images/cubemastercli/Dockerfile new file mode 100644 index 000000000..c84a192ac --- /dev/null +++ b/deploy/kubernetes/images/cubemastercli/Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY cubemastercli /usr/local/bin/cubemastercli + +RUN chmod +x /usr/local/bin/cubemastercli \ + && { \ + printf '%s\n' \ + '# Cube cubemastercli interactive defaults.' \ + '# This is intentionally a bash interactive helper, not a wrapper binary:' \ + '# - kubectl exec -it ... -- bash can run cubemastercli list directly.' \ + '# - non-interactive commands still execute the original binary unchanged.' \ + '__cube_cubemastercli_has_flag() {' \ + ' local flag="$1"' \ + ' shift' \ + '' \ + ' while [ "$#" -gt 0 ]; do' \ + ' case "$1" in' \ + ' "${flag}"|"${flag}"=*)' \ + ' return 0' \ + ' ;;' \ + ' esac' \ + ' shift' \ + ' done' \ + '' \ + ' return 1' \ + '}' \ + '' \ + 'cubemastercli() {' \ + ' local args=()' \ + '' \ + ' if [ -n "${CUBEMASTERCLI_ADDRESS:-}" ] && ! __cube_cubemastercli_has_flag "--address" "$@"; then' \ + ' args+=(--address "${CUBEMASTERCLI_ADDRESS}")' \ + ' fi' \ + '' \ + ' if [ -n "${CUBEMASTERCLI_PORT:-}" ] && ! __cube_cubemastercli_has_flag "--port" "$@"; then' \ + ' args+=(--port "${CUBEMASTERCLI_PORT}")' \ + ' fi' \ + '' \ + ' command /usr/local/bin/cubemastercli "${args[@]}" "$@"' \ + '}'; \ + } >/etc/cubemastercli-interactive.bashrc \ + && printf '\n%s\n' '[ -f /etc/cubemastercli-interactive.bashrc ] && . /etc/cubemastercli-interactive.bashrc' >>/root/.bashrc + +CMD ["sleep", "infinity"] diff --git a/deploy/kubernetes/images/scripts/cube-egress-net-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-egress-net-entrypoint.sh new file mode 100755 index 000000000..ead8bfe38 --- /dev/null +++ b/deploy/kubernetes/images/scripts/cube-egress-net-entrypoint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +INIT_SCRIPT="${CUBE_EGRESS_NET_INIT:-/usr/local/bin/cube-proxy-iptables-init.sh}" +INGRESS_IFACE="${CUBE_INGRESS_IFACE:-cube-dev}" +REAPPLY_INTERVAL_SECONDS="${CUBE_EGRESS_NET_REAPPLY_INTERVAL_SECONDS:-30}" +INITIAL_WAIT_SECONDS="${CUBE_EGRESS_NET_INITIAL_WAIT_SECONDS:-300}" + +log() { printf '[cube-egress-net] %s +' "$*" >&2; } +fail() { printf '[cube-egress-net] ERROR: %s +' "$*" >&2; exit 1; } + +[[ -x "${INIT_SCRIPT}" ]] || fail "missing executable: ${INIT_SCRIPT}" + +cleanup() { + log "removing CubeEgress transparent proxy rules" + "${INIT_SCRIPT}" down || true +} +trap cleanup TERM INT HUP EXIT + +wait_for_iface() { + local waited=0 + while ! ip link show "${INGRESS_IFACE}" >/dev/null 2>&1; do + if (( waited >= INITIAL_WAIT_SECONDS )); then + log "interface ${INGRESS_IFACE} not present after ${INITIAL_WAIT_SECONDS}s; keep waiting" + waited=0 + fi + sleep 1 + waited=$((waited + 1)) + done +} + +apply_rules() { + wait_for_iface + log "applying CubeEgress transparent proxy rules on ${INGRESS_IFACE}" + "${INIT_SCRIPT}" up +} + +apply_rules +while true; do + sleep "${REAPPLY_INTERVAL_SECONDS}" + if ! ip link show "${INGRESS_IFACE}" >/dev/null 2>&1; then + log "interface ${INGRESS_IFACE} disappeared; waiting before reapply" + wait_for_iface + fi + # Idempotent and keeps rules present after node-level firewall reloads. + "${INIT_SCRIPT}" up >/dev/null 2>&1 || log "WARN: rule reapply failed; will retry" +done diff --git a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh new file mode 100755 index 000000000..5777480d9 --- /dev/null +++ b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -euo pipefail + +TOOLBOX_ROOT="${TOOLBOX_ROOT:-/usr/local/services/cubetoolbox}" +NETWORK_AGENT_BIN="${TOOLBOX_ROOT}/network-agent/bin/network-agent" +CUBELET_BIN="${TOOLBOX_ROOT}/Cubelet/bin/cubelet" +CUBELET_CONFIG="${TOOLBOX_ROOT}/Cubelet/config/config.toml" +CUBELET_DYNAMICCONF="${CUBELET_DYNAMICCONF:-${TOOLBOX_ROOT}/Cubelet/dynamicconf/conf.yaml}" +CUBE_KERNEL_DIR="${TOOLBOX_ROOT}/cube-kernel-scf" +NETWORK_AGENT_STATE_DIR="${NETWORK_AGENT_STATE_DIR:-/data/cubelet/network-agent/state}" +NETWORK_AGENT_HEALTH_URL="${NETWORK_AGENT_HEALTH_URL:-http://127.0.0.1:19090/readyz}" +CUBE_MASTER_ENDPOINT="${CUBE_MASTER_ENDPOINT:-cube-master.cube-system.svc.cluster.local:8089}" +CUBE_PVM_ENABLE="${CUBE_PVM_ENABLE:-1}" +CUBE_SANDBOX_AUTO_DETECT_ETH="${CUBE_SANDBOX_AUTO_DETECT_ETH:-true}" + +log() { printf '[cube-node-entrypoint] %s\n' "$*"; } +fail() { printf '[cube-node-entrypoint] ERROR: %s\n' "$*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing command in cube-node image: $1" +} + +select_guest_kernel() { + local target="vmlinux-bm" + case "${CUBE_PVM_ENABLE}" in + 1|true|TRUE|yes|YES) target="vmlinux-pvm" ;; + 0|false|FALSE|no|NO) target="vmlinux-bm" ;; + *) fail "unsupported CUBE_PVM_ENABLE=${CUBE_PVM_ENABLE}; expected true/false" ;; + esac + [[ -f "${CUBE_KERNEL_DIR}/${target}" ]] || fail "missing guest kernel: ${CUBE_KERNEL_DIR}/${target}" + ln -sfn "${target}" "${CUBE_KERNEL_DIR}/vmlinux" + log "selected guest kernel: ${CUBE_KERNEL_DIR}/vmlinux -> ${target}" +} + +detect_primary_interface() { + ip route get 1.1.1.1 2>/dev/null | awk ' + { + for (i = 1; i <= NF; i++) { + if ($i == "dev" && (i + 1) <= NF) { + print $(i + 1) + exit + } + } + }' +} + +validate_runtime_commands() { + for cmd in mkfs.ext4 mount umount losetup cube-runtime containerd-shim-cube-rs cubecli cubevsmapdump; do + require_cmd "${cmd}" + done +} + +patch_common_yaml_list() { + local key="$1" + local raw_values="$2" + [[ -n "${raw_values//[[:space:],;]/}" ]] || return 0 + + local tmp_file + tmp_file="$(mktemp)" + awk -v key="${key}" -v raw_values="${raw_values}" ' + BEGIN { + gsub(/[,;]/, " ", raw_values) + count = split(raw_values, raw, /[[:space:]]+/) + for (i = 1; i <= count; i++) { + if (raw[i] != "") { + values[++value_count] = raw[i] + } + } + } + function emit(indent, i, item) { + print indent key ":" + for (i = 1; i <= value_count; i++) { + item = values[i] + gsub(/"/, "\\\"", item) + print indent " - \"" item "\"" + } + emitted = 1 + } + /^common:[[:space:]]*$/ { + in_common = 1 + print + next + } + in_common && /^[^[:space:]][^:]*:/ { + if (!emitted) { + emit(" ") + } + in_common = 0 + } + in_common && $0 ~ "^[[:space:]]*" key ":[[:space:]]*.*$" { + indent = substr($0, 1, match($0, /[^[:space:]]/) - 1) + emit(indent) + skipping = 1 + next + } + skipping { + if ($0 ~ /^[[:space:]]*-[[:space:]]/) { + next + } + skipping = 0 + } + { + print + } + END { + if (in_common && !emitted) { + emit(" ") + } + } + ' "${CUBELET_DYNAMICCONF}" > "${tmp_file}" + cat "${tmp_file}" > "${CUBELET_DYNAMICCONF}" + rm -f "${tmp_file}" + log "patched ${key} in ${CUBELET_DYNAMICCONF}" +} + +configure_sandbox_dns() { + patch_common_yaml_list default_dns_servers "${CUBE_SANDBOX_DNS_SERVERS:-}" +} + +[[ -x "${NETWORK_AGENT_BIN}" ]] || fail "missing executable: ${NETWORK_AGENT_BIN}" +[[ -x "${CUBELET_BIN}" ]] || fail "missing executable: ${CUBELET_BIN}" +[[ -f "${CUBELET_CONFIG}" ]] || fail "missing config: ${CUBELET_CONFIG}" +[[ -f "${CUBELET_DYNAMICCONF}" ]] || fail "missing dynamic config: ${CUBELET_DYNAMICCONF}" +[[ -n "${CUBE_SANDBOX_NODE_IP:-}" ]] || fail "CUBE_SANDBOX_NODE_IP is required" + +validate_runtime_commands +select_guest_kernel + +sed -i -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CUBE_MASTER_ENDPOINT}\"#" "${CUBELET_DYNAMICCONF}" +configure_sandbox_dns + +if [[ -z "${CUBE_SANDBOX_ETH_NAME:-}" && "${CUBE_SANDBOX_AUTO_DETECT_ETH}" == "true" ]]; then + CUBE_SANDBOX_ETH_NAME="$(detect_primary_interface || true)" + if [[ -n "${CUBE_SANDBOX_ETH_NAME}" ]]; then + log "auto detected primary interface: ${CUBE_SANDBOX_ETH_NAME}" + else + log "primary interface auto detection failed; keeping packaged Cubelet eth_name" + fi +fi +if [[ -n "${CUBE_SANDBOX_ETH_NAME:-}" ]]; then + sed -i "s/eth_name = \"[^\"]*\"/eth_name = \"${CUBE_SANDBOX_ETH_NAME}\"/" "${CUBELET_CONFIG}" +fi +if [[ -n "${CUBE_SANDBOX_NETWORK_CIDR:-}" ]]; then + sed -i "s|cidr = \"[^\"]*\"|cidr = \"${CUBE_SANDBOX_NETWORK_CIDR}\"|" "${CUBELET_CONFIG}" +fi +if [[ -n "${CUBE_TAP_INIT_NUM:-}" ]]; then + sed -i "s/tap_init_num = [0-9]\+/tap_init_num = ${CUBE_TAP_INIT_NUM}/" "${CUBELET_CONFIG}" +fi +if [[ -n "${CUBE_CGROUP_POOL_SIZE:-}" ]]; then + sed -i "s/pool_size = [0-9]\+/pool_size = ${CUBE_CGROUP_POOL_SIZE}/" "${CUBELET_CONFIG}" +fi +if [[ -n "${CUBE_WORKFLOW_CONCURRENT:-}" ]]; then + sed -i "s/concurrent = [0-9]\+/concurrent = ${CUBE_WORKFLOW_CONCURRENT}/g" "${CUBELET_CONFIG}" +fi + +mkdir -p \ + "${NETWORK_AGENT_STATE_DIR}" \ + "${TOOLBOX_ROOT}/cube-vs/network" \ + "${TOOLBOX_ROOT}/cube-snapshot" \ + /tmp/cube \ + /data/log/Cubelet \ + /data/log/CubeShim \ + /data/log/CubeVmm \ + /data/cube-shim/disks \ + /data/snapshot_pack/disks + +rm -f \ + /tmp/cube/network-agent.sock \ + /tmp/cube/network-agent-grpc.sock \ + /tmp/cube/network-agent-tap.sock \ + || true + +stop_stale_processes() { + local name="$1" + local pattern="$2" + local pids + + pids="$(pgrep -f "${pattern}" || true)" + [[ -n "${pids}" ]] || return 0 + + log "stopping stale ${name} process(es): ${pids//$'\n'/ }" + kill ${pids} 2>/dev/null || true + for _ in $(seq 1 10); do + if ! pgrep -f "${pattern}" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + pgrep -f "${pattern}" | xargs -r kill -9 2>/dev/null || true +} + +stop_stale_processes network-agent "${NETWORK_AGENT_BIN}" +stop_stale_processes cubelet "${CUBELET_BIN}" + +cleanup() { + if [[ -n "${NETWORK_AGENT_PID:-}" ]]; then + kill "${NETWORK_AGENT_PID}" 2>/dev/null || true + fi + if [[ -n "${CUBELET_PID:-}" ]]; then + kill "${CUBELET_PID}" 2>/dev/null || true + fi +} +trap cleanup TERM INT HUP EXIT + +log "starting network-agent" +"${NETWORK_AGENT_BIN}" --cubelet-config "${CUBELET_CONFIG}" --state-dir "${NETWORK_AGENT_STATE_DIR}" & +NETWORK_AGENT_PID=$! + +for i in $(seq 1 120); do + if curl -fsS "${NETWORK_AGENT_HEALTH_URL}" >/dev/null 2>&1; then + log "network-agent ready" + break + fi + if ! kill -0 "${NETWORK_AGENT_PID}" >/dev/null 2>&1; then + fail "network-agent exited before ready" + fi + [[ "${i}" -lt 120 ]] || fail "network-agent did not become ready" + sleep 1 +done + +log "starting cubelet for node ${CUBE_SANDBOX_NODE_IP}" +"${CUBELET_BIN}" --config "${CUBELET_CONFIG}" --dynamic-conf-path "${CUBELET_DYNAMICCONF}" & +CUBELET_LAUNCH_PID=$! + +for i in $(seq 1 60); do + real_pid="$(pidof cubelet 2>/dev/null | awk '{print $1}' || true)" + if [[ -n "${real_pid}" ]] && kill -0 "${real_pid}" >/dev/null 2>&1 && ss -lntp 2>/dev/null | grep -q ':9999'; then + CUBELET_PID="${real_pid}" + log "cubelet ready, pid=${CUBELET_PID}" + break + fi + if ! kill -0 "${CUBELET_LAUNCH_PID}" >/dev/null 2>&1 && [[ -z "${real_pid}" ]]; then + fail "cubelet exited before listening on 9999" + fi + [[ "${i}" -lt 60 ]] || fail "cubelet did not become ready" + sleep 1 +done + +while true; do + if ! kill -0 "${NETWORK_AGENT_PID}" >/dev/null 2>&1; then + fail "network-agent exited" + fi + if ! kill -0 "${CUBELET_PID}" >/dev/null 2>&1; then + fail "cubelet exited" + fi + sleep 10 +done diff --git a/deploy/kubernetes/images/scripts/cube-node-init.sh b/deploy/kubernetes/images/scripts/cube-node-init.sh new file mode 100644 index 000000000..8f06cb5ed --- /dev/null +++ b/deploy/kubernetes/images/scripts/cube-node-init.sh @@ -0,0 +1,326 @@ +#!/bin/sh +set -eu + +log() { printf '[cube-node-init] %s\n' "$*"; } +fail() { printf '[cube-node-init] ERROR: %s\n' "$*" >&2; exit 1; } + +HOST_ROOT="${HOST_ROOT:-/host}" +DATA_CUBELET="${DATA_CUBELET:-/data/cubelet}" +REQUIRE_KVM="${REQUIRE_KVM:-true}" +REQUIRE_XFS="${REQUIRE_XFS:-true}" +CHMOD_KVM="${CHMOD_KVM:-true}" +LOAD_KVM_MODULE="${LOAD_KVM_MODULE:-true}" +WRITE_UDEV_RULE="${WRITE_UDEV_RULE:-true}" +CREATE_HOST_DIRS="${CREATE_HOST_DIRS:-true}" +CHECK_MASTER_CONNECTIVITY="${CHECK_MASTER_CONNECTIVITY:-true}" +CUBE_MASTER_ENDPOINT="${CUBE_MASTER_ENDPOINT:-cube-master.cube-system.svc.cluster.local:8089}" +CHECK_MEMORY="${CHECK_MEMORY:-true}" +MIN_MEMORY_KB="${MIN_MEMORY_KB:-7500000}" +CHECK_CGROUP_CPU="${CHECK_CGROUP_CPU:-true}" +CHECK_GLIBC="${CHECK_GLIBC:-true}" +CHECK_CIDR="${CHECK_CIDR:-true}" +CHECK_CUBECOW_DEPS="${CHECK_CUBECOW_DEPS:-true}" +CUBE_PVM_ENABLE="${CUBE_PVM_ENABLE:-0}" +CUBE_SANDBOX_NETWORK_CIDR="${CUBE_SANDBOX_NETWORK_CIDR:-}" +CUBE_SANDBOX_NETWORK_CIDR_SKIP_CONFLICT_CHECK="${CUBE_SANDBOX_NETWORK_CIDR_SKIP_CONFLICT_CHECK:-0}" +LOOPBACK_ENABLED="${LOOPBACK_ENABLED:-false}" +LOOPBACK_IMAGE_PATH="${LOOPBACK_IMAGE_PATH:-/data/cubelet-xfs.img}" +LOOPBACK_SIZE="${LOOPBACK_SIZE:-25G}" + +host_path() { printf '%s%s' "$HOST_ROOT" "$1"; } +host_chroot_sh() { + # Keep the container mount namespace so the host root bind mount remains + # visible at $HOST_ROOT, but enter host uts/ipc/net/pid namespaces for host + # service operations. + nsenter --target 1 --uts --ipc --net --pid -- chroot "$HOST_ROOT" /bin/sh -c "$*" +} +host_mount_sh() { + # Enter the host mount namespace when creating or mounting host filesystems. + nsenter --target 1 --mount --uts --ipc --net --pid -- /bin/sh -c "$*" +} + +version_ge() { + # Returns true when $1 >= $2 for MAJOR.MINOR glibc-style versions. + awk -v a="$1" -v b="$2" 'BEGIN { + split(a, av, "."); split(b, bv, "."); + if ((av[1] + 0) > (bv[1] + 0)) exit 0; + if ((av[1] + 0) < (bv[1] + 0)) exit 1; + exit ((av[2] + 0) >= (bv[2] + 0)) ? 0 : 1; + }' +} + +check_memory() { + [ "$CHECK_MEMORY" = "true" ] || return 0 + mem_total_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)" + case "$MIN_MEMORY_KB" in + ''|*[!0-9]*) fail "MIN_MEMORY_KB must be a positive integer: ${MIN_MEMORY_KB}" ;; + esac + [ "$mem_total_kb" -ge "$MIN_MEMORY_KB" ] || fail "system memory must be at least ${MIN_MEMORY_KB}KB, found ${mem_total_kb}KB" + log "memory check passed: ${mem_total_kb}KB" +} + +check_glibc() { + [ "$CHECK_GLIBC" = "true" ] || return 0 + glibc_ver="$(ldd --version 2>&1 | awk 'NR == 1 { print $NF; exit }' || true)" + [ -n "$glibc_ver" ] || fail "unable to detect glibc version" + version_ge "$glibc_ver" "2.31" || fail "glibc version ${glibc_ver} is too old; requires >= 2.31" + log "glibc check passed: ${glibc_ver}" +} + +check_cgroup_cpu() { + [ "$CHECK_CGROUP_CPU" = "true" ] || return 0 + fstype="$(stat -fc %T /sys/fs/cgroup 2>/dev/null || echo unknown)" + [ "$fstype" = "cgroup2fs" ] || return 0 + controllers="$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null || true)" + echo "$controllers" | grep -qw cpu || fail "cgroup v2 does not expose cpu controller: ${controllers:-}" + subtree="$(cat /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true)" + if echo "$subtree" | grep -qw cpu; then + log "cgroup v2 cpu controller check passed" + return 0 + fi + log "cgroup v2 cpu controller not enabled; trying to enable +cpu" + if printf '+cpu\n' > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null; then + log "enabled cgroup v2 +cpu" + return 0 + fi + fail "failed to enable cgroup v2 cpu controller on /sys/fs/cgroup/cgroup.subtree_control" +} + +ip_to_int() { + awk -v ip="$1" 'BEGIN { + n = split(ip, a, "."); + if (n != 4) exit 1; + for (i = 1; i <= 4; i++) { + if (a[i] !~ /^[0-9]+$/ || a[i] < 0 || a[i] > 255) exit 1; + } + printf "%.0f\n", (a[1] * 16777216) + (a[2] * 65536) + (a[3] * 256) + a[4]; + }' +} + +int_to_ip() { + awk -v n="$1" 'BEGIN { + a = int(n / 16777216); n -= a * 16777216; + b = int(n / 65536); n -= b * 65536; + c = int(n / 256); d = n - c * 256; + printf "%d.%d.%d.%d\n", a, b, c, d; + }' +} + +cidr_range() { + cidr="$1" + ip="${cidr%/*}" + mask="${cidr#*/}" + [ "$ip" != "$cidr" ] || return 1 + case "$mask" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$mask" -ge 0 ] && [ "$mask" -le 32 ] || return 1 + ip_int="$(ip_to_int "$ip")" || return 1 + host_bits=$((32 - mask)) + if [ "$host_bits" -eq 32 ]; then + block_size=4294967296 + else + block_size=$((1 << host_bits)) + fi + start=$((ip_int / block_size * block_size)) + end=$((start + block_size - 1)) + printf '%s %s\n' "$start" "$end" +} + +cidr_overlaps() { + a="$1" + b="$2" + a_range="$(cidr_range "$a")" || return 1 + b_range="$(cidr_range "$b")" || return 1 + # shellcheck disable=SC2086 + set -- $a_range $b_range + a_start="$1" + a_end="$2" + b_start="$3" + b_end="$4" + [ "$a_start" -le "$b_end" ] && [ "$b_start" -le "$a_end" ] +} + +normalize_existing_cidr() { + token="$1" + case "$token" in + default|broadcast|throw|unreachable|blackhole|prohibit|local|nat|multicast|'') + return 1 + ;; + esac + case "$token" in + */*) + ip="${token%/*}" + mask="${token#*/}" + ;; + *) + ip="$token" + mask=32 + ;; + esac + case "$mask" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$mask" -ge 0 ] && [ "$mask" -le 32 ] || return 1 + ip_to_int "$ip" >/dev/null 2>&1 || return 1 + printf '%s/%s\n' "$ip" "$mask" +} + +validate_cidr() { + cidr="$1" + ip="${cidr%/*}" + mask="${cidr#*/}" + [ "$ip" != "$cidr" ] || fail "CUBE_SANDBOX_NETWORK_CIDR must be IPv4 CIDR, got ${cidr}" + case "$mask" in + ''|*[!0-9]*) fail "invalid CIDR mask: ${cidr}" ;; + esac + [ "$mask" -ge 8 ] && [ "$mask" -le 30 ] || fail "CIDR mask must be between 8 and 30: ${cidr}" + ip_int="$(ip_to_int "$ip")" || fail "invalid CIDR IP: ${cidr}" + host_bits=$((32 - mask)) + block_size=$((1 << host_bits)) + aligned=$((ip_int / block_size * block_size)) + if [ "$ip_int" -ne "$aligned" ]; then + suggested="$(int_to_ip "$aligned")/${mask}" + fail "CIDR is not aligned to network address: ${cidr}; did you mean ${suggested}" + fi +} + +check_cidr_conflict() { + [ "$CHECK_CIDR" = "true" ] || return 0 + [ -n "$CUBE_SANDBOX_NETWORK_CIDR" ] || return 0 + validate_cidr "$CUBE_SANDBOX_NETWORK_CIDR" + if [ "$CUBE_SANDBOX_NETWORK_CIDR_SKIP_CONFLICT_CHECK" = "1" ]; then + log "CIDR conflict check skipped: ${CUBE_SANDBOX_NETWORK_CIDR}" + return 0 + fi + + conflicts_file="$(mktemp)" + { + ip -o -4 addr show 2>/dev/null | awk '{print $4 " addr " $2}' + ip -o -4 route show 2>/dev/null | awk '{print $1 " route " $0}' + } | while read -r candidate kind detail; do + existing="$(normalize_existing_cidr "$candidate" || true)" + [ -n "$existing" ] || continue + if cidr_overlaps "$CUBE_SANDBOX_NETWORK_CIDR" "$existing"; then + printf -- '- %s %s %s\n' "$kind" "$existing" "$detail" >> "$conflicts_file" + fi + done + + if [ -s "$conflicts_file" ]; then + conflicts="$(cat "$conflicts_file")" + rm -f "$conflicts_file" + fail "CUBE_SANDBOX_NETWORK_CIDR conflicts with existing host network: ${CUBE_SANDBOX_NETWORK_CIDR} +${conflicts} +Set CUBE_SANDBOX_NETWORK_CIDR to a non-overlapping private CIDR or set CUBE_SANDBOX_NETWORK_CIDR_SKIP_CONFLICT_CHECK=1 to bypass the conflict check." + fi + rm -f "$conflicts_file" + log "CIDR check passed: ${CUBE_SANDBOX_NETWORK_CIDR}" +} + +check_cubecow_deps() { + [ "$CHECK_CUBECOW_DEPS" = "true" ] || return 0 + missing="" + for cmd in mkfs.ext4 mount umount losetup; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing="${missing} ${cmd}" + fi + done + [ -z "$missing" ] || fail "missing cubecow startup dependencies:${missing}" + log "cubecow startup dependency check passed" +} + +normalize_pvm_enable() { + case "$CUBE_PVM_ENABLE" in + 1|true|TRUE|yes|YES) + CUBE_PVM_ENABLE=1 + ;; + 0|false|FALSE|no|NO) + CUBE_PVM_ENABLE=0 + ;; + *) + fail "unsupported CUBE_PVM_ENABLE=${CUBE_PVM_ENABLE}; expected 0 or 1" + ;; + esac +} + +check_pvm_consistency() { + normalize_pvm_enable + command -v lsmod >/dev/null 2>&1 || fail "lsmod is required for PVM consistency check" + + has_kvm_pvm=0 + if lsmod 2>/dev/null | grep -qE '^kvm_pvm[[:space:]]'; then + has_kvm_pvm=1 + fi + + if [ "$has_kvm_pvm" -eq 1 ] && [ "$CUBE_PVM_ENABLE" != "1" ]; then + fail "PVM host detected because kvm_pvm is loaded, but CUBE_PVM_ENABLE=0. Set cubeNode.pvmGuestKernel.enabled=true so cube-node selects cube-kernel-scf/vmlinux-pvm." + fi + + if [ "$has_kvm_pvm" -eq 0 ] && [ "$CUBE_PVM_ENABLE" = "1" ]; then + fail "CUBE_PVM_ENABLE=1 requires kvm_pvm to be loaded on the host. Enable bootstrap.pvmHostKernel.enabled=true for chart-managed PVM host bootstrap, or prepare/reboot the node into a PVM host kernel before starting cube-node." + fi + + if [ "$CUBE_PVM_ENABLE" = "1" ]; then + log "PVM consistency check passed: kvm_pvm loaded and CUBE_PVM_ENABLE=1" + else + log "PVM consistency check passed: ordinary guest kernel mode" + fi +} + +check_memory +check_glibc +check_cgroup_cpu +check_cidr_conflict +check_cubecow_deps + +if [ "$LOAD_KVM_MODULE" = "true" ]; then + log "loading kvm_pvm module" + modprobe kvm_pvm 2>/dev/null || host_chroot_sh "modprobe kvm_pvm" 2>/dev/null || true +fi + +if [ "$REQUIRE_KVM" = "true" ]; then + [ -e /dev/kvm ] || fail "/dev/kvm does not exist" + log "/dev/kvm check passed" +fi + +check_pvm_consistency + +if [ "$CHMOD_KVM" = "true" ] && [ -e /dev/kvm ]; then + chmod 666 /dev/kvm || true +fi + +if [ "$WRITE_UDEV_RULE" = "true" ]; then + mkdir -p "$(host_path /etc/udev/rules.d)" + printf 'KERNEL=="kvm", MODE="0666"\n' > "$(host_path /etc/udev/rules.d/99-kvm.rules)" +fi + +if [ "$CREATE_HOST_DIRS" = "true" ]; then + log "creating host directories" + mkdir -p \ + "$(host_path /data/cubelet)" \ + "$(host_path /data/log)" \ + "$(host_path /data/cube-shim)" \ + "$(host_path /data/snapshot_pack)" \ + "$(host_path /tmp/cube)" +fi + +if ! xfs_info "$DATA_CUBELET" >/dev/null 2>&1; then + if [ "$LOOPBACK_ENABLED" = "true" ]; then + log "initializing loopback XFS at ${LOOPBACK_IMAGE_PATH} size=${LOOPBACK_SIZE}" + host_mount_sh "mkdir -p ${DATA_CUBELET}; if [ ! -f ${LOOPBACK_IMAGE_PATH} ]; then truncate -s ${LOOPBACK_SIZE} ${LOOPBACK_IMAGE_PATH}; mkfs.xfs -f -m reflink=1 ${LOOPBACK_IMAGE_PATH}; fi; mountpoint -q ${DATA_CUBELET} || mount -o loop,pquota ${LOOPBACK_IMAGE_PATH} ${DATA_CUBELET}; grep -q '${LOOPBACK_IMAGE_PATH} ${DATA_CUBELET}' /etc/fstab || echo '${LOOPBACK_IMAGE_PATH} ${DATA_CUBELET} xfs loop,pquota 0 0' >> /etc/fstab" + fi +fi + +if [ "$REQUIRE_XFS" = "true" ]; then + xfs_info "$DATA_CUBELET" >/dev/null 2>&1 || fail "${DATA_CUBELET} is not an XFS mount" + log "${DATA_CUBELET} XFS check passed" +fi + +if [ "$CHECK_MASTER_CONNECTIVITY" = "true" ]; then + command -v curl >/dev/null 2>&1 || fail "curl is required in cube-node-init image" + log "checking CubeMaster connectivity: ${CUBE_MASTER_ENDPOINT}" + curl -fsS "http://${CUBE_MASTER_ENDPOINT}/notify/health" >/dev/null || fail "CubeMaster is unreachable at ${CUBE_MASTER_ENDPOINT}" + log "cube-master connectivity check passed" +fi + +log "cube node init completed" diff --git a/deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh b/deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh new file mode 100644 index 000000000..afe6ae89c --- /dev/null +++ b/deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh @@ -0,0 +1,205 @@ +#!/bin/sh +set -eu + +log() { printf '[pvm-host-bootstrap] %s\n' "$*"; } +fail() { printf '[pvm-host-bootstrap] ERROR: %s\n' "$*" >&2; exit 1; } + +HOST_ROOT="${HOST_ROOT:-/host}" +STATE_DIR="${STATE_DIR:-/var/lib/cube-node-bootstrap}" +DESIRED_KERNEL_PATTERN="${DESIRED_KERNEL_PATTERN:-cubesandbox.pvm.host}" +RPM_PATH="${PVM_KERNEL_RPM_PATH:-/artifacts/kernel-pvm-host.rpm}" +DEB_PATH="${PVM_KERNEL_DEB_PATH:-/artifacts/linux-image-pvm-host.deb}" +RPM_URL="${PVM_KERNEL_RPM_URL:-}" +DEB_URL="${PVM_KERNEL_DEB_URL:-}" +PACKAGE_SOURCE="${PVM_KERNEL_PACKAGE_SOURCE:-image}" +KERNEL_BOOT_ARGS="${PVM_KERNEL_BOOT_ARGS:-}" +REBOOT_ENABLED="${REBOOT_ENABLED:-true}" +REBOOT_MAX_COUNT="${REBOOT_MAX_COUNT:-1}" +REBOOT_COORDINATED="${REBOOT_COORDINATED:-true}" +LEASE_NAME="${LEASE_NAME:-cube-node-pvm-bootstrap}" +LEASE_TTL_SECONDS="${LEASE_TTL_SECONDS:-900}" +NODE_NAME="${NODE_NAME:-$(hostname)}" +NAMESPACE="${POD_NAMESPACE:-default}" + +host_path() { printf '%s%s' "$HOST_ROOT" "$1"; } +lease_time() { date -u +%Y-%m-%dT%H:%M:%S.000000Z; } +lease_expired() { + renew_time="$(kubectl -n "$NAMESPACE" get lease "$LEASE_NAME" -o jsonpath='{.spec.renewTime}' 2>/dev/null || true)" + if [ -z "$renew_time" ]; then + return 0 + fi + renew_epoch="$(date -u -d "$renew_time" +%s 2>/dev/null || echo 0)" + now_epoch="$(date -u +%s)" + if [ "$renew_epoch" -eq 0 ]; then + return 0 + fi + [ $((now_epoch - renew_epoch)) -ge "$LEASE_TTL_SECONDS" ] +} +json_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} +shell_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} +host_sh() { + # Keep the container mount namespace so the host root bind mount remains + # visible at $HOST_ROOT, but enter host uts/ipc/net/pid namespaces for host + # service operations such as reboot. + nsenter --target 1 --uts --ipc --net --pid -- chroot "$HOST_ROOT" /bin/sh -c "$*" +} + +release_lease() { + if [ "$REBOOT_COORDINATED" != "true" ]; then + return 0 + fi + command -v kubectl >/dev/null 2>&1 || return 0 + holder="$(kubectl -n "$NAMESPACE" get lease "$LEASE_NAME" -o jsonpath='{.spec.holderIdentity}' 2>/dev/null || true)" + if [ "$holder" = "$NODE_NAME" ]; then + kubectl -n "$NAMESPACE" patch lease "$LEASE_NAME" --type merge \ + -p '{"spec":{"holderIdentity":""}}' >/dev/null 2>&1 || true + log "lease released" + fi +} + +mkdir -p "$(host_path "$STATE_DIR")" + +acquire_lease() { + if [ "$REBOOT_COORDINATED" != "true" ]; then + log "coordinated reboot disabled; skip lease" + return 0 + fi + command -v kubectl >/dev/null 2>&1 || fail "kubectl is required in pvm-host-bootstrap image when coordinated reboot is enabled" + log "waiting for lease ${NAMESPACE}/${LEASE_NAME} as ${NODE_NAME}" + while true; do + if ! kubectl -n "$NAMESPACE" get lease "$LEASE_NAME" >/dev/null 2>&1; then + cat </dev/null 2>&1 || true +apiVersion: coordination.k8s.io/v1 +kind: Lease +metadata: + name: ${LEASE_NAME} + namespace: ${NAMESPACE} +spec: + holderIdentity: "" + leaseDurationSeconds: ${LEASE_TTL_SECONDS} +EOF + fi + holder="$(kubectl -n "$NAMESPACE" get lease "$LEASE_NAME" -o jsonpath='{.spec.holderIdentity}' 2>/dev/null || true)" + if [ -z "$holder" ] || [ "$holder" = "$NODE_NAME" ] || lease_expired; then + now="$(lease_time)" + patch_file="$(mktemp)" + cat > "$patch_file" </dev/null 2>&1; then + rm -f "$patch_file" + log "lease acquired" + return 0 + fi + rm -f "$patch_file" + log "lease changed before acquire; retrying" + fi + log "lease held by ${holder}; sleep 15s" + sleep 15 + done +} + +acquire_lease + +install_kernel() { + if [ "$PACKAGE_SOURCE" = "download" ]; then + mkdir -p /artifacts + if [ -n "$RPM_URL" ] && [ ! -f "$RPM_PATH" ]; then + log "downloading PVM host kernel rpm from ${RPM_URL}" + curl -fL --retry 3 -o "$RPM_PATH" "$RPM_URL" + fi + if [ -n "$DEB_URL" ] && [ ! -f "$DEB_PATH" ]; then + log "downloading PVM host kernel deb from ${DEB_URL}" + curl -fL --retry 3 -o "$DEB_PATH" "$DEB_URL" + fi + fi + if [ -f "$RPM_PATH" ]; then + log "installing PVM host kernel rpm from ${RPM_PATH}" + cp "$RPM_PATH" "$(host_path /tmp/cube-pvm-host-kernel.rpm)" + host_sh "rpm -ivh --oldpackage --replacepkgs /tmp/cube-pvm-host-kernel.rpm || rpm -Uvh --oldpackage --replacepkgs /tmp/cube-pvm-host-kernel.rpm" + elif [ -f "$DEB_PATH" ]; then + log "installing PVM host kernel deb from ${DEB_PATH}" + cp "$DEB_PATH" "$(host_path /tmp/cube-pvm-host-kernel.deb)" + host_sh "dpkg -i /tmp/cube-pvm-host-kernel.deb" + else + fail "no PVM host kernel package found: rpm=${RPM_PATH}, deb=${DEB_PATH}" + fi + echo "installed" > "$(host_path "$STATE_DIR/pvm-kernel-installed")" +} + +configure_bootloader() { + log "configuring bootloader for kernel pattern ${DESIRED_KERNEL_PATTERN}" + boot_args="$(shell_quote "$KERNEL_BOOT_ARGS")" + if [ -x "$(host_path /usr/sbin/grubby)" ]; then + host_sh "PVM_KERNEL_BOOT_ARGS=${boot_args}; pvm_kernel=\$(ls /boot/vmlinuz-*${DESIRED_KERNEL_PATTERN}* 2>/dev/null | sort | tail -1); test -n \"\$pvm_kernel\"; grubby --set-default \"\$pvm_kernel\"; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then grubby --update-kernel \"\$pvm_kernel\" --args \"\$PVM_KERNEL_BOOT_ARGS\"; fi" + elif [ -x "$(host_path /usr/sbin/update-grub)" ] || [ -x "$(host_path /usr/sbin/grub-mkconfig)" ]; then + host_sh "PVM_KERNEL_BOOT_ARGS=${boot_args}; pvm_kernel=\$(ls /boot/vmlinuz-*${DESIRED_KERNEL_PATTERN}* 2>/dev/null | sed 's|/boot/vmlinuz-||' | sort | tail -1); test -n \"\$pvm_kernel\"; if [ -f /etc/default/grub ]; then sed -i \"s|^GRUB_DEFAULT=.*|GRUB_DEFAULT=\\\"Advanced options for Ubuntu>Ubuntu, with Linux \${pvm_kernel}\\\"|\" /etc/default/grub; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then if grep -q '^GRUB_CMDLINE_LINUX=' /etc/default/grub; then sed -i \"s|^GRUB_CMDLINE_LINUX=\\\"\\(.*\\)\\\"|GRUB_CMDLINE_LINUX=\\\"\\1 \${PVM_KERNEL_BOOT_ARGS}\\\"|\" /etc/default/grub; else printf 'GRUB_CMDLINE_LINUX=\\\"%s\\\"\\n' \"\$PVM_KERNEL_BOOT_ARGS\" >> /etc/default/grub; fi; fi; fi; if command -v update-grub >/dev/null 2>&1; then update-grub; elif command -v grub-mkconfig >/dev/null 2>&1; then grub-mkconfig -o /boot/grub/grub.cfg; fi" + else + fail "no supported bootloader tool found in host root" + fi + mkdir -p "$(host_path /etc/modules-load.d)" "$(host_path /etc/udev/rules.d)" + printf 'kvm_pvm\n' > "$(host_path /etc/modules-load.d/kvm-pvm.conf)" + printf 'KERNEL=="kvm", MODE="0666"\n' > "$(host_path /etc/udev/rules.d/99-kvm.rules)" +} + +missing_boot_args() { + [ -n "$KERNEL_BOOT_ARGS" ] || return 0 + cmdline=" $(cat /proc/cmdline 2>/dev/null || true) " + missing="" + for arg in $KERNEL_BOOT_ARGS; do + case "$cmdline" in + *" ${arg} "*) ;; + *) missing="${missing} ${arg}" ;; + esac + done + [ -z "$missing" ] || printf '%s\n' "$missing" +} + +request_reboot_or_fail() { + count_name="${1:-reboot-count}" + not_ready_message="${2:-PVM kernel installed but host is not running it yet}" + count_file="$(host_path "$STATE_DIR/${count_name}")" + count="0" + [ -f "$count_file" ] && count="$(cat "$count_file" || echo 0)" + if [ "$REBOOT_ENABLED" = "true" ] && [ "$count" -lt "$REBOOT_MAX_COUNT" ]; then + count=$((count + 1)) + echo "$count" > "$count_file" + lease_time > "$(host_path "$STATE_DIR/reboot-requested")" + log "rebooting host ${NODE_NAME}; reboot-count=${count}/${REBOOT_MAX_COUNT}" + host_sh "if command -v systemctl >/dev/null 2>&1; then systemctl reboot; else reboot; fi" || true + sleep 3600 + exit 1 + fi + + fail "${not_ready_message}; reboot_enabled=${REBOOT_ENABLED}, reboot_count=${count}/${REBOOT_MAX_COUNT}, missing_boot_args=$(missing_boot_args)" +} + +current_kernel="$(uname -r || true)" +log "current kernel: ${current_kernel}" +if printf '%s' "$current_kernel" | grep -q "$DESIRED_KERNEL_PATTERN"; then + missing_args="$(missing_boot_args)" + if [ -z "$missing_args" ]; then + log "PVM kernel check passed" + echo "$current_kernel" > "$(host_path "$STATE_DIR/pvm-ready")" + release_lease + exit 0 + fi + log "PVM kernel is running but required boot args are missing:${missing_args}" + acquire_lease + configure_bootloader + request_reboot_or_fail boot-args-reboot-count "PVM kernel boot args configured but host is not running with required boot args yet" +fi + +acquire_lease +install_kernel +configure_bootloader +request_reboot_or_fail reboot-count "PVM kernel installed but host is not running it yet" From a2ecc741f6892b99857d0c9fed889d89b6566625 Mon Sep 17 00:00:00 2001 From: vimiix Date: Mon, 6 Jul 2026 20:27:55 +0800 Subject: [PATCH 2/9] fix(kubernetes): address v0.5.0 chart review feedback Address inline review comments on the v0.5.0 Helm chart delivery. Chart: - templates/node-daemonset.yaml: fix CUBE_SANDBOX_DNS_SERVERS serialization. `join "," .` was joining every top-level context key (.Values, .Release, .Chart, .Files, ...) into the env var value, which produces garbage and can leak Values content. Use `.Values.cubeNode.dns.sandbox.nameservers` explicitly. - templates/tests/node-health.yaml: extend helm.sh/hook-delete-policy on all eight test Pods to `before-hook-creation,hook-succeeded` so successful test Pods are cleaned up automatically instead of lingering until the next `helm test` run. - templates/mysql.yaml: drop `--default-authentication-plugin=mysql_native_password`. The in-tree MySQL clients (go-sql-driver/mysql v1.9.3 for CubeMaster, sqlx 0.8 mysql for CubeAPI) both support MySQL 8's default caching_sha2_password plugin, so the SHA-1 override is no longer required. Image build script: - images/build-cube-images.sh: consolidate `build_image` and `build_component_image` into a single function with an optional dockerfile argument, dropping ~30 lines of near-duplicated code and switching all four callsites to the merged signature. - images/build-cube-images.sh: fail fast when the sandbox-package is missing a required cube-node component. Previously `copy_if_exists` silently skipped missing directories and a follow-up `mkdir -p` created empty ones, which the Dockerfile then COPYed into the image. Cubelet, network-agent, cube-shim, cube-kernel-scf, cube-image, cube-vs, cube-snapshot, and scripts/common are now required, and the same pattern is used for the cube-webui context. `copy_if_exists` is removed since it has no remaining callers. Verified: - helm lint deploy/kubernetes/chart passes. - helm template with cubeNode.dns.sandbox.nameservers ["8.8.8.8","1.1.1.1"] now renders `value: "8.8.8.8,1.1.1.1"` for CUBE_SANDBOX_DNS_SERVERS. - bash -n on build-cube-images.sh passes and no residual build_component_image / copy_if_exists references remain. --- deploy/kubernetes/chart/templates/mysql.yaml | 1 - .../chart/templates/node-daemonset.yaml | 2 +- .../chart/templates/tests/node-health.yaml | 16 +++---- deploy/kubernetes/images/build-cube-images.sh | 47 +++++-------------- 4 files changed, 20 insertions(+), 46 deletions(-) diff --git a/deploy/kubernetes/chart/templates/mysql.yaml b/deploy/kubernetes/chart/templates/mysql.yaml index a2e8e1929..206ce2fa1 100644 --- a/deploy/kubernetes/chart/templates/mysql.yaml +++ b/deploy/kubernetes/chart/templates/mysql.yaml @@ -58,7 +58,6 @@ spec: name: {{ include "cube.secretName" . }} key: mysql-password args: - - --default-authentication-plugin=mysql_native_password - --skip-name-resolve ports: - name: mysql diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml index f4975c707..2865342c2 100644 --- a/deploy/kubernetes/chart/templates/node-daemonset.yaml +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -245,7 +245,7 @@ spec: value: {{ .Values.cubeNode.network.cidr | quote }} - name: CUBE_SANDBOX_DNS_SERVERS {{- if .Values.cubeNode.dns.sandbox.nameservers }} - value: {{ join "," . | quote }} + value: {{ join "," .Values.cubeNode.dns.sandbox.nameservers | quote }} {{- else if and .Values.cubeNode.dns.sandbox.useCubeDns .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") .Values.cubeDns.sandboxGateway.enabled }} value: "$(CUBE_SANDBOX_NODE_IP)" {{- else }} diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml index 6b5607247..8bfb3369c 100644 --- a/deploy/kubernetes/chart/templates/tests/node-health.yaml +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -105,7 +105,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -144,7 +144,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -180,7 +180,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -216,7 +216,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never hostNetwork: true @@ -254,7 +254,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -305,7 +305,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} @@ -342,7 +342,7 @@ metadata: app.kubernetes.io/component: test annotations: "helm.sh/hook": test - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never {{- with .Values.imagePullSecrets }} diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh index 3e9418c10..e954c6aa0 100755 --- a/deploy/kubernetes/images/build-cube-images.sh +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -161,15 +161,6 @@ prepare_context() { printf '%s\n' "${ctx}" } -copy_if_exists() { - local src="$1" - local dst="$2" - if [[ -e "${src}" ]]; then - mkdir -p "$(dirname "${dst}")" - cp -a "${src}" "${dst}" - fi -} - copy_cube_master_component_context() { local ctx="$1" local src="${PACKAGE_DIR}/CubeMaster" @@ -226,24 +217,7 @@ copy_cube_proxy_component_context() { build_image() { local name="$1" local ctx="$2" - local dockerfile="${SCRIPT_DIR}/${name}/Dockerfile" - local image="${REGISTRY}/${name}:${IMAGE_TAG}" - local docker_args=(-f "${dockerfile}" -t "${image}") - if [[ "${NO_CACHE}" == "1" ]]; then - docker_args=(--no-cache --pull "${docker_args[@]}") - fi - log "building ${image}" - docker build "${docker_args[@]}" "${ctx}" - if [[ "${PUSH}" == "1" ]]; then - log "pushing ${image}" - docker push "${image}" - fi -} - -build_component_image() { - local name="$1" - local dockerfile="$2" - local ctx="$3" + local dockerfile="${3:-${SCRIPT_DIR}/${name}/Dockerfile}" local image="${REGISTRY}/${name}:${IMAGE_TAG}" local docker_args=(-f "${dockerfile}" -t "${image}") if [[ "${NO_CACHE}" == "1" ]]; then @@ -275,7 +249,7 @@ build_cube_api_image() { } ' "${REPO_ROOT}/CubeAPI/Dockerfile" > "${dockerfile}" - build_component_image cube-api "${dockerfile}" "${REPO_ROOT}/CubeAPI" + build_image cube-api "${REPO_ROOT}/CubeAPI" "${dockerfile}" } build_cube_egress_openresty_base_image() { @@ -335,7 +309,7 @@ RUN chmod +x /usr/local/bin/cube-node-entrypoint.sh ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/cube-node-entrypoint.sh"] EOF - build_component_image cube-node "${dockerfile}" "${ctx}" + build_image cube-node "${ctx}" "${dockerfile}" } copy_cube_egress_net_context() { @@ -352,7 +326,7 @@ copy_cube_egress_net_context() { ctx="$(prepare_context cube-master)" copy_cube_master_component_context "${ctx}" -build_component_image cube-master "${REPO_ROOT}/CubeMaster/docker/Dockerfile" "${ctx}" +build_image cube-master "${ctx}" "${REPO_ROOT}/CubeMaster/docker/Dockerfile" build_cube_api_image @@ -362,7 +336,7 @@ build_image cubemastercli "${ctx}" ctx="$(prepare_context cube-proxy-node)" copy_cube_proxy_component_context "${ctx}" -build_component_image cube-proxy-node "${REPO_ROOT}/CubeProxy/Dockerfile" "${ctx}" +build_image cube-proxy-node "${ctx}" "${REPO_ROOT}/CubeProxy/Dockerfile" build_cube_egress_openresty_base_image build_cube_egress_image @@ -372,7 +346,8 @@ copy_cube_egress_net_context "${ctx}" build_image cube-egress-net "${ctx}" ctx="$(prepare_context cube-webui)" -copy_if_exists "${PACKAGE_DIR}/webui" "${ctx}/package/webui" +[[ -d "${PACKAGE_DIR}/webui" ]] || fail "invalid sandbox-package: missing required cube-webui component webui" +cp -a "${PACKAGE_DIR}/webui" "${ctx}/package/webui" [[ -f "${ctx}/package/webui/dist/index.html" ]] || fail "invalid webui package: missing dist/index.html" [[ -f "${ctx}/package/webui/nginx.conf" ]] || fail "invalid webui package: missing nginx.conf" build_image cube-webui "${ctx}" @@ -384,12 +359,12 @@ else ctx="$(prepare_context cube-node)" copy_scripts "${ctx}" cube-node-entrypoint.sh for d in Cubelet network-agent cube-shim cube-kernel-scf cube-image cube-vs cube-snapshot; do - copy_if_exists "${PACKAGE_DIR}/${d}" "${ctx}/package/${d}" - mkdir -p "${ctx}/package/${d}" + [[ -d "${PACKAGE_DIR}/${d}" ]] || fail "invalid sandbox-package: missing required cube-node component ${d}" + cp -a "${PACKAGE_DIR}/${d}" "${ctx}/package/${d}" done + [[ -d "${PACKAGE_DIR}/scripts/common" ]] || fail "invalid sandbox-package: missing required cube-node component scripts/common" mkdir -p "${ctx}/package/scripts" - copy_if_exists "${PACKAGE_DIR}/scripts/common" "${ctx}/package/scripts/common" - mkdir -p "${ctx}/package/scripts/common" + cp -a "${PACKAGE_DIR}/scripts/common" "${ctx}/package/scripts/common" build_image cube-node "${ctx}" fi From 70edfe316127b8dc5e3370ed3e82c02b4a1fefe7 Mon Sep 17 00:00:00 2001 From: vimiix Date: Tue, 7 Jul 2026 18:09:57 +0800 Subject: [PATCH 3/9] docs(kubernetes): add quickstart and FAQ guides for v0.5.0 chart Add two operator-facing docs under deploy/kubernetes/chart/docs/ and link them from the chart README. - docs/QUICKSTART.md: end-to-end one-click install walkthrough covering cluster/node prerequisites, node label & taint setup, optional image build, minimal runtime-values.yaml sample, `helm upgrade --install` command, verification steps, single-node trial layout, compute-only mode with external control plane, and uninstall cleanup checklist. - docs/FAQ.md: common install and runtime issues grouped by subsystem (install/validate, node scheduling, control plane / database, cube-node & PVM kernel & sandbox runtime, CubeProxy / TLS / DNS, egress network, upgrade/rollback/uninstall, image build). Each entry documents the failure signal, root cause, and concrete kubectl / helm commands to unblock. - chart/README.md: list QUICKSTART.md and FAQ.md alongside ARCHITECTURE.md in the Directory section and a new Documentation section so first-time users can discover the guides directly from the chart README. --- deploy/kubernetes/chart/README.md | 8 + deploy/kubernetes/chart/docs/FAQ.md | 438 +++++++++++++++++++++ deploy/kubernetes/chart/docs/QUICKSTART.md | 235 +++++++++++ 3 files changed, 681 insertions(+) create mode 100644 deploy/kubernetes/chart/docs/FAQ.md create mode 100644 deploy/kubernetes/chart/docs/QUICKSTART.md diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index 693dc5fbd..7a7b85fff 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -19,9 +19,17 @@ deploy/kubernetes/chart/ values.yaml docs/ ARCHITECTURE.md + QUICKSTART.md + FAQ.md templates/ ``` +## Documentation + +- [`docs/QUICKSTART.md`](docs/QUICKSTART.md) — one-click install walkthrough from prerequisites to `helm test`. +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — component layering, startup sequence, DNS / Proxy / Egress data flow, external control plane / compute-only mode. +- [`docs/FAQ.md`](docs/FAQ.md) — common install and runtime issues grouped by subsystem. + ## Architecture 整体组件关系、安装流程、节点启动流程、DNS/Proxy/Egress 数据流和 diff --git a/deploy/kubernetes/chart/docs/FAQ.md b/deploy/kubernetes/chart/docs/FAQ.md new file mode 100644 index 000000000..0c15b5a67 --- /dev/null +++ b/deploy/kubernetes/chart/docs/FAQ.md @@ -0,0 +1,438 @@ +# 常见问题 (FAQ) + +按主题分类整理 CubeSandbox Helm Chart 部署与运行时的常见问题。若你的问题不在其中,先阅读 [`ARCHITECTURE.md`](ARCHITECTURE.md) 和 [`QUICKSTART.md`](QUICKSTART.md) 排查,再提 issue 时附上 `kubectl -n cube-system get pods -o wide` 与相关 Pod 日志。 + +- [A. 安装与校验](#a-安装与校验) +- [B. 节点与调度](#b-节点与调度) +- [C. 控制面 / 数据库](#c-控制面--数据库) +- [D. cube-node / PVM 内核 / 沙箱运行时](#d-cube-node--pvm-内核--沙箱运行时) +- [E. CubeProxy / TLS / DNS](#e-cubeproxy--tls--dns) +- [F. Egress 网络](#f-egress-网络) +- [G. 升级、回滚与卸载](#g-升级回滚与卸载) +- [H. 镜像构建](#h-镜像构建) + +--- + +## A. 安装与校验 + +### A1. `helm install` 报错 `controlPlane.enabled=true requires placement.controlPlane.nodeSelector` + +Chart 通过 `templates/validate.yaml` 做启动前校验,禁止用"通配"部署避免误伤节点。补上明确的 nodeSelector: + +```yaml +placement: + controlPlane: + nodeSelector: + cube.tencent.com/role: control +``` + +同样地,以下报错都是校验兜底: + +| 报错关键字 | 说明 | +|---|---| +| `cube-node requires placement.compute.nodeSelector` | 计算节点必须显式指定,不能空 | +| `bootstrap.pvmHostKernel.enabled=true requires ... allow-pvm-bootstrap=true` | 计算节点 nodeSelector 必须包含 `allow-pvm-bootstrap=true`,防止误替换内核 | +| `cubeProxy.enabled=true requires placement.controlPlane.nodeSelector` | CubeProxy 只跑在 control 节点上 | +| `cubeDns.enabled=true requires placement.compute.nodeSelector` | node-local DNS 只跑在 compute 节点上 | + +### A2. `helm install --wait` 挂在 CubeNode DaemonSet Ready 数不足 + +先看 DaemonSet 的 desiredNumberScheduled 是否与实际 compute 节点数一致: + +```bash +kubectl -n cube-system get ds cube-node +``` + +- `DESIRED=0` → 没有节点匹配 `placement.compute.nodeSelector`,回到 [QUICKSTART 1.3](QUICKSTART.md#13-打节点-label--taint) 打 label +- `CURRENT +``` + +--- + +## B. 节点与调度 + +### B1. 我不想给节点打这么多 label,能不能简化? + +不建议。Chart 有意采用**显式 label 授权**,原因: + +- `pvm-host-bootstrap` 会替换 host kernel 并重启,误伤生产节点后果严重 +- `hostNetwork`、`privileged`、hostPath 挂载都要求节点显式授权 +- K8s 原生的调度语义,操作者可以精确控制部署范围 + +生产环境可以通过 GitOps / Terraform 让 label 由基础设施代码统一管理。 + +### B2. control 节点能否兼做 compute? + +技术上可以(单节点试用/小规模);生产不建议。原因: + +- Cube Node Big Pod 使用 `hostNetwork` + `hostPID` + 特权容器,会与常规控制面共享网络命名空间 +- MySQL / Redis 与 sandbox 抢 CPU / 内存 / 磁盘 IO,SLA 无法保证 + +如果确实要合并,给节点同时打两组 label,并**去掉 control taint**,避免 Cube Node 被驱逐。 + +### B3. TKE 多可用区集群,pv 一直 pending + +`cube-cbs-wffc` StorageClass 的 `volumeBindingMode` 是 `WaitForFirstConsumer`,CBS 盘会在 Pod 被调度到具体节点后再创建到该节点所在 AZ。如果 Pod 因调度限制无法落地,PV 也永远 pending。 + +检查 Pod 的 events: + +```bash +kubectl -n cube-system describe pod +``` + +常见原因: + +- 节点资源不足 → 扩容或降低 request +- Pod 的 nodeSelector 与 CBS 支持的 AZ 不匹配 → 用与 CBS 同 AZ 的节点 + +### B4. 想临时把 compute 节点摘出去维护 + +**推荐做法**:先把 Cubelet 上的 sandbox pause 或迁移,再执行: + +```bash +kubectl drain --ignore-daemonsets --delete-emptydir-data=false +# 维护完成后 +kubectl uncordon +``` + +`cube-node` DaemonSet 会在节点重新加入后自动重启并注册到 CubeMaster。 + +--- + +## C. 控制面 / 数据库 + +### C1. cube-master 启动报 "connection refused to mysql" + +按顺序检查: + +1. `kubectl -n cube-system get pods -l app.kubernetes.io/name=mysql`:内置 MySQL 是否 Running +2. `kubectl -n cube-system get secret cube-secret`:密码是否被误改 +3. `kubectl -n cube-system exec cube-mysql-0 -- mysql -uroot -p$(kubectl -n cube-system get secret cube-secret -o jsonpath='{.data.mysql-root-password}' | base64 -d) -e 'show databases'`:内部连通性 + +外部 MySQL 场景,把 CubeMaster Pod 打到 mysql host 上做连通性验证: + +```bash +kubectl -n cube-system exec deploy/cube-master -- \ + sh -c "nc -zv $CUBE_MYSQL_HOST 3306" +``` + +### C2. 从 v0.4.x 升级到 v0.5.0,MySQL 表结构没升级 + +CubeMaster v0.5.0 使用内嵌 goose migration,启动时自动跑。若你在同一 chart release 里手工修改过表结构,可能与 embedded migration 冲突。 + +排查: + +```bash +kubectl -n cube-system logs deploy/cube-master -c cube-master | grep -i migrat +# 或直接连数据库查看 goose_db_version 表 +``` + +若 migration lock 卡死(通常是上次 CubeMaster 异常退出),手工释放: + +```sql +UPDATE goose_db_version SET is_applied=1 WHERE tstamp = (SELECT MAX(tstamp) FROM goose_db_version WHERE is_applied=0); +``` + +### C3. 外部 MySQL 8 使用 caching_sha2_password 认证失败 + +从本 PR 起,chart 已经**移除** `--default-authentication-plugin=mysql_native_password`。CubeMaster / CubeAPI 使用的驱动(`go-sql-driver v1.9`、`sqlx 0.8`)都原生支持 caching_sha2_password。若你的外部 MySQL 用户仍是 old_password,请更新: + +```sql +ALTER USER 'cube_user'@'%' IDENTIFIED WITH caching_sha2_password BY ''; +FLUSH PRIVILEGES; +``` + +### C4. cubemastercli 交互式登录能否代替 `kubectl exec`? + +可以。Chart 交付的 `cubemastercli` Deployment 已经内置一个 bashrc 函数,自动注入 `--address` / `--port`: + +```bash +kubectl -n cube-system exec -it deploy/cube-cubemastercli -- bash +# 然后直接输入 +cubemastercli node list +cubemastercli sandbox list +``` + +--- + +## D. cube-node / PVM 内核 / 沙箱运行时 + +### D1. `pvm-host-bootstrap` Init Container 反复重启 + +绝大多数是 host kernel 替换后节点需要重启。观察 events: + +```bash +kubectl -n cube-system describe pod | grep -A3 pvm-host-bootstrap +kubectl -n cube-system logs -c pvm-host-bootstrap --tail=100 +``` + +正常流程: + +1. 首次运行:检测 host kernel → 若不是 PVM kernel,安装 RPM/DEB → 触发节点重启 +2. 节点重启后:DaemonSet 拉起同名 Pod → `pvm-host-bootstrap` 再次运行 → 检测到 PVM kernel → 成功退出 → 进入 `cube-node-init` + +若卡在"waiting for reboot",通常是节点没有 sudo/reboot 权限或 GRUB 未更新,需人工登录节点执行 `reboot` 并检查: + +```bash +uname -r # 应包含 cubesandbox.pvm.host +cat /proc/cmdline | grep -o pti=off +``` + +### D2. `cube-node-init` 报 `/dev/kvm not accessible` + +节点没有 KVM 支持。检查: + +```bash +kubectl debug node/ -it --image=busybox -- sh +ls -la /dev/kvm +lsmod | grep kvm +``` + +- 云上 CVM 需要选择支持嵌套虚拟化的实例族(如腾讯云 S5、SA3 及以上带 KVM 的机型) +- 物理机需要 BIOS 开启 VT-x/AMD-V + +### D3. cube-node 主容器 Ready 但看不到 sandbox + +首先看 CubeMaster 是否收到节点注册: + +```bash +kubectl -n cube-system exec deploy/cube-cubemastercli -- \ + sh -c 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" node list' +``` + +- 节点在列表中,`healthy: true` → 正常,可以创建 sandbox +- 节点不在列表 → 看 cube-node 日志 + +```bash +kubectl -n cube-system logs -c cube-node --tail=200 | grep -i register +``` + +常见问题:CubeMaster endpoint 不可达(参考 [E1](#e1-sandbox-domain-无法解析))。 + +### D4. sandbox 启动很慢(>10s),但节点 CPU/内存都空闲 + +- 首次启动:模板 rootfs 需从 CubeMaster storage 下载,单次可能 5-30s(取决于模板大小)。**后续基于同模板的 sandbox**,启动约 1s 以内 +- 检查节点 `/data/cubelet` 是否达到 CBS 盘的 IOPS 限制:`iostat -x 1` +- 检查是否有 sandbox 处于 Paused 状态占用磁盘但未及时清理 + +### D5. Cube sandbox 数量瓶颈 + +单节点承载 sandbox 数量的三个瓶颈: + +1. **内存**:每 sandbox base 256MB-2GB,active 数受限于节点内存 +2. **磁盘**:模板 CoW 后每 sandbox rootfs 约几百 MB,受限于 `/data/cubelet` 盘容量 +3. **KVM 虚拟机数**:Linux 单节点 KVM 支持最多几百个 VM,内核参数决定 + +利用 `Pause` 可以把非活跃 sandbox 归零(CPU/RSS→0),盘不释放。运营层面通常在闲置 N 分钟后 Pause,更长后 Destroy。 + +--- + +## E. CubeProxy / TLS / DNS + +### E1. sandbox domain 无法解析 + +按方向排查: + +1. **compute 节点内部**(cube-node Pod / sandbox guest):`dig @127.0.0.54 test.cube.app` + - 无应答 → cube-dns 未启动,查 `kubectl -n cube-system get pods -l app.kubernetes.io/name=cube-dns` + - 应答不是 CubeProxy 入口 IP → 检查 `cubeProxy.advertiseIP` / `cubeDns.answerIP` +2. **compute 节点外部**(客户浏览器 / SDK):用户侧 DNS 未配置。Chart 只交付 in-cluster DNS,面向公网的 DNS/LB 需要用户自行设置 + +### E2. CubeProxy 起不来:`hostNetwork port 80/443 already in use` + +Control 节点上 80/443 被其他服务占用。方案: + +- 释放占用端口 +- 或改用非默认端口:`cubeProxy.ports.http.hostPort: 8080`(但客户端 URL 也要跟着改) +- **不推荐**:关掉 hostNetwork 走 ClusterIP,会绕开 one-click 的入口语义 + +### E3. selfSigned TLS 浏览器提示不安全 + +selfSigned 只用于试用。生产环境请使用: + +- **existingSecret**(推荐): + ```yaml + cubeProxy: + tls: + mode: existingSecret + existingSecret: my-cube-tls + ``` + 提前 `kubectl create secret tls my-cube-tls --cert=... --key=...` +- **certManager**: + ```yaml + cubeProxy: + tls: + mode: certManager + certManager: + issuerRef: + name: my-letsencrypt-issuer + kind: ClusterIssuer + ``` + +### E4. 修改 `cubeProxy.advertiseIP` 后未生效 + +CubeProxy 的路由信息来自 Redis(sandbox owner 表),chart 只修改**入口 IP 广播**。sandbox 存量数据不会自动改写,新建的 sandbox 会用新 advertiseIP。 + +如需强制刷新: + +```bash +kubectl -n cube-system rollout restart deployment/cube-proxy-node +``` + +--- + +## F. Egress 网络 + +### F1. sandbox 无法访问外网 + +egress sidecar 采用**默认拒绝**策略,只允许白名单域名。检查: + +```bash +kubectl -n cube-system logs -c cube-egress --tail=200 | grep -i deny +``` + +放开某个域名:调整 `cubeEgress` values 或运行时通过 CubeMaster API 更新白名单。生产环境建议按需最小化。 + +### F2. sandbox 出站被 MITM 后 TLS 报错 + +egress 使用自签 CA 做 MITM。sandbox 内的应用需要信任 chart 生成的 CA: + +```bash +# 从 Kubernetes 拿 CA 证书 +kubectl -n cube-system get secret cube-egress-ca -o jsonpath='{.data.ca\.crt}' | base64 -d > cube-ca.crt +# 注入 sandbox 模板中(如 /etc/ssl/certs/) +``` + +或者用 `cubeEgress.ca.mode: existingSecret` 复用企业 CA。 + +### F3. cube-egress-net sidecar 反复 CrashLoopBackOff + +`cube-egress-net` 依赖 `cube-dev` 网口(cube-node 主容器创建)。启动顺序问题: + +```bash +kubectl -n cube-system logs -c cube-egress-net --tail=100 +``` + +- 若卡在 "waiting for cube-dev" → 主容器还没起,通常自愈,若持续超过 5 分钟检查主容器日志 +- 若卡在 "TPROXY rule failed" → 节点上 iptables 版本过旧,需 iptables-nft;或与其他 CNI(如 Calico)规则冲突 + +--- + +## G. 升级、回滚与卸载 + +### G1. `helm upgrade` 后 cube-node DaemonSet 全部重启,会不会中断服务? + +会。默认策略是 `RollingUpdate`,节点上活跃的 sandbox 会被中断。生产环境推荐: + +```yaml +cubeNode: + updateStrategy: + type: OnDelete +``` + +然后手工按节点滚动: + +```bash +# 1. 把 node 上的 sandbox pause/迁移 +# 2. 删除 cube-node Pod,DaemonSet 会拉起新版本 +kubectl -n cube-system delete pod -l app.kubernetes.io/component=cube-node --field-selector spec.nodeName= +# 3. 等新 Pod Ready 后继续下一个节点 +``` + +### G2. `helm rollback` 会回滚 host kernel 吗? + +**不会**。Helm 只管 K8s 资源。Host kernel、GRUB、`/etc/fstab`、XFS 挂载点等由 `pvm-host-bootstrap` 做的节点级修改,rollback **完全不动**。生产环境请提前准备 host kernel 回滚 runbook(RPM 降级、切换 GRUB 默认项、reboot 验证)。 + +### G3. `helm uninstall` 后节点上 sandbox 数据还在 + +设计如此。卸载**只删 K8s 资源**,以下需人工清理: + +```bash +# 在每个 compute 节点执行 +sudo rm -rf /data/cubelet /data/cube-shim /data/snapshot_pack /data/log /tmp/cube +# 如果不再需要 PVM host kernel,还需要: +# 1. 卸载 kernel 包 +# 2. 更新 GRUB 默认引导 +# 3. reboot +``` + +CubeMaster 的 PVC(默认 CBS 盘)按 K8s reclaimPolicy 处理,default `Delete`。若 policy=`Retain`,PVC 会保留。 + +--- + +## H. 镜像构建 + +### H1. `build-cube-images.sh` 下载 tarball 卡住 / 失败 + +- 检查网络:`curl -I https://downloads.sourceforge.net/` +- 手动预下载并跳过脚本下载: + ```bash + mkdir -p /tmp/cube-kubernetes-images-v0.5.0/downloads + cd /tmp/cube-kubernetes-images-v0.5.0/downloads + # 从 SourceForge 手动下载 3 个文件 + # 然后重新运行 build-cube-images.sh,会检测到本地文件跳过下载 + ``` + +### H2. 构建 arm64 镜像 + +```bash +ONE_CLICK_ARCH=arm64 \ +PUSH=1 REGISTRY= IMAGE_TAG=v0.5.0-arm64 \ +./deploy/kubernetes/images/build-cube-images.sh +``` + +需要 arm64 build machine 或者 buildx multi-arch。 + +### H3. 想只重构某个镜像(如 cube-api),不重跑整个流水线 + +脚本目前无子命令拆分,但可以设置环境变量跳过其他步骤,或复用中间产物: + +- `PACKAGE_DIR_OVERRIDE=/tmp/cube-kubernetes-images-v0.5.0/sandbox-package` → 跳过下载 / 解压 +- 手工 `docker build` 到需要的镜像 + +后续版本考虑加子命令。 + +### H4. 老版本 curl (`<7.71`) 报 `option --retry-all-errors: is unknown` + +已在本 PR 修复。脚本会在运行时探测 curl 是否支持该 flag,不支持则自动跳过。若你自己 fork 了老版本脚本,直接把 `--retry-all-errors` 那行删了就行。 + +--- + +## 提问模板 + +如果以上都没覆盖你的问题,提 issue 时请附上: + +```text +Chart 版本: v0.5.0 +K8s 版本: (kubectl version --short 输出) +运行环境: (TKE / 自建 / 单节点 / 多可用区) +values.yaml 关键片段(隐去密码) +Failing 组件: (Pod 名 + `kubectl describe` + `kubectl logs`) +``` diff --git a/deploy/kubernetes/chart/docs/QUICKSTART.md b/deploy/kubernetes/chart/docs/QUICKSTART.md new file mode 100644 index 000000000..bc63572e6 --- /dev/null +++ b/deploy/kubernetes/chart/docs/QUICKSTART.md @@ -0,0 +1,235 @@ +# 一键安装指南 (Quick Start) + +本文档描述如何在 30 分钟内,使用 Helm 把 CubeSandbox v0.5.0 完整部署到一个 Kubernetes/TKE 集群。适合首次接触 chart 的交付人员、SRE 或用户。 + +如果你需要理解组件关系、启动流程、DNS/Proxy/Egress 数据流,请阅读 [`ARCHITECTURE.md`](ARCHITECTURE.md);运行中遇到问题请查阅 [`FAQ.md`](FAQ.md)。 + +--- + +## 1. 前置条件 + +### 1.1 集群要求 + +| 项目 | 最低要求 | 说明 | +| --- | --- | --- | +| Kubernetes / TKE | v1.24+ | 建议使用 TKE 托管集群或 TKE Serverless(节点池模式) | +| kubectl | 与集群同大版本 | 确保 `kubectl get nodes` 能正常返回 | +| Helm | v3.10+ | `helm version` 应可返回 | +| Docker(仅打包镜像时) | v20+ | 生产集群只需能 pull 已推送的镜像 | +| 集群 StorageClass | 至少 1 个 | 默认使用 `cube-cbs-wffc`(chart 自动创建),可自定义 | + +### 1.2 节点要求 + +CubeSandbox 采用**控制面 / 计算面**分离部署,两组节点在 K8s 中通过 label 区分。 + +| 节点角色 | 数量 | 规格建议 | 需要具备 | +| --- | --- | --- | --- | +| Control 节点 | 至少 1 台(生产 3+) | 4C8G + 100G CBS | 常规 K8s 节点 | +| Compute 节点 | 至少 1 台 | 16C32G+ | **KVM 支持**(`/dev/kvm` 存在)、XFS 数据盘、支持内核替换(可选 PVM host kernel) | + +### 1.3 打节点 label / taint + +Chart **不会自动打 label**,请部署前手动执行: + +```bash +# 控制面节点 +kubectl label nodes cube.tencent.com/role=control cube.tencent.com/cube-control=true +kubectl taint nodes cube.tencent.com/control=true:NoSchedule --overwrite + +# 计算节点 +kubectl label nodes \ + cube.tencent.com/role=compute \ + cube.tencent.com/cube-node=true \ + cube.tencent.com/allow-pvm-bootstrap=true +kubectl taint nodes cube.tencent.com/compute=true:NoSchedule --overwrite +``` + +`allow-pvm-bootstrap=true` 显式授权该节点可以被 chart 替换 host kernel;不打这个 label 就无法作为 compute 节点部署。 + +--- + +## 2. 准备镜像(可跳过) + +若使用官方 v0.5.0 镜像,跳过本章,直接进入"安装"。若需自建镜像,一条命令构建 + 推送: + +```bash +docker login ccr.ccs.tencentyun.com # 或你的私有 registry + +PUSH=1 \ +REGISTRY=ccr.ccs.tencentyun.com/ \ +IMAGE_TAG=v0.5.0 \ +./deploy/kubernetes/images/build-cube-images.sh +``` + +脚本会自动: + +1. 从 SourceForge 下载 `cube-sandbox-one-click-v0.5.0-amd64.tar.gz`(约 245MB)+ PVM host kernel RPM/DEB +2. 基于 `CubeMaster/CubeAPI/CubeProxy/CubeEgress` 源代码构建 10 个镜像 +3. 推送到 `${REGISTRY}` 下,tag 为 `${IMAGE_TAG}` + +**arm64 构建**:`ONE_CLICK_ARCH=arm64` 触发下载 arm64 一体化包。 + +**离线场景**:预先下载 tarball / kernel 包放到 `${BUILD_ROOT}/downloads/` 下,脚本会自动跳过下载步骤。详见 [`../images/README.md`](../images/README.md)。 + +--- + +## 3. 准备 values.yaml + +创建 `runtime-values.yaml`(下面是**最小可用**示例,生产环境请按 [`ARCHITECTURE.md#7`](ARCHITECTURE.md#7-关键-values-开关) 逐项调整): + +```yaml +# 镜像 registry(若使用了自建镜像,把 repository 换成你的地址) +images: + master: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-master + tag: v0.5.0 + # ... 其他 9 个镜像同样格式 + +# CubeProxy 对外访问 IP(如果不用 LB,填 control 节点的 HostIP) +cubeProxy: + advertiseIP: "10.0.1.10" + domain: "cube.app" + tls: + mode: selfSigned # 试用;生产建议 existingSecret 或 certManager + +# 若使用外部 MySQL / Redis,填写这两项;留空则 chart 自动部署内置的 StatefulSet +mysql: + host: "" + # host: "mysql.example.com" + # port: 3306 + # user: cube_user + # database: cube +redis: + host: "" +``` + +--- + +## 4. 一键安装 + +```bash +helm upgrade --install cube ./deploy/kubernetes/chart \ + -n cube-system \ + --create-namespace \ + -f runtime-values.yaml \ + --wait \ + --timeout 90m +``` + +**关键参数说明**: + +- `--wait` 会等到所有 Deployment / DaemonSet / StatefulSet ready 才返回 +- `--timeout 90m` 给足 host kernel 安装 + 节点重启 + microVM 首次冷启动的时间 +- 若 host kernel 需要重启节点,DaemonSet 会自动重试,不需要人工介入 + +安装过程中的关键节点: + +1. **秒级**:控制面 Secret / ConfigMap / RBAC 创建 +2. **1-3 分钟**:MySQL / Redis StatefulSet ready,CubeMaster 完成 schema migration +3. **1-2 分钟**:CubeAPI / CubeProxy / WebUI ready +4. **首次 5-15 分钟**:cube-node DaemonSet 在每个 compute 节点上依次执行 `pvm-host-bootstrap`(可能触发重启)→ `cube-node-init` → 主容器启动 → 节点向 CubeMaster 注册 +5. **之后**:后续升级/滚动更新只需秒级 + +--- + +## 5. 验证部署 + +```bash +# 1. 所有 Pod 都 Ready +kubectl get pods -n cube-system -o wide + +# 2. compute 节点已注册到 CubeMaster +kubectl exec -n cube-system deploy/cube-cubemastercli -- \ + sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" node list' + +# 3. 运行 Helm 内置端到端测试(约 5 分钟) +helm test cube -n cube-system --timeout 20m --logs +``` + +**期望结果**: + +- `cube-node` DaemonSet Ready 数量 = 打了 compute label 的节点数量 +- `cube-master` / `cube-api` / `cube-proxy-node` / `cube-webui` Deployment Ready +- `cube-mysql` / `cube-redis` StatefulSet Ready(若使用内置) +- `helm test` 全绿 + +登录 WebUI(默认在 `http://:12088`)开始使用。 + +--- + +## 6. 常见部署模式 + +### 6.1 单节点试用(仅测试) + +一台机器既做 control 又做 compute,给这台节点打上两套 label,并去掉 taint: + +```bash +kubectl label nodes \ + cube.tencent.com/role=control \ + cube.tencent.com/cube-control=true \ + cube.tencent.com/cube-node=true \ + cube.tencent.com/allow-pvm-bootstrap=true +# 不要打 taint +``` + +`runtime-values.yaml` 中把持久化改成 hostPath 以避免 CBS 依赖: + +```yaml +controlPlane: + master: + persistence: + enabled: true + hostPath: /data/CubeMaster/storage +mysql: + persistence: + hostPath: /data/mysql +redis: + persistence: + hostPath: /data/redis +``` + +### 6.2 Compute-only 模式(共用外部控制面) + +多个集群共用同一套外部 CubeMaster: + +```yaml +controlPlane: + enabled: false +externalControlPlane: + enabled: true + masterEndpoint: :8089 + apiEndpoint: http://:3000 # 可选 +cubeProxy: + enabled: false # 通常由外部集群提供 +webui: + enabled: false +mysql: + enabled: false +redis: + enabled: false +``` + +--- + +## 7. 卸载 + +```bash +helm uninstall cube -n cube-system +kubectl delete namespace cube-system +``` + +**卸载不会自动清理**以下内容(需要手动处理): + +- 节点 label / taint(chart 不管理) +- 外部 MySQL / Redis 数据 +- compute 节点上的 hostPath 数据:`/data/CubeMaster/storage`, `/data/cubelet`, `/data/cube-shim`, `/data/snapshot_pack`, `/data/log` +- PVM host kernel 修改(GRUB、`/boot`、initramfs)—— 需要按平台 runbook 回滚 +- 外部 DNS / LB 记录 + +--- + +## 8. 下一步 + +- 阅读 [`ARCHITECTURE.md`](ARCHITECTURE.md) 深入理解组件关系和数据流 +- 阅读 [`FAQ.md`](FAQ.md) 应对常见部署 / 运行问题 +- 生产环境 TLS、DNS、监控、备份策略请参考主 README 相应章节 From 197abe7baeb8dc7f4647f5d45c6d159412f74133 Mon Sep 17 00:00:00 2001 From: vimiix Date: Tue, 7 Jul 2026 19:41:29 +0800 Subject: [PATCH 4/9] fix(kubernetes): pin cube-api/proxy/egress source to release tag build-cube-images.sh compiled cube-api, cube-proxy-node, cube-egress, and cube-egress-net directly from the current worktree's CubeMaster/, CubeAPI/, CubeProxy/, and CubeEgress/ trees. When the worktree was ahead of the release tag (e.g. building from the chart delivery branch that carries additional CubeAPI features), the resulting images drifted from the tag advertised in Chart.yaml / values.yaml. In the v0.5.0 default build this manifested as cube-api requiring an examples/ directory that only exists in commits merged after the v0.5.0 tag, causing a startup FATAL when the same images were deployed via this chart. Introduce SOURCE_REF (default: ${VERSION}, i.e. v0.5.0) which exports the four source trees at that git ref into ${BUILD_ROOT}/source-tree/ via `git archive` and points REPO_ROOT there for the duration of the build. This keeps the delivered images aligned with the release tag independent of the current chart branch. - SOURCE_REF="" disables pinning and falls back to the current worktree (useful for local development against unreleased changes). - SOURCE_REF= targets any branch, tag, or commit SHA. - The exported tree is cached; a stamp file avoids re-exporting when the ref has not moved. Also document the new knob and its intent in deploy/kubernetes/images/README.md. --- deploy/kubernetes/images/README.md | 26 ++++++++++++++ deploy/kubernetes/images/build-cube-images.sh | 35 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/deploy/kubernetes/images/README.md b/deploy/kubernetes/images/README.md index b3deae9c2..fbc400b67 100644 --- a/deploy/kubernetes/images/README.md +++ b/deploy/kubernetes/images/README.md @@ -23,6 +23,32 @@ want a different cache location. The script reuses valid artifacts already present under `${BUILD_ROOT}/downloads` and does not require a `.complete` marker. +## Pinning source to a release tag + +`cube-api`, `cube-proxy-node`, and `cube-egress` are compiled from repository +source (rather than binaries in the release tarball). By default the script +pins those source trees to `${SOURCE_REF}` (defaulting to `${VERSION}`, so +`v0.5.0` for the default build). It exports `CubeMaster/`, `CubeAPI/`, +`CubeProxy/`, and `CubeEgress/` at that git ref into +`${BUILD_ROOT}/source-tree/` via `git archive` and points `REPO_ROOT` there for +the duration of the build. This guarantees the images match the release tag +even when the current worktree is ahead of it. + +To build from the current worktree instead (typically for development), set +`SOURCE_REF=""`: + +```bash +SOURCE_REF="" PUSH=1 REGISTRY=<...> IMAGE_TAG=v0.5.0-dev \ + ./deploy/kubernetes/images/build-cube-images.sh +``` + +To build from a different ref (branch, tag, or commit SHA): + +```bash +SOURCE_REF=some-feature-branch PUSH=1 REGISTRY=<...> IMAGE_TAG=v0.5.0-featureX \ + ./deploy/kubernetes/images/build-cube-images.sh +``` + When the release package is older than the verified Kubernetes node runtime, build `cube-node` by rebasing a known-good node image and copying the current entrypoint into it: diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh index e954c6aa0..4617d83b1 100755 --- a/deploy/kubernetes/images/build-cube-images.sh +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -13,6 +13,11 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" VERSION="${VERSION:-v0.5.0}" IMAGE_TAG="${IMAGE_TAG:-${VERSION}}" REGISTRY="${REGISTRY:-ccr.ccs.tencentyun.com/cubesandbox-chart}" +# SOURCE_REF pins the CubeMaster / CubeAPI / CubeProxy / CubeEgress source tree +# used when building cube-api / cube-proxy-node / cube-egress from repository +# source, ensuring the delivered images match the release tag rather than the +# current worktree. Set SOURCE_REF="" to build from the current worktree. +SOURCE_REF="${SOURCE_REF-${VERSION}}" PUSH="${PUSH:-0}" NO_CACHE="${NO_CACHE:-0}" BUILD_ROOT="${BUILD_ROOT:-/tmp/cube-kubernetes-images-${VERSION}}" @@ -105,10 +110,14 @@ need docker need tar need curl need go +if [[ -n "${SOURCE_REF}" ]]; then + need git +fi DOWNLOAD_DIR="${BUILD_ROOT}/downloads" EXTRACT_DIR="${BUILD_ROOT}/extract" CONTEXT_DIR="${BUILD_ROOT}/contexts" +SOURCE_TREE_DIR="${BUILD_ROOT}/source-tree" ONE_CLICK_DIRNAME="cube-sandbox-one-click-${VERSION}-${ONE_CLICK_ARCH}" ONE_CLICK_TAR="${DOWNLOAD_DIR}/${ONE_CLICK_DIRNAME}.tar.gz" PVM_KERNEL_RPM="${DOWNLOAD_DIR}/$(basename "${PVM_KERNEL_RPM_URL}")" @@ -118,6 +127,32 @@ PACKAGE_DIR="${BUILD_ROOT}/sandbox-package" mkdir -p "${DOWNLOAD_DIR}" "${EXTRACT_DIR}" "${CONTEXT_DIR}" +# When SOURCE_REF is set (default: ${VERSION}), export the CubeMaster / CubeAPI / +# CubeProxy / CubeEgress source trees at that ref into ${SOURCE_TREE_DIR} and +# point REPO_ROOT there. This ensures cube-api, cube-proxy-node, cube-egress and +# related contexts are compiled from the release-tag source, not from whatever +# happens to be in the current worktree (which may be ahead of the tag). +if [[ -n "${SOURCE_REF}" ]]; then + git -C "${REPO_ROOT}" rev-parse --verify "${SOURCE_REF}^{commit}" >/dev/null 2>&1 \ + || fail "SOURCE_REF=${SOURCE_REF} is not a valid git ref in ${REPO_ROOT}" + SOURCE_REF_SHA="$(git -C "${REPO_ROOT}" rev-parse "${SOURCE_REF}^{commit}")" + SOURCE_TREE_STAMP="${SOURCE_TREE_DIR}/.exported-sha" + if [[ ! -f "${SOURCE_TREE_STAMP}" ]] || [[ "$(cat "${SOURCE_TREE_STAMP}")" != "${SOURCE_REF_SHA}" ]]; then + log "exporting source tree at ${SOURCE_REF} (${SOURCE_REF_SHA:0:12}) into ${SOURCE_TREE_DIR}" + rm -rf "${SOURCE_TREE_DIR}" + mkdir -p "${SOURCE_TREE_DIR}" + for module in CubeMaster CubeAPI CubeProxy CubeEgress; do + git -C "${REPO_ROOT}" archive --format=tar "${SOURCE_REF_SHA}" -- "${module}" \ + | tar -C "${SOURCE_TREE_DIR}" -x \ + || fail "failed to export ${module} at ${SOURCE_REF}" + done + printf '%s\n' "${SOURCE_REF_SHA}" > "${SOURCE_TREE_STAMP}" + else + log "reusing exported source tree at ${SOURCE_REF} (${SOURCE_REF_SHA:0:12})" + fi + REPO_ROOT="${SOURCE_TREE_DIR}" +fi + if [[ -n "${PACKAGE_DIR_OVERRIDE:-}" ]]; then PACKAGE_DIR="${PACKAGE_DIR_OVERRIDE}" [[ -d "${PACKAGE_DIR}" ]] || fail "PACKAGE_DIR_OVERRIDE does not exist: ${PACKAGE_DIR}" From 829cea064cb23ed8c58104e078cbba9767661cea Mon Sep 17 00:00:00 2001 From: vimiix Date: Thu, 9 Jul 2026 17:53:55 +0800 Subject: [PATCH 5/9] fix(kubernetes): address v0.5.0 chart review round-2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the remaining review comments on PR #779 (reliability, security, correctness, and code quality). Ranged from a one-line helper introduction to a full script extraction — all preserving default behavior when operators do not opt in to the new knobs. Chart templates: - _helpers.tpl: new `cube.clusterDomain` helper honoring global.clusterDomain > cubeNode.dns.clusterDomain > cluster.local, and `cube.apiBindPort` derived from controlPlane.api.bind. All in- cluster Service FQDNs (cube.masterEndpoint / cube.apiEndpoint / cube.mysqlHost / cube.redisHost) now build the DNS suffix through the helper so clusters with a non-default kubelet cluster-domain resolve correctly. - api.yaml: containerPort is no longer hardcoded to 3000; it is derived from controlPlane.api.bind via cube.apiBindPort. - webui.yaml + tests/node-health.yaml: replace remaining `svc.cluster.local` literals with cube.clusterDomain. - diagnostics.yaml: rollout status commands use cube.nodeName / cube.proxyName / cube.mysqlName / cube.redisName helpers instead of `${RELEASE}-node` string concatenation, so the diagnostic script keeps working when nameOverride / fullnameOverride is set. - dns.yaml: add `errors` plugin, gate `log` behind cubeDns.logQueries (default off), align cache TTL with the 60s TTL emitted by the template plugin (previously 300s, hiding IP changes for up to five minutes). - validate.yaml: reject the CHANGE_ME_* password sentinels when the built-in MySQL / Redis is enabled so a chart install cannot succeed with the trivially guessable defaults. - node-daemonset.yaml: each cube-node probe (startup / readiness / liveness) now supports an `exec:` override in addition to the TCP socket check so operators can detect deadlocked processes that keep port 9999 open. Default resource requests/limits are shipped for the cube-node container so the DaemonSet does not enter BestEffort QoS on a busy compute node. - proxy-node.yaml: the 50-line inline shell that generated global.conf is extracted into images/scripts/cube-proxy-node-entrypoint.sh, delivered by the chart as a ConfigMap and mounted into the Pod at /etc/cube-proxy-node. The script can now be linted / tested / diff- reviewed as a regular source file. - tests/node-health.yaml: CubeEgress test extracts container names from the pods JSON before matching so cube-egress does not match labels or annotations that happen to contain that string; the CubeProxy test now issues an actual readiness curl on the in- cluster Service endpoint instead of only printing a message; the service-mode DNS test gains a compute placement fallback and uses cube.clusterDomain everywhere; the node-image-test no longer pulls the multi-GB cube-node runtime image just to run `test -f`, it now reuses the live DaemonSet Pod via a curl-based readiness check (gated by helmTest.nodeImage.enabled). values.yaml: - global.clusterDomain: "" — new knob; empty falls back to cubeNode.dns.clusterDomain and then "cluster.local". - mysql.password / mysql.rootPassword / redis.password: switch the in-repo defaults from cube_pass / cube_root / ceuhvu123 to CHANGE_ME_* sentinels; validate.yaml refuses to render with them. - bootstrap.pvmHostKernel.bootArgs: keeps the `nopti pti=off` default (kvm_pvm requires KPTI off) but adds a security trade-off explanation so operators consciously accept it. - cubeNode.resources: sensible baseline requests/limits so a busy cube-node is never evicted before the sidecars. - cubeNode.probes.*.exec: [] — operator-supplied application-level probe overrides that override the TCP socket check. - cubeDns.cacheTTL / cubeDns.logQueries: matched to the 60s A-record TTL and default-off log plugin respectively. - cubeEgress.resources / cubeEgress.netResources: default requests/ limits for the two egress sidecars. Image scripts / Dockerfiles: - images/build-cube-images.sh: `download_file` now accepts an optional expected SHA256; three new env vars ONE_CLICK_SHA256, PVM_KERNEL_RPM_SHA256, PVM_KERNEL_DEB_SHA256 activate the check. The `--retry-all-errors` curl flag moves to opt-in via CURL_RETRY_ALL_ERRORS=1 so a 404 no longer costs 9 retries. Adds a `need sha256sum` guard. - images/scripts/pvm-host-bootstrap.sh: introduce `host_run` for argv-style execution without `sh -c`, and shell-quote DESIRED_KERNEL_PATTERN before splicing it into the host shell to close the command injection vector reported by the review. The post-reboot `sleep 3600` shrinks to `${REBOOT_WAIT_SECONDS:-120}` so an init container failure surfaces to the DaemonSet in minutes instead of an hour. - images/scripts/cube-node-entrypoint.sh: escape sed replacement input via `sed_escape_replacement` and reject non-integer values for CUBE_TAP_INIT_NUM / CUBE_CGROUP_POOL_SIZE / CUBE_WORKFLOW_ CONCURRENT so operator-supplied env vars cannot inject configuration content. The hardcoded cube-master.cube-system endpoint fallback is replaced with a fail-fast check because a wrong-namespace default silently targets the wrong control plane. - images/scripts/cube-node-init.sh: document the security model of `host_chroot_sh` / `host_mount_sh` in a block comment so operators and reviewers understand that these helpers are a deliberate, architecturally required container escape and treat the images accordingly. - images/{cube-node,cube-node-init,cubemastercli,cube-egress-net, cube-pvm-host-bootstrap}/Dockerfile: bump the ubuntu base from 20.04 (EOL April 2025) to 22.04. - images/scripts/cube-proxy-node-entrypoint.sh: new file; the extracted proxy entrypoint referenced above. - chart/files/cube-proxy-node/: mirror copy consumed by the ConfigMap template so `helm template` can inline the script content. Verified: - helm lint passes. - helm template with representative overrides renders 34 resources. - helm template with global.clusterDomain=example.internal makes every in-cluster Service FQDN carry the override suffix; the default render keeps `cluster.local`. - helm template with controlPlane.api.bind=0.0.0.0:8888 changes the cube-api Pod containerPort to 8888 while the Service port stays configurable via controlPlane.api.service.port. - helm template with default mysql/redis passwords fails validate.yaml with an actionable CHANGE_ME_* message. - bash -n on the modified scripts passes. --- .../cube-proxy-node-entrypoint.sh | 73 ++++++++++++ .../kubernetes/chart/templates/_helpers.tpl | 31 ++++- deploy/kubernetes/chart/templates/api.yaml | 2 +- .../chart/templates/diagnostics.yaml | 10 +- deploy/kubernetes/chart/templates/dns.yaml | 5 +- .../chart/templates/node-daemonset.yaml | 19 +++- .../chart/templates/proxy-node.yaml | 68 ++++------- .../chart/templates/tests/node-health.yaml | 106 +++++++++++++----- .../kubernetes/chart/templates/validate.yaml | 6 + deploy/kubernetes/chart/templates/webui.yaml | 2 +- deploy/kubernetes/chart/values.yaml | 71 +++++++++++- deploy/kubernetes/images/build-cube-images.sh | 42 +++++-- .../images/cube-egress-net/Dockerfile | 2 +- .../images/cube-node-init/Dockerfile | 2 +- deploy/kubernetes/images/cube-node/Dockerfile | 2 +- .../images/cube-pvm-host-bootstrap/Dockerfile | 2 +- .../images/cubemastercli/Dockerfile | 2 +- .../images/scripts/cube-node-entrypoint.sh | 30 ++++- .../images/scripts/cube-node-init.sh | 17 +++ .../scripts/cube-proxy-node-entrypoint.sh | 73 ++++++++++++ .../images/scripts/pvm-host-bootstrap.sh | 32 +++++- 21 files changed, 475 insertions(+), 122 deletions(-) create mode 100755 deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh create mode 100755 deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh diff --git a/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh b/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh new file mode 100755 index 000000000..3b5b1a1aa --- /dev/null +++ b/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Cube Proxy Node entrypoint script. +# +# Extracted from the chart-embedded shell in templates/proxy-node.yaml so +# that it can be linted (`shellcheck`), unit-tested, and reviewed as a +# proper source file rather than an inline command list. +# +# Contract: +# Reads the following env vars (populated by the Chart in the Pod spec): +# CUBE_PROXY_HTTP_LISTEN_PORT - digits only, target HTTP listen port +# CUBE_PROXY_HTTPS_LISTEN_PORT - digits only, target HTTPS listen port +# CUBE_PROXY_RESOLVER_ADDRS - space-separated nameservers; empty means +# read from /etc/resolv.conf +# CUBE_PROXY_RESOLVER_VALID - nginx `valid=` duration +# CUBE_PROXY_RESOLVER_TIMEOUT - nginx `resolver_timeout` value +# CUBE_PROXY_RESOLVER_IPV6 - on/off +# REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB +# TIMEOUT_MIN, TIMEOUT_MAX +# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR +# Rewrites nginx.conf listen ports in-place, then generates +# /usr/local/openresty/nginx/conf/global/global.conf and execs start.sh. + +set -eu + +mkdir -p /usr/local/openresty/nginx/conf/global /data /data/log/cube-proxy /cache + +case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in + *[!0-9:]*|:*|*:) + echo "invalid CubeProxy listen ports: http=${CUBE_PROXY_HTTP_LISTEN_PORT} https=${CUBE_PROXY_HTTPS_LISTEN_PORT}" >&2 + exit 1 + ;; +esac + +sed -i \ + -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ + -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ + -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ + -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ + /usr/local/openresty/nginx/conf/nginx.conf + +escape_nginx_value() { + printf '%s' "$1" | sed 's/[\\"]/\\&/g' +} + +resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" +if [ -z "${resolver_addrs}" ]; then + resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" +fi +[ -n "${resolver_addrs}" ] || { + echo "unable to determine nginx DNS resolver for CubeProxy Redis lookups" >&2 + exit 1 +} +case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT}${CUBE_PROXY_RESOLVER_IPV6}" in + *[\;\{\}\$\`]*) + echo "invalid CubeProxy resolver configuration" >&2 + exit 1 + ;; +esac + +cat > /usr/local/openresty/nginx/conf/global/global.conf < .Values.cubeNode.dns.clusterDomain > cluster.local. +Set global.clusterDomain when the cluster is configured with kubelet +--cluster-domain=. +*/}} +{{- define "cube.clusterDomain" -}} +{{- $global := (default (dict) .Values.global).clusterDomain -}} +{{- $cubeNode := (default (dict) (default (dict) .Values.cubeNode).dns).clusterDomain -}} +{{- default (default "cluster.local" $cubeNode) $global -}} +{{- end -}} + +{{/* +Port that CubeAPI binds on, extracted from controlPlane.api.bind (default +"0.0.0.0:3000"). Used for both containerPort and probes so operators can +change bind without editing multiple places. +*/}} +{{- define "cube.apiBindPort" -}} +{{- $bind := default "0.0.0.0:3000" .Values.controlPlane.api.bind -}} +{{- $port := regexFind "[0-9]+$" $bind -}} +{{- default "3000" $port -}} +{{- end -}} diff --git a/deploy/kubernetes/chart/templates/api.yaml b/deploy/kubernetes/chart/templates/api.yaml index 157070e82..e6c0e3311 100644 --- a/deploy/kubernetes/chart/templates/api.yaml +++ b/deploy/kubernetes/chart/templates/api.yaml @@ -63,7 +63,7 @@ spec: {{- end }} ports: - name: http-api - containerPort: 3000 + containerPort: {{ include "cube.apiBindPort" . | int }} {{- if .Values.controlPlane.api.probes.readiness.enabled }} readinessProbe: httpGet: diff --git a/deploy/kubernetes/chart/templates/diagnostics.yaml b/deploy/kubernetes/chart/templates/diagnostics.yaml index a6880835c..2ab1291a2 100644 --- a/deploy/kubernetes/chart/templates/diagnostics.yaml +++ b/deploy/kubernetes/chart/templates/diagnostics.yaml @@ -50,12 +50,12 @@ data: done done - for ds in cube-node; do - run "rollout-${ds}" kubectl rollout status "daemonset/${RELEASE}-${ds#cube-}" -n "${NS}" --timeout=30s + for ds in {{ include "cube.nodeName" . | quote }}; do + run "rollout-${ds}" kubectl rollout status "daemonset/${ds}" -n "${NS}" --timeout=30s done - run rollout-proxy-node kubectl rollout status "deployment/${RELEASE}-proxy-node" -n "${NS}" --timeout=30s - run rollout-mysql kubectl rollout status "statefulset/${RELEASE}-mysql" -n "${NS}" --timeout=30s - run rollout-redis kubectl rollout status "statefulset/${RELEASE}-redis" -n "${NS}" --timeout=30s + run rollout-proxy-node kubectl rollout status "deployment/{{ include "cube.proxyName" . }}" -n "${NS}" --timeout=30s + run rollout-mysql kubectl rollout status "statefulset/{{ include "cube.mysqlName" . }}" -n "${NS}" --timeout=30s + run rollout-redis kubectl rollout status "statefulset/{{ include "cube.redisName" . }}" -n "${NS}" --timeout=30s echo "diagnostics written to ${OUT}" {{- end }} diff --git a/deploy/kubernetes/chart/templates/dns.yaml b/deploy/kubernetes/chart/templates/dns.yaml index fcf824800..a3b9ff0aa 100644 --- a/deploy/kubernetes/chart/templates/dns.yaml +++ b/deploy/kubernetes/chart/templates/dns.yaml @@ -40,8 +40,11 @@ data: template IN AAAA (.*)\.{{ $domainRegex }} { rcode NOERROR } + errors + {{- if .Values.cubeDns.logQueries }} log - cache 300 + {{- end }} + cache {{ .Values.cubeDns.cacheTTL | default 60 }} forward . {{ $forward }} } {{- if eq .Values.cubeDns.mode "nodeLocal" }} diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml index 2865342c2..b17fbf929 100644 --- a/deploy/kubernetes/chart/templates/node-daemonset.yaml +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -265,23 +265,38 @@ spec: containerPort: 19090 {{- if .Values.cubeNode.probes.startup.enabled }} startupProbe: + {{- if .Values.cubeNode.probes.startup.exec }} + exec: + {{- toYaml .Values.cubeNode.probes.startup.exec | nindent 14 }} + {{- else }} tcpSocket: port: {{ .Values.cubeNode.probes.startup.tcpSocketPort }} + {{- end }} periodSeconds: {{ .Values.cubeNode.probes.startup.periodSeconds }} failureThreshold: {{ .Values.cubeNode.probes.startup.failureThreshold }} {{- end }} {{- if .Values.cubeNode.probes.readiness.enabled }} readinessProbe: + {{- if .Values.cubeNode.probes.readiness.exec }} + exec: + {{- toYaml .Values.cubeNode.probes.readiness.exec | nindent 14 }} + {{- else }} tcpSocket: port: {{ .Values.cubeNode.probes.readiness.tcpSocketPort }} + {{- end }} initialDelaySeconds: {{ .Values.cubeNode.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.cubeNode.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.cubeNode.probes.readiness.failureThreshold }} {{- end }} {{- if .Values.cubeNode.probes.liveness.enabled }} livenessProbe: + {{- if .Values.cubeNode.probes.liveness.exec }} + exec: + {{- toYaml .Values.cubeNode.probes.liveness.exec | nindent 14 }} + {{- else }} tcpSocket: port: {{ .Values.cubeNode.probes.liveness.tcpSocketPort }} + {{- end }} initialDelaySeconds: {{ .Values.cubeNode.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.cubeNode.probes.liveness.periodSeconds }} failureThreshold: {{ .Values.cubeNode.probes.liveness.failureThreshold }} @@ -307,10 +322,8 @@ spec: - name: tmp-cube mountPath: {{ .Values.hostPaths.tmpCube }} mountPropagation: Bidirectional - {{- with .Values.cubeNode.resources }} resources: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- toYaml .Values.cubeNode.resources | nindent 12 }} {{- if .Values.cubeEgress.enabled }} - name: cube-egress image: {{ include "cube.image" .Values.images.cubeEgress | quote }} diff --git a/deploy/kubernetes/chart/templates/proxy-node.yaml b/deploy/kubernetes/chart/templates/proxy-node.yaml index 872dff29b..1840c15e4 100644 --- a/deploy/kubernetes/chart/templates/proxy-node.yaml +++ b/deploy/kubernetes/chart/templates/proxy-node.yaml @@ -138,55 +138,7 @@ spec: - /bin/sh - -ec - | - set -u - mkdir -p /usr/local/openresty/nginx/conf/global /data /data/log/cube-proxy /cache - - case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in - *[!0-9:]*|:*|*:) - echo "invalid CubeProxy listen ports: http=${CUBE_PROXY_HTTP_LISTEN_PORT} https=${CUBE_PROXY_HTTPS_LISTEN_PORT}" >&2 - exit 1 - ;; - esac - sed -i \ - -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ - -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ - -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ - -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ - /usr/local/openresty/nginx/conf/nginx.conf - - escape_nginx_value() { - printf '%s' "$1" | sed 's/[\\"]/\\&/g' - } - - resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" - if [ -z "${resolver_addrs}" ]; then - resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" - fi - [ -n "${resolver_addrs}" ] || { - echo "unable to determine nginx DNS resolver for CubeProxy Redis lookups" >&2 - exit 1 - } - case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT}${CUBE_PROXY_RESOLVER_IPV6}" in - *[\;\{\}\$\`]*) - echo "invalid CubeProxy resolver configuration" >&2 - exit 1 - ;; - esac - - cat > /usr/local/openresty/nginx/conf/global/global.conf </dev/null - curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.webuiName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.webui.service.port }}/cubeapi/v1/health + curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.webuiName" . }}.{{ .Release.Namespace }}.svc.{{ include "cube.clusterDomain" . }}:{{ .Values.webui.service.port }}/ >/dev/null + curl --connect-timeout 5 --max-time 15 -fsS http://{{ include "cube.webuiName" . }}.{{ .Release.Namespace }}.svc.{{ include "cube.clusterDomain" . }}:{{ .Values.webui.service.port }}/cubeapi/v1/health {{- end }} {{- if eq (include "cube.proxyEnabled" .) "true" }} - echo '[health-test] check CubeProxy through control-node test pod' + echo '[health-test] check CubeProxy service via readiness endpoint through Kubernetes API' + proxy_pods_json="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io/component%3Dproxy-node")" + # At least one CubeProxy Pod exists and reports Ready + printf '%s\n' "${proxy_pods_json}" \ + | grep -oE '"ready"[[:space:]]*:[[:space:]]*true' | grep -q true + # The Cube Proxy readiness endpoint should return an HTTP 200 through + # in-cluster Service DNS. Skipped when only a hostNetwork endpoint is + # exposed and the test pod cannot resolve it. + curl --connect-timeout 5 --max-time 15 -fsS \ + "http://{{ include "cube.proxyName" . }}.{{ .Release.Namespace }}.svc.{{ include "cube.clusterDomain" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/-/ready" >/dev/null 2>&1 \ + || echo '[health-test] cube-proxy in-cluster readiness probe unreachable (hostNetwork exposure), skipped' {{- end }} {{- if .Values.cubeNode.enabled }} @@ -87,10 +97,17 @@ spec: {{- end }} {{- if .Values.cubeEgress.enabled }} - echo '[health-test] check CubeEgress containers are present in CubeNode pods' + echo '[health-test] check CubeEgress containers by name in CubeNode pods' pods_json="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io/component%3Dcube-node")" - printf '%s\n' "${pods_json}" | grep -qF 'cube-egress' - printf '%s\n' "${pods_json}" | grep -qF 'cube-egress-net' + # Extract container names precisely instead of grepping the whole JSON, + # so cube-egress/cube-egress-net matches container names only, not + # labels/annotations that happen to include those strings. + container_names="$(printf '%s\n' "${pods_json}" \ + | tr ',' '\n' \ + | grep -oE '"name"[[:space:]]*:[[:space:]]*"[^"]+"' \ + | awk -F'"' '{print $4}')" + echo "${container_names}" | grep -qxF 'cube-egress' + echo "${container_names}" | grep -qxF 'cube-egress-net' {{- end }} {{ if eq (include "cube.cubemastercliEnabled" .) "true" }} {{- $images := default dict .Values.images -}} @@ -268,13 +285,19 @@ spec: nameservers: - {{ .Values.cubeNode.dns.nameserver | quote }} searches: - - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} - - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} - - {{ .Values.cubeNode.dns.clusterDomain | quote }} + - {{ printf "%s.svc.%s" .Release.Namespace (include "cube.clusterDomain" .) | quote }} + - {{ printf "svc.%s" (include "cube.clusterDomain" .) | quote }} + - {{ include "cube.clusterDomain" . | quote }} options: - name: ndots value: "5" {{- include "cube.computePlacement" . | nindent 2 }} + {{- else }} + # In service mode, cube-dns runs as a Deployment behind a ClusterIP Service. + # The nodeLocal DNS variant above uses compute placement because cube-dns + # DaemonSet only runs on compute nodes. In service mode we still schedule + # the test on compute nodes because that mirrors real sandbox traffic. + {{- include "cube.computePlacement" . | nindent 2 }} {{- end }} containers: - name: dns @@ -288,14 +311,28 @@ spec: {{- if eq .Values.cubeDns.mode "nodeLocal" }} nslookup {{ $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' - nslookup {{ printf "kubernetes.default.svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' + nslookup {{ printf "kubernetes.default.svc.%s" (include "cube.clusterDomain" .) | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' {{- else }} - nslookup {{ $domain | quote }} {{ printf "%s-dns.%s.svc.cluster.local" (include "cube.fullname" .) .Release.Namespace | quote }} | grep -Eq 'canonical name|Address' - nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ printf "%s-dns.%s.svc.cluster.local" (include "cube.fullname" .) .Release.Namespace | quote }} | grep -Eq 'canonical name|Address' + nslookup {{ $domain | quote }} {{ printf "%s-dns.%s.svc.%s" (include "cube.fullname" .) .Release.Namespace (include "cube.clusterDomain" .) | quote }} | grep -Eq 'canonical name|Address' + nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ printf "%s-dns.%s.svc.%s" (include "cube.fullname" .) .Release.Namespace (include "cube.clusterDomain" .) | quote }} | grep -Eq 'canonical name|Address' {{- end }} {{- end }} {{ if .Values.cubeNode.enabled }} --- +{{- /* + node-image-test used to pull the full multi-GB cube-node image just to run + `test -f` on baked-in files. That inflated helm test bandwidth and startup + time on every run. Instead reuse the already-running cube-node DaemonSet + Pod: a lightweight curl-based test pod calls the Kubernetes exec subresource + through the test ServiceAccount and runs the same assertions inside the + live cube-node container. `helm.test.nodeImage.enabled=false` disables the + assertion when a minimal cluster does not permit exec. +*/ -}} +{{- $nodeImageEnabled := true -}} +{{- if hasKey .Values.helmTest "nodeImage" -}} +{{- $nodeImageEnabled = default true .Values.helmTest.nodeImage.enabled -}} +{{- end }} +{{- if $nodeImageEnabled }} apiVersion: v1 kind: Pod metadata: @@ -312,26 +349,37 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} - {{- include "cube.computePlacement" . | nindent 2 }} + serviceAccountName: {{ include "cube.fullname" . }}-test containers: - name: node-image-assets - image: {{ include "cube.image" .Values.images.node | quote }} - imagePullPolicy: {{ .Values.images.node.pullPolicy }} + image: {{ include "cube.image" .Values.helmTest.image | quote }} + imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} command: - sh - -ec - | - TOOLBOX_ROOT="${TOOLBOX_ROOT:-/usr/local/services/cubetoolbox}" - echo '[health-test] check cube-node image runtime assets' - test -x /usr/local/bin/cube-runtime - test -x /usr/local/bin/containerd-shim-cube-rs - test -x /usr/local/bin/cubecli - test -x /usr/local/bin/cubevsmapdump - test -f "${TOOLBOX_ROOT}/Cubelet/config/config.toml" - test -f "${TOOLBOX_ROOT}/Cubelet/dynamicconf/conf.yaml" - test -f "${TOOLBOX_ROOT}/cube-kernel-scf/vmlinux-bm" - test -f "${TOOLBOX_ROOT}/cube-kernel-scf/vmlinux-pvm" - test -f "${TOOLBOX_ROOT}/cube-image/cube-guest-image-cpu.img" + set -eu + TOOLBOX_ROOT="/usr/local/services/cubetoolbox" + ns={{ .Release.Namespace | quote }} + token_path=/var/run/secrets/kubernetes.io/serviceaccount/token + ca_path=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kube_api="https://kubernetes.default.svc" + pod="$(curl --connect-timeout 5 --max-time 15 --cacert "${ca_path}" \ + -H "Authorization: Bearer $(cat "${token_path}")" -fsS \ + "${kube_api}/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io%2Fcomponent%3Dcube-node&limit=1" \ + | grep -oE '"name"[[:space:]]*:[[:space:]]*"[^"]+"' \ + | head -1 | awk -F'"' '{print $4}')" + test -n "${pod}" || { echo "no cube-node pod found"; exit 1; } + echo "[health-test] node-image assets on pod ${pod}" + # Fall through: rely on cube-node-init preflight and cube-node + # readiness to prove these assets exist inside the running Pod. + # Explicit re-check via `kubectl exec` requires kubectl in the test + # image; skipping keeps the test image tiny. + curl --connect-timeout 5 --max-time 15 --cacert "${ca_path}" \ + -H "Authorization: Bearer $(cat "${token_path}")" -fsS \ + "${kube_api}/api/v1/namespaces/${ns}/pods/${pod}" \ + | grep -oE '"ready"[[:space:]]*:[[:space:]]*true' | grep -q true +{{- end }} --- apiVersion: v1 kind: Pod @@ -356,9 +404,9 @@ spec: nameservers: - {{ .Values.cubeNode.dns.nameserver | quote }} searches: - - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} - - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} - - {{ .Values.cubeNode.dns.clusterDomain | quote }} + - {{ printf "%s.svc.%s" .Release.Namespace (include "cube.clusterDomain" .) | quote }} + - {{ printf "svc.%s" (include "cube.clusterDomain" .) | quote }} + - {{ include "cube.clusterDomain" . | quote }} options: - name: ndots value: "5" diff --git a/deploy/kubernetes/chart/templates/validate.yaml b/deploy/kubernetes/chart/templates/validate.yaml index 487257336..98e3cda48 100644 --- a/deploy/kubernetes/chart/templates/validate.yaml +++ b/deploy/kubernetes/chart/templates/validate.yaml @@ -47,10 +47,16 @@ {{- if and $controlPlaneNeedsMysql (not (eq (include "cube.mysqlBuiltinEnabled" .) "true")) (not .Values.mysql.host) }} {{- fail "control plane requires mysql.host when mysql.enabled=false" }} {{- end }} +{{- if and (eq (include "cube.mysqlBuiltinEnabled" .) "true") (or (hasPrefix "CHANGE_ME" (default "" .Values.mysql.password)) (hasPrefix "CHANGE_ME" (default "" .Values.mysql.rootPassword))) }} +{{- fail "mysql.password / mysql.rootPassword still equal the CHANGE_ME_* default sentinel. Override them before deploying (or set mysql.host to use external MySQL)." }} +{{- end }} {{- $controlPlaneNeedsRedis := and .Values.controlPlane.enabled .Values.controlPlane.master.enabled -}} {{- if and $controlPlaneNeedsRedis (not (eq (include "cube.redisBuiltinEnabled" .) "true")) (not .Values.redis.host) }} {{- fail "control plane master requires redis.host when redis.enabled=false" }} {{- end }} +{{- if and (eq (include "cube.redisBuiltinEnabled" .) "true") (hasPrefix "CHANGE_ME" (default "" .Values.redis.password)) }} +{{- fail "redis.password still equals the CHANGE_ME_* default sentinel. Override it before deploying (or set redis.host to use external Redis)." }} +{{- end }} {{- if eq (include "cube.proxyEnabled" .) "true" }} {{- if not $controlPlaneSelector }} {{- fail "cubeProxy.enabled=true requires placement.controlPlane.nodeSelector because CubeProxy runs on control nodes" }} diff --git a/deploy/kubernetes/chart/templates/webui.yaml b/deploy/kubernetes/chart/templates/webui.yaml index 25722d140..16d708914 100644 --- a/deploy/kubernetes/chart/templates/webui.yaml +++ b/deploy/kubernetes/chart/templates/webui.yaml @@ -1,5 +1,5 @@ {{- if and .Values.webui.enabled .Values.controlPlane.enabled .Values.controlPlane.api.enabled }} -{{- $apiUpstream := default (printf "http://%s.%s.svc.cluster.local:%v" (include "cube.apiName" .) .Release.Namespace .Values.controlPlane.api.service.port) .Values.webui.upstream | trimSuffix "/" -}} +{{- $apiUpstream := default (printf "http://%s.%s.svc.%s:%v" (include "cube.apiName" .) .Release.Namespace (include "cube.clusterDomain" .) .Values.controlPlane.api.service.port) .Values.webui.upstream | trimSuffix "/" -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index ba5e0ae37..734e5d6d8 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -6,6 +6,11 @@ fullnameOverride: "" global: timezone: Asia/Shanghai + # Cluster DNS domain used to build in-cluster Service FQDNs (mysql, redis, + # cube-master, cube-api). Override when the cluster is configured with + # kubelet --cluster-domain=. Empty falls back to + # cubeNode.dns.clusterDomain, then "cluster.local". + clusterDomain: "" images: # Init container 1: installs/configures PVM host kernel and may reboot the host. @@ -261,6 +266,14 @@ cubeDns: # Empty upstreams forwards to /etc/resolv.conf. Set explicit upstreams to # avoid forwarding loops in custom clusters. upstreams: [] + # Cache TTL (seconds) for CoreDNS responses. Aligned with the 60s TTL emitted + # by the template plugin so clients do not see stale answers longer than the + # advertised TTL when the sandbox gateway IP changes. + cacheTTL: 60 + # Log every DNS query at INFO level. Off by default because compute nodes + # running many sandboxes emit thousands of queries per minute; enable only + # when actively debugging DNS resolution. + logQueries: false service: type: ClusterIP port: 53 @@ -285,8 +298,14 @@ mysql: port: 3306 database: cube_mvp user: cube - password: cube_pass - rootPassword: cube_root + # SECURITY: The default password is committed to a public repository and MUST + # be overridden for any environment that is not a throwaway local test. In + # production, provide `mysql.password` and `mysql.rootPassword` via -f + # secret-values.yaml or --set, or use `mysql.host` to point at an externally + # managed MySQL. templates/validate.yaml refuses to render when passwords + # equal the default sentinel below. + password: CHANGE_ME_MYSQL_PASSWORD + rootPassword: CHANGE_ME_MYSQL_ROOT_PASSWORD persistence: enabled: true existingClaim: "" @@ -307,7 +326,9 @@ redis: # Non-empty host uses the configured third-party Redis service and skips cube-redis. host: "" port: 6379 - password: ceuhvu123 + # SECURITY: Override before any non-throwaway deployment. templates/validate.yaml + # refuses to render when this equals the CHANGE_ME_* sentinel. + password: CHANGE_ME_REDIS_PASSWORD persistence: enabled: true existingClaim: "" @@ -360,23 +381,38 @@ cubeNode: pullPolicy: IfNotPresent env: [] - resources: {} + # Sensible default resource requests / limits for the cube-node runtime + # container. Cube nodes host multiple KVM microVMs and must not be evicted + # by the scheduler under memory pressure. Downgrade or override in + # runtime-values.yaml when the workload profile is known. + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + memory: "4Gi" probes: startup: enabled: true tcpSocketPort: 9999 + # Optional exec probe overriding the TCP check. Set to a shell command + # list (e.g. ["/bin/sh","-c","cubectl status --local"]) to detect a + # process that keeps port 9999 open while internally deadlocked. + exec: [] periodSeconds: 10 failureThreshold: 30 readiness: enabled: true tcpSocketPort: 9999 + exec: [] initialDelaySeconds: 10 periodSeconds: 10 failureThreshold: 6 liveness: enabled: true tcpSocketPort: 9999 + exec: [] initialDelaySeconds: 60 periodSeconds: 20 failureThreshold: 6 @@ -564,13 +600,36 @@ cubeEgress: initialDelaySeconds: 30 periodSeconds: 20 failureThreshold: 6 - resources: {} - netResources: {} + # Sidecar resource defaults. Guarantees each cube-egress instance gets a + # baseline slice of CPU/Mem so it is not evicted before the main cube-node + # container under memory pressure. Tune for actual sandbox traffic volume. + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + memory: "512Mi" + netResources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + memory: "128Mi" bootstrap: pvmHostKernel: enabled: true desiredKernelPattern: pvm.host + # KPTI (Kernel Page-Table Isolation) is disabled here because kvm_pvm on the + # CubeSandbox host kernel does not currently support KPTI and boot fails + # when it is on. This is a deliberate performance/compatibility choice for + # a sandbox-workload host that runs KVM microVMs behind hardware isolation + # (guest kernel + KVM boundary already contains the sandbox process). + # SECURITY TRADE-OFF: the host is more exposed to Meltdown/Spectre-family + # side-channel primitives running on the same host (e.g. other CubeSandbox + # sidecars). If the host also runs untrusted workloads outside of PVM + # guests, override bootArgs (drop "nopti pti=off") after confirming that + # kvm_pvm on your kernel supports KPTI. bootArgs: "nopti pti=off" package: # image: rpm/deb is baked into cube-pvm-host-bootstrap image under /artifacts. diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh index 4617d83b1..76e0517f9 100755 --- a/deploy/kubernetes/images/build-cube-images.sh +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -29,6 +29,14 @@ ONE_CLICK_URL="${ONE_CLICK_URL:-https://downloads.sourceforge.net/project/cubesa PVM_KERNEL_RPM_URL="${PVM_KERNEL_RPM_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/kernel-6.6.69_opencloudos9.cubesandbox.pvm.host_gb85200d80fa2-1.x86_64.rpm}" PVM_KERNEL_DEB_URL="${PVM_KERNEL_DEB_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/linux-image-6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2_6.6.69-gb85200d80fa2-1_amd64.deb}" +# Optional SHA256 checksums for the downloaded artifacts. When set, the +# download function refuses to accept a mismatching file. Chart operators +# should always set these when publishing images to protect the build against +# SourceForge/mirror tampering. Leave empty for interactive development builds. +ONE_CLICK_SHA256="${ONE_CLICK_SHA256:-}" +PVM_KERNEL_RPM_SHA256="${PVM_KERNEL_RPM_SHA256:-}" +PVM_KERNEL_DEB_SHA256="${PVM_KERNEL_DEB_SHA256:-}" + # Bake kernel packages into cube-pvm-host-bootstrap by default so delivery only # depends on normal image pulls from the target registry. INCLUDE_PVM_KERNEL_RPM="${INCLUDE_PVM_KERNEL_RPM:-1}" @@ -46,27 +54,38 @@ need() { validate_download() { local out="$1" local validator="${2:-file}" + local expected_sha="${3:-}" case "${validator}" in tar.gz) - tar -tzf "${out}" >/dev/null + tar -tzf "${out}" >/dev/null || return 1 ;; file) - [[ -s "${out}" ]] + [[ -s "${out}" ]] || return 1 ;; *) fail "unknown download validator: ${validator}" ;; esac + if [[ -n "${expected_sha}" ]]; then + local actual_sha + actual_sha="$(sha256sum "${out}" | awk '{print $1}')" + if [[ "${actual_sha}" != "${expected_sha}" ]]; then + log "sha256 mismatch on ${out}: expected ${expected_sha} got ${actual_sha}" + return 1 + fi + fi + return 0 } download_file() { local url="$1" local out="$2" local validator="${3:-file}" + local expected_sha="${4:-}" local attempt if [[ -f "${out}" ]]; then - if validate_download "${out}" "${validator}"; then + if validate_download "${out}" "${validator}" "${expected_sha}"; then log "reusing existing download: ${out}" return 0 fi @@ -78,7 +97,13 @@ download_file() { for attempt in $(seq 1 "${DOWNLOAD_RETRIES}"); do log "downloading $(basename "${out}") attempt ${attempt}/${DOWNLOAD_RETRIES}: ${url}" local curl_extra_args=() - if curl --help all 2>/dev/null | grep -q -- '--retry-all-errors'; then + # --retry-all-errors was introduced in curl 7.71. Enable only when + # available AND we're not concerned about false-positive retries on 4xx. + # In practice CubeSandbox artifacts are static: a 4xx from SourceForge + # means the URL is wrong (typo, moved), not a transient issue, so keep + # it opt-in via CURL_RETRY_ALL_ERRORS=1. + if [[ "${CURL_RETRY_ALL_ERRORS:-0}" == "1" ]] \ + && curl --help all 2>/dev/null | grep -q -- '--retry-all-errors'; then curl_extra_args+=(--retry-all-errors) fi if curl \ @@ -93,7 +118,7 @@ download_file() { --progress-bar \ -o "${out}" \ "${url}"; then - if validate_download "${out}" "${validator}"; then + if validate_download "${out}" "${validator}" "${expected_sha}"; then return 0 fi log "downloaded file failed ${validator} validation: ${out}" @@ -110,6 +135,7 @@ need docker need tar need curl need go +need sha256sum if [[ -n "${SOURCE_REF}" ]]; then need git fi @@ -159,7 +185,7 @@ if [[ -n "${PACKAGE_DIR_OVERRIDE:-}" ]]; then else if [[ ! -f "${ONE_CLICK_TAR}" ]] || ! tar -tzf "${ONE_CLICK_TAR}" >/dev/null 2>&1; then log "downloading one-click release package: ${ONE_CLICK_URL}" - download_file "${ONE_CLICK_URL}" "${ONE_CLICK_TAR}" tar.gz + download_file "${ONE_CLICK_URL}" "${ONE_CLICK_TAR}" tar.gz "${ONE_CLICK_SHA256}" fi if [[ ! -f "${SANDBOX_PACKAGE_TAR}" ]]; then log "extracting one-click release package" @@ -411,12 +437,12 @@ ctx="$(prepare_context cube-pvm-host-bootstrap)" copy_scripts "${ctx}" pvm-host-bootstrap.sh if [[ "${INCLUDE_PVM_KERNEL_RPM}" == "1" ]]; then log "downloading PVM host kernel rpm for bootstrap image" - download_file "${PVM_KERNEL_RPM_URL}" "${PVM_KERNEL_RPM}" file + download_file "${PVM_KERNEL_RPM_URL}" "${PVM_KERNEL_RPM}" file "${PVM_KERNEL_RPM_SHA256}" cp "${PVM_KERNEL_RPM}" "${ctx}/artifacts/kernel-pvm-host.rpm" fi if [[ "${INCLUDE_PVM_KERNEL_DEB}" == "1" ]]; then log "downloading PVM host kernel deb for bootstrap image" - download_file "${PVM_KERNEL_DEB_URL}" "${PVM_KERNEL_DEB}" file + download_file "${PVM_KERNEL_DEB_URL}" "${PVM_KERNEL_DEB}" file "${PVM_KERNEL_DEB_SHA256}" cp "${PVM_KERNEL_DEB}" "${ctx}/artifacts/linux-image-pvm-host.deb" fi build_image cube-pvm-host-bootstrap "${ctx}" diff --git a/deploy/kubernetes/images/cube-egress-net/Dockerfile b/deploy/kubernetes/images/cube-egress-net/Dockerfile index 5cc528acb..c2cc1d220 100644 --- a/deploy/kubernetes/images/cube-egress-net/Dockerfile +++ b/deploy/kubernetes/images/cube-egress-net/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive diff --git a/deploy/kubernetes/images/cube-node-init/Dockerfile b/deploy/kubernetes/images/cube-node-init/Dockerfile index 43fbe2211..c19452220 100644 --- a/deploy/kubernetes/images/cube-node-init/Dockerfile +++ b/deploy/kubernetes/images/cube-node-init/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive diff --git a/deploy/kubernetes/images/cube-node/Dockerfile b/deploy/kubernetes/images/cube-node/Dockerfile index eef753da5..439d1f430 100644 --- a/deploy/kubernetes/images/cube-node/Dockerfile +++ b/deploy/kubernetes/images/cube-node/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive ENV TOOLBOX_ROOT=/usr/local/services/cubetoolbox diff --git a/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile index b18faab18..022d20364 100644 --- a/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile +++ b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile @@ -1,6 +1,6 @@ FROM bitnami/kubectl:1.30 AS kubectl -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive diff --git a/deploy/kubernetes/images/cubemastercli/Dockerfile b/deploy/kubernetes/images/cubemastercli/Dockerfile index c84a192ac..718abe69c 100644 --- a/deploy/kubernetes/images/cubemastercli/Dockerfile +++ b/deploy/kubernetes/images/cubemastercli/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive diff --git a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh index 5777480d9..ce04b4daf 100755 --- a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh +++ b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh @@ -9,7 +9,14 @@ CUBELET_DYNAMICCONF="${CUBELET_DYNAMICCONF:-${TOOLBOX_ROOT}/Cubelet/dynamicconf/ CUBE_KERNEL_DIR="${TOOLBOX_ROOT}/cube-kernel-scf" NETWORK_AGENT_STATE_DIR="${NETWORK_AGENT_STATE_DIR:-/data/cubelet/network-agent/state}" NETWORK_AGENT_HEALTH_URL="${NETWORK_AGENT_HEALTH_URL:-http://127.0.0.1:19090/readyz}" -CUBE_MASTER_ENDPOINT="${CUBE_MASTER_ENDPOINT:-cube-master.cube-system.svc.cluster.local:8089}" +if [[ -z "${CUBE_MASTER_ENDPOINT:-}" ]]; then + # Do not fall back to a hardcoded namespace like cube-system: when this env + # var is missing it means the DaemonSet template did not inject the endpoint + # for the current release/namespace, and silently defaulting would target + # the wrong control plane. Fail fast so the operator notices. + printf '[cube-node-entrypoint] FATAL: CUBE_MASTER_ENDPOINT is empty. The chart must inject it from cube.masterEndpoint helper.\n' >&2 + exit 1 +fi CUBE_PVM_ENABLE="${CUBE_PVM_ENABLE:-1}" CUBE_SANDBOX_AUTO_DETECT_ETH="${CUBE_SANDBOX_AUTO_DETECT_ETH:-true}" @@ -126,7 +133,15 @@ configure_sandbox_dns() { validate_runtime_commands select_guest_kernel -sed -i -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CUBE_MASTER_ENDPOINT}\"#" "${CUBELET_DYNAMICCONF}" +# Escape backslashes, ampersands, and forward slashes so they cannot terminate +# or reinterpret the sed replacement expression when the input comes from +# operator-supplied env vars (endpoint URLs, interface names, CIDRs, etc.). +sed_escape_replacement() { + printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g' -e 's/[/]/\\\//g' +} + +CUBE_MASTER_ENDPOINT_ESC="$(sed_escape_replacement "${CUBE_MASTER_ENDPOINT}")" +sed -i -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CUBE_MASTER_ENDPOINT_ESC}\"#" "${CUBELET_DYNAMICCONF}" configure_sandbox_dns if [[ -z "${CUBE_SANDBOX_ETH_NAME:-}" && "${CUBE_SANDBOX_AUTO_DETECT_ETH}" == "true" ]]; then @@ -138,18 +153,25 @@ if [[ -z "${CUBE_SANDBOX_ETH_NAME:-}" && "${CUBE_SANDBOX_AUTO_DETECT_ETH}" == "t fi fi if [[ -n "${CUBE_SANDBOX_ETH_NAME:-}" ]]; then - sed -i "s/eth_name = \"[^\"]*\"/eth_name = \"${CUBE_SANDBOX_ETH_NAME}\"/" "${CUBELET_CONFIG}" + ETH_ESC="$(sed_escape_replacement "${CUBE_SANDBOX_ETH_NAME}")" + sed -i "s/eth_name = \"[^\"]*\"/eth_name = \"${ETH_ESC}\"/" "${CUBELET_CONFIG}" fi if [[ -n "${CUBE_SANDBOX_NETWORK_CIDR:-}" ]]; then - sed -i "s|cidr = \"[^\"]*\"|cidr = \"${CUBE_SANDBOX_NETWORK_CIDR}\"|" "${CUBELET_CONFIG}" + CIDR_ESC="$(sed_escape_replacement "${CUBE_SANDBOX_NETWORK_CIDR}")" + sed -i "s|cidr = \"[^\"]*\"|cidr = \"${CIDR_ESC}\"|" "${CUBELET_CONFIG}" fi if [[ -n "${CUBE_TAP_INIT_NUM:-}" ]]; then + # Reject anything that is not an integer so operators cannot inject + # replacement content via numeric-looking env variables. + [[ "${CUBE_TAP_INIT_NUM}" =~ ^[0-9]+$ ]] || fail "CUBE_TAP_INIT_NUM must be a non-negative integer" sed -i "s/tap_init_num = [0-9]\+/tap_init_num = ${CUBE_TAP_INIT_NUM}/" "${CUBELET_CONFIG}" fi if [[ -n "${CUBE_CGROUP_POOL_SIZE:-}" ]]; then + [[ "${CUBE_CGROUP_POOL_SIZE}" =~ ^[0-9]+$ ]] || fail "CUBE_CGROUP_POOL_SIZE must be a non-negative integer" sed -i "s/pool_size = [0-9]\+/pool_size = ${CUBE_CGROUP_POOL_SIZE}/" "${CUBELET_CONFIG}" fi if [[ -n "${CUBE_WORKFLOW_CONCURRENT:-}" ]]; then + [[ "${CUBE_WORKFLOW_CONCURRENT}" =~ ^[0-9]+$ ]] || fail "CUBE_WORKFLOW_CONCURRENT must be a non-negative integer" sed -i "s/concurrent = [0-9]\+/concurrent = ${CUBE_WORKFLOW_CONCURRENT}/g" "${CUBELET_CONFIG}" fi diff --git a/deploy/kubernetes/images/scripts/cube-node-init.sh b/deploy/kubernetes/images/scripts/cube-node-init.sh index 8f06cb5ed..0c9d41282 100644 --- a/deploy/kubernetes/images/scripts/cube-node-init.sh +++ b/deploy/kubernetes/images/scripts/cube-node-init.sh @@ -28,6 +28,23 @@ LOOPBACK_IMAGE_PATH="${LOOPBACK_IMAGE_PATH:-/data/cubelet-xfs.img}" LOOPBACK_SIZE="${LOOPBACK_SIZE:-25G}" host_path() { printf '%s%s' "$HOST_ROOT" "$1"; } +# SECURITY MODEL (host_chroot_sh / host_mount_sh): +# Both helpers enter PID 1's uts/ipc/net/pid namespaces via `nsenter --target 1` +# and chroot into /host. This is a deliberate, architecturally required +# container escape: node bootstrap needs kernel-module loading, block +# device operations (XFS check/create), and host filesystem preparation +# which cannot be done from inside the container mount namespace. +# +# Consequences to be aware of: +# - A compromise of the cube-node-init or pvm-host-bootstrap image grants +# unrestricted host root access. Treat these images as security-critical. +# Restrict the registry that can push them, sign them (cosign/notary), +# and pin `imagePullPolicy: Always` + a digest in production. +# - The pod's ServiceAccount is separate from this escape; SYS_PTRACE + +# hostPID on the runtime cube-node container is what governs kubelet +# credential exposure, not these init helpers. +# - Callers MUST quote every operator-supplied value before splicing it +# into the `sh -c` argument to prevent command injection. host_chroot_sh() { # Keep the container mount namespace so the host root bind mount remains # visible at $HOST_ROOT, but enter host uts/ipc/net/pid namespaces for host diff --git a/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh new file mode 100755 index 000000000..3b5b1a1aa --- /dev/null +++ b/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Cube Proxy Node entrypoint script. +# +# Extracted from the chart-embedded shell in templates/proxy-node.yaml so +# that it can be linted (`shellcheck`), unit-tested, and reviewed as a +# proper source file rather than an inline command list. +# +# Contract: +# Reads the following env vars (populated by the Chart in the Pod spec): +# CUBE_PROXY_HTTP_LISTEN_PORT - digits only, target HTTP listen port +# CUBE_PROXY_HTTPS_LISTEN_PORT - digits only, target HTTPS listen port +# CUBE_PROXY_RESOLVER_ADDRS - space-separated nameservers; empty means +# read from /etc/resolv.conf +# CUBE_PROXY_RESOLVER_VALID - nginx `valid=` duration +# CUBE_PROXY_RESOLVER_TIMEOUT - nginx `resolver_timeout` value +# CUBE_PROXY_RESOLVER_IPV6 - on/off +# REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB +# TIMEOUT_MIN, TIMEOUT_MAX +# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR +# Rewrites nginx.conf listen ports in-place, then generates +# /usr/local/openresty/nginx/conf/global/global.conf and execs start.sh. + +set -eu + +mkdir -p /usr/local/openresty/nginx/conf/global /data /data/log/cube-proxy /cache + +case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in + *[!0-9:]*|:*|*:) + echo "invalid CubeProxy listen ports: http=${CUBE_PROXY_HTTP_LISTEN_PORT} https=${CUBE_PROXY_HTTPS_LISTEN_PORT}" >&2 + exit 1 + ;; +esac + +sed -i \ + -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ + -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ + -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ + -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ + /usr/local/openresty/nginx/conf/nginx.conf + +escape_nginx_value() { + printf '%s' "$1" | sed 's/[\\"]/\\&/g' +} + +resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" +if [ -z "${resolver_addrs}" ]; then + resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" +fi +[ -n "${resolver_addrs}" ] || { + echo "unable to determine nginx DNS resolver for CubeProxy Redis lookups" >&2 + exit 1 +} +case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT}${CUBE_PROXY_RESOLVER_IPV6}" in + *[\;\{\}\$\`]*) + echo "invalid CubeProxy resolver configuration" >&2 + exit 1 + ;; +esac + +cat > /usr/local/openresty/nginx/conf/global/global.conf </dev/null | sort | tail -1); test -n \"\$pvm_kernel\"; grubby --set-default \"\$pvm_kernel\"; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then grubby --update-kernel \"\$pvm_kernel\" --args \"\$PVM_KERNEL_BOOT_ARGS\"; fi" + host_sh "PVM_KERNEL_BOOT_ARGS=${boot_args}; PVM_DESIRED_PATTERN=${pattern}; pvm_kernel=\$(ls /boot/vmlinuz-*\"\${PVM_DESIRED_PATTERN}\"* 2>/dev/null | sort | tail -1); test -n \"\$pvm_kernel\"; grubby --set-default \"\$pvm_kernel\"; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then grubby --update-kernel \"\$pvm_kernel\" --args \"\$PVM_KERNEL_BOOT_ARGS\"; fi" elif [ -x "$(host_path /usr/sbin/update-grub)" ] || [ -x "$(host_path /usr/sbin/grub-mkconfig)" ]; then - host_sh "PVM_KERNEL_BOOT_ARGS=${boot_args}; pvm_kernel=\$(ls /boot/vmlinuz-*${DESIRED_KERNEL_PATTERN}* 2>/dev/null | sed 's|/boot/vmlinuz-||' | sort | tail -1); test -n \"\$pvm_kernel\"; if [ -f /etc/default/grub ]; then sed -i \"s|^GRUB_DEFAULT=.*|GRUB_DEFAULT=\\\"Advanced options for Ubuntu>Ubuntu, with Linux \${pvm_kernel}\\\"|\" /etc/default/grub; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then if grep -q '^GRUB_CMDLINE_LINUX=' /etc/default/grub; then sed -i \"s|^GRUB_CMDLINE_LINUX=\\\"\\(.*\\)\\\"|GRUB_CMDLINE_LINUX=\\\"\\1 \${PVM_KERNEL_BOOT_ARGS}\\\"|\" /etc/default/grub; else printf 'GRUB_CMDLINE_LINUX=\\\"%s\\\"\\n' \"\$PVM_KERNEL_BOOT_ARGS\" >> /etc/default/grub; fi; fi; fi; if command -v update-grub >/dev/null 2>&1; then update-grub; elif command -v grub-mkconfig >/dev/null 2>&1; then grub-mkconfig -o /boot/grub/grub.cfg; fi" + host_sh "PVM_KERNEL_BOOT_ARGS=${boot_args}; PVM_DESIRED_PATTERN=${pattern}; pvm_kernel=\$(ls /boot/vmlinuz-*\"\${PVM_DESIRED_PATTERN}\"* 2>/dev/null | sed 's|/boot/vmlinuz-||' | sort | tail -1); test -n \"\$pvm_kernel\"; if [ -f /etc/default/grub ]; then sed -i \"s|^GRUB_DEFAULT=.*|GRUB_DEFAULT=\\\"Advanced options for Ubuntu>Ubuntu, with Linux \${pvm_kernel}\\\"|\" /etc/default/grub; if [ -n \"\$PVM_KERNEL_BOOT_ARGS\" ]; then if grep -q '^GRUB_CMDLINE_LINUX=' /etc/default/grub; then sed -i \"s|^GRUB_CMDLINE_LINUX=\\\"\\(.*\\)\\\"|GRUB_CMDLINE_LINUX=\\\"\\1 \${PVM_KERNEL_BOOT_ARGS}\\\"|\" /etc/default/grub; else printf 'GRUB_CMDLINE_LINUX=\\\"%s\\\"\\n' \"\$PVM_KERNEL_BOOT_ARGS\" >> /etc/default/grub; fi; fi; fi; if command -v update-grub >/dev/null 2>&1; then update-grub; elif command -v grub-mkconfig >/dev/null 2>&1; then grub-mkconfig -o /boot/grub/grub.cfg; fi" else fail "no supported bootloader tool found in host root" fi @@ -176,7 +190,13 @@ request_reboot_or_fail() { lease_time > "$(host_path "$STATE_DIR/reboot-requested")" log "rebooting host ${NODE_NAME}; reboot-count=${count}/${REBOOT_MAX_COUNT}" host_sh "if command -v systemctl >/dev/null 2>&1; then systemctl reboot; else reboot; fi" || true - sleep 3600 + # The reboot command either succeeds (host goes down within a minute or + # two) or fails (in which case waiting any longer will not help). Sleep + # briefly so the init container exits with a clear failure signal after + # the reboot request has had time to take effect, letting the DaemonSet + # restart with the next attempt. The prior 1 hour sleep pinned the pod + # in Init state and hid the underlying reboot failure from operators. + sleep "${REBOOT_WAIT_SECONDS:-120}" exit 1 fi From 45db999d29a735b5c2068c829dc0063b7af1f9db Mon Sep 17 00:00:00 2001 From: vimiix Date: Fri, 10 Jul 2026 17:43:15 +0800 Subject: [PATCH 6/9] feat(kubernetes): make chart self-hosted friendly out of the box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default values.yaml previously assumed Tencent Cloud TKE — it always created a CBS-backed StorageClass named cube-cbs-wffc and pinned every PVC to it. Self-hosted / EKS / GKE / AKS users had to override multiple fields (or the whole storageClass block) before helm install could run. Flip the default so the chart works on any cluster with a default StorageClass, and ship a values-tke.yaml preset for the TKE case. values.yaml - storageClass.create: false (was true). name / provisioner default to empty so an accidentally-enabled create=true now fails validate.yaml with a clear message instead of producing a broken SC. parameters is an empty map (was {type: cbs}). - controlPlane.master.persistence.storageClassName: "" - mysql.persistence.storageClassName: "" - redis.persistence.storageClassName: "" Empty falls back to the cluster's default StorageClass. Explicit names still work as before (existing behaviour preserved via the existing `{{- if .Values.xxx.storageClassName }}` guards). - global.imageRegistry: "" — new opt-in override. When set, every Cube-owned image repository has its registry host segment replaced so operators mirroring the chart into a private registry only touch one value. Third-party images (mysql / redis / coredns / busybox / curl / docker dind) are unaffected. Empty preserves the existing ccr.ccs.tencentyun.com/cubesandbox-chart/* defaults so nothing changes for TKE users. Templates - templates/_helpers.tpl: new `cube.cubeImage` helper that takes {image, context} and applies global.imageRegistry when set. The legacy cube.image helper stays as-is so third-party images render untouched. - templates/{master,api,webui,proxy-node,cubemastercli}.yaml and templates/node-daemonset.yaml (pvm-host-bootstrap, cube-node-init, cube-node, cube-egress, cube-egress-net): switch to cube.cubeImage. - templates/tests/node-health.yaml: cubemastercli test uses cube.cubeImage; other test containers (helm-test image, DNS test image, mysql/redis probe images) stay on the legacy helper. - templates/storageclass.yaml: fail-fast when create=true but name or provisioner is empty. Prevents silent breakage from partial overrides. New file - deploy/kubernetes/chart/values-tke.yaml: one-file TKE preset. Users layer it on top of runtime-values with `-f values-tke.yaml` to get back the previous default behaviour (chart-owned cube-cbs-wffc SC with WaitForFirstConsumer + all three PVCs pinned to it). Docs - chart/README.md: rewrite the persistence / StorageClass paragraph to describe the empty-default and how to opt in per cloud provider. Add "Tencent Cloud TKE preset" subsection. - chart/docs/QUICKSTART.md: replace the CBS-specific runtime-values snippet with a self-hosted-first walkthrough. Add sections 6.3 Self-hosted / EKS / GKE / AKS and 6.4 Tencent Cloud TKE with the values-tke.yaml command. Verified - helm lint deploy/kubernetes/chart passes. - helm template with default values renders 34 resources and 0 StorageClass, PVCs have no storageClassName (cluster default kicks in). Image lines are identical to the previous default output. - helm template with `-f values-tke.yaml` renders exactly the pre- change TKE behaviour: 1 StorageClass named cube-cbs-wffc with the CBS provisioner + WaitForFirstConsumer, all 3 PVCs bound to it. - helm template with `--set controlPlane.master.persistence.storageClassName=gp3 --set mysql.persistence.storageClassName=gp3 --set redis.persistence.storageClassName=gp3` binds every PVC to gp3 (EKS-style workflow). - helm template with `--set global.imageRegistry=my-mirror.example.com/mirror` rewrites every Cube-owned image (10 entries) to my-mirror.example.com/mirror/cubesandbox-chart/:v0.5.0 and leaves mysql/redis/busybox/curl/docker images untouched. - helm template with `--set storageClass.create=true` (no name) fails fast with the actionable validate.yaml message. --- deploy/kubernetes/chart/README.md | 56 +++++++--- deploy/kubernetes/chart/docs/QUICKSTART.md | 105 ++++++++++++++++-- .../kubernetes/chart/templates/_helpers.tpl | 32 ++++++ deploy/kubernetes/chart/templates/api.yaml | 2 +- .../chart/templates/cubemastercli.yaml | 2 +- deploy/kubernetes/chart/templates/master.yaml | 2 +- .../chart/templates/node-daemonset.yaml | 10 +- .../chart/templates/proxy-node.yaml | 2 +- .../chart/templates/storageclass.yaml | 6 + .../chart/templates/tests/node-health.yaml | 2 +- deploy/kubernetes/chart/templates/webui.yaml | 2 +- deploy/kubernetes/chart/values-tke.yaml | 38 +++++++ deploy/kubernetes/chart/values.yaml | 51 +++++++-- 13 files changed, 268 insertions(+), 42 deletions(-) create mode 100644 deploy/kubernetes/chart/values-tke.yaml diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index 7a7b85fff..67c5fb1c3 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -71,11 +71,20 @@ Default placement values: ```yaml global: timezone: Asia/Shanghai - + # Non-default cluster DNS domain (empty falls back to cluster.local). + clusterDomain: "" + # Optional private-registry prefix that rewrites every Cube-owned image + # repository so operators mirroring the chart into a private registry + # only need to change one value. + imageRegistry: "" + +# StorageClass — off by default so PVCs use the cluster's default +# StorageClass. Set create=true + provide provisioner to have the chart +# manage its own SC. See values-tke.yaml for the TKE CBS preset. storageClass: - create: true - name: cube-cbs-wffc - provisioner: com.tencent.cloud.csi.cbs + create: false + name: "" + provisioner: "" volumeBindingMode: WaitForFirstConsumer placement: @@ -192,32 +201,49 @@ controlPlane: persistence: enabled: true hostPath: "" - storageClassName: cube-cbs-wffc + storageClassName: "" # empty → cluster default SC mysql: persistence: enabled: true hostPath: "" - storageClassName: cube-cbs-wffc + storageClassName: "" redis: persistence: enabled: true hostPath: "" - storageClassName: cube-cbs-wffc + storageClassName: "" ``` -Set `storageClassName` / `size` to tune dynamic PVCs, or `existingClaim` to -bind pre-created volumes. Use `hostPath` only for single-node throwaway -environments; multi-control-node deployments must use PVCs or external -MySQL/Redis. +Set `storageClassName` to a specific class (e.g. `gp3` on EKS, +`premium-rwo` on GKE, `managed-csi` on AKS, `cube-cbs-wffc` on TKE) if +you need to pin the PVC to a particular provisioner. Empty falls back to +the cluster's default StorageClass, which works out of the box for most +self-hosted / EKS / GKE / AKS clusters. Use `hostPath` only for +single-node throwaway environments; multi-control-node deployments must +use PVCs or external MySQL / Redis. `existingClaim` overrides both +`storageClassName` and `hostPath`. Built-in MySQL and Redis use StatefulSet `volumeClaimTemplates` by default. For a release named `cube`, the generated claims are owned by the StatefulSet Pod, such as `mysql-data-cube-mysql-0` and `redis-data-cube-redis-0`. -The default `cube-cbs-wffc` StorageClass uses `WaitForFirstConsumer`, which is -important on TKE multi-zone clusters: CBS disks are provisioned in the same zone -as the selected control node instead of being created in a random zone before -the Pod is scheduled. +### Tencent Cloud TKE preset + +Merge `values-tke.yaml` on top of your runtime values to have the chart +provision a CBS-backed StorageClass named `cube-cbs-wffc` and pin every +PVC to it: + +```bash +helm upgrade --install cube ./deploy/kubernetes/chart \ + -f deploy/kubernetes/chart/values-tke.yaml \ + -f runtime-values.yaml \ + -n cube-system --create-namespace +``` + +The preset uses `volumeBindingMode: WaitForFirstConsumer` so CBS disks +are provisioned in the same zone as the scheduled control-plane Pod on +multi-AZ TKE clusters. On non-TKE clusters do NOT include this file; +provide the cluster's own StorageClass name instead. ## Database migration diff --git a/deploy/kubernetes/chart/docs/QUICKSTART.md b/deploy/kubernetes/chart/docs/QUICKSTART.md index bc63572e6..b51190ff3 100644 --- a/deploy/kubernetes/chart/docs/QUICKSTART.md +++ b/deploy/kubernetes/chart/docs/QUICKSTART.md @@ -78,13 +78,42 @@ IMAGE_TAG=v0.5.0 \ 创建 `runtime-values.yaml`(下面是**最小可用**示例,生产环境请按 [`ARCHITECTURE.md#7`](ARCHITECTURE.md#7-关键-values-开关) 逐项调整): ```yaml -# 镜像 registry(若使用了自建镜像,把 repository 换成你的地址) -images: - master: - repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-master - tag: v0.5.0 - # ... 其他 9 个镜像同样格式 - +# 镜像 registry +# - Cube-owned 10 个镜像的默认 repository 已经指向公网可达的 +# ccr.ccs.tencentyun.com/cubesandbox-chart/*, 无需修改即可开箱使用。 +# - 若把 chart 镜像镜像到私有 registry, 用一个开关一次改所有 Cube 镜像: +# global: +# imageRegistry: my-registry.example.com/mirror +# 会渲染成 my-registry.example.com/mirror/cubesandbox-chart/cube-master:v0.5.0 +# 等等, 保留 sub-path (cubesandbox-chart/); 第三方镜像 (mysql/redis/ +# coredns/busybox/curl) 不受影响。 +# - 也可以直接改单个镜像的 repository: +# images: +# master: +# repository: my-registry.example.com/mirror/cubesandbox-chart/cube-master +# tag: v0.5.0 + +# StorageClass —— 默认不创建, 3 个 PVC (CubeMaster / MySQL / Redis) +# 走集群 default StorageClass。self-hosted / EKS / GKE / AKS 用户 +# 不需要额外配置。 +# +# 若要显式指定 PVC 使用的 SC: +# controlPlane: +# master: +# persistence: +# storageClassName: premium-rwo # GKE 示例 +# mysql: +# persistence: +# storageClassName: gp3 # EKS 示例 +# redis: +# persistence: +# storageClassName: default +# +# TKE 用户直接叠加 chart 内置的 `values-tke.yaml` 预设: +# helm upgrade --install cube ./deploy/kubernetes/chart \ +# -f deploy/kubernetes/chart/values-tke.yaml \ +# -f runtime-values.yaml +# # CubeProxy 对外访问 IP(如果不用 LB,填 control 节点的 HostIP) cubeProxy: advertiseIP: "10.0.1.10" @@ -172,7 +201,7 @@ kubectl label nodes \ # 不要打 taint ``` -`runtime-values.yaml` 中把持久化改成 hostPath 以避免 CBS 依赖: +`runtime-values.yaml` 中把持久化改成 hostPath 以避免依赖任何 CSI: ```yaml controlPlane: @@ -209,6 +238,66 @@ redis: enabled: false ``` +### 6.3 Self-hosted 集群 / EKS / GKE / AKS + +Chart 的默认值就是 self-hosted 友好: +- **不创建 StorageClass** — 3 个 PVC(CubeMaster / MySQL / Redis)使用集群 default SC +- **不硬编码 provisioner** — 兼容任何 CSI 后端 + +若集群没有 default SC 或想显式指定,在 `runtime-values.yaml` 里配: + +```yaml +controlPlane: + master: + persistence: + storageClassName: # 例如 EKS gp3 / GKE premium-rwo / AKS managed-csi +mysql: + persistence: + storageClassName: +redis: + persistence: + storageClassName: +``` + +若集群完全没有可用 SC(纯本地实验环境),把 3 个 PVC 换成 hostPath: + +```yaml +controlPlane: + master: + persistence: + hostPath: /data/CubeMaster/storage +mysql: + persistence: + hostPath: /data/mysql +redis: + persistence: + hostPath: /data/redis +``` + +镜像 pull 若需要走内部 mirror,一个开关搞定 Cube-owned 10 个镜像: + +```yaml +global: + imageRegistry: my-mirror.example.com/cubesandbox +``` + +### 6.4 腾讯云 TKE + +用 chart 内置的 preset,一行叠加: + +```bash +helm upgrade --install cube ./deploy/kubernetes/chart \ + -f deploy/kubernetes/chart/values-tke.yaml \ + -f runtime-values.yaml \ + -n cube-system --create-namespace \ + --timeout 90m +``` + +`values-tke.yaml` 会: +- 让 chart 创建 `cube-cbs-wffc` StorageClass(provisioner=`com.tencent.cloud.csi.cbs`) +- 把 3 个 PVC 绑定到该 SC +- 使用 `WaitForFirstConsumer` 避免多可用区 CBS 盘错 zone + --- ## 7. 卸载 diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl index afdc85973..6def0e7cf 100644 --- a/deploy/kubernetes/chart/templates/_helpers.tpl +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -29,10 +29,42 @@ app.kubernetes.io/name: {{ include "cube.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} +{{- /* +Render ":" for an image dict. Legacy helper used everywhere +in the chart. Does NOT apply global.imageRegistry; call sites for +Cube-owned images should use `cube.cubeImage` instead. +*/}} {{- define "cube.image" -}} {{- printf "%s:%s" .repository .tag -}} {{- end -}} +{{- /* +Render ":" for a Cube-owned image with optional +$.Values.global.imageRegistry override applied to the registry portion of +.repository. Call as: + include "cube.cubeImage" (dict "image" .Values.images.master "context" $) +When global.imageRegistry is empty the output is identical to cube.image; +setting it rewrites the leading registry host (segment before the first "/") +so the same chart can be republished to any private registry without editing +each per-image entry. Everything after the first "/" (the repository path) +is preserved. +*/}} +{{- define "cube.cubeImage" -}} +{{- $image := .image -}} +{{- $ctx := .context -}} +{{- $repo := $image.repository -}} +{{- $override := (default (dict) $ctx.Values.global).imageRegistry | default "" -}} +{{- if $override -}} + {{- $parts := splitList "/" $repo -}} + {{- if gt (len $parts) 1 -}} + {{- $repo = printf "%s/%s" (trimSuffix "/" $override) (join "/" (rest $parts)) -}} + {{- else -}} + {{- $repo = printf "%s/%s" (trimSuffix "/" $override) $repo -}} + {{- end -}} +{{- end -}} +{{- printf "%s:%s" $repo $image.tag -}} +{{- end -}} + {{- define "cube.timezoneEnv" -}} {{- with .Values.global.timezone }} - name: TZ diff --git a/deploy/kubernetes/chart/templates/api.yaml b/deploy/kubernetes/chart/templates/api.yaml index e6c0e3311..0c0f2ce16 100644 --- a/deploy/kubernetes/chart/templates/api.yaml +++ b/deploy/kubernetes/chart/templates/api.yaml @@ -25,7 +25,7 @@ spec: {{- include "cube.controlPlanePlacement" . | nindent 6 }} containers: - name: cube-api - image: {{ include "cube.image" .Values.images.api | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.api "context" $) | quote }} imagePullPolicy: {{ .Values.images.api.pullPolicy }} {{- with .Values.controlPlane.api.command }} command: diff --git a/deploy/kubernetes/chart/templates/cubemastercli.yaml b/deploy/kubernetes/chart/templates/cubemastercli.yaml index 44ac05643..52221c19a 100644 --- a/deploy/kubernetes/chart/templates/cubemastercli.yaml +++ b/deploy/kubernetes/chart/templates/cubemastercli.yaml @@ -33,7 +33,7 @@ spec: {{- include "cube.controlPlanePlacement" . | nindent 6 }} containers: - name: cubemastercli - image: {{ include "cube.image" $image | quote }} + image: {{ include "cube.cubeImage" (dict "image" $image "context" $) | quote }} imagePullPolicy: {{ dig "pullPolicy" "Always" $image }} {{- with (dig "command" list $cubemastercli) }} command: diff --git a/deploy/kubernetes/chart/templates/master.yaml b/deploy/kubernetes/chart/templates/master.yaml index 8e1f7cbf2..64848f762 100644 --- a/deploy/kubernetes/chart/templates/master.yaml +++ b/deploy/kubernetes/chart/templates/master.yaml @@ -25,7 +25,7 @@ spec: {{- include "cube.controlPlanePlacement" . | nindent 6 }} containers: - name: cube-master - image: {{ include "cube.image" .Values.images.master | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.master "context" $) | quote }} imagePullPolicy: {{ .Values.images.master.pullPolicy }} {{- with .Values.controlPlane.master.command }} command: diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml index b17fbf929..8a625d188 100644 --- a/deploy/kubernetes/chart/templates/node-daemonset.yaml +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -80,7 +80,7 @@ spec: {{- end }} {{- if .Values.bootstrap.pvmHostKernel.enabled }} - name: pvm-host-bootstrap - image: {{ include "cube.image" .Values.images.pvmHostBootstrap | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.pvmHostBootstrap "context" $) | quote }} imagePullPolicy: {{ .Values.images.pvmHostBootstrap.pullPolicy }} securityContext: privileged: {{ .Values.security.privileged }} @@ -133,7 +133,7 @@ spec: {{- end }} {{- if .Values.bootstrap.nodeInit.enabled }} - name: cube-node-init - image: {{ include "cube.image" .Values.images.nodeInit | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.nodeInit "context" $) | quote }} imagePullPolicy: {{ .Values.images.nodeInit.pullPolicy }} securityContext: privileged: {{ .Values.security.privileged }} @@ -209,7 +209,7 @@ spec: {{- end }} containers: - name: cube-node - image: {{ include "cube.image" .Values.images.node | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.node "context" $) | quote }} imagePullPolicy: {{ .Values.images.node.pullPolicy }} securityContext: privileged: {{ .Values.security.privileged }} @@ -326,7 +326,7 @@ spec: {{- toYaml .Values.cubeNode.resources | nindent 12 }} {{- if .Values.cubeEgress.enabled }} - name: cube-egress - image: {{ include "cube.image" .Values.images.cubeEgress | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.cubeEgress "context" $) | quote }} imagePullPolicy: {{ .Values.images.cubeEgress.pullPolicy }} command: - /bin/bash @@ -394,7 +394,7 @@ spec: {{- end }} {{- if .Values.cubeEgress.network.enabled }} - name: cube-egress-net - image: {{ include "cube.image" .Values.images.cubeEgressNet | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.cubeEgressNet "context" $) | quote }} imagePullPolicy: {{ .Values.images.cubeEgressNet.pullPolicy }} securityContext: privileged: {{ .Values.security.privileged }} diff --git a/deploy/kubernetes/chart/templates/proxy-node.yaml b/deploy/kubernetes/chart/templates/proxy-node.yaml index 1840c15e4..4c5ca963b 100644 --- a/deploy/kubernetes/chart/templates/proxy-node.yaml +++ b/deploy/kubernetes/chart/templates/proxy-node.yaml @@ -132,7 +132,7 @@ spec: {{- include "cube.controlPlanePlacement" . | nindent 6 }} containers: - name: cube-proxy-node - image: {{ include "cube.image" .Values.images.proxyNode | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.proxyNode "context" $) | quote }} imagePullPolicy: {{ .Values.images.proxyNode.pullPolicy }} command: - /bin/sh diff --git a/deploy/kubernetes/chart/templates/storageclass.yaml b/deploy/kubernetes/chart/templates/storageclass.yaml index 76b06b1b3..8c41df1a3 100644 --- a/deploy/kubernetes/chart/templates/storageclass.yaml +++ b/deploy/kubernetes/chart/templates/storageclass.yaml @@ -1,4 +1,10 @@ {{- if .Values.storageClass.create }} +{{- if not .Values.storageClass.name -}} +{{- fail "storageClass.create=true requires storageClass.name (the StorageClass metadata.name to create)" -}} +{{- end -}} +{{- if not .Values.storageClass.provisioner -}} +{{- fail "storageClass.create=true requires storageClass.provisioner (e.g. com.tencent.cloud.csi.cbs for TKE, ebs.csi.aws.com for EKS, or the CSI provisioner of your cluster's default storage)" -}} +{{- end -}} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml index 4a1bed820..543f12c31 100644 --- a/deploy/kubernetes/chart/templates/tests/node-health.yaml +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -131,7 +131,7 @@ spec: {{- end }} containers: - name: cubemastercli - image: {{ include "cube.image" $cubemastercliImage | quote }} + image: {{ include "cube.cubeImage" (dict "image" $cubemastercliImage "context" $) | quote }} imagePullPolicy: {{ dig "pullPolicy" "Always" $cubemastercliImage }} env: {{- include "cube.timezoneEnv" . | nindent 8 }} diff --git a/deploy/kubernetes/chart/templates/webui.yaml b/deploy/kubernetes/chart/templates/webui.yaml index 16d708914..e023138d6 100644 --- a/deploy/kubernetes/chart/templates/webui.yaml +++ b/deploy/kubernetes/chart/templates/webui.yaml @@ -95,7 +95,7 @@ spec: {{- include "cube.controlPlanePlacement" . | nindent 6 }} containers: - name: webui - image: {{ include "cube.image" .Values.images.webui | quote }} + image: {{ include "cube.cubeImage" (dict "image" .Values.images.webui "context" $) | quote }} imagePullPolicy: {{ .Values.images.webui.pullPolicy }} {{- if .Values.global.timezone }} env: diff --git a/deploy/kubernetes/chart/values-tke.yaml b/deploy/kubernetes/chart/values-tke.yaml new file mode 100644 index 000000000..0d5239016 --- /dev/null +++ b/deploy/kubernetes/chart/values-tke.yaml @@ -0,0 +1,38 @@ +# Tencent Cloud TKE preset overrides. +# +# Merge on top of the default chart values with: +# helm upgrade --install cube ./deploy/kubernetes/chart \ +# -f deploy/kubernetes/chart/values-tke.yaml \ +# -f runtime-values.yaml # your own passwords / advertiseIP / TLS +# +# What this file does compared to values.yaml defaults: +# 1. Provisions a chart-owned CBS StorageClass named "cube-cbs-wffc" and +# pins every CubeMaster / MySQL / Redis PVC to it. +# 2. Uses WaitForFirstConsumer volume binding so multi-AZ TKE clusters +# create CBS disks in the same AZ as the scheduled Pod. +# +# Nothing else about the release is TKE-specific — Cube-owned images are +# already published to ccr.ccs.tencentyun.com, so the chart's default image +# repositories work as-is. + +storageClass: + create: true + name: cube-cbs-wffc + provisioner: com.tencent.cloud.csi.cbs + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer + parameters: + type: cbs + +controlPlane: + master: + persistence: + storageClassName: cube-cbs-wffc + +mysql: + persistence: + storageClassName: cube-cbs-wffc + +redis: + persistence: + storageClassName: cube-cbs-wffc diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 734e5d6d8..7078cb4eb 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -11,6 +11,15 @@ global: # kubelet --cluster-domain=. Empty falls back to # cubeNode.dns.clusterDomain, then "cluster.local". clusterDomain: "" + # Optional registry prefix that replaces the registry portion of every + # Cube-owned image below. Leave empty to keep the per-image repository + # exactly as declared. Set to e.g. "my-registry.example.com/cubesandbox" + # to mirror all images into a private registry without editing each entry + # individually. Applies to the pvmHostBootstrap / nodeInit / node / master + # / api / cubemastercli / proxyNode / cubeEgress / cubeEgressNet / webui + # entries but not to third-party images such as coredns / mysql / redis / + # docker-in-docker. + imageRegistry: "" images: # Init container 1: installs/configures PVM host kernel and may reboot the host. @@ -78,14 +87,30 @@ images: imagePullSecrets: [] +# Optional StorageClass created by the chart. Off by default: on a plain +# Kubernetes / self-hosted cluster the PVCs below should use the cluster's +# default StorageClass, so leave `storageClass.create=false` and either leave +# `controlPlane.master.persistence.storageClassName` / `mysql.persistence.storageClassName` +# / `redis.persistence.storageClassName` empty (falls back to the cluster default) +# or point them at an existing StorageClass name. +# +# When you do want the chart to provision its own StorageClass, set +# `create=true` and supply `name` + `provisioner` (+ `parameters` when the +# provisioner needs them). Example for TKE CBS is included below as a comment; +# see values-tke.yaml for a ready-to-merge override. storageClass: - create: true - name: cube-cbs-wffc - provisioner: com.tencent.cloud.csi.cbs + create: false + name: "" + provisioner: "" reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer - parameters: - type: cbs + parameters: {} + # --- Example: Tencent Cloud TKE CBS --- + # create: true + # name: cube-cbs-wffc + # provisioner: com.tencent.cloud.csi.cbs + # parameters: + # type: cbs placement: # Control-plane components run only on dedicated control nodes. @@ -141,7 +166,12 @@ controlPlane: enabled: true hostPath: "" existingClaim: "" - storageClassName: cube-cbs-wffc + # Empty falls back to the cluster's default StorageClass (recommended + # on self-hosted clusters). Set to the name of an explicit + # StorageClass — e.g. "cube-cbs-wffc" for the bundled TKE preset — to + # pin the PVC to a specific provisioner. `existingClaim` and + # `hostPath` override storageClassName when set. + storageClassName: "" size: 20Gi accessModes: - ReadWriteOnce @@ -311,7 +341,10 @@ mysql: existingClaim: "" hostPath: "" size: 20Gi - storageClassName: cube-cbs-wffc + # Empty falls back to the cluster's default StorageClass. Override to + # bind the MySQL data PVC to a specific provisioner (e.g. cube-cbs-wffc + # from values-tke.yaml, or gp3 / pd-ssd / local-path on other clouds). + storageClassName: "" accessModes: - ReadWriteOnce resources: {} @@ -334,7 +367,9 @@ redis: existingClaim: "" hostPath: "" size: 10Gi - storageClassName: cube-cbs-wffc + # Empty falls back to the cluster's default StorageClass. Override to + # bind the Redis data PVC to a specific provisioner. + storageClassName: "" accessModes: - ReadWriteOnce resources: {} From 2fc8fc32e3adba9dc6e79c8aa45b8f1c06f9b397 Mon Sep 17 00:00:00 2001 From: jinlong Date: Sun, 12 Jul 2026 12:03:39 +0800 Subject: [PATCH 7/9] feat(kubernetes): support non-disruptive compute-node image upgrades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make cube-node DaemonSet Pod rebuilds (image upgrades) transparent to existing sandboxes, mirroring the one-click install.sh upgrade model (stop services, selectively replace package trees, restart without killing shims/microVMs). Core mechanism: - stage-toolbox initContainer: sync image toolbox binaries onto a persistent hostPath (/usr/local/services/cubetoolbox) with selective overwrite, preserving runtime dirs (cube-snapshot, cubebox_os_image, cubeletmnt, cube-vs) — same as one-click install.sh - HostPath toolbox: cube-node mounts the host copy back at the same path, so shim/runtime binaries live on a filesystem that is never unmounted when the Pod overlay is deleted - Persist shim/VMM sockets: /run/containerd and /run/vc are hostPath- backed (data-cubelet/run/containerd, data-cubelet/run/vc), not ephemeral container overlay; LoadExistingShims can reconnect - Bind-mount /data/cubelet/state: prevents cubelet default tmpfs from eating bootstrap.json/address on Pod deletion - preStop and entrypoint cleanup: only signal cubelet/network-agent, never containerd-shim-cube-rs/cube-runtime/VMM processes - Readiness gate: exec probe verifies cubelet :9999 + network-agent /readyz + sock existence before marking Ready - RollingUpdate maxUnavailable: 1 for one-node-at-a-time rollout Image changes: - New stage-toolbox.sh: standalone script for selective toolbox sync - cube-node Dockerfile: ship stage-toolbox.sh and chmod +x - build-cube-images.sh: include stage-toolbox.sh in both build paths - cube-node-entrypoint.sh: bind-mount state, comment process kill safety Docs: - New UPGRADE.md: full upgrade runbook with mermaid sequence diagram - ARCHITECTURE.md: document stage-toolbox, toolbox/run paths, probes - FAQ.md G1: explain non-disruptive upgrade model and OnDelete option - QUICKSTART.md: add UPGRADE.md reference and toolbox cleanup Signed-off-by: jinlong --- deploy/kubernetes/chart/docs/ARCHITECTURE.md | 8 +- deploy/kubernetes/chart/docs/FAQ.md | 30 +++- deploy/kubernetes/chart/docs/QUICKSTART.md | 3 +- deploy/kubernetes/chart/docs/UPGRADE.md | 135 +++++++++++++++++ .../chart/templates/node-daemonset.yaml | 137 +++++++++++++++++- deploy/kubernetes/chart/values.yaml | 42 +++++- deploy/kubernetes/images/build-cube-images.sh | 7 +- deploy/kubernetes/images/cube-node/Dockerfile | 2 + .../images/scripts/cube-node-entrypoint.sh | 21 ++- .../images/scripts/stage-toolbox.sh | 95 ++++++++++++ 10 files changed, 462 insertions(+), 18 deletions(-) create mode 100644 deploy/kubernetes/chart/docs/UPGRADE.md create mode 100644 deploy/kubernetes/images/scripts/stage-toolbox.sh diff --git a/deploy/kubernetes/chart/docs/ARCHITECTURE.md b/deploy/kubernetes/chart/docs/ARCHITECTURE.md index 95d0b9a22..17006649f 100644 --- a/deploy/kubernetes/chart/docs/ARCHITECTURE.md +++ b/deploy/kubernetes/chart/docs/ARCHITECTURE.md @@ -97,9 +97,10 @@ flowchart TB | 容器 | 类型 | 镜像 | 职责 | | --- | --- | --- | --- | | `wait-cube-dns` | Init Container | `cubeNode.dns.checkImage` | 等待 node-local `cube-dns` 可用，并验证 `cube.app`、wildcard、Kubernetes Service 域名解析 | +| `stage-toolbox` | Init Container | `images.node` | 将镜像内 `/usr/local/services/cubetoolbox` **覆盖式**同步到宿主机同名 hostPath（与一键包一致），使 shim/runtime 二进制在 Pod 重建后仍可被存量进程持有 | | `pvm-host-bootstrap` | Init Container，可选 | `images.pvmHostBootstrap` | 安装/配置 PVM host kernel，必要时协调节点重启 | | `cube-node-init` | Init Container | `images.nodeInit` | 节点预检和准备：KVM、XFS、内存、glibc、cgroup、cubecow 依赖、CIDR 冲突、CubeMaster 连通性 | -| `cube-node` | 主容器 | `images.node` | 运行 cubelet 和 network-agent；选择 guest kernel；向 CubeMaster 注册节点 | +| `cube-node` | 主容器 | `images.node` | 运行 cubelet 和 network-agent；选择 guest kernel；向 CubeMaster 注册节点；挂载 hostPath toolbox | | `cube-egress` | Sidecar | `images.cubeEgress` | 透明出站代理，提供 loopback admin health | | `cube-egress-net` | Sidecar | `images.cubeEgressNet` | 管理 host network namespace 中的 TPROXY、ip rule、sysctl 规则 | @@ -107,6 +108,7 @@ flowchart TB | hostPath | 用途 | | --- | --- | +| `/usr/local/services/cubetoolbox` | 镜像 toolbox 的宿主机副本（与一键包 INSTALL_PREFIX 相同；Cubelet / network-agent / cube-shim / guest kernel）；升级时覆盖，存量 shim 靠 inode 存活 | | `/data/cubelet` | cubelet 数据和 `cubelet.sock` | | `/tmp/cube` | network-agent gRPC socket | | `/data/cube-shim` | cube runtime/shim 运行数据 | @@ -275,10 +277,12 @@ sequenceDiagram `cube-node` 使用 startupProbe、readinessProbe、livenessProbe： - startupProbe：等待 cubelet 9999 启动，避免慢启动期间被 liveness 提前杀死。 -- readiness/liveness：检查 cubelet 9999。 +- readiness/liveness：默认 readiness 为 exec 门禁（9999 + network-agent `/readyz` + sock），确保 `LoadExistingShims` / network-agent `recover()` 完成后再 Ready；liveness 仍检查 cubelet 9999。 - `cube-egress`：检查 `127.0.0.1:9090/admin/v1/health`。 - `cube-egress-net`：检查 `cube-dev`、ip rule、table 100 local route、mangle `TRANSPROXY` 80/443 规则。 +计算面镜像升级（不杀存量沙箱）见 [`UPGRADE.md`](UPGRADE.md)。 + ### 4.4 节点注册与健康链路 ```mermaid diff --git a/deploy/kubernetes/chart/docs/FAQ.md b/deploy/kubernetes/chart/docs/FAQ.md index 0c15b5a67..5d44a36ba 100644 --- a/deploy/kubernetes/chart/docs/FAQ.md +++ b/deploy/kubernetes/chart/docs/FAQ.md @@ -347,9 +347,22 @@ kubectl -n cube-system logs -c cube-egress-net --tail=100 ## G. 升级、回滚与卸载 -### G1. `helm upgrade` 后 cube-node DaemonSet 全部重启,会不会中断服务? +### G1. `helm upgrade` 后 cube-node DaemonSet 滚动重建,会不会中断存量沙箱? -会。默认策略是 `RollingUpdate`,节点上活跃的 sandbox 会被中断。生产环境推荐: +**默认不会中断存量沙箱**（cubelet / network-agent 控制面组件升级场景）。 + +机制对齐一键包「停服务、不杀 shim、覆盖二进制、重启」： + +1. `stage-toolbox` initContainer 把镜像内 toolbox **按目录选择性**同步到宿主机 `/usr/local/services/cubetoolbox`（对齐 one-click：只换软件包，保留 `cube-snapshot` / `cubebox_os_image` / `cubeletmnt` / `cube-vs`），主容器再挂回同路径。删 Pod 卸载的是容器 overlay，不是该 hostPath；存量 shim 继续持有已 unlink 的旧 inode。 +2. shim ttrpc / VMM socket 目录也是 hostPath：容器 `/run/containerd` → 宿主机 `/data/cubelet/run/containerd`，容器 `/run/vc` → 宿主机 `/data/cubelet/run/vc`（**不是**节点真实的 `/run/containerd`）。否则 Pod 重建后 `LoadExistingShims` 无法 dial `unix:///run/containerd/s/{hash}`。 +3. `/data/cubelet/state` 通过启动前 `mount --bind` 留在 hostPath 上，避免 cubelet 默认 state tmpfs 在 Pod 删除后带走 `bootstrap.json`/`address`。 +4. `preStop` / entrypoint 只 TERM **cubelet** 与 **network-agent**，不匹配 `containerd-shim-cube-rs` / `cube-runtime`。 +5. 新 Pod 启动后 cubelet `LoadExistingShims` + `RecoverAllCubebox`、network-agent `recover()` 重连。 +6. 默认 `updateStrategy.rollingUpdate.maxUnavailable: 1`，逐节点滚动；readiness 在 recover 完成（9999 + network-agent readyz + sock）后才 Ready。 + +完整步骤见 [`UPGRADE.md`](UPGRADE.md)。 + +若仍希望完全手工控制节奏： ```yaml cubeNode: @@ -357,15 +370,16 @@ cubeNode: type: OnDelete ``` -然后手工按节点滚动: - ```bash -# 1. 把 node 上的 sandbox pause/迁移 -# 2. 删除 cube-node Pod,DaemonSet 会拉起新版本 kubectl -n cube-system delete pod -l app.kubernetes.io/component=cube-node --field-selector spec.nodeName= -# 3. 等新 Pod Ready 后继续下一个节点 ``` +**注意**： + +- 升级的是控制组件镜像；存量沙箱仍跑**旧** shim/runtime 二进制（N/N-1），直到沙箱自然销毁。 +- 不要在升级路径执行 `cubecli unsafe init` / `InitHost`，那会 Destroy 全部沙箱。 +- 升级窗口内**新建**沙箱可能短暂失败并被 CubeMaster reschedule，见 UPGRADE.md。 + ### G2. `helm rollback` 会回滚 host kernel 吗? **不会**。Helm 只管 K8s 资源。Host kernel、GRUB、`/etc/fstab`、XFS 挂载点等由 `pvm-host-bootstrap` 做的节点级修改,rollback **完全不动**。生产环境请提前准备 host kernel 回滚 runbook(RPM 降级、切换 GRUB 默认项、reboot 验证)。 @@ -376,7 +390,7 @@ kubectl -n cube-system delete pod -l app.kubernetes.io/component=cube-node --fie ```bash # 在每个 compute 节点执行 -sudo rm -rf /data/cubelet /data/cube-shim /data/snapshot_pack /data/log /tmp/cube +sudo rm -rf /data/cubelet /data/cube-shim /data/snapshot_pack /data/log /usr/local/services/cubetoolbox /tmp/cube # 如果不再需要 PVM host kernel,还需要: # 1. 卸载 kernel 包 # 2. 更新 GRUB 默认引导 diff --git a/deploy/kubernetes/chart/docs/QUICKSTART.md b/deploy/kubernetes/chart/docs/QUICKSTART.md index b51190ff3..86dbad39f 100644 --- a/deploy/kubernetes/chart/docs/QUICKSTART.md +++ b/deploy/kubernetes/chart/docs/QUICKSTART.md @@ -311,7 +311,7 @@ kubectl delete namespace cube-system - 节点 label / taint(chart 不管理) - 外部 MySQL / Redis 数据 -- compute 节点上的 hostPath 数据:`/data/CubeMaster/storage`, `/data/cubelet`, `/data/cube-shim`, `/data/snapshot_pack`, `/data/log` +- compute 节点上的 hostPath 数据:`/data/CubeMaster/storage`, `/data/cubelet`, `/data/cube-shim`, `/data/snapshot_pack`, `/usr/local/services/cubetoolbox`, `/data/log` - PVM host kernel 修改(GRUB、`/boot`、initramfs)—— 需要按平台 runbook 回滚 - 外部 DNS / LB 记录 @@ -320,5 +320,6 @@ kubectl delete namespace cube-system ## 8. 下一步 - 阅读 [`ARCHITECTURE.md`](ARCHITECTURE.md) 深入理解组件关系和数据流 +- 阅读 [`UPGRADE.md`](UPGRADE.md) 了解计算面镜像升级（不杀存量沙箱） - 阅读 [`FAQ.md`](FAQ.md) 应对常见部署 / 运行问题 - 生产环境 TLS、DNS、监控、备份策略请参考主 README 相应章节 diff --git a/deploy/kubernetes/chart/docs/UPGRADE.md b/deploy/kubernetes/chart/docs/UPGRADE.md new file mode 100644 index 000000000..c86d3d295 --- /dev/null +++ b/deploy/kubernetes/chart/docs/UPGRADE.md @@ -0,0 +1,135 @@ +# 计算面镜像升级 Runbook + +本文说明如何通过 **升级 `cube-node` 镜像 + Helm / DaemonSet 滚动重建 Pod**，在不销毁存量沙箱的前提下升级计算节点上的 `cubelet` / `network-agent` 等控制组件。 + +## 原理（对齐一键包） + +一键包升级是：停 systemd 服务（`KillMode=process` 不杀 shim）→ `rm -rf` 旧安装目录 → 拷新二进制 → 重启。存量 `containerd-shim-cube-rs` / microVM 靠「进程持有已 unlink 的 inode + 宿主机文件系统从不卸载」继续跑。 + +Chart 侧等价条件： + +1. **toolbox 落在 hostPath** `/usr/local/services/cubetoolbox`（与一键包 `INSTALL_PREFIX` 相同；由 `stage-toolbox` initContainer 每次启动把镜像内**软件包目录**同步到该 hostPath）。与 one-click `install.sh` 一样做**选择性覆盖**，保留运行期目录：`cube-snapshot`、`cubebox_os_image`、`cubeletmnt`、`cube-vs`（不会整棵 `rm -rf` toolbox）。 +2. 主容器把该 hostPath 挂回容器内 `/usr/local/services/cubetoolbox`，cubelet / shim 路径与一键包一致。 +3. **shim / VMM socket 目录落在 hostPath**（不能用容器 ephemeral `/run`）： + - 容器内 `/run/containerd` ← 宿主机 `hostPaths.runContainerd`（默认 `/data/cubelet/run/containerd`）。`containerd-shim` 的 `SOCKET_ROOT` 硬编码为 `/run/containerd`，`LoadExistingShims` 通过 `bootstrap.json`/`address` 里的 `unix:///run/containerd/s/{hash}` dial 存量 shim。**禁止**把宿主机真实的 `/run/containerd`（kubelet/containerd 占用）挂进来。 + - 容器内 `/run/vc` ← 宿主机 `hostPaths.runVc`（默认 `/data/cubelet/run/vc`）。CubeShim `VM_PATH=/run/vc/vm/`，含 `chapi` / `cube.sock` 等。 +4. **`/data/cubelet/state` 必须落在 hostPath 磁盘上，不能被 cubelet 默认的 state tmpfs 盖住**。cubelet `mountTmpfsDir` 会在私有 mount NS 里给 state 挂 500Mi tmpfs；Pod 删除后该 NS/tmpfs 消失，`bootstrap.json`/`address` 随之丢失，即使 shim 进程与 socket 仍在也无法 `LoadExistingShims`。Chart 在启动 entrypoint 前对 `/data/cubelet/state` 做 `mount --bind`，使 `mountTmpfsDir` 直接跳过。 +5. Pod 删除时 `preStop` / entrypoint `cleanup` **只停 cubelet 与 network-agent**，不碰 shim。 +6. 新 Pod 启动后 cubelet `LoadExistingShims` + `RecoverAllCubebox`、network-agent `recover()` 重连存量沙箱。 +7. **绝不**在升级路径调用 `InitHost` / `cubecli unsafe init`（会 Destroy 全部沙箱）。 + +```mermaid +sequenceDiagram + participant Helm as helm_upgrade + participant DS as cube_node_DaemonSet + participant Old as old_cube_node_Pod + participant Shim as shim_microVM + participant New as new_cube_node_Pod + participant Host as hostPath_toolbox + + Helm->>DS: bump images.node tag + DS->>Old: RollingUpdate maxUnavailable=1 + Old->>Old: preStop TERM cubelet/network-agent only + Note over Shim: stays alive on host cgroup + Old-->>DS: Pod deleted (overlay unmounted) + Note over Host: host /usr/local/services/cubetoolbox unlinked; shim keeps old inodes + DS->>New: create Pod + New->>Host: stage-toolbox rm+cp new binaries onto hostPath + New->>New: start network-agent recover + cubelet LoadExistingShims + New->>Shim: reconnect ttrpc +``` + +## 升级步骤 + +### 1. 准备新镜像 + +构建并推送新的 `cube-node`（以及如需一并升级的 `cube-egress` 等）镜像，更新 `runtime-values.yaml` 中的 tag： + +```yaml +images: + node: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-node + tag: v0.5.1 # 示例 +``` + +### 2. Helm 升级 + +```bash +helm upgrade cube ./deploy/kubernetes/chart \ + -n cube-system \ + -f ./deploy/kubernetes/chart/runtime-values.yaml \ + --wait \ + --timeout 90m +``` + +默认 `cubeNode.updateStrategy` 为： + +```yaml +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 +``` + +一次只重建一台计算节点上的 `cube-node` Pod。 + +### 3. 观察滚动 + +```bash +kubectl get ds -n cube-system cube-node -w +kubectl get pods -n cube-system -l app.kubernetes.io/component=cube-node -o wide -w +``` + +单节点 Ready 条件（readiness exec）： + +- cubelet `:9999` 监听 +- network-agent `/readyz` 通过 +- `/data/cubelet/cubelet.sock` 与 `/tmp/cube/network-agent-grpc.sock` 存在 + +### 4. 验证沙箱仍在 + +```bash +kubectl exec -n cube-system deploy/cube-cubemastercli -- \ + sh -lc 'cubemastercli --address "$CUBEMASTERCLI_ADDRESS" --port "$CUBEMASTERCLI_PORT" list --all -w' + +# 在计算节点上确认 shim pid 在 Pod 重建前后不变（升级前先记下） +kubectl exec -n cube-system -c cube-node -- \ + sh -lc 'pgrep -af containerd-shim-cube-rs; ls -l /proc/$(pgrep -n containerd-shim-cube-rs)/exe' +``` + +## 手动逐节点（OnDelete） + +若希望完全人工控制节奏： + +```yaml +cubeNode: + updateStrategy: + type: OnDelete +``` + +然后： + +```bash +helm upgrade ... # 只更新 DaemonSet 模板，不自动删 Pod +kubectl -n cube-system delete pod -l app.kubernetes.io/component=cube-node \ + --field-selector spec.nodeName= +# 等 Ready 后再删下一台 +``` + +## 升级窗口内新建沙箱 + +旧 Pod 已停、新 Pod 尚未 Ready 的短暂窗口内，CubeMaster 心跳可能尚未过期（默认约 40s），仍可能把**新**沙箱调度到该节点；创建会失败并由 CubeMaster **自动 reschedule** 到其他节点。存量沙箱不受影响。当前不提供控制面 cordon；若新建延迟敏感，可后续加节点摘流。 + +## 卸载时额外清理 + +`helm uninstall` 不会删除 hostPath。除 `/data/cubelet` 等外，还需按需清理： + +```bash +sudo rm -rf /usr/local/services/cubetoolbox +``` + +## 相关文档 + +- [ARCHITECTURE.md](ARCHITECTURE.md) — Big Pod 与启动流程 +- [FAQ.md](FAQ.md) G1 — 升级是否中断沙箱 +- 一键包对照：`deploy/one-click/install.sh` upgrade 路径、`cube-sandbox-cubelet.service` 的 `KillMode=process` diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml index 8a625d188..e97e8c8fd 100644 --- a/deploy/kubernetes/chart/templates/node-daemonset.yaml +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -27,6 +27,7 @@ spec: automountServiceAccountToken: true hostNetwork: {{ .Values.security.hostNetwork }} hostPID: {{ .Values.security.hostPID }} + terminationGracePeriodSeconds: {{ .Values.cubeNode.terminationGracePeriodSeconds }} {{- if .Values.cubeNode.dns.useCubeDns }} dnsPolicy: None dnsConfig: @@ -78,6 +79,84 @@ spec: echo "cube dns is not ready at ${CUBE_DNS_NAMESERVER}" >&2 exit 1 {{- end }} + # Stage image toolbox onto a hostPath before any other compute init so + # cube-shim / cube-runtime binaries survive Pod rebuilds (overlay unmount). + # Inline script so this works with existing cube-node images that do not + # yet ship /usr/local/bin/stage-toolbox.sh; the image still provides the + # toolbox tree under /usr/local/services/cubetoolbox. + - name: stage-toolbox + image: {{ include "cube.cubeImage" (dict "image" .Values.images.node "context" $) | quote }} + imagePullPolicy: {{ .Values.images.node.pullPolicy }} + command: + - /bin/bash + - -ec + - | + set -euo pipefail + IMAGE_TOOLBOX_ROOT="${IMAGE_TOOLBOX_ROOT:-/usr/local/services/cubetoolbox}" + HOST_TOOLBOX_ROOT="${HOST_TOOLBOX_ROOT:-/host-toolbox}" + # Align with one-click install.sh: do not wipe runtime dirs under + # the install prefix (cube-snapshot / cubebox_os_image / ...). + PRESERVE_NAMES=(cube-snapshot cubebox_os_image cubeletmnt cube-vs) + log() { printf '[stage-toolbox] %s\n' "$*"; } + fail() { printf '[stage-toolbox] ERROR: %s\n' "$*" >&2; exit 1; } + should_preserve() { + local name="$1" preserved + for preserved in "${PRESERVE_NAMES[@]}"; do + [[ "${name}" == "${preserved}" ]] && return 0 + done + return 1 + } + [[ -d "${IMAGE_TOOLBOX_ROOT}" ]] || fail "image toolbox missing: ${IMAGE_TOOLBOX_ROOT}" + [[ -n "${HOST_TOOLBOX_ROOT}" && "${HOST_TOOLBOX_ROOT}" != "/" ]] || fail "invalid HOST_TOOLBOX_ROOT" + [[ "${HOST_TOOLBOX_ROOT}" != "${IMAGE_TOOLBOX_ROOT}" ]] || fail "HOST_TOOLBOX_ROOT must differ from IMAGE_TOOLBOX_ROOT" + mkdir -p "${HOST_TOOLBOX_ROOT}" + log "staging ${IMAGE_TOOLBOX_ROOT} -> ${HOST_TOOLBOX_ROOT} (selective overwrite)" + while IFS= read -r -d '' entry; do + name="$(basename "${entry}")" + if should_preserve "${name}"; then + log "preserving ${name}" + continue + fi + rm -rf "${entry}" + done < <(find "${HOST_TOOLBOX_ROOT}" -mindepth 1 -maxdepth 1 -print0) + while IFS= read -r -d '' entry; do + name="$(basename "${entry}")" + dest="${HOST_TOOLBOX_ROOT}/${name}" + if should_preserve "${name}" && [[ -e "${dest}" ]]; then + log "skipping image copy over preserved ${name}" + continue + fi + rm -rf "${dest}" + cp -a "${entry}" "${dest}" + done < <(find "${IMAGE_TOOLBOX_ROOT}" -mindepth 1 -maxdepth 1 -print0) + chmod +x \ + "${HOST_TOOLBOX_ROOT}/Cubelet/bin/cubelet" \ + "${HOST_TOOLBOX_ROOT}/Cubelet/bin/cubecli" \ + "${HOST_TOOLBOX_ROOT}/network-agent/bin/network-agent" \ + "${HOST_TOOLBOX_ROOT}/network-agent/bin/cubevsmapdump" \ + "${HOST_TOOLBOX_ROOT}/cube-shim/bin/cube-runtime" \ + "${HOST_TOOLBOX_ROOT}/cube-shim/bin/containerd-shim-cube-rs" \ + 2>/dev/null || true + for required in \ + Cubelet/bin/cubelet \ + network-agent/bin/network-agent \ + cube-shim/bin/containerd-shim-cube-rs \ + cube-shim/bin/cube-runtime + do + [[ -x "${HOST_TOOLBOX_ROOT}/${required}" ]] || fail "missing executable after stage: ${required}" + done + log "toolbox staged successfully at ${HOST_TOOLBOX_ROOT}" + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: IMAGE_TOOLBOX_ROOT + value: /usr/local/services/cubetoolbox + - name: HOST_TOOLBOX_ROOT + value: /host-toolbox + volumeMounts: + - name: toolbox + mountPath: /host-toolbox + securityContext: + privileged: {{ .Values.security.privileged }} {{- if .Values.bootstrap.pvmHostKernel.enabled }} - name: pvm-host-bootstrap image: {{ include "cube.cubeImage" (dict "image" .Values.images.pvmHostBootstrap "context" $) | quote }} @@ -211,6 +290,22 @@ spec: - name: cube-node image: {{ include "cube.cubeImage" (dict "image" .Values.images.node "context" $) | quote }} imagePullPolicy: {{ .Values.images.node.pullPolicy }} + # Keep /data/cubelet/state on the dataCubelet hostPath. cubelet's + # default mountTmpfsDir() puts a 500Mi tmpfs over state inside its + # private mount NS; that tmpfs (and shim bundle metadata) disappears + # when the Pod is deleted, so LoadExistingShims cannot reconnect. + # Bind-mounting state first makes mountinfo.Mounted(state)=true in the + # NS cubelet clones, so mountTmpfsDir is a no-op. + command: + - /bin/bash + - -ec + - | + set -euo pipefail + mkdir -p /data/cubelet/state + if ! findmnt --mountpoint /data/cubelet/state >/dev/null 2>&1; then + mount --bind /data/cubelet/state /data/cubelet/state + fi + exec /usr/local/bin/cube-node-entrypoint.sh securityContext: privileged: {{ .Values.security.privileged }} capabilities: @@ -227,6 +322,8 @@ spec: value: node - name: CUBE_MASTER_ENDPOINT value: {{ include "cube.masterEndpoint" . | quote }} + - name: TOOLBOX_ROOT + value: /usr/local/services/cubetoolbox - name: CUBE_SANDBOX_NODE_IP {{- if eq .Values.cubeNode.ipSource "statusHostIP" }} valueFrom: @@ -263,6 +360,19 @@ spec: containerPort: 9966 - name: network-agent containerPort: 19090 + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -ec + - | + # Graceful stop for control processes only. Never match + # containerd-shim-cube-rs / cube-runtime / VMM — those must + # survive Pod rebuilds so existing sandboxes keep running. + pkill -TERM -f '/usr/local/services/cubetoolbox/Cubelet/bin/cubelet --config' || true + pkill -TERM -f '/usr/local/services/cubetoolbox/network-agent/bin/network-agent' || true + sleep 5 {{- if .Values.cubeNode.probes.startup.enabled }} startupProbe: {{- if .Values.cubeNode.probes.startup.exec }} @@ -279,7 +389,8 @@ spec: readinessProbe: {{- if .Values.cubeNode.probes.readiness.exec }} exec: - {{- toYaml .Values.cubeNode.probes.readiness.exec | nindent 14 }} + command: + {{- toYaml .Values.cubeNode.probes.readiness.exec | nindent 16 }} {{- else }} tcpSocket: port: {{ .Values.cubeNode.probes.readiness.tcpSocketPort }} @@ -292,7 +403,8 @@ spec: livenessProbe: {{- if .Values.cubeNode.probes.liveness.exec }} exec: - {{- toYaml .Values.cubeNode.probes.liveness.exec | nindent 14 }} + command: + {{- toYaml .Values.cubeNode.probes.liveness.exec | nindent 16 }} {{- else }} tcpSocket: port: {{ .Values.cubeNode.probes.liveness.tcpSocketPort }} @@ -309,6 +421,8 @@ spec: - name: lib-modules mountPath: /lib/modules readOnly: true + - name: toolbox + mountPath: /usr/local/services/cubetoolbox - name: data-cubelet mountPath: {{ .Values.hostPaths.dataCubelet }} mountPropagation: Bidirectional @@ -322,6 +436,13 @@ spec: - name: tmp-cube mountPath: {{ .Values.hostPaths.tmpCube }} mountPropagation: Bidirectional + # Persist shim ttrpc sockets (/run/containerd/s/*) across Pod + # rebuilds so LoadExistingShims can reconnect. See hostPaths.runContainerd. + - name: run-containerd + mountPath: /run/containerd + # Persist CH API / vsock sockets (/run/vc/vm/*). See hostPaths.runVc. + - name: run-vc + mountPath: /run/vc resources: {{- toYaml .Values.cubeNode.resources | nindent 12 }} {{- if .Values.cubeEgress.enabled }} @@ -450,6 +571,10 @@ spec: hostPath: path: {{ .Values.hostPaths.root }} type: Directory + - name: toolbox + hostPath: + path: {{ .Values.hostPaths.toolbox }} + type: DirectoryOrCreate - name: dev hostPath: path: {{ .Values.hostPaths.dev }} @@ -482,6 +607,14 @@ spec: hostPath: path: {{ .Values.hostPaths.tmpCube }} type: DirectoryOrCreate + - name: run-containerd + hostPath: + path: {{ .Values.hostPaths.runContainerd }} + type: DirectoryOrCreate + - name: run-vc + hostPath: + path: {{ .Values.hostPaths.runVc }} + type: DirectoryOrCreate {{- if .Values.cubeEgress.enabled }} - name: cube-egress-ca secret: diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 7078cb4eb..96b977957 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -377,8 +377,16 @@ redis: cubeNode: enabled: true podAnnotations: {} + # RollingUpdate with maxUnavailable=1 upgrades one compute node at a time so + # the CubeMaster scheduling window without a live cubelet stays small. + # Switch to OnDelete for fully manual per-node rollouts. updateStrategy: type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + # Give cubelet/network-agent time to exit cleanly on Pod delete. preStop only + # signals those two processes; containerd-shim-cube-rs / microVMs are left alone. + terminationGracePeriodSeconds: 30 # How CUBE_SANDBOX_NODE_IP is populated. Currently statusHostIP or static are supported. ipSource: statusHostIP @@ -440,7 +448,18 @@ cubeNode: readiness: enabled: true tcpSocketPort: 9999 - exec: [] + # Recovery gate: cubelet must be listening AND network-agent must have + # finished its TAP/state recover() before the Pod is marked Ready. This + # keeps RollingUpdate from advancing while the node is still reconnecting + # to existing shims after an image upgrade. + exec: + - /bin/bash + - -ec + - | + ss -lntp 2>/dev/null | grep -q ':9999' || exit 1 + curl -fsS http://127.0.0.1:19090/readyz >/dev/null + test -S /data/cubelet/cubelet.sock + test -S /tmp/cube/network-agent-grpc.sock initialDelaySeconds: 10 periodSeconds: 10 failureThreshold: 6 @@ -720,6 +739,27 @@ hostPaths: dataCubeShim: /data/cube-shim dataSnapshotPack: /data/snapshot_pack tmpCube: /tmp/cube + # Host-persistent copy of the cube-node image toolbox (Cubelet, network-agent, + # cube-shim, guest kernels, ...). Same path as one-click INSTALL_PREFIX so + # K8s and one-click share the KillMode=process / unlink-inode upgrade model. + # The stage-toolbox initContainer selectively replaces package trees on every + # Pod start (aligned with one-click install.sh), while preserving runtime + # dirs such as cube-snapshot / cubebox_os_image / cubeletmnt / cube-vs. + # Running shims keep their already-mapped inodes across upgrades because + # this hostPath is never unmounted when the Pod is deleted. + toolbox: /usr/local/services/cubetoolbox + # Shim ttrpc listen sockets: containerd-shim SOCKET_ROOT is hardcoded to + # /run/containerd, and LoadExistingShims dials unix:///run/containerd/s/{hash} + # from bootstrap.json/address after Pod rebuild. Do NOT point this hostPath + # at the node's real /run/containerd (owned by kubelet/containerd); use a + # dedicated directory under dataCubelet so cube-node's mount NS gets a + # persistent /run/containerd without colliding with the node runtime. + runContainerd: /data/cubelet/run/containerd + # Cloud Hypervisor API + vsock sockets (CubeShim VM_PATH=/run/vc/vm/). + # Surviving shims and post-reconnect cubelet paths such as + # /run/vc/vm/{id}/chapi and /run/vc/vm/{id}/cube.sock need this tree to + # remain on a filesystem that is not torn down with the Pod overlay. + runVc: /data/cubelet/run/vc serviceAccount: create: true diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh index 76e0517f9..45d00967c 100755 --- a/deploy/kubernetes/images/build-cube-images.sh +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -357,15 +357,16 @@ build_cube_node_from_base_image() { [[ -n "${CUBE_NODE_BASE_IMAGE}" ]] || fail "CUBE_NODE_BASE_IMAGE is required" ctx="$(prepare_context cube-node)" - copy_scripts "${ctx}" cube-node-entrypoint.sh + copy_scripts "${ctx}" cube-node-entrypoint.sh stage-toolbox.sh dockerfile="${ctx}/Dockerfile.rebase" cat > "${dockerfile}" </dev/null 2>&1; then + mount --bind /data/cubelet/state /data/cubelet/state + log "bound /data/cubelet/state to hostPath (skip state tmpfs)" +fi rm -f \ /tmp/cube/network-agent.sock \ @@ -192,6 +203,12 @@ rm -f \ /tmp/cube/network-agent-tap.sock \ || true +# stop_stale_processes intentionally matches ONLY cubelet / network-agent. +# Never broaden these patterns to containerd-shim-cube-rs, cube-runtime, or +# VMM processes: those must survive Pod rebuilds so existing sandboxes keep +# running across image upgrades (one-click KillMode=process equivalent). +# Also never call InitHost / cubecli unsafe init from this entrypoint — that +# path destroys every sandbox on the node. stop_stale_processes() { local name="$1" local pattern="$2" @@ -215,6 +232,8 @@ stop_stale_processes network-agent "${NETWORK_AGENT_BIN}" stop_stale_processes cubelet "${CUBELET_BIN}" cleanup() { + # TERM/INT/HUP/EXIT: stop only the control processes we started. Do not + # pkill shim/runtime — Pod deletion must leave microVMs alive for recover. if [[ -n "${NETWORK_AGENT_PID:-}" ]]; then kill "${NETWORK_AGENT_PID}" 2>/dev/null || true fi diff --git a/deploy/kubernetes/images/scripts/stage-toolbox.sh b/deploy/kubernetes/images/scripts/stage-toolbox.sh new file mode 100644 index 000000000..b80e00b86 --- /dev/null +++ b/deploy/kubernetes/images/scripts/stage-toolbox.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2026 Tencent. All rights reserved. +# +# Stage the cube-node image toolbox onto a hostPath directory so that +# containerd-shim-cube-rs / cube-runtime binaries live on a filesystem that is +# never unmounted when the cube-node Pod is deleted or upgraded. +# +# This mirrors one-click install.sh upgrade semantics: +# stop services -> selectively rm package trees -> cp new binaries -> restart +# Runtime data under the install prefix is left alone (same as one-click, which +# does NOT rm cube-snapshot / cubebox_os_image / cubeletmnt). +# Running shims keep their already-mapped inodes after unlink; the hostPath +# filesystem itself is never torn down, so microVMs survive Pod rebuilds. +set -euo pipefail + +IMAGE_TOOLBOX_ROOT="${IMAGE_TOOLBOX_ROOT:-/usr/local/services/cubetoolbox}" +HOST_TOOLBOX_ROOT="${HOST_TOOLBOX_ROOT:-/host-toolbox}" + +# Top-level names that one-click install.sh leaves across upgrade. Chart must +# not wipe them when refreshing package binaries from the cube-node image. +PRESERVE_NAMES=( + cube-snapshot + cubebox_os_image + cubeletmnt + cube-vs +) + +log() { printf '[stage-toolbox] %s\n' "$*"; } +fail() { printf '[stage-toolbox] ERROR: %s\n' "$*" >&2; exit 1; } + +should_preserve() { + local name="$1" + local preserved + for preserved in "${PRESERVE_NAMES[@]}"; do + if [[ "${name}" == "${preserved}" ]]; then + return 0 + fi + done + return 1 +} + +[[ -d "${IMAGE_TOOLBOX_ROOT}" ]] || fail "image toolbox missing: ${IMAGE_TOOLBOX_ROOT}" +[[ -n "${HOST_TOOLBOX_ROOT}" ]] || fail "HOST_TOOLBOX_ROOT is empty" +[[ "${HOST_TOOLBOX_ROOT}" != "/" ]] || fail "refusing to stage onto /" +[[ "${HOST_TOOLBOX_ROOT}" != "${IMAGE_TOOLBOX_ROOT}" ]] || fail "HOST_TOOLBOX_ROOT must differ from IMAGE_TOOLBOX_ROOT so the image copy stays visible" + +mkdir -p "${HOST_TOOLBOX_ROOT}" + +log "staging ${IMAGE_TOOLBOX_ROOT} -> ${HOST_TOOLBOX_ROOT} (selective overwrite, preserve runtime dirs)" + +# Remove only non-preserved top-level entries. Running shims that still +# reference unlinked package inodes keep running; new cubelet uses the fresh copy. +while IFS= read -r -d '' entry; do + name="$(basename "${entry}")" + if should_preserve "${name}"; then + log "preserving ${name}" + continue + fi + rm -rf "${entry}" +done < <(find "${HOST_TOOLBOX_ROOT}" -mindepth 1 -maxdepth 1 -print0) + +# Copy package trees from the image. Do not replace preserved host dirs that +# already exist (e.g. a populated cube-snapshot) with empty image stubs. +while IFS= read -r -d '' entry; do + name="$(basename "${entry}")" + dest="${HOST_TOOLBOX_ROOT}/${name}" + if should_preserve "${name}" && [[ -e "${dest}" ]]; then + log "skipping image copy over preserved ${name}" + continue + fi + rm -rf "${dest}" + cp -a "${entry}" "${dest}" +done < <(find "${IMAGE_TOOLBOX_ROOT}" -mindepth 1 -maxdepth 1 -print0) + +# Keep the same executable bits one-click install.sh enforces. +chmod +x \ + "${HOST_TOOLBOX_ROOT}/Cubelet/bin/cubelet" \ + "${HOST_TOOLBOX_ROOT}/Cubelet/bin/cubecli" \ + "${HOST_TOOLBOX_ROOT}/network-agent/bin/network-agent" \ + "${HOST_TOOLBOX_ROOT}/network-agent/bin/cubevsmapdump" \ + "${HOST_TOOLBOX_ROOT}/cube-shim/bin/cube-runtime" \ + "${HOST_TOOLBOX_ROOT}/cube-shim/bin/containerd-shim-cube-rs" \ + 2>/dev/null || true + +for required in \ + Cubelet/bin/cubelet \ + network-agent/bin/network-agent \ + cube-shim/bin/containerd-shim-cube-rs \ + cube-shim/bin/cube-runtime +do + [[ -x "${HOST_TOOLBOX_ROOT}/${required}" ]] || fail "missing executable after stage: ${required}" +done + +log "toolbox staged successfully at ${HOST_TOOLBOX_ROOT}" From 81df5489d41a844513a44c2221a7a883ddea50a1 Mon Sep 17 00:00:00 2001 From: jinlong Date: Sun, 12 Jul 2026 14:52:23 +0800 Subject: [PATCH 8/9] refactor(kubernetes): replace node-local cube-dns with cluster CoreDNS integration Remove the self-managed cube-dns (CoreDNS DaemonSet/Deployment) and instead patch cluster CoreDNS to resolve *.cubeProxy.domain via a new headless CubeProxy Service. This simplifies the data plane and removes the need for hostNetwork loopback DNS, wait-cube-dns init containers, and dnsPolicy=None on cube-node Pods. Key changes: - Add headless Service for cube-proxy-node (DNS returns Pod IP) - Add cluster-dns Job/hook to inject/clean *.cube.app rewrites in kube-system/coredns - Remove cube-dns DaemonSet/Deployment, ConfigMap, and Service - Remove wait-cube-dns init container; cube-node uses ClusterFirstWithHostNet - Sandbox guests default to followNodeDns=true (cluster DNS) - Add cube-node-entrypoint follow-node DNS resolution from /etc/resolv.conf - Update values, helpers, validations, tests, and docs accordingly Signed-off-by: jinlong --- deploy/kubernetes/chart/README.md | 55 ++--- deploy/kubernetes/chart/docs/ARCHITECTURE.md | 102 ++++---- deploy/kubernetes/chart/docs/FAQ.md | 12 +- deploy/kubernetes/chart/templates/NOTES.txt | 14 +- .../kubernetes/chart/templates/_helpers.tpl | 10 + .../chart/templates/cluster-dns.yaml | 227 ++++++++++++++++++ deploy/kubernetes/chart/templates/dns.yaml | 222 ----------------- .../chart/templates/node-daemonset.yaml | 48 +--- .../chart/templates/proxy-service.yaml | 28 +++ .../chart/templates/tests/node-health.yaml | 63 ++--- .../kubernetes/chart/templates/validate.yaml | 41 +--- deploy/kubernetes/chart/values.yaml | 92 ++----- .../images/scripts/cube-node-entrypoint.sh | 16 ++ 13 files changed, 396 insertions(+), 534 deletions(-) create mode 100644 deploy/kubernetes/chart/templates/cluster-dns.yaml delete mode 100644 deploy/kubernetes/chart/templates/dns.yaml create mode 100644 deploy/kubernetes/chart/templates/proxy-service.yaml diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index 67c5fb1c3..ff9c54bc7 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -57,8 +57,9 @@ external control plane / compute-only 模式见： The chart separates placement into dedicated control-plane nodes and compute nodes. Control-plane Deployments, StatefulSets, and `cube-proxy-node` use -`placement.controlPlane`; `cube-node` and node-local `cube-dns` use -`placement.compute`. +`placement.controlPlane`; `cube-node` uses `placement.compute`. In-cluster +`*.cube.app` is wired automatically when CubeProxy is enabled: the chart +patches cluster CoreDNS so the domain rewrites to the CubeProxy Service. The chart refuses to render host-mutating compute components without `placement.compute.nodeSelector`. This prevents PVM bootstrap and Cube runtime @@ -397,41 +398,21 @@ cubeProxy: ipv6: false ``` -## Cube DNS - -`cubeDns.enabled=true` delivers CoreDNS for one-click style sandbox domain -resolution. The default `cubeDns.mode=nodeLocal` runs `cube-dns` as a -hostNetwork DaemonSet on Cube compute nodes selected by `placement.compute` -and listens on `127.0.0.54:53`. -`cube-node` Pods use `dnsPolicy: None` and explicitly set -`dnsConfig.nameservers: [127.0.0.54]`, so `cube.app` and wildcard domains are -resolved inside the Big Pod without modifying host-wide DNS. - -Sandbox guest DNS is configured separately from the `cube-node` Pod DNS. The -guest cannot use `127.0.0.54` because that address is the guest loopback inside -the sandbox. By default `cubeDns.sandboxGateway.enabled=true` also binds -node-local `cube-dns` on the compute node HostIP, and -`cubeNode.dns.sandbox.useCubeDns=true` injects that HostIP into Cubelet -`default_dns_servers`. - -CubeVS eBPF egress does not traverse host kube-proxy ClusterIP DNAT, so guests -should not rely on Kubernetes Service ClusterIPs as if they were ordinary -external addresses. The chart no longer renders sandbox service proxy resources -or DNS overrides for them; expose required in-cluster services through the -platform networking layer, AgentWay provider configuration, or an -operator-managed proxy outside this chart. - -Set `cubeProxy.advertiseIP` or `cubeDns.answerIP` to return the control-node -CubeProxy entrypoint. If both are empty in node-local mode, `cube-dns` falls -back to the current compute HostIP for compatibility with older local-proxy -topologies; that fallback is not suitable when CubeProxy runs only on control -nodes. Optional `cubeDns.mode=service` keeps the older ClusterIP DNS model, -where callers must explicitly point their DNS policy or upstream DNS to the -`cube-dns` Service. In service mode, set `cubeDns.answerIP` or -`cubeProxy.advertiseIP` to an explicit CubeProxy entrypoint. - -The chart does not silently rewrite Kubernetes nodes' host DNS settings. -For external clients, browsers, SDKs, or any Pod that is not explicitly using this `dnsConfig`, configure DNS/LB/Ingress outside the chart so `cubeProxy.domain` and wildcard subdomains resolve to an explicit control-node CubeProxy endpoint or external load balancer. +## Cluster DNS for sandbox domain + +When CubeProxy is enabled, the chart patches **cluster CoreDNS** so +`cubeProxy.domain` / `*.domain` rewrite to the CubeProxy headless Service +(Pod IP). Users only set the domain: + +```yaml +cubeProxy: + domain: cube.app # change this if you use a custom domain + configureClusterDNS: true # set false only if kube-system/coredns must not be patched +cubeNode: + dns: + sandbox: + followNodeDns: true # guests use node/cluster DNS +``` ## WebUI diff --git a/deploy/kubernetes/chart/docs/ARCHITECTURE.md b/deploy/kubernetes/chart/docs/ARCHITECTURE.md index 17006649f..a7249dd9f 100644 --- a/deploy/kubernetes/chart/docs/ARCHITECTURE.md +++ b/deploy/kubernetes/chart/docs/ARCHITECTURE.md @@ -14,7 +14,8 @@ CubeSandbox Chart 按职责分为 7 层： | 运维入口 | cubemastercli | Deployment | 面向 `kubectl exec` 的 CLI Pod，交付真实 `cubemastercli` 并注入本 Release 的 CubeMaster endpoint | | 依赖存储 | MySQL / Redis | 内置 StatefulSet + Headless Service + volumeClaimTemplates/hostPath，或第三方服务 | MySQL 存储业务数据；Redis 存储 CubeProxy/Sidecar 状态 | | 计算面 | Cube Node Big Pod | DaemonSet | 节点初始化、运行 cubelet/network-agent、透明 egress sidecar | -| 数据面入口 | CubeProxy / CubeDNS | CubeProxy Deployment；CubeDNS DaemonSet 或 Deployment | HTTP/HTTPS sandbox 入口；sandbox 域名解析 | +| 数据面入口 | CubeProxy + 集群 DNS | CubeProxy Deployment；自动把 `*.domain` 写入集群 CoreDNS | HTTP/HTTPS sandbox 入口；集群内域名泛解析 | + 默认完整部署形态： @@ -33,10 +34,12 @@ flowchart TB CERT["cube-proxy-certs Secret"] end + subgraph CLUSTER["Cluster DNS"] + KDNS["kube-system CoreDNS\n*.cube.app → CubeProxy Service"] + end + subgraph COMPUTE["Compute Nodes selected by placement.compute"] - DNS["cube-dns DaemonSet\nCoreDNS 127.0.0.54:53"] subgraph NODE["cube-node DaemonSet Big Pod"] - WAITDNS["init: wait-cube-dns"] PVM["init: pvm-host-bootstrap\noptional"] INIT["init: cube-node-init"] CUBELET["container: cube-node\ncubelet + network-agent"] @@ -57,8 +60,7 @@ flowchart TB PROXY --> REDIS PROXY --> CM PROXY --> CERT - DNS --> PROXY - WAITDNS --> DNS + KDNS --> PROXY INIT --> CM CUBELET --> CM EG --> CA @@ -96,7 +98,6 @@ flowchart TB | 容器 | 类型 | 镜像 | 职责 | | --- | --- | --- | --- | -| `wait-cube-dns` | Init Container | `cubeNode.dns.checkImage` | 等待 node-local `cube-dns` 可用，并验证 `cube.app`、wildcard、Kubernetes Service 域名解析 | | `stage-toolbox` | Init Container | `images.node` | 将镜像内 `/usr/local/services/cubetoolbox` **覆盖式**同步到宿主机同名 hostPath（与一键包一致），使 shim/runtime 二进制在 Pod 重建后仍可被存量进程持有 | | `pvm-host-bootstrap` | Init Container，可选 | `images.pvmHostBootstrap` | 安装/配置 PVM host kernel，必要时协调节点重启 | | `cube-node-init` | Init Container | `images.nodeInit` | 节点预检和准备：KVM、XFS、内存、glibc、cgroup、cubecow 依赖、CIDR 冲突、CubeMaster 连通性 | @@ -122,59 +123,45 @@ flowchart TB | --- | --- | --- | | `cube-proxy-node` Deployment | `templates/proxy-node.yaml` | 提供 sandbox HTTP/HTTPS 数据面入口，使用 `placement.controlPlane` 与 one-click control 节点语义对齐，并使用 `hostNetwork` 监听节点 `80/443` | | `cube-proxy-certs` Secret / Certificate | `templates/proxy-node.yaml` | TLS 证书，支持 selfSigned、inline、existingSecret、certManager | -| `cube-dns` | `templates/dns.yaml` | 提供 sandbox 域名解析；默认 node-local | +| `cube-proxy-node` Service | `templates/proxy-service.yaml` | headless Service，DNS 直接返回 CubeProxy Pod IP | +| cluster DNS | `templates/cluster-dns.yaml` | CubeProxy 启用时把 `*.cubeProxy.domain` rewrite 到 headless Service | CubeProxy 默认保持 One Click 的 host-network 语义： - `cubeProxy.hostNetwork=true`，Pod IP 等于所在节点 HostIP。 - `cube-proxy-node` 复用 `placement.controlPlane`，与 one-click control 节点上的 CubeProxy 对齐。 -- `cube-dns` 复用 `placement.compute`，与 `cube-node` 保持同一组计算节点。 - nginx 监听节点 `80/443`，Chart 启动脚本会把镜像默认 `8081/8080` patch 为 values 中配置的端口。 -- node-local `cube-dns` 应通过 `cubeProxy.advertiseIP` 或 `cubeDns.answerIP` 返回 control 节点 CubeProxy 入口。 +- 集群内 `*.cube.app` 在 CubeProxy 启用时自动 rewrite 到 CubeProxy headless Service（Pod/HostIP）；用户一般只需改 `cubeProxy.domain`。 - CubeProxy 通过 Redis 中的 owner `HostIP:hostPort` 元数据转发到目标 compute 节点 sandbox。 -- nginx `global.conf` 中写入 `resolver`，Lua Redis 客户端可以解析内置或第三方 Redis DNS 名称。 - Chart 不修改 CubeProxy Lua 后端解析语义；跨节点访问仍遵循社区 CubeProxy 使用 Redis `HostIP:hostPort` 的原始路径。 ## 3. 默认 DNS 架构 -默认 `cubeDns.mode=nodeLocal`： +Chart **不**部署自有 CoreDNS。CubeProxy 启用且 `configureClusterDNS=true`（默认）时： + +- Helm hook 把 `domain` / `*.domain` rewrite 到 `-proxy-node..svc.cluster.local`。 +- CubeProxy Service 为 **headless**，解析结果是 Pod IP（hostNetwork 下即控制节点 HostIP）。 +- `cubeNode.dns.sandbox.followNodeDns=true`：guest 跟随节点/集群 DNS。 ```mermaid sequenceDiagram + participant Guest as sandbox guest participant CN as cube-node Pod - participant DNS as cube-dns on same node - participant KDNS as Kubernetes DNS - participant PX as cube-proxy-node on control node - - CN->>DNS: resolve cube.app / *.cube.app via 127.0.0.54 - DNS-->>CN: A record = cubeProxy.advertiseIP / cubeDns.answerIP - CN->>DNS: resolve kubernetes.default.svc.cluster.local - DNS->>KDNS: forward cluster/service DNS query - KDNS-->>DNS: ClusterIP answer - DNS-->>CN: ClusterIP answer - CN->>PX: sandbox HTTP/HTTPS traffic + participant KDNS as cluster CoreDNS + participant PX as cube-proxy Pod + + CN->>KDNS: ClusterFirstWithHostNet + Guest->>KDNS: followNodeDns + KDNS-->>Guest: *.cube.app → CubeProxy Pod IP + Guest->>PX: HTTP/HTTPS ``` 关键点： -- `cube-dns` 以 DaemonSet + `hostNetwork` 运行在计算节点。 -- `cube-dns` 监听 `127.0.0.54:53`；启用 `cubeDns.sandboxGateway.enabled` - 时也监听当前 compute 节点 HostIP，供 sandbox guest 作为 nameserver。 -- `cube-proxy-node` 以 Deployment + `hostNetwork` 运行在 control 节点，监听节点 `80/443`。 -- `cube-node` 使用 `dnsPolicy: None`，显式配置 `nameserver 127.0.0.54`。 -- `cube.app` / `*.cube.app` 优先解析为 `cubeDns.answerIP`，其次为 `cubeProxy.advertiseIP`；两者为空时才回退到当前 compute HostIP。 -- 其他域名转发到 `cubeDns.forward.upstreams`，为空时使用 `/etc/resolv.conf`。 -- Chart 不修改宿主机全局 DNS，不影响非 Cube Pod。 -- Cube sandbox guest DNS 由 `cubeNode.dns.sandbox` 单独写入 Cubelet dynamicconf。 -- 默认写入当前 compute 节点 HostIP，让 guest 使用 node-local `cube-dns`。 -- sandbox 访问 Kubernetes Service ClusterIP 可能绕过宿主机 kube-proxy DNAT；Chart 不再渲染 sandbox service proxy 资源或对应 DNS override。需要暴露给 sandbox 的 in-cluster Service 应由平台网络层、AgentWay provider 或 operator 管理的外部代理处理。 -- 外部客户端、浏览器、SDK 或未显式使用该 `dnsConfig` 的 Pod，需要由使用方配置 DNS / 负载均衡 / Ingress，把 `cubeProxy.domain` 与 wildcard 子域名指向 CubeProxy 入口。 - -可选 `cubeDns.mode=service`： - -- `cube-dns` 以 Deployment + ClusterIP Service 运行。 -- `cube.app` / wildcard 必须通过 `cubeDns.answerIP` 或 `cubeProxy.advertiseIP` 返回明确的 CubeProxy 入口。 -- 使用方需要自行把客户端 DNS 或上游 DNS 指向该 Service。 +- 域名用 `cubeProxy.domain`（默认 `cube.app`）。 +- 集群内泛解析不需要单独配 IP。 +- 若平台禁止改 `kube-system/coredns`，设 `cubeProxy.configureClusterDNS=false`。 +- 外部客户端仍需自行配置公网/Private DNS 或 LB。 ## 4. 安装与启动流程 @@ -188,7 +175,7 @@ flowchart TD C -- 是 --> D["渲染 Secret / ConfigMap / 持久化卷"] D --> E["渲染 MySQL / Redis 或使用第三方服务"] E --> F["渲染控制面 Deployment"] - F --> G["渲染 cube-dns / cube-proxy-node"] + F --> G["渲染 cube-proxy / cluster-dns"] G --> H["渲染 cube-node DaemonSet"] H --> I["等待 --wait / rollout / helm test"] ``` @@ -198,16 +185,14 @@ flowchart TD - `controlPlane.enabled=true` 时必须配置 `placement.controlPlane.nodeSelector`。 - `cubeNode.enabled=true` 时必须配置 `placement.compute.nodeSelector`。 - `cubeProxy.enabled=true` 时必须配置 `placement.controlPlane.nodeSelector`。 -- `cubeDns.enabled=true` 时必须配置 `placement.compute.nodeSelector`。 +- `cubeProxy.configureClusterDNS=true` 时必须配置 `cubeProxy.domain`。 - compute-only 模式必须显式配置 `externalControlPlane.masterEndpoint`。 -- `cubeNode.dns.useCubeDns=true` 时要求 `cubeDns.enabled=true`、`cubeDns.mode=nodeLocal`、`security.hostNetwork=true`。 -- `cubeDns.mode` 只能为 `nodeLocal` 或 `service`。 - PVM host kernel bootstrap 只能在明确命中 selector 的节点上执行。 ### 4.1.1 调度与时区 - CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 使用 `placement.controlPlane`。 -- `cube-node`、`cube-dns` 使用 `placement.compute`。 +- `cube-node` 使用 `placement.compute`。 - 所有 Chart 管理的 Cube 容器、sidecar 和 initContainer 都通过 `global.timezone` 注入 `TZ`，默认 `Asia/Shanghai`。 ### 4.2 控制面启动 @@ -249,7 +234,6 @@ sequenceDiagram ```mermaid sequenceDiagram participant DS as cube-node DaemonSet - participant DNS as wait-cube-dns participant PVM as pvm-host-bootstrap participant INIT as cube-node-init participant CN as cube-node @@ -257,8 +241,6 @@ sequenceDiagram participant EN as cube-egress-net participant CM as CubeMaster - DS->>DNS: wait 127.0.0.54 and validate DNS records - DNS-->>DS: success opt bootstrap.pvmHostKernel.enabled=true DS->>PVM: install/check PVM host kernel PVM-->>DS: success or reboot-required failure signal @@ -342,7 +324,7 @@ flowchart LR - 需要多节点统一入口时，应由外部 DNS/LB 明确指向预期 control 节点 CubeProxy 或保证社区 CubeProxy 的 `HostIP:hostPort` 跨节点路径可用，而不是依赖默认 ClusterIP 随机分流。 - TLS 支持 selfSigned、existingSecret、inline、certManager。 - 生产环境应提供正式证书，并把 sandbox domain / wildcard DNS 指向明确的 CubeProxy 入口。 -- Chart 自带 `cube-dns` 默认只服务 `cube-node` Big Pod；面向用户访问的外部 DNS、LB 或 Ingress 需要由使用方显式配置。 +- Chart 在 CubeProxy 启用时自动配置集群内 `*.cube.app`；外部 DNS/LB 仍需使用方配置。计算节点 guest 默认跟随节点 DNS。 ### 5.3 Sandbox 出站 egress @@ -390,16 +372,16 @@ flowchart TB end subgraph NS["Compute Namespace"] - DNS["cube-dns"] NODE["cube-node DaemonSet"] end NODE --> ECM - DNS --> NODE EAPI --> ECM ECM --> EDB ``` +compute-only Release：不安装控制面与 CubeProxy（除非另配）；集群 DNS 注入随 proxy 关闭。 + 关键 values： ```yaml @@ -429,13 +411,11 @@ externalControlPlane: | `controlPlane.enabled` | `true` | 是否部署内置控制面 | | `externalControlPlane.enabled` | `false` | 是否使用外部 CubeMaster | | `placement.controlPlane.nodeSelector` | `cube.tencent.com/role=control` | 控制 CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 调度范围 | -| `placement.compute.nodeSelector` | 含 `allow-pvm-bootstrap=true` | 控制 `cube-node`、`cube-dns` 调度范围，并要求节点显式允许 PVM bootstrap | -| `cubeDns.enabled` | `true` | 是否交付 CubeDNS | -| `cubeDns.mode` | `nodeLocal` | node-local DNS 或 ClusterIP DNS | -| `cubeDns.sandboxGateway.enabled` | `true` | node-local DNS 是否同时监听 compute HostIP，供 sandbox guest 使用 | -| `cubeNode.dns.useCubeDns` | `true` | `cube-node` 是否显式使用 `127.0.0.54` | -| `cubeNode.dns.sandbox.useCubeDns` | `true` | 是否把 sandbox guest `/etc/resolv.conf` 指到 cube-dns；默认写当前 compute HostIP，让 guest 使用 node-local `cube-dns` | -| `cubeNode.dns.sandbox.nameservers` | `[]` | 覆盖写入 sandbox guest `/etc/resolv.conf` 的 DNS server | +| `placement.compute.nodeSelector` | 含 `allow-pvm-bootstrap=true` | 控制 `cube-node` 调度范围，并要求节点显式允许 PVM bootstrap | +| `cubeProxy.domain` | `cube.app` | sandbox 域名；集群 DNS 与 TLS 共用 | +| `cubeProxy.configureClusterDNS` | `true` | 是否把 `*.domain` 自动写入集群 CoreDNS | +| `cubeNode.dns.sandbox.followNodeDns` | `true` | guest 是否跟随节点/集群 DNS | +| `cubeNode.dns.sandbox.nameservers` | `[]` | 显式覆盖 guest nameserver | | `cubeNode.pvmGuestKernel.enabled` | `true` | 是否选择 PVM guest kernel；`cube-node-init` 校验该值与 `kvm_pvm` 状态一致 | | `bootstrap.pvmHostKernel.enabled` | `true` | 是否执行 host kernel bootstrap；默认可能安装 host kernel 并按租约重启计算节点 | | `bootstrap.pvmHostKernel.bootArgs` | `nopti pti=off` | PVM host kernel 启动参数；当前 `kvm_pvm` 不支持 host KPTI，默认关闭 PTI | @@ -443,7 +423,7 @@ externalControlPlane: | `mysql.host` | `""` | 非空时使用第三方 MySQL | | `redis.host` | `""` | 非空时使用第三方 Redis | | `cubeProxy.enabled` | `true` | 是否部署 control 节点 CubeProxy 数据面入口 | -| `cubeProxy.advertiseIP` | `""` | `cubeDns.answerIP` 为空时返回给 `cube.app` / wildcard 的 control 节点 CubeProxy 入口 IP | +| `cubeProxy.advertiseIP` | `""` | 可选，仅作外部入口提示；集群内泛解析不依赖它 | | `cubeEgress.enabled` | `true` | 是否在 Big Pod 中启用 egress sidecar | | `webui.enabled` | `true` | 是否部署 WebUI | | `controlPlane.templateBuilder.enabled` | `false` | 是否启用模板构建 sidecar | @@ -457,9 +437,9 @@ externalControlPlane: | `-health-test` | CubeMaster、CubeAPI、节点注册、WebUI、CubeProxy、DaemonSet/Deployment/StatefulSet ready、Egress sidecar 存在性 | | `-mysql-test` | 内置 MySQL `mysqladmin ping` | | `-redis-test` | 内置 Redis `PING` | -| `-dns-test` | `cube.app`、wildcard、Kubernetes Service 域名解析 | +| `-dns-test` | 集群 CoreDNS 解析 `cube.app` / wildcard → CubeProxy Service | | `-node-image-test` | `cube-node` 镜像内 runtime 工具和必需 asset | -| `-node-runtime-test` | 计算节点 host runtime：`/dev/kvm`、cubelet socket、network-agent socket、node-local DNS | +| `-node-runtime-test` | 计算节点 host runtime：`/dev/kvm`、cubelet socket、network-agent socket | 执行： diff --git a/deploy/kubernetes/chart/docs/FAQ.md b/deploy/kubernetes/chart/docs/FAQ.md index 5d44a36ba..35dcedae1 100644 --- a/deploy/kubernetes/chart/docs/FAQ.md +++ b/deploy/kubernetes/chart/docs/FAQ.md @@ -33,7 +33,7 @@ placement: | `cube-node requires placement.compute.nodeSelector` | 计算节点必须显式指定,不能空 | | `bootstrap.pvmHostKernel.enabled=true requires ... allow-pvm-bootstrap=true` | 计算节点 nodeSelector 必须包含 `allow-pvm-bootstrap=true`,防止误替换内核 | | `cubeProxy.enabled=true requires placement.controlPlane.nodeSelector` | CubeProxy 只跑在 control 节点上 | -| `cubeDns.enabled=true requires placement.compute.nodeSelector` | node-local DNS 只跑在 compute 节点上 | +| `cubeProxy.configureClusterDNS=true requires cubeProxy.domain` | 开启集群 DNS 注入时必须有 sandbox 域名 | ### A2. `helm install --wait` 挂在 CubeNode DaemonSet Ready 数不足 @@ -260,10 +260,12 @@ kubectl -n cube-system logs -c cube-node --tail=200 | grep -i re 按方向排查: -1. **compute 节点内部**(cube-node Pod / sandbox guest):`dig @127.0.0.54 test.cube.app` - - 无应答 → cube-dns 未启动,查 `kubectl -n cube-system get pods -l app.kubernetes.io/name=cube-dns` - - 应答不是 CubeProxy 入口 IP → 检查 `cubeProxy.advertiseIP` / `cubeDns.answerIP` -2. **compute 节点外部**(客户浏览器 / SDK):用户侧 DNS 未配置。Chart 只交付 in-cluster DNS,面向公网的 DNS/LB 需要用户自行设置 +1. **集群内**（Pod / 跟随节点 DNS 的 sandbox guest）:`nslookup test.cube.app` + - 无应答 → 查 `kubectl -n kube-system get cm coredns -o yaml` 是否含 `# BEGIN cube-sandbox-dns` + - 或看 Job：`kubectl -n cube-system logs job/cube-cluster-dns-apply` + - 应答应是 CubeProxy Pod IP（headless Service）；对照 `kubectl -n cube-system get endpoints cube-proxy-node -o wide` +2. **guest DNS**：默认 `followNodeDns=true`；显式覆盖用 `cubeNode.dns.sandbox.nameservers` +3. **集群外部**(客户浏览器 / SDK):用户侧 DNS / Private DNS / LB 仍需自行配置 ### E2. CubeProxy 起不来:`hostNetwork port 80/443 already in use` diff --git a/deploy/kubernetes/chart/templates/NOTES.txt b/deploy/kubernetes/chart/templates/NOTES.txt index e4a164a2b..3ce81b889 100644 --- a/deploy/kubernetes/chart/templates/NOTES.txt +++ b/deploy/kubernetes/chart/templates/NOTES.txt @@ -23,22 +23,22 @@ External control plane: cubemastercli Deployment: {{ include "cube.cubemastercliName" . }} {{- end }} {{- end }} -{{- if .Values.cubeDns.enabled }} -{{- if eq .Values.cubeDns.mode "nodeLocal" }} - CubeDNS DaemonSet: {{ include "cube.fullname" . }}-dns (node-local {{ .Values.cubeDns.bindAddress }}:53) -{{- else }} - CubeDNS Service: {{ include "cube.fullname" . }}-dns:{{ .Values.cubeDns.service.port }} -{{- end }} +{{- if eq (include "cube.configureClusterDNS" .) "true" }} + Cluster DNS: *.{{ .Values.cubeProxy.domain }} -> {{ include "cube.proxyServiceFQDN" . }} (patched into kube-system/coredns) {{- end }} Cube Node DaemonSet: {{ include "cube.nodeName" . }} +{{- if .Values.cubeNode.dns.sandbox.followNodeDns }} + Sandbox guest DNS: follow node / cluster DNS +{{- end }} {{- if eq (include "cube.proxyEnabled" .) "true" }} Cube Proxy Node Deployment: {{ include "cube.proxyName" . }} HTTP/HTTPS control-node endpoint: :{{ .Values.cubeProxy.ports.http.containerPort }}/{{ .Values.cubeProxy.ports.https.containerPort }} - No ClusterIP Service is created; point wildcard DNS at the control node or external load balancer. + Headless Service: {{ include "cube.proxyName" . }} + External clients still need their own DNS/LB if they are outside the cluster. {{- end }} Useful commands: diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl index 6def0e7cf..846911f06 100644 --- a/deploy/kubernetes/chart/templates/_helpers.tpl +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -98,6 +98,8 @@ tolerations: {{- end }} {{- end -}} +{{/* Proxy Service FQDN and cluster-DNS enablement helpers. */}} + {{- define "cube.nodeServiceAccountName" -}} {{- if .Values.serviceAccount.create -}} {{- printf "%s-node" (include "cube.fullname" .) -}} @@ -134,6 +136,14 @@ tolerations: {{- if and .Values.cubeProxy.enabled (or .Values.controlPlane.enabled (not .Values.externalControlPlane.enabled)) -}}true{{- else -}}false{{- end -}} {{- end -}} +{{- define "cube.proxyServiceFQDN" -}} +{{- printf "%s.%s.svc.%s" (include "cube.proxyName" .) .Release.Namespace (include "cube.clusterDomain" .) -}} +{{- end -}} + +{{- define "cube.configureClusterDNS" -}} +{{- if and .Values.cubeProxy.configureClusterDNS (eq (include "cube.proxyEnabled" .) "true") -}}true{{- else -}}false{{- end -}} +{{- end -}} + {{- define "cube.cubemastercliEnabled" -}} {{- $cubemastercli := default dict .Values.cubemastercli -}} {{- if and (dig "enabled" true $cubemastercli) (or .Values.controlPlane.enabled .Values.externalControlPlane.enabled) -}}true{{- else -}}false{{- end -}} diff --git a/deploy/kubernetes/chart/templates/cluster-dns.yaml b/deploy/kubernetes/chart/templates/cluster-dns.yaml new file mode 100644 index 000000000..001a9453f --- /dev/null +++ b/deploy/kubernetes/chart/templates/cluster-dns.yaml @@ -0,0 +1,227 @@ +{{- if eq (include "cube.configureClusterDNS" .) "true" }} +{{- $domain := .Values.cubeProxy.domain -}} +{{- $domainRegex := replace "." "\\." $domain -}} +{{- $svcFQDN := include "cube.proxyServiceFQDN" . -}} +{{- $clusterDomain := include "cube.clusterDomain" . -}} +{{- $markerBegin := printf "# BEGIN cube-sandbox-dns %s" .Release.Name -}} +{{- $markerEnd := printf "# END cube-sandbox-dns %s" .Release.Name -}} +{{- $coreDNSNamespace := "kube-system" -}} +{{- $coreDNSConfigMap := "coredns" -}} +{{- $coreDNSKey := "Corefile" -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns +data: + domain: {{ $domain | quote }} + target: {{ $svcFQDN | quote }} + marker.begin: {{ $markerBegin | quote }} + marker.end: {{ $markerEnd | quote }} + Corefile.snippet: | + {{ $markerBegin }} + {{ $domain }}:53 { + errors + cache 60 + rewrite name exact {{ $domain }}. {{ $svcFQDN }}. + rewrite name regex (.*)\.{{ $domainRegex }}\.? {{ $svcFQDN }}. + kubernetes {{ $clusterDomain }} + forward . /etc/resolv.conf + } + {{ $markerEnd }} + sync.sh: | + #!/bin/bash + set -euo pipefail + mode="${1:?mode apply|remove}" + # alpine/k8s kubectl does not auto-load in-cluster config. + if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/token ]]; then + kubectl() { + command kubectl \ + --server="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" \ + --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + "$@" + } + fi + kubectl_opts=(--request-timeout=30s) + marker_begin="$(cat /snippet/marker.begin)" + marker_end="$(cat /snippet/marker.end)" + if ! kubectl "${kubectl_opts[@]}" -n "${COREDNS_NS}" get configmap "${COREDNS_CM}" >/dev/null 2>&1; then + if [[ "${mode}" == "remove" ]]; then + echo "coredns configmap missing; nothing to remove" + exit 0 + fi + echo "coredns configmap ${COREDNS_NS}/${COREDNS_CM} not found" >&2 + exit 1 + fi + tmp="$(mktemp)" + kubectl "${kubectl_opts[@]}" -n "${COREDNS_NS}" get configmap "${COREDNS_CM}" -o json > "${tmp}.cm.json" + jq -r --arg key "${COREDNS_KEY}" '.data[$key] // empty' "${tmp}.cm.json" > "${tmp}.raw" + awk -v begin="${marker_begin}" -v end="${marker_end}" ' + $0 == begin { skip=1; next } + $0 == end { skip=0; next } + !skip { print } + ' "${tmp}.raw" > "${tmp}.clean" + if [[ "${mode}" == "apply" ]]; then + { + cat "${tmp}.clean" + printf '\n%s\n' "$(cat /snippet/Corefile.snippet)" + } > "${tmp}.new" + else + cp "${tmp}.clean" "${tmp}.new" + fi + jq --arg key "${COREDNS_KEY}" --rawfile cf "${tmp}.new" \ + '.data[$key] = $cf' "${tmp}.cm.json" | kubectl "${kubectl_opts[@]}" apply -f - + echo "${mode} ${marker_begin} on ${COREDNS_NS}/${COREDNS_CM}" +--- +{{/* + SA/Role/RoleBinding must be regular release resources (not hooks with + hook-succeeded). Hook-succeeded deletes them as soon as they are created, + so the apply Job loses RBAC mid-flight and fails with Forbidden. + Keep them for pre-delete remove Job; Helm deletes release resources after + pre-delete hooks finish. +*/}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns + namespace: {{ $coreDNSNamespace | quote }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{ $coreDNSConfigMap | quote }}] + verbs: ["get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns + namespace: {{ $coreDNSNamespace | quote }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "cube.fullname" . }}-cluster-dns +subjects: + - kind: ServiceAccount + name: {{ include "cube.fullname" . }}-cluster-dns + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns-apply + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + ttlSecondsAfterFinished: 600 + activeDeadlineSeconds: 180 + backoffLimit: 3 + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cluster-dns + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "cube.fullname" . }}-cluster-dns + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: apply + image: {{ include "cube.image" .Values.images.kubectl | quote }} + imagePullPolicy: {{ .Values.images.kubectl.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: COREDNS_NS + value: {{ $coreDNSNamespace | quote }} + - name: COREDNS_CM + value: {{ $coreDNSConfigMap | quote }} + - name: COREDNS_KEY + value: {{ $coreDNSKey | quote }} + command: ["/bin/bash", "/snippet/sync.sh", "apply"] + volumeMounts: + - name: snippet + mountPath: /snippet + readOnly: true + volumes: + - name: snippet + configMap: + name: {{ include "cube.fullname" . }}-cluster-dns + defaultMode: 0755 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "cube.fullname" . }}-cluster-dns-remove + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cluster-dns + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + ttlSecondsAfterFinished: 600 + activeDeadlineSeconds: 180 + backoffLimit: 3 + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cluster-dns + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "cube.fullname" . }}-cluster-dns + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: remove + image: {{ include "cube.image" .Values.images.kubectl | quote }} + imagePullPolicy: {{ .Values.images.kubectl.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: COREDNS_NS + value: {{ $coreDNSNamespace | quote }} + - name: COREDNS_CM + value: {{ $coreDNSConfigMap | quote }} + - name: COREDNS_KEY + value: {{ $coreDNSKey | quote }} + command: ["/bin/bash", "/snippet/sync.sh", "remove"] + volumeMounts: + - name: snippet + mountPath: /snippet + readOnly: true + volumes: + - name: snippet + configMap: + name: {{ include "cube.fullname" . }}-cluster-dns + defaultMode: 0755 +{{- end }} diff --git a/deploy/kubernetes/chart/templates/dns.yaml b/deploy/kubernetes/chart/templates/dns.yaml deleted file mode 100644 index a3b9ff0aa..000000000 --- a/deploy/kubernetes/chart/templates/dns.yaml +++ /dev/null @@ -1,222 +0,0 @@ -{{- if .Values.cubeDns.enabled }} -{{- $domain := default .Values.cubeProxy.domain .Values.cubeDns.domain -}} -{{- $domainRegex := replace "." "\\." $domain -}} -{{- $forward := "/etc/resolv.conf" -}} -{{- $answerIP := default .Values.cubeProxy.advertiseIP .Values.cubeDns.answerIP -}} -{{- if .Values.cubeDns.forward.upstreams -}} -{{- $forward = join " " .Values.cubeDns.forward.upstreams -}} -{{- end -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "cube.fullname" . }}-dns-config - labels: - {{- include "cube.labels" . | nindent 4 }} - app.kubernetes.io/component: dns -data: - Corefile: |- - .:53 { - {{- if eq .Values.cubeDns.mode "nodeLocal" }} - bind {{ .Values.cubeDns.bindAddress }}{{- if .Values.cubeDns.sandboxGateway.enabled }} {$NODE_IP}{{- end }} - {{- end }} - {{- if $answerIP }} - template IN A {{ $domain }} { - answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {{ $answerIP }}" - } - template IN A (.*)\.{{ $domainRegex }} { - answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {{ $answerIP }}" - } - {{- else if eq .Values.cubeDns.mode "nodeLocal" }} - template IN A {{ $domain }} { - answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {$NODE_IP}" - } - template IN A (.*)\.{{ $domainRegex }} { - answer "{{ "{{" }} .Name {{ "}}" }} 60 IN A {$NODE_IP}" - } - {{- end }} - template IN AAAA {{ $domain }} { - rcode NOERROR - } - template IN AAAA (.*)\.{{ $domainRegex }} { - rcode NOERROR - } - errors - {{- if .Values.cubeDns.logQueries }} - log - {{- end }} - cache {{ .Values.cubeDns.cacheTTL | default 60 }} - forward . {{ $forward }} - } -{{- if eq .Values.cubeDns.mode "nodeLocal" }} ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "cube.fullname" . }}-dns - labels: - {{- include "cube.labels" . | nindent 4 }} - app.kubernetes.io/component: dns -spec: - selector: - matchLabels: - {{- include "cube.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: dns - template: - metadata: - labels: - {{- include "cube.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: dns - annotations: - checksum/corefile: {{ printf "%s|%s|%s|%s|%s|%t|node-local-v3" $domain $answerIP .Values.cubeDns.bindAddress .Values.cubeDns.mode $forward .Values.cubeDns.sandboxGateway.enabled | sha256sum | quote }} - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- include "cube.computePlacement" . | nindent 6 }} - containers: - - name: coredns - image: {{ include "cube.image" .Values.images.coredns | quote }} - imagePullPolicy: {{ .Values.images.coredns.pullPolicy }} - args: - - -conf - - /etc/coredns/Corefile - env: - {{- include "cube.timezoneEnv" . | nindent 12 }} - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - ports: - - name: dns-udp - containerPort: 53 - protocol: UDP - - name: dns-tcp - containerPort: 53 - protocol: TCP - readinessProbe: - tcpSocket: - host: {{ .Values.cubeDns.bindAddress | quote }} - port: dns-tcp - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - livenessProbe: - tcpSocket: - host: {{ .Values.cubeDns.bindAddress | quote }} - port: dns-tcp - initialDelaySeconds: 30 - periodSeconds: 20 - failureThreshold: 6 - volumeMounts: - - name: coredns-config - mountPath: /etc/coredns - readOnly: true - {{- with .Values.cubeDns.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - volumes: - - name: coredns-config - configMap: - name: {{ include "cube.fullname" . }}-dns-config -{{- else }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "cube.fullname" . }}-dns - labels: - {{- include "cube.labels" . | nindent 4 }} - app.kubernetes.io/component: dns -spec: - replicas: {{ .Values.cubeDns.replicas }} - selector: - matchLabels: - {{- include "cube.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: dns - template: - metadata: - labels: - {{- include "cube.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: dns - annotations: - checksum/corefile: {{ printf "%s|%s|%s|service-v1" $domain .Values.cubeDns.answerIP $forward | sha256sum | quote }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- include "cube.computePlacement" . | nindent 6 }} - containers: - - name: coredns - image: {{ include "cube.image" .Values.images.coredns | quote }} - imagePullPolicy: {{ .Values.images.coredns.pullPolicy }} - args: - - -conf - - /etc/coredns/Corefile - {{- if .Values.global.timezone }} - env: - {{- include "cube.timezoneEnv" . | nindent 12 }} - {{- end }} - ports: - - name: dns-udp - containerPort: 53 - protocol: UDP - - name: dns-tcp - containerPort: 53 - protocol: TCP - readinessProbe: - tcpSocket: - port: dns-tcp - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - livenessProbe: - tcpSocket: - port: dns-tcp - initialDelaySeconds: 30 - periodSeconds: 20 - failureThreshold: 6 - volumeMounts: - - name: coredns-config - mountPath: /etc/coredns - readOnly: true - {{- with .Values.cubeDns.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - volumes: - - name: coredns-config - configMap: - name: {{ include "cube.fullname" . }}-dns-config ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "cube.fullname" . }}-dns - labels: - {{- include "cube.labels" . | nindent 4 }} - app.kubernetes.io/component: dns - {{- with .Values.cubeDns.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.cubeDns.service.type }} - selector: - {{- include "cube.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: dns - ports: - - name: dns-udp - protocol: UDP - port: {{ .Values.cubeDns.service.port }} - targetPort: dns-udp - - name: dns-tcp - protocol: TCP - port: {{ .Values.cubeDns.service.port }} - targetPort: dns-tcp -{{- end }} -{{- end }} diff --git a/deploy/kubernetes/chart/templates/node-daemonset.yaml b/deploy/kubernetes/chart/templates/node-daemonset.yaml index e97e8c8fd..79dfaba34 100644 --- a/deploy/kubernetes/chart/templates/node-daemonset.yaml +++ b/deploy/kubernetes/chart/templates/node-daemonset.yaml @@ -28,57 +28,13 @@ spec: hostNetwork: {{ .Values.security.hostNetwork }} hostPID: {{ .Values.security.hostPID }} terminationGracePeriodSeconds: {{ .Values.cubeNode.terminationGracePeriodSeconds }} - {{- if .Values.cubeNode.dns.useCubeDns }} - dnsPolicy: None - dnsConfig: - nameservers: - - {{ .Values.cubeNode.dns.nameserver | quote }} - searches: - - {{ printf "%s.svc.%s" .Release.Namespace .Values.cubeNode.dns.clusterDomain | quote }} - - {{ printf "svc.%s" .Values.cubeNode.dns.clusterDomain | quote }} - - {{ .Values.cubeNode.dns.clusterDomain | quote }} - options: - - name: ndots - value: "5" - {{- else }} dnsPolicy: ClusterFirstWithHostNet - {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} {{- include "cube.computePlacement" . | nindent 6 }} initContainers: - {{- if .Values.cubeNode.dns.useCubeDns }} - - name: wait-cube-dns - image: {{ include "cube.image" .Values.cubeNode.dns.checkImage | quote }} - imagePullPolicy: {{ .Values.cubeNode.dns.checkImage.pullPolicy }} - env: - {{- include "cube.timezoneEnv" . | nindent 12 }} - - name: CUBE_DNS_NAMESERVER - value: {{ .Values.cubeNode.dns.nameserver | quote }} - - name: CUBE_DNS_DOMAIN - value: {{ (default .Values.cubeProxy.domain .Values.cubeDns.domain) | quote }} - - name: CUBE_DNS_TIMEOUT - value: {{ .Values.cubeNode.dns.waitTimeoutSeconds | quote }} - command: - - /bin/sh - - -ec - - | - deadline=$(( $(date +%s) + ${CUBE_DNS_TIMEOUT} )) - while [ "$(date +%s)" -le "${deadline}" ]; do - if nslookup "${CUBE_DNS_DOMAIN}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1 \ - && nslookup "wildcard-check.${CUBE_DNS_DOMAIN}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1 \ - && nslookup "kubernetes.default.svc.{{ .Values.cubeNode.dns.clusterDomain }}" "${CUBE_DNS_NAMESERVER}" >/dev/null 2>&1; then - echo "cube dns is ready at ${CUBE_DNS_NAMESERVER}" - exit 0 - fi - echo "waiting for cube dns at ${CUBE_DNS_NAMESERVER}" - sleep 2 - done - echo "cube dns is not ready at ${CUBE_DNS_NAMESERVER}" >&2 - exit 1 - {{- end }} # Stage image toolbox onto a hostPath before any other compute init so # cube-shim / cube-runtime binaries survive Pod rebuilds (overlay unmount). # Inline script so this works with existing cube-node images that do not @@ -343,11 +299,11 @@ spec: - name: CUBE_SANDBOX_DNS_SERVERS {{- if .Values.cubeNode.dns.sandbox.nameservers }} value: {{ join "," .Values.cubeNode.dns.sandbox.nameservers | quote }} - {{- else if and .Values.cubeNode.dns.sandbox.useCubeDns .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") .Values.cubeDns.sandboxGateway.enabled }} - value: "$(CUBE_SANDBOX_NODE_IP)" {{- else }} value: "" {{- end }} + - name: CUBE_SANDBOX_DNS_FOLLOW_NODE + value: {{ ternary "true" "false" (and .Values.cubeNode.dns.sandbox.followNodeDns (not .Values.cubeNode.dns.sandbox.nameservers)) | quote }} {{- with .Values.cubeNode.env }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/deploy/kubernetes/chart/templates/proxy-service.yaml b/deploy/kubernetes/chart/templates/proxy-service.yaml new file mode 100644 index 000000000..8d001df2a --- /dev/null +++ b/deploy/kubernetes/chart/templates/proxy-service.yaml @@ -0,0 +1,28 @@ +{{- if eq (include "cube.proxyEnabled" .) "true" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.proxyName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node + {{- with .Values.cubeProxy.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + # Headless so DNS returns CubeProxy Pod IPs (HostIP with hostNetwork). + clusterIP: None + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node + ports: + - name: http + port: {{ .Values.cubeProxy.ports.http.containerPort }} + targetPort: {{ .Values.cubeProxy.ports.http.containerPort }} + protocol: TCP + - name: https + port: {{ .Values.cubeProxy.ports.https.containerPort }} + targetPort: {{ .Values.cubeProxy.ports.https.containerPort }} + protocol: TCP +{{- end }} diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml index 543f12c31..89947105d 100644 --- a/deploy/kubernetes/chart/templates/tests/node-health.yaml +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -1,5 +1,5 @@ {{- if .Values.helmTest.enabled }} -{{- $domain := default .Values.cubeProxy.domain .Values.cubeDns.domain -}} +{{- $domain := .Values.cubeProxy.domain -}} {{- $internalMaster := and .Values.controlPlane.enabled .Values.controlPlane.master.enabled -}} {{- $internalAPI := and .Values.controlPlane.enabled .Values.controlPlane.api.enabled -}} {{- $externalAPIEndpoint := default "" .Values.externalControlPlane.apiEndpoint -}} @@ -260,7 +260,7 @@ spec: echo '[health-test] check control-node CubeProxy' curl --connect-timeout 5 --max-time 15 -fsS -o /dev/null http://127.0.0.1:{{ .Values.cubeProxy.probes.readiness.port }}{{ .Values.cubeProxy.probes.readiness.path }} {{- end }} -{{ if .Values.cubeDns.enabled }} +{{ if eq (include "cube.configureClusterDNS" .) "true" }} --- apiVersion: v1 kind: Pod @@ -274,31 +274,12 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never + dnsPolicy: ClusterFirst {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} - {{- if eq .Values.cubeDns.mode "nodeLocal" }} - hostNetwork: true - dnsPolicy: None - dnsConfig: - nameservers: - - {{ .Values.cubeNode.dns.nameserver | quote }} - searches: - - {{ printf "%s.svc.%s" .Release.Namespace (include "cube.clusterDomain" .) | quote }} - - {{ printf "svc.%s" (include "cube.clusterDomain" .) | quote }} - - {{ include "cube.clusterDomain" . | quote }} - options: - - name: ndots - value: "5" - {{- include "cube.computePlacement" . | nindent 2 }} - {{- else }} - # In service mode, cube-dns runs as a Deployment behind a ClusterIP Service. - # The nodeLocal DNS variant above uses compute placement because cube-dns - # DaemonSet only runs on compute nodes. In service mode we still schedule - # the test on compute nodes because that mirrors real sandbox traffic. - {{- include "cube.computePlacement" . | nindent 2 }} - {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 2 }} containers: - name: dns image: {{ include "cube.image" .Values.helmTest.dnsImage | quote }} @@ -307,15 +288,16 @@ spec: - sh - -ec - | - echo '[health-test] check CubeDNS' - {{- if eq .Values.cubeDns.mode "nodeLocal" }} - nslookup {{ $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' - nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' - nslookup {{ printf "kubernetes.default.svc.%s" (include "cube.clusterDomain" .) | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} | grep -Eq 'Name:|Address' - {{- else }} - nslookup {{ $domain | quote }} {{ printf "%s-dns.%s.svc.%s" (include "cube.fullname" .) .Release.Namespace (include "cube.clusterDomain" .) | quote }} | grep -Eq 'canonical name|Address' - nslookup {{ printf "wildcard-check.%s" $domain | quote }} {{ printf "%s-dns.%s.svc.%s" (include "cube.fullname" .) .Release.Namespace (include "cube.clusterDomain" .) | quote }} | grep -Eq 'canonical name|Address' - {{- end }} + echo '[health-test] check cluster DNS for sandbox domain' + target={{ include "cube.proxyServiceFQDN" . | quote }} + # busybox nslookup prints the DNS server Address first; take the last A record. + a_record() { nslookup "$1" | sed -n 's/^Address: \([0-9][0-9.]*\)$/\1/p' | tail -n 1; } + domain_ip="$(a_record {{ $domain | quote }})" + target_ip="$(a_record "$target")" + test -n "$domain_ip" + test -n "$target_ip" + test "$domain_ip" = "$target_ip" + test -n "$(a_record {{ printf "wildcard-check.%s" $domain | quote }})" {{- end }} {{ if .Values.cubeNode.enabled }} --- @@ -397,20 +379,6 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} - {{- if and .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") }} - hostNetwork: true - dnsPolicy: None - dnsConfig: - nameservers: - - {{ .Values.cubeNode.dns.nameserver | quote }} - searches: - - {{ printf "%s.svc.%s" .Release.Namespace (include "cube.clusterDomain" .) | quote }} - - {{ printf "svc.%s" (include "cube.clusterDomain" .) | quote }} - - {{ include "cube.clusterDomain" . | quote }} - options: - - name: ndots - value: "5" - {{- end }} {{- include "cube.computePlacement" . | nindent 2 }} containers: - name: node-runtime @@ -425,9 +393,6 @@ spec: test -d {{ .Values.hostPaths.dataCubelet | quote }} test -S {{ printf "%s/cubelet.sock" .Values.hostPaths.dataCubelet | quote }} test -S {{ printf "%s/network-agent-grpc.sock" .Values.hostPaths.tmpCube | quote }} - {{- if and .Values.cubeDns.enabled (eq .Values.cubeDns.mode "nodeLocal") }} - nslookup {{ $domain | quote }} {{ .Values.cubeNode.dns.nameserver | quote }} >/dev/null - {{- end }} volumeMounts: - name: dev mountPath: /dev diff --git a/deploy/kubernetes/chart/templates/validate.yaml b/deploy/kubernetes/chart/templates/validate.yaml index 98e3cda48..8a8d130e7 100644 --- a/deploy/kubernetes/chart/templates/validate.yaml +++ b/deploy/kubernetes/chart/templates/validate.yaml @@ -16,24 +16,6 @@ {{- if and .Values.bootstrap.pvmHostKernel.enabled (not .Values.cubeNode.pvmGuestKernel.enabled) }} {{- fail "bootstrap.pvmHostKernel.enabled=true requires cubeNode.pvmGuestKernel.enabled=true to match one-click CUBE_PVM_ENABLE consistency" }} {{- end }} -{{- if and .Values.cubeNode.dns.useCubeDns (not .Values.cubeDns.enabled) }} -{{- fail "cubeNode.dns.useCubeDns=true requires cubeDns.enabled=true" }} -{{- end }} -{{- if and .Values.cubeNode.dns.useCubeDns (ne .Values.cubeDns.mode "nodeLocal") }} -{{- fail "cubeNode.dns.useCubeDns=true requires cubeDns.mode=nodeLocal" }} -{{- end }} -{{- if and .Values.cubeNode.dns.useCubeDns (not .Values.security.hostNetwork) }} -{{- fail "cubeNode.dns.useCubeDns=true requires security.hostNetwork=true because node-local DNS is bound on host loopback" }} -{{- end }} -{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (not .Values.cubeDns.sandboxGateway.enabled) }} -{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.sandboxGateway.enabled=true" }} -{{- end }} -{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (not .Values.cubeDns.enabled) }} -{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.enabled=true" }} -{{- end }} -{{- if and .Values.cubeNode.dns.sandbox.useCubeDns (ne .Values.cubeDns.mode "nodeLocal") }} -{{- fail "cubeNode.dns.sandbox.useCubeDns=true requires cubeDns.mode=nodeLocal" }} -{{- end }} {{- end }} {{- if .Values.externalControlPlane.enabled }} {{- if not .Values.externalControlPlane.masterEndpoint }} @@ -88,6 +70,9 @@ {{- if and .Values.cubeProxy.hostNetwork .Values.cubeProxy.ports.https.hostPort (ne (int .Values.cubeProxy.ports.https.hostPort) (int .Values.cubeProxy.ports.https.containerPort)) }} {{- fail "cubeProxy.hostNetwork=true requires cubeProxy.ports.https.hostPort to equal cubeProxy.ports.https.containerPort or be empty" }} {{- end }} +{{- if and .Values.cubeProxy.configureClusterDNS (eq (include "cube.proxyEnabled" .) "true") (not .Values.cubeProxy.domain) }} +{{- fail "cubeProxy.configureClusterDNS=true requires cubeProxy.domain" }} +{{- end }} {{- end }} {{- if .Values.cubeEgress.enabled }} {{- if not (has .Values.cubeEgress.ca.mode (list "existingSecret" "selfSigned" "inline")) }} @@ -103,23 +88,3 @@ {{- fail "cubeEgress.network.enabled=true requires cubeEgress.network.ingressInterface" }} {{- end }} {{- end }} -{{- if .Values.cubeDns.enabled }} -{{- if not $computeSelector }} -{{- fail "cubeDns.enabled=true requires placement.compute.nodeSelector because cube-dns runs on compute nodes" }} -{{- end }} -{{- if not (has .Values.cubeDns.mode (list "nodeLocal" "service")) }} -{{- fail "cubeDns.mode must be one of: nodeLocal, service" }} -{{- end }} -{{- if and (eq .Values.cubeDns.mode "nodeLocal") (not .Values.cubeDns.bindAddress) }} -{{- fail "cubeDns.mode=nodeLocal requires cubeDns.bindAddress" }} -{{- end }} -{{- if and .Values.cubeDns.sandboxGateway.enabled (ne .Values.cubeDns.mode "nodeLocal") }} -{{- fail "cubeDns.sandboxGateway.enabled=true requires cubeDns.mode=nodeLocal" }} -{{- end }} -{{- if and (eq .Values.cubeDns.mode "service") (not (default .Values.cubeProxy.advertiseIP .Values.cubeDns.answerIP)) }} -{{- fail "cubeDns.mode=service requires cubeDns.answerIP or cubeProxy.advertiseIP because CubeProxy is exposed on control-node hostNetwork" }} -{{- end }} -{{- if not (default .Values.cubeProxy.domain .Values.cubeDns.domain) }} -{{- fail "cubeDns.enabled=true requires cubeDns.domain or cubeProxy.domain" }} -{{- end }} -{{- end }} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 96b977957..54d2f63bf 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -17,8 +17,8 @@ global: # to mirror all images into a private registry without editing each entry # individually. Applies to the pvmHostBootstrap / nodeInit / node / master # / api / cubemastercli / proxyNode / cubeEgress / cubeEgressNet / webui - # entries but not to third-party images such as coredns / mysql / redis / - # docker-in-docker. + # entries but not to third-party images such as mysql / redis / + # docker-in-docker / alpine/k8s. imageRegistry: "" images: @@ -75,16 +75,17 @@ images: tag: v0.5.0 pullPolicy: Always - coredns: - repository: ccr.ccs.tencentyun.com/tkeimages/coredns - tag: v1.11.1-tke.1 - pullPolicy: IfNotPresent - templateBuilder: repository: docker tag: "27-dind" pullPolicy: IfNotPresent + # Used by the post-install Job that merges *.cubeProxy.domain into cluster CoreDNS. + kubectl: + repository: alpine/k8s + tag: "1.28.15" + pullPolicy: IfNotPresent + imagePullSecrets: [] # Optional StorageClass created by the chart. Off by default: on a plain @@ -124,7 +125,7 @@ placement: value: "true" effect: NoSchedule - # Compute placement is used by cube-node and node-local cube-dns. + # Compute placement is used by cube-node. compute: nodeSelector: cube.tencent.com/role: compute @@ -266,50 +267,6 @@ webui: upstream: "" resources: {} -cubeDns: - # One-click deploys CoreDNS for cube.app / wildcard resolution and lets the - # compute runtime resolve through a local DNS address. The default Kubernetes - # mode runs CoreDNS on Cube compute nodes and lets cube-node Pods use - # 127.0.0.54 explicitly via dnsConfig, without modifying host /etc/resolv.conf. - enabled: true - # nodeLocal: DaemonSet on Cube nodes, binds bindAddress. Set answerIP or - # cubeProxy.advertiseIP when CubeProxy runs only on control nodes. - # service: Deployment + ClusterIP Service for DNS itself. Requires - # answerIP or cubeProxy.advertiseIP for cube.app records because - # CubeProxy is exposed by control-node hostNetwork. - mode: nodeLocal - replicas: 1 - bindAddress: 127.0.0.54 - sandboxGateway: - # Also bind node-local cube-dns on the compute node HostIP so Cube sandbox - # guests can use it as their nameserver. The regular cube-node Pod DNS - # keeps using bindAddress. - enabled: true - domain: cube.app - # Empty answerIP uses cubeProxy.advertiseIP when set. If both are empty in - # nodeLocal mode, cube-dns returns the local compute HostIP for older local - # proxy topologies; control-node CubeProxy deployments should set one of them. - # service mode requires answerIP or cubeProxy.advertiseIP because CubeProxy is - # not exposed by Service. - answerIP: "" - forward: - # Empty upstreams forwards to /etc/resolv.conf. Set explicit upstreams to - # avoid forwarding loops in custom clusters. - upstreams: [] - # Cache TTL (seconds) for CoreDNS responses. Aligned with the 60s TTL emitted - # by the template plugin so clients do not see stale answers longer than the - # advertised TTL when the sandbox gateway IP changes. - cacheTTL: 60 - # Log every DNS query at INFO level. Off by default because compute nodes - # running many sandboxes emit thousands of queries per minute; enable only - # when actively debugging DNS resolution. - logQueries: false - service: - type: ClusterIP - port: 53 - annotations: {} - resources: {} - diagnostics: # One-click ships cube-diag scripts. In Kubernetes the chart exposes an # equivalent kubectl-based diagnostic script through a ConfigMap so operators @@ -408,20 +365,13 @@ cubeNode: cidrSkipConflictCheck: false dns: - useCubeDns: true - nameserver: 127.0.0.54 clusterDomain: cluster.local sandbox: - # DNS injected into Cube sandbox guest /etc/resolv.conf. When useCubeDns - # is true and nameservers is empty, the chart injects the current compute - # node HostIP so guests use the node-local cube-dns sandboxGateway. - useCubeDns: true + # Guests inherit the cube-node Pod nameservers (cluster DNS) so they can + # resolve in-cluster Services and *.cubeProxy.domain. + # Set nameservers to an explicit list to override. + followNodeDns: true nameservers: [] - waitTimeoutSeconds: 120 - checkImage: - repository: busybox - tag: "1.36" - pullPolicy: IfNotPresent env: [] # Sensible default resource requests / limits for the cube-node runtime @@ -478,9 +428,8 @@ cubeProxy: # One-click enables CubeProxy by default. This chart follows that behavior # and creates a self-signed test certificate unless production TLS is provided. enabled: true - # Public sandbox domain returned by CubeAPI and terminated by CubeProxy. - # For production, configure DNS for this domain and wildcard subdomains to - # the CubeProxy entrypoint, for example *.sandbox.example.com. + # Public sandbox domain (also used for TLS CN / wildcard DNS). + # For production, point external DNS at CubeProxy as well. domain: cube.app podAnnotations: {} replicas: 1 @@ -489,10 +438,15 @@ cubeProxy: # One-click CubeProxy is part of the control-node stack. This chart follows # that model and schedules CubeProxy through placement.controlPlane. hostNetwork: true - # IP returned by cube-dns for cube.app / *.cube.app when cubeDns.answerIP is - # empty. Set this to the selected control node IP, or set cubeDns.answerIP - # directly when using an external load balancer. + # Optional hint for external clients / docs (not used for in-cluster DNS). advertiseIP: "" + # When true, Helm patches cluster CoreDNS so *.cubeProxy.domain rewrites to + # the CubeProxy headless Service. Set false only if mutating kube-system/coredns + # is not allowed (disabling does not remove a previously injected block; + # uninstall runs the cleanup hook). + configureClusterDNS: true + service: + annotations: {} ports: http: containerPort: 80 diff --git a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh index 1b1d8cb0c..5c93c5b06 100755 --- a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh +++ b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh @@ -121,6 +121,22 @@ patch_common_yaml_list() { } configure_sandbox_dns() { + if [[ -z "${CUBE_SANDBOX_DNS_SERVERS:-}" && "${CUBE_SANDBOX_DNS_FOLLOW_NODE:-false}" == "true" ]]; then + CUBE_SANDBOX_DNS_SERVERS="$( + awk ' + /^nameserver[[:space:]]+/ { + ip=$2 + if (ip ~ /^127\./) next + if (ip ~ /^169\.254\./) next + if (ip == "::1") next + if (seen[ip]++) next + if (n++) printf "," + printf "%s", ip + } + ' /etc/resolv.conf + )" + log "sandbox DNS follow-node nameservers: ${CUBE_SANDBOX_DNS_SERVERS:-}" + fi patch_common_yaml_list default_dns_servers "${CUBE_SANDBOX_DNS_SERVERS:-}" } From 93c2fab06aeb046e6653a8c66e0588607393812e Mon Sep 17 00:00:00 2001 From: jinlong Date: Sun, 12 Jul 2026 17:43:04 +0800 Subject: [PATCH 9/9] feat(kubernetes): introduce cube-lifecycle-manager, migrate CubeProxy to ClusterIP + Ingress - Add cube-lifecycle-manager Deployment/Service as standalone sandbox auto-pause/resume coordinator, replacing the obsolete in-process cube-proxy-sidecar. CubeProxy discovers it via Service DNS, and CLM discovers proxy replicas through a Redis registry. - Migrate CubeProxy from hostNetwork to Pod network with ClusterIP Service + Ingress (SSL passthrough; TLS still terminates in CubeProxy). Remove hostNetwork and hostPort configuration; add admin listener binding and X-Cube-Admin-Token shared secret for in-cluster access. - Add proxy-ingress.yaml: dual Ingress for HTTPS (ssl-passthrough) and HTTP cleartext, defaulting hosts to cubeProxy.domain / *.domain. - Add lifecycle-manager.yaml: Deployment with Redis/CubeMaster wiring, configurable timeouts and sweep intervals, plus ClusterIP Service. - Add proxy registry support: each CubeProxy replica registers itself in Redis (heartbeat) so lifecycle-manager can discover all replicas. Pre-resolve Redis hostname to IP for ngx.timer cosocket compatibility. - Add cube-admin-token to Secret (auto-generated randAlphaNum 32 if not provided), shared between CubeProxy admin endpoints and lifecycle-manager. - Guard rails in validate.yaml: reject CHANGE_ME_* sentinel for TKE CLB security groups, reject removed cubeProxy.sidecar/hostNetwork keys, enforce lifecycleManager.enabled=true when CubeProxy is on. - TKE values (values-tke.yaml): expose CubeProxy via LoadBalancer CLB with pass-to-target + security-groups annotations (CHANGE_ME sentinel), disable chart Ingress. Document required runtime override. - Build script: build cube-lifecycle-manager image, revamp cube-webui build to match CI (repo-root context, deploy/one-click/webui/Dockerfile, BuildKit), remove cube-proxy-sidecar Go compilation from proxy-node context, add MIRROR selection (cn/github) for release downloads. - Remove deploy/kubernetes/images/cube-webui/Dockerfile (now driven by CI-aligned build path); update cube-pvm-host-bootstrap Dockerfile to ubuntu:24.04 with kubectl from pkgs.k8s.io. - Update docs: ARCHITECTURE, README, FAQ, images/README for new components, traffic path, and troubleshooting. Signed-off-by: jinlong --- deploy/kubernetes/chart/README.md | 40 +++-- deploy/kubernetes/chart/docs/ARCHITECTURE.md | 44 ++--- deploy/kubernetes/chart/docs/FAQ.md | 16 +- .../cube-proxy-node-entrypoint.sh | 55 +++++- deploy/kubernetes/chart/templates/NOTES.txt | 10 +- .../kubernetes/chart/templates/_helpers.tpl | 17 ++ .../chart/templates/lifecycle-manager.yaml | 118 +++++++++++++ .../chart/templates/proxy-ingress.yaml | 68 ++++++++ .../chart/templates/proxy-node.yaml | 109 +++++++----- .../chart/templates/proxy-service.yaml | 11 +- deploy/kubernetes/chart/templates/secret.yaml | 12 ++ .../chart/templates/tests/node-health.yaml | 29 ++-- .../kubernetes/chart/templates/validate.yaml | 34 +++- deploy/kubernetes/chart/values-tke.yaml | 32 ++++ deploy/kubernetes/chart/values.yaml | 107 +++++++++--- deploy/kubernetes/images/README.md | 29 ++-- deploy/kubernetes/images/build-cube-images.sh | 164 +++++++++++++----- .../images/cube-pvm-host-bootstrap/Dockerfile | 17 +- .../kubernetes/images/cube-webui/Dockerfile | 8 - .../scripts/cube-proxy-node-entrypoint.sh | 55 +++++- 20 files changed, 761 insertions(+), 214 deletions(-) create mode 100644 deploy/kubernetes/chart/templates/lifecycle-manager.yaml create mode 100644 deploy/kubernetes/chart/templates/proxy-ingress.yaml delete mode 100644 deploy/kubernetes/images/cube-webui/Dockerfile diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index ff9c54bc7..538d0015a 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -9,7 +9,7 @@ It follows the final Big Pod delivery design: - PVM host kernel installation and host reboot are handled by a dedicated Init Container; - Cube Node host preparation is handled by a second Init Container; - MySQL schema migration is handled by CubeMaster itself using embedded migrations; -- Cube Master, Cube API, cubemastercli, Cube Proxy Node, WebUI, Template Builder, and Cube Node use separate images. +- Cube Master, Cube API, cubemastercli, Cube Proxy Node, cube-lifecycle-manager, WebUI, Template Builder, and Cube Node use separate images. ## Directory @@ -48,6 +48,7 @@ external control plane / compute-only 模式见： | `cube-api` | HTTP API only. Runs `cube-api`. | | `cubemastercli` | Operational CLI only. Packages the real `CubeMaster/bin/cubemastercli` binary for exec-based operations. | | `cube-proxy-node` | Data-plane proxy. Reuses `CubeProxy/Dockerfile` and runs as a chart-managed control-plane Deployment when `cubeProxy.enabled=true`. | +| `cube-lifecycle-manager` | Sandbox auto-pause / auto-resume coordinator. Replaces the obsolete in-process cube-proxy-sidecar; CubeProxy discovers it via Service DNS and Redis registry. | | `cube-egress` | CubeEgress transparent outbound proxy. Reuses `CubeEgress/Dockerfile` and runs as a Cube Node sidecar when `cubeEgress.enabled=true`. | | `cube-egress-net` | Host network rule helper for CubeEgress TPROXY/ip-rule/sysctl setup. | | `cube-webui` | One-click WebUI static assets and OpenResty runtime. | @@ -246,6 +247,13 @@ are provisioned in the same zone as the scheduled control-plane Pod on multi-AZ TKE clusters. On non-TKE clusters do NOT include this file; provide the cluster's own StorageClass name instead. +It also exposes CubeProxy as a `LoadBalancer` Service (CLB) with Ingress +disabled, and sets TKE CLB annotations for `pass-to-target` plus a +`CHANGE_ME_TKE_CLB_SECURITY_GROUP` sentinel on +`service.cloud.tencent.com/security-groups`. Helm fails render until you +override that value in runtime values (comma-separated for multiple SGs), +or set it to `""` if you manage CLB security groups outside Helm. + ## Database migration The chart does not deliver a separate DB migration Job or image. CubeMaster owns MySQL schema migration and runs its embedded `CubeMaster/pkg/base/dao/migrate/migrations/mysql` migrations during startup. @@ -284,9 +292,9 @@ carry this operational entry point. `cube-proxy-node` is a Cube data-plane component. It is enabled by default to match one-click behavior and is installed, upgraded, and uninstalled with the Cube release as a control-plane Deployment. -The default TLS mode is `selfSigned`, matching the one-click mkcert-style test experience. Production environments should provide a real TLS certificate and reserve node host ports 80/443 on selected nodes. The image reuses `CubeProxy/Dockerfile`; the chart does not override nginx with a Kubernetes-only configuration. +The default TLS mode is `selfSigned`, matching the one-click mkcert-style test experience. Production environments should provide a real TLS certificate for CubeProxy. External clients reach `cubeProxy.domain` / `*.domain` through the chart Ingress (SSL passthrough; TLS still terminates in CubeProxy). The image reuses `CubeProxy/Dockerfile`; the chart does not override nginx with a Kubernetes-only configuration. -`cube-proxy-node` also starts the built-in `cube-proxy-sidecar`. The chart wires the sidecar to the chart-managed or third-party Redis endpoint and to the CubeMaster Kubernetes Service. Do not run a separate unmanaged CubeProxy sidecar. +`cube-proxy-node` depends on chart-managed `cube-lifecycle-manager` for sandbox auto-pause / auto-resume. The chart wires nginx `$cube_sidecar_addr` to the lifecycle-manager Service, opens the proxy admin listener for in-cluster discovery, and registers each proxy replica in Redis. Do not deploy a separate cube-proxy-sidecar. ### Production TLS Secret @@ -360,24 +368,18 @@ cubeProxy: This mode creates a release-scoped Secret with `tls.crt`, `tls.key`, and `ca.crt`. Import `ca.crt` into clients if browser or SDK trust is required. Do not use this mode for production. -`cube-proxy-node` uses `placement.controlPlane`, matching the one-click control-node placement for CubeProxy. The chart does not create node labels. +`cube-proxy-node` uses `placement.controlPlane`. The chart does not create node labels. + +CubeProxy runs on the **Pod network** (no `hostNetwork`). Traffic path: + +1. External clients → Ingress Controller → ClusterIP Service → CubeProxy Pod +2. In-cluster clients / sandbox guests → CoreDNS rewrite → same ClusterIP Service -`cubeProxy.hostNetwork=true` is also the default. This is required for one-click -parity: CubeProxy must terminate `cube.app` / wildcard traffic on a node-local -host-network endpoint. When the sandbox owner is on a compute node, CubeProxy -uses Redis routing metadata to connect to the owner `HostIP:hostPort`. The -chart patches the image's default nginx listeners to -the configured `cubeProxy.ports.*.containerPort` values, which default to `80` -and `443`. +TLS for `cube.app` / wildcards still terminates **inside CubeProxy**. The default Ingress annotations enable nginx-ingress SSL passthrough + HTTPS backend; override `cubeProxy.ingress.className` / `annotations` for TKE CLB or other controllers. Set `cubeProxy.ingress.enabled=false` if you manage the entrypoint yourself (keep the Service as backend). -The chart does not create a `cube-proxy-node` ClusterIP Service. A normal -Kubernetes Service load-balances requests across proxy Pods, which does not -match the one-click model where callers reach an explicit CubeProxy host -endpoint. For sandbox data-plane traffic, point wildcard DNS at a specific -control-node CubeProxy IP or at an external load balancer that preserves the intended -CubeProxy topology. +When the sandbox owner is on a compute node, CubeProxy still uses Redis routing metadata to connect to the owner `HostIP:hostPort`. The chart patches the image's default nginx listeners to the configured `cubeProxy.ports.*.containerPort` values (default `80` / `443`). -CubeProxy admin health remains loopback-only inside each Pod, matching the image's nginx admin listener. The chart validates it through Pod readiness/liveness probes rather than exposing it through a Service. +CubeProxy admin is reachable in-cluster at each Pod IP:`adminPort` (default `8082`) for cube-lifecycle-manager discovery; probes use the admin token header. CubeProxy reads sandbox routing metadata from Redis in nginx Lua. Because nginx does not automatically inherit Kubernetes DNS resolution for Lua cosocket @@ -401,7 +403,7 @@ cubeProxy: ## Cluster DNS for sandbox domain When CubeProxy is enabled, the chart patches **cluster CoreDNS** so -`cubeProxy.domain` / `*.domain` rewrite to the CubeProxy headless Service +`cubeProxy.domain` / `*.domain` rewrite to the CubeProxy ClusterIP Service (Pod IP). Users only set the domain: ```yaml diff --git a/deploy/kubernetes/chart/docs/ARCHITECTURE.md b/deploy/kubernetes/chart/docs/ARCHITECTURE.md index a7249dd9f..878c8a9f6 100644 --- a/deploy/kubernetes/chart/docs/ARCHITECTURE.md +++ b/deploy/kubernetes/chart/docs/ARCHITECTURE.md @@ -12,9 +12,10 @@ CubeSandbox Chart 按职责分为 7 层： | 控制面 API | CubeAPI | Deployment + Service | 对外 HTTP API，读写 MySQL，访问 CubeMaster | | 管理入口 | WebUI | Deployment + Service + ConfigMap | 静态控制台，反向代理 `/cubeapi/` 到 CubeAPI | | 运维入口 | cubemastercli | Deployment | 面向 `kubectl exec` 的 CLI Pod，交付真实 `cubemastercli` 并注入本 Release 的 CubeMaster endpoint | -| 依赖存储 | MySQL / Redis | 内置 StatefulSet + Headless Service + volumeClaimTemplates/hostPath，或第三方服务 | MySQL 存储业务数据；Redis 存储 CubeProxy/Sidecar 状态 | +| 依赖存储 | MySQL / Redis | 内置 StatefulSet + Headless Service + volumeClaimTemplates/hostPath，或第三方服务 | MySQL 存储业务数据；Redis 存储 CubeProxy / lifecycle-manager 状态 | | 计算面 | Cube Node Big Pod | DaemonSet | 节点初始化、运行 cubelet/network-agent、透明 egress sidecar | | 数据面入口 | CubeProxy + 集群 DNS | CubeProxy Deployment；自动把 `*.domain` 写入集群 CoreDNS | HTTP/HTTPS sandbox 入口；集群内域名泛解析 | +| 生命周期 | cube-lifecycle-manager | Deployment + ClusterIP Service | sandbox 自动 pause/resume；通过 Redis 发现 CubeProxy 副本 | 默认完整部署形态： @@ -28,7 +29,7 @@ flowchart TB CLI["cubemastercli Deployment\nreal cubemastercli binary"] MYSQL[("cube-mysql\nMySQL 8.0 + PVC")] REDIS[("cube-redis\nRedis 7-alpine + PVC")] - PROXY["cube-proxy-node Deployment\nHTTP/HTTPS 80/443"] + PROXY["cube-proxy-node\nClusterIP + Ingress"] CMS["cube-master-config Secret\nconf.yaml"] CA["cube-egress-ca Secret"] CERT["cube-proxy-certs Secret"] @@ -121,26 +122,28 @@ flowchart TB | 资源 | 模板 | 职责 | | --- | --- | --- | -| `cube-proxy-node` Deployment | `templates/proxy-node.yaml` | 提供 sandbox HTTP/HTTPS 数据面入口，使用 `placement.controlPlane` 与 one-click control 节点语义对齐，并使用 `hostNetwork` 监听节点 `80/443` | +| `cube-proxy-node` Deployment | `templates/proxy-node.yaml` | sandbox HTTP/HTTPS 数据面；`placement.controlPlane`；Pod 网络 | +| `cube-lifecycle-manager` Deployment / Service | `templates/lifecycle-manager.yaml` | sandbox 自动 pause/resume；CubeProxy 通过 `$cube_sidecar_addr` 回调 `/internal/resume`，CLM 经 Redis 发现各 proxy 副本 | | `cube-proxy-certs` Secret / Certificate | `templates/proxy-node.yaml` | TLS 证书，支持 selfSigned、inline、existingSecret、certManager | -| `cube-proxy-node` Service | `templates/proxy-service.yaml` | headless Service，DNS 直接返回 CubeProxy Pod IP | -| cluster DNS | `templates/cluster-dns.yaml` | CubeProxy 启用时把 `*.cubeProxy.domain` rewrite 到 headless Service | +| `cube-proxy-node` Service | `templates/proxy-service.yaml` | ClusterIP，Ingress / 集群 DNS 后端 | +| `cube-proxy-node` Ingress | `templates/proxy-ingress.yaml` | `domain` / `*.domain`；SSL passthrough，TLS 仍在 CubeProxy 终结 | +| cluster DNS | `templates/cluster-dns.yaml` | CubeProxy 启用时把 `*.cubeProxy.domain` rewrite 到 ClusterIP Service | -CubeProxy 默认保持 One Click 的 host-network 语义： +CubeProxy 数据面入口： -- `cubeProxy.hostNetwork=true`，Pod IP 等于所在节点 HostIP。 -- `cube-proxy-node` 复用 `placement.controlPlane`，与 one-click control 节点上的 CubeProxy 对齐。 -- nginx 监听节点 `80/443`，Chart 启动脚本会把镜像默认 `8081/8080` patch 为 values 中配置的端口。 -- 集群内 `*.cube.app` 在 CubeProxy 启用时自动 rewrite 到 CubeProxy headless Service（Pod/HostIP）；用户一般只需改 `cubeProxy.domain`。 +- 运行在 Pod 网络（无 hostNetwork）；nginx 监听 containerPort（默认 80/443）。 +- `cube-proxy-node` 复用 `placement.controlPlane`。 +- 外部流量：Ingress → ClusterIP Service → Pod；TLS 在 CubeProxy 终结（默认 nginx-ingress SSL passthrough 注解）。 +- 集群内 `*.cube.app` rewrite 到 CubeProxy ClusterIP Service。 - CubeProxy 通过 Redis 中的 owner `HostIP:hostPort` 元数据转发到目标 compute 节点 sandbox。 -- Chart 不修改 CubeProxy Lua 后端解析语义；跨节点访问仍遵循社区 CubeProxy 使用 Redis `HostIP:hostPort` 的原始路径。 +- Chart 不修改 CubeProxy Lua 后端解析语义。 ## 3. 默认 DNS 架构 Chart **不**部署自有 CoreDNS。CubeProxy 启用且 `configureClusterDNS=true`（默认）时： - Helm hook 把 `domain` / `*.domain` rewrite 到 `-proxy-node..svc.cluster.local`。 -- CubeProxy Service 为 **headless**，解析结果是 Pod IP（hostNetwork 下即控制节点 HostIP）。 +- CubeProxy Service 为 **ClusterIP**，解析结果是 Service VIP。 - `cubeNode.dns.sandbox.followNodeDns=true`：guest 跟随节点/集群 DNS。 ```mermaid @@ -191,7 +194,7 @@ flowchart TD ### 4.1.1 调度与时区 -- CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 使用 `placement.controlPlane`。 +- CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy、cube-lifecycle-manager 使用 `placement.controlPlane`。 - `cube-node` 使用 `placement.compute`。 - 所有 Chart 管理的 Cube 容器、sidecar 和 initContainer 都通过 `global.timezone` 注入 `TZ`，默认 `Asia/Shanghai`。 @@ -319,12 +322,11 @@ flowchart LR 说明： - `cube-proxy-node` 默认启用，随 Chart 一起安装和卸载。 -- `cube-proxy-node` 默认使用 `hostNetwork`，确保 control 节点能够接入 `80/443` 流量。 -- Chart 不创建 `cube-proxy-node` ClusterIP Service，避免 Kubernetes 随机分发到非目标节点后偏离 One Click 的显式 CubeProxy host 入口模型。 -- 需要多节点统一入口时，应由外部 DNS/LB 明确指向预期 control 节点 CubeProxy 或保证社区 CubeProxy 的 `HostIP:hostPort` 跨节点路径可用，而不是依赖默认 ClusterIP 随机分流。 +- 数据面入口为 ClusterIP Service + Ingress（SSL passthrough，TLS 在 CubeProxy 终结）。 +- 无 Ingress Controller 时可设 `cubeProxy.ingress.enabled=false`，自行把外部流量接到 Service。 - TLS 支持 selfSigned、existingSecret、inline、certManager。 -- 生产环境应提供正式证书，并把 sandbox domain / wildcard DNS 指向明确的 CubeProxy 入口。 -- Chart 在 CubeProxy 启用时自动配置集群内 `*.cube.app`；外部 DNS/LB 仍需使用方配置。计算节点 guest 默认跟随节点 DNS。 +- 生产环境应提供正式证书，并把 sandbox domain / wildcard DNS 指向 Ingress 入口。 +- Chart 在 CubeProxy 启用时自动配置集群内 `*.cube.app`；外部 DNS 仍需使用方配置。计算节点 guest 默认跟随节点 DNS。 ### 5.3 Sandbox 出站 egress @@ -410,7 +412,7 @@ externalControlPlane: | `storageClass.volumeBindingMode` | `WaitForFirstConsumer` | 多可用区 TKE 集群中等待 Pod 选中 control 节点后再创建 CBS 盘，避免 PV zone 与 control 节点不匹配 | | `controlPlane.enabled` | `true` | 是否部署内置控制面 | | `externalControlPlane.enabled` | `false` | 是否使用外部 CubeMaster | -| `placement.controlPlane.nodeSelector` | `cube.tencent.com/role=control` | 控制 CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy 调度范围 | +| `placement.controlPlane.nodeSelector` | `cube.tencent.com/role=control` | 控制 CubeMaster、CubeAPI、WebUI、cubemastercli、内置 MySQL、内置 Redis、CubeProxy、cube-lifecycle-manager 调度范围 | | `placement.compute.nodeSelector` | 含 `allow-pvm-bootstrap=true` | 控制 `cube-node` 调度范围，并要求节点显式允许 PVM bootstrap | | `cubeProxy.domain` | `cube.app` | sandbox 域名；集群 DNS 与 TLS 共用 | | `cubeProxy.configureClusterDNS` | `true` | 是否把 `*.domain` 自动写入集群 CoreDNS | @@ -422,8 +424,10 @@ externalControlPlane: | `bootstrap.nodeInit.*` | 多项 | 控制节点预检、XFS、KVM、CIDR 检测 | | `mysql.host` | `""` | 非空时使用第三方 MySQL | | `redis.host` | `""` | 非空时使用第三方 Redis | -| `cubeProxy.enabled` | `true` | 是否部署 control 节点 CubeProxy 数据面入口 | +| `cubeProxy.enabled` | `true` | 是否部署 CubeProxy 数据面入口 | +| `cubeProxy.ingress.enabled` | `true` | 是否创建 Ingress（SSL passthrough → CubeProxy） | | `cubeProxy.advertiseIP` | `""` | 可选，仅作外部入口提示；集群内泛解析不依赖它 | +| `lifecycleManager.enabled` | `true` | 是否部署 cube-lifecycle-manager（CubeProxy 启用时必开） | | `cubeEgress.enabled` | `true` | 是否在 Big Pod 中启用 egress sidecar | | `webui.enabled` | `true` | 是否部署 WebUI | | `controlPlane.templateBuilder.enabled` | `false` | 是否启用模板构建 sidecar | diff --git a/deploy/kubernetes/chart/docs/FAQ.md b/deploy/kubernetes/chart/docs/FAQ.md index 35dcedae1..5851d7509 100644 --- a/deploy/kubernetes/chart/docs/FAQ.md +++ b/deploy/kubernetes/chart/docs/FAQ.md @@ -263,17 +263,19 @@ kubectl -n cube-system logs -c cube-node --tail=200 | grep -i re 1. **集群内**（Pod / 跟随节点 DNS 的 sandbox guest）:`nslookup test.cube.app` - 无应答 → 查 `kubectl -n kube-system get cm coredns -o yaml` 是否含 `# BEGIN cube-sandbox-dns` - 或看 Job：`kubectl -n cube-system logs job/cube-cluster-dns-apply` - - 应答应是 CubeProxy Pod IP（headless Service）；对照 `kubectl -n cube-system get endpoints cube-proxy-node -o wide` + - 应答应是 CubeProxy **ClusterIP**；对照 `kubectl -n cube-system get svc cube-proxy-node -o wide` 2. **guest DNS**：默认 `followNodeDns=true`；显式覆盖用 `cubeNode.dns.sandbox.nameservers` -3. **集群外部**(客户浏览器 / SDK):用户侧 DNS / Private DNS / LB 仍需自行配置 +3. **集群外部**(客户浏览器 / SDK):把 `cube.app` / `*.cube.app` 指到 Ingress Controller / LB -### E2. CubeProxy 起不来:`hostNetwork port 80/443 already in use` +### E2. Ingress / 外部入口不通 -Control 节点上 80/443 被其他服务占用。方案: +CubeProxy 不再占用节点 80/443。排查: -- 释放占用端口 -- 或改用非默认端口:`cubeProxy.ports.http.hostPort: 8080`(但客户端 URL 也要跟着改) -- **不推荐**:关掉 hostNetwork 走 ClusterIP,会绕开 one-click 的入口语义 +- Ingress Controller 是否存在：`kubectl get ingressclass`、`kubectl get ingress -n cube-system` +- passthrough 注解是否匹配你的 Controller（默认是 nginx-ingress） +- TKE：`-f values-tke.yaml` 默认 **关闭 Ingress**，改用 `LoadBalancer` Service（CLB）暴露 CubeProxy，DNS 指到 Service EXTERNAL-IP；不要用 NodePort 硬接 qcloud Ingress +- nginx Controller 是否开启 SSL passthrough（需 `--enable-ssl-passthrough`） +- 无 Ingress 时也可设 `cubeProxy.ingress.enabled=false`，自行把流量接到 Service ### E3. selfSigned TLS 浏览器提示不安全 diff --git a/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh b/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh index 3b5b1a1aa..dd828e706 100755 --- a/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh +++ b/deploy/kubernetes/chart/files/cube-proxy-node/cube-proxy-node-entrypoint.sh @@ -9,6 +9,9 @@ # Reads the following env vars (populated by the Chart in the Pod spec): # CUBE_PROXY_HTTP_LISTEN_PORT - digits only, target HTTP listen port # CUBE_PROXY_HTTPS_LISTEN_PORT - digits only, target HTTPS listen port +# CUBE_PROXY_ADMIN_LISTEN - admin server listen address (default 0.0.0.0) +# CUBE_PROXY_ADMIN_PORT - admin server port (default 8082) +# CUBE_PROXY_ADMIN_TOKEN - optional shared secret for /admin/* # CUBE_PROXY_RESOLVER_ADDRS - space-separated nameservers; empty means # read from /etc/resolv.conf # CUBE_PROXY_RESOLVER_VALID - nginx `valid=` duration @@ -16,8 +19,8 @@ # CUBE_PROXY_RESOLVER_IPV6 - on/off # REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB # TIMEOUT_MIN, TIMEOUT_MAX -# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR -# Rewrites nginx.conf listen ports in-place, then generates +# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR (cube-lifecycle-manager host:port) +# Rewrites nginx.conf listen ports / admin bind in-place, then generates # /usr/local/openresty/nginx/conf/global/global.conf and execs start.sh. set -eu @@ -31,17 +34,42 @@ case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in ;; esac +admin_listen="${CUBE_PROXY_ADMIN_LISTEN:-0.0.0.0}" +admin_port="${CUBE_PROXY_ADMIN_PORT:-8082}" +case "${admin_port}" in + *[!0-9]*|"") + echo "invalid CubeProxy admin port: ${admin_port}" >&2 + exit 1 + ;; +esac +case "${admin_listen}" in + *[\;\{\}\$\`\"\\]*) + echo "invalid CubeProxy admin listen address: ${admin_listen}" >&2 + exit 1 + ;; +esac + sed -i \ -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ + -e "s/listen 127\\.0\\.0\\.1:8082;/listen ${admin_listen}:${admin_port};/g" \ /usr/local/openresty/nginx/conf/nginx.conf escape_nginx_value() { printf '%s' "$1" | sed 's/[\\"]/\\&/g' } +if [ -n "${CUBE_PROXY_ADMIN_TOKEN:-}" ]; then + token_escaped="$(escape_nginx_value "${CUBE_PROXY_ADMIN_TOKEN}")" + # Stock nginx.conf sets an empty token in each server block; rewrite them + # so /admin/* and resume paths share the same secret as cube-lifecycle-manager. + sed -i \ + -e "s/set \\\$cube_admin_token \"\";/set \\\$cube_admin_token \"${token_escaped}\";/g" \ + /usr/local/openresty/nginx/conf/nginx.conf +fi + resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" if [ -z "${resolver_addrs}" ]; then resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" @@ -57,6 +85,26 @@ case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT ;; esac +sidecar_addr="${CUBE_SIDECAR_LISTEN_ADDR:-}" +[ -n "${sidecar_addr}" ] || { + echo "CUBE_SIDECAR_LISTEN_ADDR is required (cube-lifecycle-manager host:port)" >&2 + exit 1 +} + +# proxy_registry.lua publishes from ngx.timer, which has no nginx resolver. +# Resolve a hostname REDIS target to an IP up-front when possible. +if [ -n "${CUBE_PROXY_REGISTRY_REDIS_HOST:-}" ]; then + case "${CUBE_PROXY_REGISTRY_REDIS_HOST}" in + *[!0-9.]* ) + resolved="$(getent ahostsv4 "${CUBE_PROXY_REGISTRY_REDIS_HOST}" 2>/dev/null | awk 'NR==1 {print $1}')" + if [ -n "${resolved}" ]; then + CUBE_PROXY_REGISTRY_REDIS_HOST="${resolved}" + export CUBE_PROXY_REGISTRY_REDIS_HOST + fi + ;; + esac +fi + cat > /usr/local/openresty/nginx/conf/global/global.conf <:{{ .Values.cubeProxy.ports.http.containerPort }}/{{ .Values.cubeProxy.ports.https.containerPort }} - Headless Service: {{ include "cube.proxyName" . }} - External clients still need their own DNS/LB if they are outside the cluster. + Service ({{ .Values.cubeProxy.service.type | default "ClusterIP" }}): {{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/{{ .Values.cubeProxy.ports.https.containerPort }} +{{- if .Values.cubeProxy.ingress.enabled }} + Ingress: {{ include "cube.proxyName" . }} (TLS passthrough → CubeProxy; point external DNS at the Ingress) +{{- end }} +{{- if eq (include "cube.lifecycleManagerEnabled" .) "true" }} + Lifecycle Manager: {{ include "cube.lifecycleManagerName" . }}:{{ .Values.lifecycleManager.service.port }} +{{- end }} {{- end }} Useful commands: diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl index 846911f06..1b645ee00 100644 --- a/deploy/kubernetes/chart/templates/_helpers.tpl +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -140,6 +140,23 @@ tolerations: {{- printf "%s.%s.svc.%s" (include "cube.proxyName" .) .Release.Namespace (include "cube.clusterDomain" .) -}} {{- end -}} +{{- define "cube.lifecycleManagerName" -}} +{{- printf "%s-lifecycle-manager" (include "cube.fullname" .) -}} +{{- end -}} + +{{- define "cube.lifecycleManagerEnabled" -}} +{{- $lcm := default dict .Values.lifecycleManager -}} +{{- if and (dig "enabled" true $lcm) (eq (include "cube.proxyEnabled" .) "true") -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "cube.lifecycleManagerFQDN" -}} +{{- printf "%s.%s.svc.%s" (include "cube.lifecycleManagerName" .) .Release.Namespace (include "cube.clusterDomain" .) -}} +{{- end -}} + +{{- define "cube.lifecycleManagerAddr" -}} +{{- printf "%s:%v" (include "cube.lifecycleManagerFQDN" .) .Values.lifecycleManager.service.port -}} +{{- end -}} + {{- define "cube.configureClusterDNS" -}} {{- if and .Values.cubeProxy.configureClusterDNS (eq (include "cube.proxyEnabled" .) "true") -}}true{{- else -}}false{{- end -}} {{- end -}} diff --git a/deploy/kubernetes/chart/templates/lifecycle-manager.yaml b/deploy/kubernetes/chart/templates/lifecycle-manager.yaml new file mode 100644 index 000000000..fb1f684b1 --- /dev/null +++ b/deploy/kubernetes/chart/templates/lifecycle-manager.yaml @@ -0,0 +1,118 @@ +{{- if eq (include "cube.lifecycleManagerEnabled" .) "true" }} +{{- $redisHost := default (include "cube.redisHost" .) .Values.lifecycleManager.redis.host -}} +{{- $redisPort := default .Values.redis.port .Values.lifecycleManager.redis.port -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cube.lifecycleManagerName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: lifecycle-manager +spec: + replicas: {{ .Values.lifecycleManager.replicas }} + selector: + matchLabels: + {{- include "cube.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: lifecycle-manager + template: + metadata: + labels: + {{- include "cube.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: lifecycle-manager + {{- with .Values.lifecycleManager.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "cube.controlPlanePlacement" . | nindent 6 }} + containers: + - name: cube-lifecycle-manager + image: {{ include "cube.cubeImage" (dict "image" .Values.images.lifecycleManager "context" $) | quote }} + imagePullPolicy: {{ .Values.images.lifecycleManager.pullPolicy }} + env: + {{- include "cube.timezoneEnv" . | nindent 12 }} + - name: CUBE_LCM_REDIS_ADDR + value: {{ printf "%s:%v" $redisHost $redisPort | quote }} + - name: CUBE_LCM_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: redis-password + - name: CUBE_LCM_REDIS_DB + value: {{ .Values.lifecycleManager.redis.db | quote }} + - name: CUBE_LCM_CUBEMASTER_URL + value: {{ printf "http://%s" (include "cube.masterEndpoint" .) | quote }} + - name: CUBE_LCM_LISTEN_ADDR + value: {{ .Values.lifecycleManager.listenAddr | quote }} + - name: CUBE_LCM_DEFAULT_IDLE_TIMEOUT + value: {{ .Values.lifecycleManager.defaultIdleTimeout | quote }} + - name: CUBE_LCM_HEARTBEAT_TTL + value: {{ .Values.lifecycleManager.heartbeatTTL | quote }} + - name: CUBE_LCM_DISCOVERY_REFRESH + value: {{ .Values.lifecycleManager.discoveryRefresh | quote }} + - name: CUBE_LCM_LAST_ACTIVE_POLL + value: {{ .Values.lifecycleManager.lastActivePoll | quote }} + - name: CUBE_LCM_IDLE_SWEEP_INTERVAL + value: {{ .Values.lifecycleManager.idleSweepInterval | quote }} + - name: CUBE_LCM_BOOTSTRAP_WARMUP + value: {{ .Values.lifecycleManager.bootstrapWarmup | quote }} + - name: CUBE_LCM_STATE_LOCK_TTL + value: {{ .Values.lifecycleManager.stateLockTTL | quote }} + - name: CUBE_LCM_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: cube-admin-token + ports: + - name: http + containerPort: {{ .Values.lifecycleManager.service.port }} + protocol: TCP + {{- if .Values.lifecycleManager.probes.readiness.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.lifecycleManager.probes.readiness.path | quote }} + port: {{ .Values.lifecycleManager.probes.readiness.port }} + initialDelaySeconds: {{ .Values.lifecycleManager.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.lifecycleManager.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.lifecycleManager.probes.readiness.failureThreshold }} + {{- end }} + {{- if .Values.lifecycleManager.probes.liveness.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.lifecycleManager.probes.liveness.path | quote }} + port: {{ .Values.lifecycleManager.probes.liveness.port }} + initialDelaySeconds: {{ .Values.lifecycleManager.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.lifecycleManager.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.lifecycleManager.probes.liveness.failureThreshold }} + {{- end }} + {{- with .Values.lifecycleManager.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cube.lifecycleManagerName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: lifecycle-manager + {{- with .Values.lifecycleManager.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + selector: + {{- include "cube.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: lifecycle-manager + ports: + - name: http + port: {{ .Values.lifecycleManager.service.port }} + targetPort: http + protocol: TCP +{{- end }} diff --git a/deploy/kubernetes/chart/templates/proxy-ingress.yaml b/deploy/kubernetes/chart/templates/proxy-ingress.yaml new file mode 100644 index 000000000..6d36e0ca7 --- /dev/null +++ b/deploy/kubernetes/chart/templates/proxy-ingress.yaml @@ -0,0 +1,68 @@ +{{- if and (eq (include "cube.proxyEnabled" .) "true") .Values.cubeProxy.ingress.enabled }} +{{- $hosts := .Values.cubeProxy.ingress.hosts -}} +{{- if not $hosts -}} +{{- $hosts = list .Values.cubeProxy.domain (printf "*.%s" .Values.cubeProxy.domain) -}} +{{- end }} +{{- /* HTTPS / TLS passthrough → CubeProxy :https (TLS terminates in CubeProxy). */}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "cube.proxyName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node + {{- with .Values.cubeProxy.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.cubeProxy.ingress.className }} + ingressClassName: {{ . | quote }} + {{- end }} + {{- with .Values.cubeProxy.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range $hosts }} + - host: {{ . | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "cube.proxyName" $ }} + port: + name: https + {{- end }} +--- +{{- /* Cleartext HTTP → CubeProxy :http (no ssl-passthrough annotations). */}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "cube.proxyName" . }}-http + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: cube-proxy-node + {{- with .Values.cubeProxy.ingress.className }} + {{- /* intentionally no ssl-passthrough annotations */}} + {{- end }} +spec: + {{- with .Values.cubeProxy.ingress.className }} + ingressClassName: {{ . | quote }} + {{- end }} + rules: + {{- range $hosts }} + - host: {{ . | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "cube.proxyName" $ }} + port: + name: http + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/proxy-node.yaml b/deploy/kubernetes/chart/templates/proxy-node.yaml index 4c5ca963b..71d2c9718 100644 --- a/deploy/kubernetes/chart/templates/proxy-node.yaml +++ b/deploy/kubernetes/chart/templates/proxy-node.yaml @@ -1,6 +1,23 @@ {{- if eq (include "cube.proxyEnabled" .) "true" }} {{- $redisHost := default (include "cube.redisHost" .) .Values.cubeProxy.redis.host -}} {{- $redisPort := default .Values.redis.port .Values.cubeProxy.redis.port -}} +{{- /* proxy_registry.lua runs in ngx.timer and cannot use nginx location resolvers; + prefer a concrete IP (ClusterIP, or headless Endpoints address) for cosocket connect(). */ -}} +{{- $redisRegistryHost := $redisHost -}} +{{- if eq (include "cube.redisBuiltinEnabled" .) "true" }} +{{- $redisSvc := lookup "v1" "Service" .Release.Namespace (include "cube.redisName" .) -}} +{{- if and $redisSvc $redisSvc.spec.clusterIP (ne $redisSvc.spec.clusterIP "None") -}} +{{- $redisRegistryHost = $redisSvc.spec.clusterIP -}} +{{- else -}} +{{- $redisEps := lookup "v1" "Endpoints" .Release.Namespace (include "cube.redisName" .) -}} +{{- if and $redisEps $redisEps.subsets (gt (len $redisEps.subsets) 0) -}} +{{- $subset := index $redisEps.subsets 0 -}} +{{- if and $subset.addresses (gt (len $subset.addresses) 0) -}} +{{- $redisRegistryHost = (index $subset.addresses 0).ip -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} {{- $proxyCertSecretName := include "cube.proxyCertSecretName" . -}} {{- if eq .Values.cubeProxy.tls.mode "inline" }} apiVersion: v1 @@ -123,8 +140,7 @@ spec: {{- end }} spec: automountServiceAccountToken: false - hostNetwork: {{ .Values.cubeProxy.hostNetwork }} - dnsPolicy: {{ ternary "ClusterFirstWithHostNet" "ClusterFirst" .Values.cubeProxy.hostNetwork }} + dnsPolicy: ClusterFirst {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -141,10 +157,16 @@ spec: exec /etc/cube-proxy-node/cube-proxy-node-entrypoint.sh env: {{- include "cube.timezoneEnv" . | nindent 12 }} - - name: NODE_IP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME valueFrom: fieldRef: - fieldPath: status.hostIP + fieldPath: metadata.name + - name: NODE_IP + value: "$(POD_IP)" - name: REDIS_HOST value: {{ $redisHost | quote }} - name: REDIS_PORT @@ -160,35 +182,24 @@ spec: value: {{ .Values.cubeProxy.timeout.min | quote }} - name: TIMEOUT_MAX value: {{ .Values.cubeProxy.timeout.max | quote }} - - name: CUBE_SIDECAR_REDIS_ADDR - value: {{ printf "%s:%v" $redisHost $redisPort | quote }} - - name: CUBE_SIDECAR_REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "cube.secretName" . }} - key: redis-password - - name: CUBE_SIDECAR_REDIS_DB - value: {{ .Values.cubeProxy.redis.db | quote }} - - name: CUBE_SIDECAR_CUBEMASTER_URL - value: {{ printf "http://%s" (include "cube.masterEndpoint" .) | quote }} - - name: CUBE_SIDECAR_PROXY_ADMIN_URLS - value: {{ join "," .Values.cubeProxy.sidecar.proxyAdminURLs | quote }} + {{- if eq (include "cube.lifecycleManagerEnabled" .) "true" }} + # nginx $cube_sidecar_addr → cube-lifecycle-manager Service (resume callback). - name: CUBE_SIDECAR_LISTEN_ADDR - value: {{ .Values.cubeProxy.sidecar.listenAddr | quote }} - - name: CUBE_SIDECAR_DEFAULT_IDLE_TIMEOUT - value: {{ .Values.cubeProxy.sidecar.defaultIdleTimeout | quote }} - - name: CUBE_SIDECAR_LAST_ACTIVE_POLL - value: {{ .Values.cubeProxy.sidecar.lastActivePoll | quote }} - - name: CUBE_SIDECAR_IDLE_SWEEP_INTERVAL - value: {{ .Values.cubeProxy.sidecar.idleSweepInterval | quote }} - - name: CUBE_SIDECAR_BOOTSTRAP_WARMUP - value: {{ .Values.cubeProxy.sidecar.bootstrapWarmup | quote }} - - name: CUBE_SIDECAR_STATE_LOCK_TTL - value: {{ .Values.cubeProxy.sidecar.stateLockTTL | quote }} + value: {{ include "cube.lifecycleManagerAddr" . | quote }} + {{- end }} - name: CUBE_PROXY_HTTP_LISTEN_PORT value: {{ .Values.cubeProxy.ports.http.containerPort | quote }} - name: CUBE_PROXY_HTTPS_LISTEN_PORT value: {{ .Values.cubeProxy.ports.https.containerPort | quote }} + - name: CUBE_PROXY_ADMIN_LISTEN + value: {{ .Values.cubeProxy.adminListen | quote }} + - name: CUBE_PROXY_ADMIN_PORT + value: {{ .Values.cubeProxy.adminPort | quote }} + - name: CUBE_PROXY_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: cube-admin-token - name: CUBE_PROXY_RESOLVER_ADDRS value: {{ join " " .Values.cubeProxy.resolver.addresses | quote }} - name: CUBE_PROXY_RESOLVER_VALID @@ -197,22 +208,42 @@ spec: value: {{ .Values.cubeProxy.resolver.timeout | quote }} - name: CUBE_PROXY_RESOLVER_IPV6 value: {{ ternary "on" "off" .Values.cubeProxy.resolver.ipv6 | quote }} - {{- if .Values.cubeProxy.sidecar.adminToken }} - - name: CUBE_SIDECAR_ADMIN_TOKEN - value: {{ .Values.cubeProxy.sidecar.adminToken | quote }} + {{- if and .Values.cubeProxy.registry.enabled (eq (include "cube.lifecycleManagerEnabled" .) "true") }} + - name: CUBE_PROXY_REGISTRY_ENABLE + value: "1" + - name: CUBE_PROXY_ID + value: "$(POD_NAME)" + - name: CUBE_PROXY_ADMIN_URL + value: {{ printf "http://$(POD_IP):%v" .Values.cubeProxy.adminPort | quote }} + - name: CUBE_PROXY_RESUME_URL + value: {{ printf "http://$(POD_IP):%v" .Values.cubeProxy.adminPort | quote }} + - name: CUBE_PROXY_NODE_IP + value: "$(POD_IP)" + - name: CUBE_PROXY_VERSION + value: {{ .Values.images.proxyNode.tag | quote }} + - name: CUBE_PROXY_HEARTBEAT_INTERVAL_MS + value: {{ .Values.cubeProxy.registry.heartbeatIntervalMs | quote }} + - name: CUBE_PROXY_REGISTRY_REDIS_HOST + value: {{ $redisRegistryHost | quote }} + - name: CUBE_PROXY_REGISTRY_REDIS_PORT + value: {{ $redisPort | quote }} + - name: CUBE_PROXY_REGISTRY_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: redis-password + - name: CUBE_PROXY_REGISTRY_REDIS_DB + value: {{ .Values.cubeProxy.redis.db | quote }} {{- end }} ports: - name: http containerPort: {{ .Values.cubeProxy.ports.http.containerPort }} - {{- if .Values.cubeProxy.ports.http.hostPort }} - hostPort: {{ .Values.cubeProxy.ports.http.hostPort }} - {{- end }} protocol: TCP - name: https containerPort: {{ .Values.cubeProxy.ports.https.containerPort }} - {{- if .Values.cubeProxy.ports.https.hostPort }} - hostPort: {{ .Values.cubeProxy.ports.https.hostPort }} - {{- end }} + protocol: TCP + - name: admin + containerPort: {{ .Values.cubeProxy.adminPort }} protocol: TCP {{- if .Values.cubeProxy.probes.readiness.enabled }} readinessProbe: @@ -220,7 +251,7 @@ spec: command: - /bin/sh - -ec - - {{ printf "curl -fsS http://127.0.0.1:%v%s >/dev/null" .Values.cubeProxy.probes.readiness.port .Values.cubeProxy.probes.readiness.path | quote }} + - {{ printf "curl -fsS -H \"X-Cube-Admin-Token: ${CUBE_PROXY_ADMIN_TOKEN}\" http://127.0.0.1:%v%s >/dev/null" .Values.cubeProxy.probes.readiness.port .Values.cubeProxy.probes.readiness.path | quote }} initialDelaySeconds: {{ .Values.cubeProxy.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.cubeProxy.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.cubeProxy.probes.readiness.failureThreshold }} @@ -231,7 +262,7 @@ spec: command: - /bin/sh - -ec - - {{ printf "curl -fsS http://127.0.0.1:%v%s >/dev/null" .Values.cubeProxy.probes.liveness.port .Values.cubeProxy.probes.liveness.path | quote }} + - {{ printf "curl -fsS -H \"X-Cube-Admin-Token: ${CUBE_PROXY_ADMIN_TOKEN}\" http://127.0.0.1:%v%s >/dev/null" .Values.cubeProxy.probes.liveness.port .Values.cubeProxy.probes.liveness.path | quote }} initialDelaySeconds: {{ .Values.cubeProxy.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.cubeProxy.probes.liveness.periodSeconds }} failureThreshold: {{ .Values.cubeProxy.probes.liveness.failureThreshold }} diff --git a/deploy/kubernetes/chart/templates/proxy-service.yaml b/deploy/kubernetes/chart/templates/proxy-service.yaml index 8d001df2a..b24d23679 100644 --- a/deploy/kubernetes/chart/templates/proxy-service.yaml +++ b/deploy/kubernetes/chart/templates/proxy-service.yaml @@ -11,18 +11,21 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - # Headless so DNS returns CubeProxy Pod IPs (HostIP with hostNetwork). - clusterIP: None + type: {{ .Values.cubeProxy.service.type | default "ClusterIP" }} selector: {{- include "cube.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: cube-proxy-node ports: - name: http port: {{ .Values.cubeProxy.ports.http.containerPort }} - targetPort: {{ .Values.cubeProxy.ports.http.containerPort }} + targetPort: http protocol: TCP - name: https port: {{ .Values.cubeProxy.ports.https.containerPort }} - targetPort: {{ .Values.cubeProxy.ports.https.containerPort }} + targetPort: https + protocol: TCP + - name: admin + port: {{ .Values.cubeProxy.adminPort }} + targetPort: admin protocol: TCP {{- end }} diff --git a/deploy/kubernetes/chart/templates/secret.yaml b/deploy/kubernetes/chart/templates/secret.yaml index 2f3366c0d..db9c5c591 100644 --- a/deploy/kubernetes/chart/templates/secret.yaml +++ b/deploy/kubernetes/chart/templates/secret.yaml @@ -1,4 +1,15 @@ {{- if eq (include "cube.secretEnabled" .) "true" }} +{{- $adminToken := "" -}} +{{- if .Values.lifecycleManager.adminToken -}} +{{- $adminToken = .Values.lifecycleManager.adminToken -}} +{{- else -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace (include "cube.secretName" .) -}} +{{- if and $existing (index $existing.data "cube-admin-token") -}} +{{- $adminToken = index $existing.data "cube-admin-token" | b64dec -}} +{{- else -}} +{{- $adminToken = randAlphaNum 32 -}} +{{- end -}} +{{- end -}} apiVersion: v1 kind: Secret metadata: @@ -10,4 +21,5 @@ stringData: mysql-root-password: {{ .Values.mysql.rootPassword | quote }} mysql-password: {{ .Values.mysql.password | quote }} redis-password: {{ .Values.redis.password | quote }} + cube-admin-token: {{ $adminToken | quote }} {{- end }} diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml index 89947105d..e9fb30ec2 100644 --- a/deploy/kubernetes/chart/templates/tests/node-health.yaml +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -76,17 +76,14 @@ spec: {{- end }} {{- if eq (include "cube.proxyEnabled" .) "true" }} - echo '[health-test] check CubeProxy service via readiness endpoint through Kubernetes API' - proxy_pods_json="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io/component%3Dproxy-node")" - # At least one CubeProxy Pod exists and reports Ready + echo '[health-test] check CubeProxy Deployment' + proxy_pods_json="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io/component%3Dcube-proxy-node")" printf '%s\n' "${proxy_pods_json}" \ | grep -oE '"ready"[[:space:]]*:[[:space:]]*true' | grep -q true - # The Cube Proxy readiness endpoint should return an HTTP 200 through - # in-cluster Service DNS. Skipped when only a hostNetwork endpoint is - # exposed and the test pod cannot resolve it. - curl --connect-timeout 5 --max-time 15 -fsS \ - "http://{{ include "cube.proxyName" . }}.{{ .Release.Namespace }}.svc.{{ include "cube.clusterDomain" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/-/ready" >/dev/null 2>&1 \ - || echo '[health-test] cube-proxy in-cluster readiness probe unreachable (hostNetwork exposure), skipped' + echo '[health-test] check CubeProxy ClusterIP Service HTTP' + curl --connect-timeout 5 --max-time 15 -fsS -o /dev/null \ + "http://{{ include "cube.proxyName" . }}.{{ .Release.Namespace }}.svc.{{ include "cube.clusterDomain" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/" \ + || echo '[health-test] cube-proxy Service HTTP soft-failed (may require Host header / TLS path)' {{- end }} {{- if .Values.cubeNode.enabled }} @@ -236,8 +233,6 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: restartPolicy: Never - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 4 }} @@ -249,16 +244,16 @@ spec: imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} env: {{- include "cube.timezoneEnv" . | nindent 8 }} - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP command: - sh - -ec - | - echo '[health-test] check control-node CubeProxy' - curl --connect-timeout 5 --max-time 15 -fsS -o /dev/null http://127.0.0.1:{{ .Values.cubeProxy.probes.readiness.port }}{{ .Values.cubeProxy.probes.readiness.path }} + echo '[health-test] check CubeProxy via ClusterIP Service' + curl --connect-timeout 5 --max-time 15 -fsS -o /dev/null \ + "http://{{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/" \ + || curl --connect-timeout 5 --max-time 15 -k -fsS -o /dev/null \ + "https://{{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.ports.https.containerPort }}/" \ + || { echo "cube-proxy Service unreachable"; exit 1; } {{- end }} {{ if eq (include "cube.configureClusterDNS" .) "true" }} --- diff --git a/deploy/kubernetes/chart/templates/validate.yaml b/deploy/kubernetes/chart/templates/validate.yaml index 8a8d130e7..2be5be890 100644 --- a/deploy/kubernetes/chart/templates/validate.yaml +++ b/deploy/kubernetes/chart/templates/validate.yaml @@ -58,21 +58,39 @@ {{- if and (eq .Values.cubeProxy.tls.mode "certManager") (not .Values.cubeProxy.tls.certManager.issuerRef.name) }} {{- fail "cubeProxy.tls.mode=certManager requires cubeProxy.tls.certManager.issuerRef.name" }} {{- end }} -{{- if .Values.cubeProxy.sidecar.adminToken }} -{{- fail "cubeProxy.sidecar.adminToken is not supported until CubeProxy nginx admin server token injection is chart-managed; leave it empty" }} +{{- $clbSecurityGroups := index (default dict .Values.cubeProxy.service.annotations) "service.cloud.tencent.com/security-groups" | default "" -}} +{{- if and (eq (.Values.cubeProxy.service.type | default "ClusterIP") "LoadBalancer") (hasPrefix "CHANGE_ME" $clbSecurityGroups) }} +{{- fail "cubeProxy.service.annotations[\"service.cloud.tencent.com/security-groups\"] still equals the CHANGE_ME_* sentinel from values-tke.yaml. Override it with your VPC security group id(s) (comma-separated), or set it to \"\" if you manage CLB security groups outside Helm." }} {{- end }} -{{- if not .Values.cubeProxy.hostNetwork }} -{{- fail "cubeProxy.hostNetwork=true is required to keep CubeProxy aligned with one-click control-node ingress semantics" }} +{{- if hasKey .Values.cubeProxy "sidecar" }} +{{- fail "cubeProxy.sidecar was removed; lifecycle coordination is handled by lifecycleManager (cube-lifecycle-manager). Remove cubeProxy.sidecar from your values." }} {{- end }} -{{- if and .Values.cubeProxy.hostNetwork .Values.cubeProxy.ports.http.hostPort (ne (int .Values.cubeProxy.ports.http.hostPort) (int .Values.cubeProxy.ports.http.containerPort)) }} -{{- fail "cubeProxy.hostNetwork=true requires cubeProxy.ports.http.hostPort to equal cubeProxy.ports.http.containerPort or be empty" }} +{{- if hasKey .Values.cubeProxy "hostNetwork" }} +{{- fail "cubeProxy.hostNetwork was removed; CubeProxy always runs on the Pod network behind a ClusterIP Service + Ingress. Remove cubeProxy.hostNetwork from your values." }} {{- end }} -{{- if and .Values.cubeProxy.hostNetwork .Values.cubeProxy.ports.https.hostPort (ne (int .Values.cubeProxy.ports.https.hostPort) (int .Values.cubeProxy.ports.https.containerPort)) }} -{{- fail "cubeProxy.hostNetwork=true requires cubeProxy.ports.https.hostPort to equal cubeProxy.ports.https.containerPort or be empty" }} +{{- if and .Values.lifecycleManager.adminToken (lt (len .Values.lifecycleManager.adminToken) 16) }} +{{- fail "lifecycleManager.adminToken must be empty for auto-generation or at least 16 characters when explicitly set" }} {{- end }} {{- if and .Values.cubeProxy.configureClusterDNS (eq (include "cube.proxyEnabled" .) "true") (not .Values.cubeProxy.domain) }} {{- fail "cubeProxy.configureClusterDNS=true requires cubeProxy.domain" }} {{- end }} +{{- if and .Values.cubeProxy.ingress.enabled (not .Values.cubeProxy.domain) (not .Values.cubeProxy.ingress.hosts) }} +{{- fail "cubeProxy.ingress.enabled=true requires cubeProxy.domain or cubeProxy.ingress.hosts" }} +{{- end }} +{{- if ne (include "cube.lifecycleManagerEnabled" .) "true" }} +{{- fail "cubeProxy.enabled=true requires lifecycleManager.enabled=true (cube-lifecycle-manager replaces the obsolete cube-proxy-sidecar)" }} +{{- end }} +{{- if eq (include "cube.lifecycleManagerEnabled" .) "true" }} +{{- if not (include "cube.masterEndpoint" .) }} +{{- fail "lifecycleManager.enabled=true requires a CubeMaster endpoint (controlPlane.master or externalControlPlane.masterEndpoint)" }} +{{- end }} +{{- if and (not (eq (include "cube.redisBuiltinEnabled" .) "true")) (not .Values.redis.host) (not .Values.lifecycleManager.redis.host) (not .Values.cubeProxy.redis.host) }} +{{- fail "lifecycleManager.enabled=true requires redis.host, lifecycleManager.redis.host, or cubeProxy.redis.host when redis.enabled=false" }} +{{- end }} +{{- if not .Values.images.lifecycleManager.repository }} +{{- fail "lifecycleManager.enabled=true requires images.lifecycleManager.repository" }} +{{- end }} +{{- end }} {{- end }} {{- if .Values.cubeEgress.enabled }} {{- if not (has .Values.cubeEgress.ca.mode (list "existingSecret" "selfSigned" "inline")) }} diff --git a/deploy/kubernetes/chart/values-tke.yaml b/deploy/kubernetes/chart/values-tke.yaml index 0d5239016..043e8574c 100644 --- a/deploy/kubernetes/chart/values-tke.yaml +++ b/deploy/kubernetes/chart/values-tke.yaml @@ -10,6 +10,14 @@ # pins every CubeMaster / MySQL / Redis PVC to it. # 2. Uses WaitForFirstConsumer volume binding so multi-AZ TKE clusters # create CBS disks in the same AZ as the scheduled Pod. +# 3. Exposes CubeProxy via a TKE CLB (LoadBalancer Service) and disables +# chart Ingress. qcloud Ingress is L7 and only accepts NodePort/LB +# backends; NodePort is a poor fit for TLS-in-CubeProxy. Point +# cube.app / *.cube.app DNS at the Service EXTERNAL-IP instead. +# 4. Sets TKE CLB Service annotations for security-group binding. The +# security-groups value defaults to CHANGE_ME_TKE_CLB_SECURITY_GROUP; +# helm fails render until you override it in a local values file (or +# set it to "" if you manage CLB SGs outside Helm). # # Nothing else about the release is TKE-specific — Cube-owned images are # already published to ccr.ccs.tencentyun.com, so the chart's default image @@ -36,3 +44,27 @@ mysql: redis: persistence: storageClassName: cube-cbs-wffc + +cubeProxy: + service: + type: LoadBalancer + # TKE CLB annotations. Override security-groups in your runtime values — + # do not commit real SG ids into this preset. Helm refuses to render while + # the CHANGE_ME_* sentinel remains + # + # Example runtime override: + # cubeProxy: + # service: + # annotations: + # service.cloud.tencent.com/security-groups: "sg-xxxxxxxx" + # # multiple: "sg-aaa,sg-bbb" + # # Or clear if you manage CLB SGs outside Helm: + # # service.cloud.tencent.com/security-groups: "" + annotations: + # CLB → backend default pass-through; pair with security-groups below. + # See: https://cloud.tencent.com/document/product/457/51258 + service.cloud.tencent.com/pass-to-target: "true" + # REPLACE via -f override with your VPC security group id(s). + service.cloud.tencent.com/security-groups: "CHANGE_ME_TKE_CLB_SECURITY_GROUP" + ingress: + enabled: false diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 54d2f63bf..4508662ae 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -16,9 +16,9 @@ global: # exactly as declared. Set to e.g. "my-registry.example.com/cubesandbox" # to mirror all images into a private registry without editing each entry # individually. Applies to the pvmHostBootstrap / nodeInit / node / master - # / api / cubemastercli / proxyNode / cubeEgress / cubeEgressNet / webui - # entries but not to third-party images such as mysql / redis / - # docker-in-docker / alpine/k8s. + # / api / cubemastercli / proxyNode / lifecycleManager / cubeEgress / + # cubeEgressNet / webui entries but not to third-party images such as mysql / + # redis / docker-in-docker / alpine/k8s. imageRegistry: "" images: @@ -60,6 +60,11 @@ images: tag: v0.5.0 pullPolicy: Always + lifecycleManager: + repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-lifecycle-manager + tag: v0.5.0 + pullPolicy: Always + cubeEgress: repository: ccr.ccs.tencentyun.com/cubesandbox-chart/cube-egress tag: v0.5.0 @@ -267,6 +272,50 @@ webui: upstream: "" resources: {} +lifecycleManager: + # Standalone sandbox auto-pause / auto-resume coordinator. Replaces the + # obsolete in-process cube-proxy-sidecar. Enabled by default when CubeProxy + # is enabled; CubeProxy points $cube_sidecar_addr at this Service and + # registers itself in Redis for CLM discovery. + enabled: true + replicas: 1 + podAnnotations: {} + listenAddr: "0.0.0.0:8083" + service: + port: 8083 + annotations: {} + redis: + # Empty host uses redis.host or chart-managed cube-redis. + host: "" + port: "" + db: 0 + defaultIdleTimeout: 5m + heartbeatTTL: 15s + discoveryRefresh: 3s + lastActivePoll: 5s + idleSweepInterval: 5s + bootstrapWarmup: 30s + stateLockTTL: 60s + # Shared with CubeProxy admin endpoints (X-Cube-Admin-Token). Empty means + # the chart auto-generates and persists one in the release Secret. + adminToken: "" + probes: + readiness: + enabled: true + path: /readyz + port: 8083 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + liveness: + enabled: true + path: /healthz + port: 8083 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + resources: {} + diagnostics: # One-click ships cube-diag scripts. In Kubernetes the chart exposes an # equivalent kubectl-based diagnostic script through a ConfigMap so operators @@ -427,33 +476,47 @@ cubeProxy: # # One-click enables CubeProxy by default. This chart follows that behavior # and creates a self-signed test certificate unless production TLS is provided. + # Traffic reaches CubeProxy via ClusterIP Service + Ingress (TLS still + # terminates in CubeProxy; Ingress uses SSL passthrough / HTTPS backend). enabled: true - # Public sandbox domain (also used for TLS CN / wildcard DNS). - # For production, point external DNS at CubeProxy as well. + # Public sandbox domain (also used for TLS CN / wildcard DNS / Ingress hosts). + # For production, point external DNS at the Ingress entrypoint. domain: cube.app podAnnotations: {} replicas: 1 strategy: type: RollingUpdate - # One-click CubeProxy is part of the control-node stack. This chart follows - # that model and schedules CubeProxy through placement.controlPlane. - hostNetwork: true + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 # Optional hint for external clients / docs (not used for in-cluster DNS). advertiseIP: "" # When true, Helm patches cluster CoreDNS so *.cubeProxy.domain rewrites to - # the CubeProxy headless Service. Set false only if mutating kube-system/coredns + # the CubeProxy ClusterIP Service. Set false only if mutating kube-system/coredns # is not allowed (disabling does not remove a previously injected block; # uninstall runs the cleanup hook). configureClusterDNS: true service: + type: ClusterIP annotations: {} + ingress: + # Requires a cluster Ingress Controller. Set false to manage the entrypoint + # yourself (still keep the ClusterIP Service as the backend). + enabled: true + className: "" + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # Empty hosts use cubeProxy.domain and "*.cubeProxy.domain". + hosts: [] + # Passthrough mode usually leaves this empty; override when the controller + # requires an Ingress TLS block. + tls: [] ports: http: containerPort: 80 - hostPort: "" https: containerPort: 443 - hostPort: "" redis: # Empty host uses redis.host or chart-managed cube-redis. host: "" @@ -506,20 +569,14 @@ cubeProxy: # Used only when tls.mode=inline. cert: "" key: "" - sidecar: - # CubeProxy image starts cube-proxy-sidecar from CubeProxy/Dockerfile. - # These values wire it to Kubernetes Services instead of the one-click - # loopback defaults. - listenAddr: 127.0.0.1:8083 - proxyAdminURLs: - - http://127.0.0.1:8082 - defaultIdleTimeout: 5m - lastActivePoll: 5s - idleSweepInterval: 5s - bootstrapWarmup: 30s - stateLockTTL: 60s - # Leave empty until CubeProxy nginx admin server also has token injection. - adminToken: "" + # Admin HTTP server (stock nginx.conf listens on 127.0.0.1:8082). The chart + # rewrites this so cube-lifecycle-manager can reach /admin/* from other Pods. + adminListen: "0.0.0.0" + adminPort: 8082 + # Redis registry so cube-lifecycle-manager can discover each proxy replica. + registry: + enabled: true + heartbeatIntervalMs: 5000 resolver: # CubeProxy nginx Lua Redis client needs an nginx resolver when Redis is # configured by DNS name (the default Kubernetes Service DNS name). diff --git a/deploy/kubernetes/images/README.md b/deploy/kubernetes/images/README.md index fbc400b67..42e0301e8 100644 --- a/deploy/kubernetes/images/README.md +++ b/deploy/kubernetes/images/README.md @@ -25,14 +25,15 @@ and does not require a `.complete` marker. ## Pinning source to a release tag -`cube-api`, `cube-proxy-node`, and `cube-egress` are compiled from repository -source (rather than binaries in the release tarball). By default the script -pins those source trees to `${SOURCE_REF}` (defaulting to `${VERSION}`, so -`v0.5.0` for the default build). It exports `CubeMaster/`, `CubeAPI/`, -`CubeProxy/`, and `CubeEgress/` at that git ref into -`${BUILD_ROOT}/source-tree/` via `git archive` and points `REPO_ROOT` there for -the duration of the build. This guarantees the images match the release tag -even when the current worktree is ahead of it. +`cube-api`, `cube-proxy-node`, `cube-egress`, `cube-lifecycle-manager`, and +`cube-webui` are compiled from repository source (rather than binaries in the +release tarball). By default the script pins those source trees to +`${SOURCE_REF}` (defaulting to `${VERSION}`, so `v0.5.0` for the default +build). It exports `CubeMaster/`, `CubeAPI/`, `CubeProxy/`, `CubeEgress/`, +`cube-lifecycle-manager/`, `web/`, and `deploy/one-click/webui/` at that git +ref into `${BUILD_ROOT}/source-tree/` via `git archive` and points `REPO_ROOT` +there for the duration of the build. This guarantees the images match the +release tag even when the current worktree is ahead of it. To build from the current worktree instead (typically for development), set `SOURCE_REF=""`: @@ -75,7 +76,8 @@ entrypoint behavior. release-package `CubeMaster/bin/cubemastercli` binary and minimal runtime dependencies. It is separate from `cube-master` and `cube-node` so runtime image responsibilities remain clean. -- `cube-proxy-node` is built from `CubeProxy/Dockerfile`; no duplicate Dockerfile is kept here. +- `cube-proxy-node` is built from `CubeProxy/Dockerfile`; no duplicate Dockerfile is kept here. Auto-pause/resume is **not** baked into this image — use the standalone `cube-lifecycle-manager` image instead of the retired `cube-proxy-sidecar`. +- `cube-lifecycle-manager` is built from `cube-lifecycle-manager/Dockerfile`; no duplicate Dockerfile is kept here. - `cube-egress` is built from `CubeEgress/Dockerfile`; no duplicate Dockerfile is kept here. Its `cube-egress/openresty:1.29.2.5-tproxy` base image is built first from `CubeEgress/openresty/Dockerfile`, because that patched OpenResty base is part of the upstream CubeEgress build chain rather than a public pull-only dependency. The build script also tags that local base as `cube-sandbox-cn.tencentcloudcr.com/cube-sandbox/openresty-tproxy`, matching @@ -86,9 +88,12 @@ entrypoint behavior. `CubeEgress/scripts/cube-proxy-iptables-init.sh` plus a small idempotent entrypoint that waits for `cube-dev`, applies rules, and removes them on termination. -- `cube-webui` packages the one-click `webui/dist` static assets and reuses - the one-click WebUI nginx layout. The chart mounts a rendered nginx config so - `/cubeapi/` proxies to the chart-managed CubeAPI Service. +- `cube-webui` is built exactly like CI (`.github/workflows/release-docker-images.yml`): + context = repository root, file = `deploy/one-click/webui/Dockerfile`, with + `OPENRESTY_BASE_IMAGE` / `CUBE_VERSION` / `CUBE_COMMIT` / `CUBE_BUILD_TIME`. + Requires BuildKit (`DOCKER_BUILDKIT=1`) for the adjacent + `Dockerfile.dockerignore`. The chart may still mount a ConfigMap nginx.conf + over the image default at runtime. - The template builder sidecar uses a dind image by default; no duplicate Dockerfile is kept here. The Helm chart stays under `deploy/kubernetes/chart`; image build logic stays here to avoid coupling chart templates with image construction. diff --git a/deploy/kubernetes/images/build-cube-images.sh b/deploy/kubernetes/images/build-cube-images.sh index 45d00967c..f991a8912 100755 --- a/deploy/kubernetes/images/build-cube-images.sh +++ b/deploy/kubernetes/images/build-cube-images.sh @@ -9,30 +9,62 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +WORKTREE_ROOT="${REPO_ROOT}" VERSION="${VERSION:-v0.5.0}" IMAGE_TAG="${IMAGE_TAG:-${VERSION}}" REGISTRY="${REGISTRY:-ccr.ccs.tencentyun.com/cubesandbox-chart}" -# SOURCE_REF pins the CubeMaster / CubeAPI / CubeProxy / CubeEgress source tree -# used when building cube-api / cube-proxy-node / cube-egress from repository -# source, ensuring the delivered images match the release tag rather than the -# current worktree. Set SOURCE_REF="" to build from the current worktree. +# SOURCE_REF pins the CubeMaster / CubeAPI / CubeProxy / CubeEgress / +# cube-lifecycle-manager / web / deploy/one-click/webui source tree used when +# building cube-api / cube-proxy-node / cube-egress / cube-lifecycle-manager / +# cube-webui from repository source, ensuring the delivered images match the +# release tag rather than the current worktree. Set SOURCE_REF="" to build from +# the current worktree. SOURCE_REF="${SOURCE_REF-${VERSION}}" PUSH="${PUSH:-0}" NO_CACHE="${NO_CACHE:-0}" BUILD_ROOT="${BUILD_ROOT:-/tmp/cube-kubernetes-images-${VERSION}}" CUBE_NODE_BASE_IMAGE="${CUBE_NODE_BASE_IMAGE:-}" CUBE_EGRESS_OPENRESTY_BASE_IMAGE="${CUBE_EGRESS_OPENRESTY_BASE_IMAGE:-cube-sandbox-cn.tencentcloudcr.com/cube-sandbox/openresty-tproxy}" +# CubeProxy / webui OpenResty bases. Private TCR defaults need auth; override to +# a reachable image (same knobs CI passes as OPENRESTY_BASE_IMAGE). +CUBE_PROXY_BASE_IMAGE="${CUBE_PROXY_BASE_IMAGE:-openresty/openresty:1.21.4.1-6-alpine-fat}" +OPENRESTY_BASE_IMAGE="${OPENRESTY_BASE_IMAGE:-${CUBE_PROXY_BASE_IMAGE}}" +# Match CI release-docker-images.yml metadata build-args for webui / LCM. +CUBE_COMMIT="${CUBE_COMMIT:-$(git -C "${WORKTREE_ROOT}" rev-parse HEAD 2>/dev/null || echo unknown)}" +CUBE_BUILD_TIME="${CUBE_BUILD_TIME:-$(date -u +'%Y-%m-%dT%H:%M:%SZ')}" +# webui Dockerfile.dockerignore requires BuildKit (per-Dockerfile ignore files). +export DOCKER_BUILDKIT="${DOCKER_BUILDKIT:-1}" ONE_CLICK_ARCH="${ONE_CLICK_ARCH:-amd64}" -ONE_CLICK_URL="${ONE_CLICK_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/cube-sandbox-one-click-${VERSION}-${ONE_CLICK_ARCH}.tar.gz}" -PVM_KERNEL_RPM_URL="${PVM_KERNEL_RPM_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/kernel-6.6.69_opencloudos9.cubesandbox.pvm.host_gb85200d80fa2-1.x86_64.rpm}" -PVM_KERNEL_DEB_URL="${PVM_KERNEL_DEB_URL:-https://downloads.sourceforge.net/project/cubesandbox.mirror/${VERSION}/linux-image-6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2_6.6.69-gb85200d80fa2-1_amd64.deb}" +# MIRROR selects the release download origin for one-click + PVM host packages: +# cn -> https://cnb.cool/CubeSandbox/CubeSandbox/-/releases (China) +# "" -> https://github.com/TencentCloud/CubeSandbox/releases (default / overseas) +# Explicit ONE_CLICK_URL / PVM_KERNEL_*_URL overrides still win. +MIRROR="${MIRROR:-}" +ONE_CLICK_ARTIFACT="${ONE_CLICK_ARTIFACT:-cube-sandbox-one-click-${VERSION}-${ONE_CLICK_ARCH}.tar.gz}" +PVM_KERNEL_RPM_ARTIFACT="${PVM_KERNEL_RPM_ARTIFACT:-kernel-6.6.69_opencloudos9.cubesandbox.pvm.host_gb85200d80fa2-1.x86_64.rpm}" +PVM_KERNEL_DEB_ARTIFACT="${PVM_KERNEL_DEB_ARTIFACT:-linux-image-6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2_6.6.69-gb85200d80fa2-1_amd64.deb}" +case "${MIRROR}" in + cn|CN|cnb|CNB) + RELEASE_DOWNLOAD_BASE="https://cnb.cool/CubeSandbox/CubeSandbox/-/releases/download/${VERSION}" + ;; + ""|github|GITHUB|gh|GH) + RELEASE_DOWNLOAD_BASE="https://github.com/TencentCloud/CubeSandbox/releases/download/${VERSION}" + ;; + *) + printf '[build-cube-images] ERROR: unsupported MIRROR=%s (use cn or github)\n' "${MIRROR}" >&2 + exit 1 + ;; +esac +ONE_CLICK_URL="${ONE_CLICK_URL:-${RELEASE_DOWNLOAD_BASE}/${ONE_CLICK_ARTIFACT}}" +PVM_KERNEL_RPM_URL="${PVM_KERNEL_RPM_URL:-${RELEASE_DOWNLOAD_BASE}/${PVM_KERNEL_RPM_ARTIFACT}}" +PVM_KERNEL_DEB_URL="${PVM_KERNEL_DEB_URL:-${RELEASE_DOWNLOAD_BASE}/${PVM_KERNEL_DEB_ARTIFACT}}" # Optional SHA256 checksums for the downloaded artifacts. When set, the # download function refuses to accept a mismatching file. Chart operators # should always set these when publishing images to protect the build against -# SourceForge/mirror tampering. Leave empty for interactive development builds. +# release-mirror tampering. Leave empty for interactive development builds. ONE_CLICK_SHA256="${ONE_CLICK_SHA256:-}" PVM_KERNEL_RPM_SHA256="${PVM_KERNEL_RPM_SHA256:-}" PVM_KERNEL_DEB_SHA256="${PVM_KERNEL_DEB_SHA256:-}" @@ -152,27 +184,32 @@ SANDBOX_PACKAGE_TAR="${EXTRACT_DIR}/${ONE_CLICK_DIRNAME}/assets/package/sandbox- PACKAGE_DIR="${BUILD_ROOT}/sandbox-package" mkdir -p "${DOWNLOAD_DIR}" "${EXTRACT_DIR}" "${CONTEXT_DIR}" +log "release download base (${MIRROR:-github}): ${RELEASE_DOWNLOAD_BASE}" # When SOURCE_REF is set (default: ${VERSION}), export the CubeMaster / CubeAPI / -# CubeProxy / CubeEgress source trees at that ref into ${SOURCE_TREE_DIR} and -# point REPO_ROOT there. This ensures cube-api, cube-proxy-node, cube-egress and -# related contexts are compiled from the release-tag source, not from whatever -# happens to be in the current worktree (which may be ahead of the tag). +# CubeProxy / CubeEgress / cube-lifecycle-manager / web / deploy/one-click/webui +# trees at that ref into ${SOURCE_TREE_DIR} and point REPO_ROOT there. This +# ensures cube-api, cube-proxy-node, cube-egress, cube-lifecycle-manager, +# cube-webui and related contexts are compiled from the release-tag source, not +# from whatever happens to be in the current worktree (which may be ahead of +# the tag). if [[ -n "${SOURCE_REF}" ]]; then - git -C "${REPO_ROOT}" rev-parse --verify "${SOURCE_REF}^{commit}" >/dev/null 2>&1 \ - || fail "SOURCE_REF=${SOURCE_REF} is not a valid git ref in ${REPO_ROOT}" - SOURCE_REF_SHA="$(git -C "${REPO_ROOT}" rev-parse "${SOURCE_REF}^{commit}")" + git -C "${WORKTREE_ROOT}" rev-parse --verify "${SOURCE_REF}^{commit}" >/dev/null 2>&1 \ + || fail "SOURCE_REF=${SOURCE_REF} is not a valid git ref in ${WORKTREE_ROOT}" + SOURCE_REF_SHA="$(git -C "${WORKTREE_ROOT}" rev-parse "${SOURCE_REF}^{commit}")" SOURCE_TREE_STAMP="${SOURCE_TREE_DIR}/.exported-sha" - if [[ ! -f "${SOURCE_TREE_STAMP}" ]] || [[ "$(cat "${SOURCE_TREE_STAMP}")" != "${SOURCE_REF_SHA}" ]]; then + SOURCE_EXPORT_SET="CubeMaster CubeAPI CubeProxy CubeEgress cube-lifecycle-manager web deploy/one-click/webui" + if [[ ! -f "${SOURCE_TREE_STAMP}" ]] \ + || [[ "$(cat "${SOURCE_TREE_STAMP}")" != "${SOURCE_REF_SHA}"$'\n'"${SOURCE_EXPORT_SET}" ]]; then log "exporting source tree at ${SOURCE_REF} (${SOURCE_REF_SHA:0:12}) into ${SOURCE_TREE_DIR}" rm -rf "${SOURCE_TREE_DIR}" mkdir -p "${SOURCE_TREE_DIR}" - for module in CubeMaster CubeAPI CubeProxy CubeEgress; do - git -C "${REPO_ROOT}" archive --format=tar "${SOURCE_REF_SHA}" -- "${module}" \ + for module in ${SOURCE_EXPORT_SET}; do + git -C "${WORKTREE_ROOT}" archive --format=tar "${SOURCE_REF_SHA}" -- "${module}" \ | tar -C "${SOURCE_TREE_DIR}" -x \ || fail "failed to export ${module} at ${SOURCE_REF}" done - printf '%s\n' "${SOURCE_REF_SHA}" > "${SOURCE_TREE_STAMP}" + printf '%s\n%s\n' "${SOURCE_REF_SHA}" "${SOURCE_EXPORT_SET}" > "${SOURCE_TREE_STAMP}" else log "reusing exported source tree at ${SOURCE_REF} (${SOURCE_REF_SHA:0:12})" fi @@ -249,13 +286,12 @@ copy_cubemastercli_context() { copy_cube_proxy_component_context() { local ctx="$1" local src="${REPO_ROOT}/CubeProxy" - local sidecar_src="${src}/sidecar" - local sidecar_out="${ctx}/bin/cube-proxy-sidecar" [[ -f "${src}/nginx.conf" ]] || fail "missing CubeProxy nginx.conf" [[ -d "${src}/conf/includes" ]] || fail "missing CubeProxy conf/includes" - [[ -f "${sidecar_src}/go.mod" ]] || fail "missing CubeProxy sidecar source" + # Auto-pause/resume moved out of cube-proxy-sidecar into the standalone + # cube-lifecycle-manager image. cube-proxy-node is nginx/OpenResty only. cp -a "${src}/lua" "${ctx}/lua" mkdir -p "${ctx}/conf" cp -a "${src}/conf/includes" "${ctx}/conf/includes" @@ -263,24 +299,76 @@ copy_cube_proxy_component_context() { cp "${src}/rotate_nginx_log.sh" "${ctx}/rotate_nginx_log.sh" cp "${src}/root" "${ctx}/root" cp "${src}/start.sh" "${ctx}/start.sh" +} + +build_cube_lifecycle_manager_image() { + local src="${REPO_ROOT}/cube-lifecycle-manager" + [[ -f "${src}/Dockerfile" ]] || fail "missing cube-lifecycle-manager Dockerfile" + [[ -f "${src}/go.mod" ]] || fail "missing cube-lifecycle-manager go.mod" + build_image cube-lifecycle-manager "${src}" "${src}/Dockerfile" \ + --build-arg "CUBE_VERSION=${IMAGE_TAG}" \ + --build-arg "CUBE_COMMIT=${CUBE_COMMIT}" \ + --build-arg "CUBE_BUILD_TIME=${CUBE_BUILD_TIME}" +} + +# Same as .github/workflows/release-docker-images.yml for component "webui": +# context=., file=deploy/one-click/webui/Dockerfile, OPENRESTY_BASE_IMAGE + CUBE_*. +build_cube_webui_image() { + local dockerfile="${REPO_ROOT}/deploy/one-click/webui/Dockerfile" + local image="${REGISTRY}/cube-webui:${IMAGE_TAG}" + local docker_args=( + -f "${dockerfile}" + -t "${image}" + --build-arg "CUBE_VERSION=${IMAGE_TAG}" + --build-arg "CUBE_COMMIT=${CUBE_COMMIT}" + --build-arg "CUBE_BUILD_TIME=${CUBE_BUILD_TIME}" + --build-arg "OPENRESTY_BASE_IMAGE=${OPENRESTY_BASE_IMAGE}" + --build-arg "CUBE_PROXY_BASE_IMAGE=${CUBE_PROXY_BASE_IMAGE}" + ) + [[ -f "${dockerfile}" ]] || fail "missing webui Dockerfile: ${dockerfile}" + [[ -f "${REPO_ROOT}/web/package.json" ]] || fail "missing web/ frontend sources in ${REPO_ROOT}" + if [[ "${NO_CACHE}" == "1" ]]; then + docker_args=(--no-cache --pull "${docker_args[@]}") + fi + log "building ${image} from ${dockerfile} (context=${REPO_ROOT}, base=${OPENRESTY_BASE_IMAGE})" + docker build "${docker_args[@]}" "${REPO_ROOT}" + if [[ "${PUSH}" == "1" ]]; then + log "pushing ${image}" + docker push "${image}" + fi +} - mkdir -p "${ctx}/bin" - log "building CubeProxy sidecar for cube-proxy-node image context" - ( - cd "${sidecar_src}" - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ - go build -trimpath -tags 'netgo osusergo' -ldflags '-s -w' \ - -o "${sidecar_out}" ./cmd/sidecar +build_cube_proxy_node_image() { + local ctx="$1" + local dockerfile="${REPO_ROOT}/CubeProxy/Dockerfile" + local image="${REGISTRY}/cube-proxy-node:${IMAGE_TAG}" + local docker_args=( + -f "${dockerfile}" + -t "${image}" + --build-arg "CUBE_PROXY_BASE_IMAGE=${CUBE_PROXY_BASE_IMAGE}" ) - chmod +x "${sidecar_out}" + if [[ "${NO_CACHE}" == "1" ]]; then + docker_args=(--no-cache --pull "${docker_args[@]}") + fi + log "building ${image} from ${dockerfile} (base=${CUBE_PROXY_BASE_IMAGE})" + docker build "${docker_args[@]}" "${ctx}" + if [[ "${PUSH}" == "1" ]]; then + log "pushing ${image}" + docker push "${image}" + fi } build_image() { local name="$1" local ctx="$2" - local dockerfile="${3:-${SCRIPT_DIR}/${name}/Dockerfile}" + shift 2 + local dockerfile="${SCRIPT_DIR}/${name}/Dockerfile" + if [[ $# -gt 0 && "$1" != --* ]]; then + dockerfile="$1" + shift + fi local image="${REGISTRY}/${name}:${IMAGE_TAG}" - local docker_args=(-f "${dockerfile}" -t "${image}") + local docker_args=(-f "${dockerfile}" -t "${image}" "$@") if [[ "${NO_CACHE}" == "1" ]]; then docker_args=(--no-cache --pull "${docker_args[@]}") fi @@ -398,7 +486,9 @@ build_image cubemastercli "${ctx}" ctx="$(prepare_context cube-proxy-node)" copy_cube_proxy_component_context "${ctx}" -build_image cube-proxy-node "${ctx}" "${REPO_ROOT}/CubeProxy/Dockerfile" +build_cube_proxy_node_image "${ctx}" + +build_cube_lifecycle_manager_image build_cube_egress_openresty_base_image build_cube_egress_image @@ -407,12 +497,7 @@ ctx="$(prepare_context cube-egress-net)" copy_cube_egress_net_context "${ctx}" build_image cube-egress-net "${ctx}" -ctx="$(prepare_context cube-webui)" -[[ -d "${PACKAGE_DIR}/webui" ]] || fail "invalid sandbox-package: missing required cube-webui component webui" -cp -a "${PACKAGE_DIR}/webui" "${ctx}/package/webui" -[[ -f "${ctx}/package/webui/dist/index.html" ]] || fail "invalid webui package: missing dist/index.html" -[[ -f "${ctx}/package/webui/nginx.conf" ]] || fail "invalid webui package: missing nginx.conf" -build_image cube-webui "${ctx}" +build_cube_webui_image if [[ -n "${CUBE_NODE_BASE_IMAGE}" ]]; then log "building cube-node by rebasing ${CUBE_NODE_BASE_IMAGE}" @@ -455,6 +540,7 @@ Built CubeSandbox images: ${REGISTRY}/cube-api:${IMAGE_TAG} ${REGISTRY}/cubemastercli:${IMAGE_TAG} ${REGISTRY}/cube-proxy-node:${IMAGE_TAG} + ${REGISTRY}/cube-lifecycle-manager:${IMAGE_TAG} ${REGISTRY}/cube-egress:${IMAGE_TAG} ${REGISTRY}/cube-egress-net:${IMAGE_TAG} ${REGISTRY}/cube-webui:${IMAGE_TAG} diff --git a/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile index 022d20364..0baad7b09 100644 --- a/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile +++ b/deploy/kubernetes/images/cube-pvm-host-bootstrap/Dockerfile @@ -1,27 +1,30 @@ -FROM bitnami/kubectl:1.30 AS kubectl - -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive +ARG KUBECTL_CHANNEL=v1.30 RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + gnupg \ kmod \ rpm \ tini \ util-linux \ + && curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBECTL_CHANNEL}/deb/Release.key" \ + | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBECTL_CHANNEL}/deb/ /" \ + > /etc/apt/sources.list.d/kubernetes.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends kubectl \ && rm -rf /var/lib/apt/lists/* -COPY --from=kubectl /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl COPY scripts/pvm-host-bootstrap.sh /scripts/pvm-host-bootstrap.sh # Optional kernel packages. Put files under build context artifacts/ to build a fully self-contained image. COPY artifacts/ /artifacts/ -RUN chmod +x \ - /usr/local/bin/kubectl \ - /scripts/pvm-host-bootstrap.sh +RUN chmod +x /scripts/pvm-host-bootstrap.sh ENTRYPOINT ["/usr/bin/tini", "-s", "--", "/scripts/pvm-host-bootstrap.sh"] diff --git a/deploy/kubernetes/images/cube-webui/Dockerfile b/deploy/kubernetes/images/cube-webui/Dockerfile deleted file mode 100644 index d32e324f1..000000000 --- a/deploy/kubernetes/images/cube-webui/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM cube-sandbox-image.tencentcloudcr.com/opensource/openresty:1.21.4.1-6-alpine-fat - -COPY package/webui/dist /usr/share/nginx/html -COPY package/webui/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf - -EXPOSE 80 - -CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"] diff --git a/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh index 3b5b1a1aa..dd828e706 100755 --- a/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh +++ b/deploy/kubernetes/images/scripts/cube-proxy-node-entrypoint.sh @@ -9,6 +9,9 @@ # Reads the following env vars (populated by the Chart in the Pod spec): # CUBE_PROXY_HTTP_LISTEN_PORT - digits only, target HTTP listen port # CUBE_PROXY_HTTPS_LISTEN_PORT - digits only, target HTTPS listen port +# CUBE_PROXY_ADMIN_LISTEN - admin server listen address (default 0.0.0.0) +# CUBE_PROXY_ADMIN_PORT - admin server port (default 8082) +# CUBE_PROXY_ADMIN_TOKEN - optional shared secret for /admin/* # CUBE_PROXY_RESOLVER_ADDRS - space-separated nameservers; empty means # read from /etc/resolv.conf # CUBE_PROXY_RESOLVER_VALID - nginx `valid=` duration @@ -16,8 +19,8 @@ # CUBE_PROXY_RESOLVER_IPV6 - on/off # REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB # TIMEOUT_MIN, TIMEOUT_MAX -# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR -# Rewrites nginx.conf listen ports in-place, then generates +# NODE_IP, CUBE_SIDECAR_LISTEN_ADDR (cube-lifecycle-manager host:port) +# Rewrites nginx.conf listen ports / admin bind in-place, then generates # /usr/local/openresty/nginx/conf/global/global.conf and execs start.sh. set -eu @@ -31,17 +34,42 @@ case "${CUBE_PROXY_HTTP_LISTEN_PORT}:${CUBE_PROXY_HTTPS_LISTEN_PORT}" in ;; esac +admin_listen="${CUBE_PROXY_ADMIN_LISTEN:-0.0.0.0}" +admin_port="${CUBE_PROXY_ADMIN_PORT:-8082}" +case "${admin_port}" in + *[!0-9]*|"") + echo "invalid CubeProxy admin port: ${admin_port}" >&2 + exit 1 + ;; +esac +case "${admin_listen}" in + *[\;\{\}\$\`\"\\]*) + echo "invalid CubeProxy admin listen address: ${admin_listen}" >&2 + exit 1 + ;; +esac + sed -i \ -e "s/listen 8081 reuseport;/listen ${CUBE_PROXY_HTTP_LISTEN_PORT} reuseport;/g" \ -e "s/listen 8080 ssl reuseport;/listen ${CUBE_PROXY_HTTPS_LISTEN_PORT} ssl reuseport;/g" \ -e "s/set \\\$host_proxy_port 8081;/set \\\$host_proxy_port ${CUBE_PROXY_HTTP_LISTEN_PORT};/g" \ -e "s/set \\\$host_proxy_port 8080;/set \\\$host_proxy_port ${CUBE_PROXY_HTTPS_LISTEN_PORT};/g" \ + -e "s/listen 127\\.0\\.0\\.1:8082;/listen ${admin_listen}:${admin_port};/g" \ /usr/local/openresty/nginx/conf/nginx.conf escape_nginx_value() { printf '%s' "$1" | sed 's/[\\"]/\\&/g' } +if [ -n "${CUBE_PROXY_ADMIN_TOKEN:-}" ]; then + token_escaped="$(escape_nginx_value "${CUBE_PROXY_ADMIN_TOKEN}")" + # Stock nginx.conf sets an empty token in each server block; rewrite them + # so /admin/* and resume paths share the same secret as cube-lifecycle-manager. + sed -i \ + -e "s/set \\\$cube_admin_token \"\";/set \\\$cube_admin_token \"${token_escaped}\";/g" \ + /usr/local/openresty/nginx/conf/nginx.conf +fi + resolver_addrs="${CUBE_PROXY_RESOLVER_ADDRS:-}" if [ -z "${resolver_addrs}" ]; then resolver_addrs="$(awk '/^nameserver[[:space:]]+/ { printf "%s ", $2 }' /etc/resolv.conf | sed 's/[[:space:]]*$//')" @@ -57,6 +85,26 @@ case "${resolver_addrs}${CUBE_PROXY_RESOLVER_VALID}${CUBE_PROXY_RESOLVER_TIMEOUT ;; esac +sidecar_addr="${CUBE_SIDECAR_LISTEN_ADDR:-}" +[ -n "${sidecar_addr}" ] || { + echo "CUBE_SIDECAR_LISTEN_ADDR is required (cube-lifecycle-manager host:port)" >&2 + exit 1 +} + +# proxy_registry.lua publishes from ngx.timer, which has no nginx resolver. +# Resolve a hostname REDIS target to an IP up-front when possible. +if [ -n "${CUBE_PROXY_REGISTRY_REDIS_HOST:-}" ]; then + case "${CUBE_PROXY_REGISTRY_REDIS_HOST}" in + *[!0-9.]* ) + resolved="$(getent ahostsv4 "${CUBE_PROXY_REGISTRY_REDIS_HOST}" 2>/dev/null | awk 'NR==1 {print $1}')" + if [ -n "${resolved}" ]; then + CUBE_PROXY_REGISTRY_REDIS_HOST="${resolved}" + export CUBE_PROXY_REGISTRY_REDIS_HOST + fi + ;; + esac +fi + cat > /usr/local/openresty/nginx/conf/global/global.conf <