diff --git a/api/Taskfile.dist.yaml b/api/Taskfile.dist.yaml index dccea27f26..8781f15677 100644 --- a/api/Taskfile.dist.yaml +++ b/api/Taskfile.dist.yaml @@ -15,7 +15,9 @@ tasks: deps: - _ensure:k8s-codegen-mod - _ensure:k8s-kube-openapi - cmd: ./scripts/update-codegen.sh all + cmds: + - ./scripts/update-codegen.sh all + - task: format:yaml generate:core: desc: "Regenerate code for core components." @@ -40,7 +42,17 @@ tasks: desc: "Regenerate crds" deps: - _ensure:k8s-controller-gen - cmd: ./scripts/update-codegen.sh crds + cmds: + - ./scripts/update-codegen.sh crds + - task: format:yaml + + format:yaml: + desc: "Format non-templated YAML files, e.g. CRDs" + cmds: + - | + cd ../ && docker run --rm \ + -v ./:/tmp/virt ghcr.io/deckhouse/virtualization/prettier:3.2.5 \ + sh -c "cd /tmp/virt ; prettier -w \"**/*.yaml\" \"**/*.yml\"" _ci:verify-gen: desc: "Check generated files are up-to-date." diff --git a/api/core/v1alpha2/block_device.go b/api/core/v1alpha2/block_device.go index 5751315791..4e7eb743e4 100644 --- a/api/core/v1alpha2/block_device.go +++ b/api/core/v1alpha2/block_device.go @@ -18,19 +18,33 @@ package v1alpha2 type BlockDeviceSpecRef struct { Kind BlockDeviceKind `json:"kind"` - Name string `json:"name"` + // The name of attached resource. + Name string `json:"name"` } type BlockDeviceStatusRef struct { - Kind BlockDeviceKind `json:"kind"` - Name string `json:"name"` - Size string `json:"size"` - Target string `json:"target"` - Attached bool `json:"attached"` - Hotplugged bool `json:"hotplugged,omitempty"` - VirtualMachineBlockDeviceAttachmentName string `json:"virtualMachineBlockDeviceAttachmentName,omitempty"` + Kind BlockDeviceKind `json:"kind"` + // The name of attached resource. + Name string `json:"name"` + // The size of attached block device. + Size string `json:"size"` + // The block device is attached to the virtual machine. + Attached bool `json:"attached"` + // The name of attached block device. + // +kubebuilder:example=sda + Target string `json:"target,omitempty"` + // Block device is attached via hot plug connection. + Hotplugged bool `json:"hotplugged,omitempty"` + // The name of the `VirtualMachineBlockDeviceAttachment` resource that defines hot plug disk connection to the virtual machine. + VirtualMachineBlockDeviceAttachmentName string `json:"virtualMachineBlockDeviceAttachmentName,omitempty"` } +// The BlockDeviceKind is a type of the block device. Options are: +// +// * `ClusterVirtualImage` — Use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. +// * `VirtualImage` — Use `VirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. +// * `VirtualDisk` — Use `VirtualDisk` as the disk. This type is always mounted in RW mode. +// +kubebuilder:validation:Enum:={ClusterVirtualImage,VirtualImage,VirtualDisk} type BlockDeviceKind string const ( diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 80208e5c6a..eac7643fd1 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -29,7 +29,27 @@ const ( VirtualMachineResource = "virtualmachines" ) -// VirtualMachine specifies configuration of the virtual machine. +// VirtualMachine describes the configuration and status of a virtual machine (VM). +// For a running VM, parameter changes can only be applied after the VM is rebooted, except for the following parameters (they are applied on the fly): +// - `.metadata.labels`. +// - `.metadata.annotations`. +// - `.spec.disruptions.restartApprovalMode`. +// - `.spec.disruptions.runPolicy`. +// +// +kubebuilder:object:root=true +// +kubebuilder:metadata:labels={heritage=deckhouse,module=virtualization} +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={all,virtualization},scope=Namespaced,shortName={vm,vms},singular=virtualmachine +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="The phase of the virtual machine." +// +kubebuilder:printcolumn:name="Cores",priority=1,type="string",JSONPath=".spec.cpu.cores",description="The number of cores of the virtual machine." +// +kubebuilder:printcolumn:name="CoreFraction",priority=1,type="string",JSONPath=".spec.cpu.coreFraction",description="Virtual machine core fraction." +// +kubebuilder:printcolumn:name="Memory",priority=1,type="string",JSONPath=".spec.memory.size",description="The amount of memory of the virtual machine." +// +kubebuilder:printcolumn:name="Need restart",priority=1,type="string",JSONPath=".status.conditions[?(@.type=='AwaitingRestartToApplyConfiguration')].status",description="A restart of the virtual machine is required." +// +kubebuilder:printcolumn:name="Agent",priority=1,type="string",JSONPath=".status.conditions[?(@.type=='AgentReady')].status",description="Agent status." +// +kubebuilder:printcolumn:name="Migratable",priority=1,type="string",JSONPath=".status.conditions[?(@.type=='Migratable')].status",description="Is it possible to migrate a virtual machine." +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.nodeName",description="The node where the virtual machine is running." +// +kubebuilder:printcolumn:name="IPAddress",type="string",JSONPath=".status.ipAddress",description="The IP address of the virtual machine." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time of creation resource." // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type VirtualMachine struct { @@ -41,48 +61,64 @@ type VirtualMachine struct { } type VirtualMachineSpec struct { - // RunPolicy is a power-on behaviour of the VM. - RunPolicy RunPolicy `json:"runPolicy"` + // +kubebuilder:default:="AlwaysOnUnlessStoppedManually" + RunPolicy RunPolicy `json:"runPolicy,omitempty"` - // VirtualMachineIPAddress specifies a name for the associated - // `VirtualMachineIPAddress` resource. Defaults to `{vm name}`. + // Name for the associated `virtualMachineIPAddress` resource. + // Specified when it is necessary to use a previously created IP address of the VM. + // If not explicitly specified, by default a `virtualMachineIPAddress` resource is created for the VM with a name similar to the VM resource (`.metadata.name`). VirtualMachineIPAddress string `json:"virtualMachineIPAddressName,omitempty"` - // TopologySpreadConstraints specifies how to spread matching pods among the given topology. TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` - // Affinity is a group of affinity scheduling rules. Affinity *VMAffinity `json:"affinity,omitempty"` // NodeSelector must match a node's labels for the VM to be scheduled on that node. - // +optional + // [The same](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) as in the pods `spec.nodeSelector` parameter in Kubernetes. NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // PriorityClassName - PriorityClassName string `json:"priorityClassName"` + // PriorityClassName [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) as in the pods `spec.priorityClassName` parameter in Kubernetes. + PriorityClassName string `json:"priorityClassName,omitempty"` // Tolerations define rules to tolerate node taints. + // The same](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) as in the pods `spec.tolerations` parameter in Kubernetes. Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - // Disruptions define an approval mode to apply disruptive (dangerous) changes. - Disruptions *Disruptions `json:"disruptions"` + // +kubebuilder:default:={"restartApprovalMode": "Manual"} + Disruptions *Disruptions `json:"disruptions,omitempty"` - // TerminationGracePeriodSeconds + // Grace period observed after signalling a VM to stop after which the VM is force terminated. + // +kubebuilder:default:=60 TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` - // EnableParavirtualization flag disables virtio for virtual machine. - // Default value is true, so omitempty is not specified. - EnableParavirtualization bool `json:"enableParavirtualization"` - - OsType OsType `json:"osType,omitempty"` - Bootloader BootloaderType `json:"bootloader,omitempty"` - VirtualMachineClassName string `json:"virtualMachineClassName,omitempty"` - CPU CPUSpec `json:"cpu"` - Memory MemorySpec `json:"memory"` - BlockDeviceRefs []BlockDeviceSpecRef `json:"blockDeviceRefs"` - Provisioning *Provisioning `json:"provisioning"` + // Use the `virtio` bus to connect virtual devices of the VM. Set false to disable `virtio` for this VM. + // Note: To use paravirtualization mode, some operating systems require the appropriate drivers to be installed. + // +kubebuilder:default:=true + EnableParavirtualization bool `json:"enableParavirtualization,omitempty"` + + // +kubebuilder:default:="Generic" + OsType OsType `json:"osType,omitempty"` + // +kubebuilder:default:="BIOS" + Bootloader BootloaderType `json:"bootloader,omitempty"` + // Name of the `VirtualMachineClass` resource describing the requirements for a virtual CPU, memory and the resource allocation policy and node placement policies for virtual machines. + VirtualMachineClassName string `json:"virtualMachineClassName"` + CPU CPUSpec `json:"cpu"` + Memory MemorySpec `json:"memory"` + // List of block devices that can be mounted by disks belonging to the virtual machine. + // The order of booting is determined by the order in the list. + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=16 + BlockDeviceRefs []BlockDeviceSpecRef `json:"blockDeviceRefs"` + Provisioning *Provisioning `json:"provisioning,omitempty"` } +// RunPolicy parameter defines the VM startup policy +// * `AlwaysOn` - after creation the VM is always in a running state, even in case of its shutdown by OS means. +// * `AlwaysOff` - after creation the VM is always in the off state. +// * `Manual` - after creation the VM is switched off, the VM state (switching on/off) is controlled via sub-resources or OS means. +// * `AlwaysOnUnlessStoppedManually` - after creation the VM is always in a running state, even in case of its shutdown by means of the OS, the VM can be shut down using the corresponding subresource. +// +// +kubebuilder:validation:Enum={AlwaysOn,AlwaysOff,Manual,AlwaysOnUnlessStoppedManually} type RunPolicy string const ( @@ -92,14 +128,23 @@ const ( AlwaysOnUnlessStoppedManually RunPolicy = "AlwaysOnUnlessStoppedManually" ) +// The OsType parameter allows you to select the type of used OS, for which a VM with an optimal set of required virtual devices and parameters will be created. +// +// * Windows - for Microsoft Windows family operating systems. +// * Generic - for other types of OS. +// +kubebuilder:validation:Enum={Windows,Generic} type OsType string const ( - Windows OsType = "Windows" - LegacyWindows OsType = "LegacyWindows" - GenericOs OsType = "Generic" + Windows OsType = "Windows" + GenericOs OsType = "Generic" ) +// The BootloaderType defines bootloader for VM. +// * BIOS - use legacy BIOS. +// * EFI - use Unified Extensible Firmware (EFI/UEFI). +// * EFIWithSecureBoot - use UEFI/EFI with SecureBoot support. +// +kubebuilder:validation:Enum={BIOS,EFI,EFIWithSecureBoot} type BootloaderType string const ( @@ -108,15 +153,25 @@ const ( EFIWithSecureBoot BootloaderType = "EFIWithSecureBoot" ) +// CPUSpec specifies the CPU settings for the VM. type CPUSpec struct { - Cores int `json:"cores"` - CoreFraction string `json:"coreFraction"` + // Specifies the number of cores inside the VM. The value must be greater or equal 1. + // +kubebuilder:validation:Format:=int32 + // +kubebuilder:validation:Minimum=1 + Cores int `json:"cores"` + // Guaranteed share of CPU that will be allocated to the VM. Specified as a percentage. + // +kubebuilder:default:="100%" + // +kubebuilder:validation:Enum:={"5%", "10%", "25%", "50%", "100%"} + CoreFraction string `json:"coreFraction,omitempty"` } +// MemorySpec specifies the memory settings for the VM. type MemorySpec struct { Size resource.Quantity `json:"size"` } +// RestartApprovalMode defines a restart approving mode: Manual or Automatic. +// +kubebuilder:validation:Enum={Manual,Automatic} type RestartApprovalMode string const ( @@ -124,20 +179,36 @@ const ( Manual RestartApprovalMode = "Manual" ) +// Disruptions describes the policy for applying changes that require rebooting the VM +// Changes to some VM configuration settings require a reboot of the VM to apply them. This policy allows you to specify the behavior of how the VM will respond to such changes. type Disruptions struct { - // RestartApprovalMode defines a restart approving mode: Manual or Automatic. - RestartApprovalMode RestartApprovalMode `json:"restartApprovalMode"` + RestartApprovalMode RestartApprovalMode `json:"restartApprovalMode,omitempty"` } +// Provisioning is a block allows you to configure the provisioning script for the VM. +// +// +kubebuilder:validation:XValidation:rule="self.type == 'UserData' ? has(self.userData) && !has(self.userDataRef) && !has(self.sysprepRef) : true",message="UserData cannot have userDataRef or sysprepRef." +// +kubebuilder:validation:XValidation:rule="self.type == 'UserDataRef' ? has(self.userDataRef) && !has(self.userData) && !has(self.sysprepRef) : true",message="UserDataRef cannot have userData or sysprepRef." +// +kubebuilder:validation:XValidation:rule="self.type == 'SysprepRef' ? has(self.sysprepRef) && !has(self.userData) && !has(self.userDataRef) : true",message="SysprepRef cannot have userData or userDataRef." type Provisioning struct { - Type ProvisioningType `json:"type"` - UserData string `json:"userData,omitempty"` - UserDataRef *UserDataRef `json:"userDataRef,omitempty"` - SysprepRef *SysprepRef `json:"sysprepRef,omitempty"` + Type ProvisioningType `json:"type"` + // Inline cloud-init userdata script. + UserData string `json:"userData,omitempty"` + UserDataRef *UserDataRef `json:"userDataRef,omitempty"` + SysprepRef *SysprepRef `json:"sysprepRef,omitempty"` } +// UserDataRef is reference to an existing resource with a cloud-init script. +// Resource structure for userDataRef type: +// * `.data.userData`. type UserDataRef struct { - Kind UserDataRefKind `json:"kind"` + // The kind of existing cloud-init automation resource. + // The following options are supported: + // - Secret + // + // +kubebuilder:validation:Enum:={Secret} + // +kubebuilder:default:="Secret" + Kind UserDataRefKind `json:"kind,omitempty"` Name string `json:"name"` } @@ -147,8 +218,17 @@ const ( UserDataRefKindSecret UserDataRefKind = "Secret" ) +// SysprepRef is reference to an existing Windows sysprep automation. +// Resource structure for the SysprepRef type: +// * `.data.autounattend.xml`. +// * `.data.unattend.xml`. type SysprepRef struct { - Kind SysprepRefKind `json:"kind"` + // The kind of existing Windows sysprep automation resource. + // The following options are supported: + // - Secret + // +kubebuilder:validation:Enum:={Secret} + // +kubebuilder:default:="Secret" + Kind SysprepRefKind `json:"kind,omitempty"` Name string `json:"name"` } @@ -159,58 +239,83 @@ const ( ) type VirtualMachineStatus struct { - Phase MachinePhase `json:"phase"` - Node string `json:"nodeName"` - VirtualMachineIPAddress string `json:"virtualMachineIPAddressName"` - IPAddress string `json:"ipAddress"` - BlockDeviceRefs []BlockDeviceStatusRef `json:"blockDeviceRefs"` - GuestOSInfo virtv1.VirtualMachineInstanceGuestOSInfo `json:"guestOSInfo"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - Stats *VirtualMachineStats `json:"stats,omitempty"` - MigrationState *VirtualMachineMigrationState `json:"migrationState,omitempty"` - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // RestartAwaitingChanges holds operations to be manually approved - // before applying to the virtual machine spec. - // - // Change operation has these fields: - // - // operation enum(add|remove|replace) - // path string - // currentValue any (bool|int|string|struct|array of structs) - // desiredValue any (bool|int|string|struct|array of structs) - // - // Such 'any' type can't be described using the OpenAPI v3 schema. - // The workaround is to declare a whole change operation structure - // using 'type: object' and 'x-kubernetes-preserve-fields: true'. + Phase MachinePhase `json:"phase"` + // The name of the node on which the VM is currently running. + Node string `json:"nodeName"` + // Name of `virtualMachineIPAddressName` holding the ip address of the VirtualMachine. + VirtualMachineIPAddress string `json:"virtualMachineIPAddressName"` + // IP address of VM. + IPAddress string `json:"ipAddress"` + // The list of attached block device attachments. + BlockDeviceRefs []BlockDeviceStatusRef `json:"blockDeviceRefs,omitempty"` + GuestOSInfo virtv1.VirtualMachineInstanceGuestOSInfo `json:"guestOSInfo,omitempty"` + // Detailed state of the virtual machine lifecycle. + Conditions []metav1.Condition `json:"conditions,omitempty"` + // VirtualMachine statistics. + Stats *VirtualMachineStats `json:"stats,omitempty"` + // Migration info. + MigrationState *VirtualMachineMigrationState `json:"migrationState,omitempty"` + // Generating a resource that was last processed by the controller. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + /* + Change operation has these fields: + * operation enum(add|remove|replace) + * path string + * currentValue any (bool|int|string|struct|array of structs) + * desiredValue any (bool|int|string|struct|array of structs) + Such 'any' type can't be described using the OpenAPI v3 schema. + The workaround is to declare a whole change operation structure + using 'type: object' and 'x-kubernetes-preserve-fields: true'. + */ + + // RestartAwaitingChanges holds operations to be manually approved before applying to the virtual machine spec. RestartAwaitingChanges []apiextensionsv1.JSON `json:"restartAwaitingChanges,omitempty"` + // List of virtual machine pods. + VirtualMachinePods []VirtualMachinePod `json:"virtualMachinePods,omitempty"` + Resources ResourcesStatus `json:"resources,omitempty"` } type VirtualMachineStats struct { - PhasesTransitions []VirtualMachinePhaseTransitionTimestamp `json:"phasesTransitions,omitempty"` - LaunchTimeDuration VirtualMachineLaunchTimeDuration `json:"launchTimeDuration,omitempty"` + // The history of phases. + PhasesTransitions []VirtualMachinePhaseTransitionTimestamp `json:"phasesTransitions,omitempty"` + // Launch information. + LaunchTimeDuration VirtualMachineLaunchTimeDuration `json:"launchTimeDuration,omitempty"` } // VirtualMachinePhaseTransitionTimestamp gives a timestamp in relation to when a phase is set on a vm. type VirtualMachinePhaseTransitionTimestamp struct { Phase MachinePhase `json:"phase,omitempty"` // PhaseTransitionTimestamp is the timestamp of when the phase change occurred + // +kubebuilder:validation:Format:=date-time + // +nullable Timestamp metav1.Time `json:"timestamp,omitempty"` } type VirtualMachineLaunchTimeDuration struct { + // The waiting time for dependent resources. pending -> starting. + // +nullable WaitingForDependencies *metav1.Duration `json:"waitingForDependencies,omitempty"` + // The waiting time for the virtual machine to start. starting -> running. + // +nullable VirtualMachineStarting *metav1.Duration `json:"virtualMachineStarting,omitempty"` - GuestOSAgentStarting *metav1.Duration `json:"guestOSAgentStarting,omitempty"` + // The waiting time for the guestOsAgent to start. running -> running with guestOSAgent. + // +nullable + GuestOSAgentStarting *metav1.Duration `json:"guestOSAgentStarting,omitempty"` } type VirtualMachineMigrationState struct { - StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` - EndTimestamp *metav1.Time `json:"endTimestamp,omitempty"` - Target VirtualMachineLocation `json:"target,omitempty"` - Source VirtualMachineLocation `json:"source,omitempty"` - Result MigrationResult `json:"result,omitempty"` + // Migration start time. + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + // Migration end time. + EndTimestamp *metav1.Time `json:"endTimestamp,omitempty"` + Target VirtualMachineLocation `json:"target,omitempty"` + Source VirtualMachineLocation `json:"source,omitempty"` + Result MigrationResult `json:"result,omitempty"` } +// MigrationResult defines a migration result +// +kubebuilder:validation:Enum:={"Succeeded","Failed",""} type MigrationResult string const ( @@ -219,10 +324,52 @@ const ( ) type VirtualMachineLocation struct { + // The name of the node on which the VM is currently migrating. Node string `json:"node,omitempty"` - Pod string `json:"pod,omitempty"` + // The name of the pod where the VM is currently being migrated. + Pod string `json:"pod,omitempty"` +} + +type VirtualMachinePod struct { + // Name of virtual machine pod. + Name string `json:"name"` + // Current working pod. + Active bool `json:"active"` } +// ResourcesStatus defines resource usage statistics. +type ResourcesStatus struct { + CPU CPUStatus `json:"cpu,omitempty"` + Memory MemoryStatus `json:"memory,omitempty"` +} + +// CPUStatus defines statistics about the CPU resource usage. +type CPUStatus struct { + // Current number of cores inside the VM. + Cores int `json:"cores"` + // Current CoreFraction. + CoreFraction string `json:"coreFraction,omitempty"` + // Requested cores. + RequestedCores resource.Quantity `json:"requestedCores,omitempty"` + // runtime overhead. + RuntimeOverhead resource.Quantity `json:"runtimeOverhead,omitempty"` +} + +// MemoryStatus defines statistics about the Memory resource usage. +type MemoryStatus struct { + // Current memory size. + Size resource.Quantity `json:"size"` + // Memory runtime overhead. + RuntimeOverhead resource.Quantity `json:"runtimeOverhead,omitempty"` +} + +// MachinePhase defines current phase of the virtual machine: +// * `Pending` - The process of starting the VM is in progress. +// * `Running` - VM is running. +// * `Degraded` - An error occurred during the startup process or while the VM is running. +// * `Terminating` - The VM is currently in the process of shutting down. +// * `Stopped` - The VM is stopped. +// +kubebuilder:validation:Enum:={Pending,Running,Terminating,Stopped,Stopping,Starting,Migrating,Pause,Degraded} type MachinePhase string const ( @@ -242,9 +389,18 @@ const ( type VirtualMachineList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` - Items []VirtualMachine `json:"items"` + + // Items provides a list of VirtualMachines + Items []VirtualMachine `json:"items"` } +// ProvisioningType parameter defines the type of provisioning script: +// +// Parameters supported for using the provisioning script: +// * UserData - use the cloud-init in the .spec.provisioning.UserData section. +// * UserDataRef - use a cloud-init script that resides in a different resource. +// * SysprepRef - Use a Windows Automation script that resides in a different resource. +// More information: https://cloudinit.readthedocs.io/en/latest/reference/examples.html type ProvisioningType string const ( diff --git a/api/core/v1alpha2/virtual_machine_class.go b/api/core/v1alpha2/virtual_machine_class.go index 415bde1dce..a590e1926c 100644 --- a/api/core/v1alpha2/virtual_machine_class.go +++ b/api/core/v1alpha2/virtual_machine_class.go @@ -35,7 +35,7 @@ const ( // +kubebuilder:object:root=true // +kubebuilder:metadata:labels={heritage=deckhouse,module=virtualization} // +kubebuilder:subresource:status -// +kubebuilder:resource:categories=virtualization,scope=Cluster,shortName={vmc,vmcs,vmclass,vmclasses},singular=virtualmachineclass +// +kubebuilder:resource:categories={virtualization},scope=Cluster,shortName={vmc,vmcs,vmclass,vmclasses},singular=virtualmachineclass // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="VirtualMachineClass phase." // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time of creation resource." // +genclient @@ -55,7 +55,7 @@ type VirtualMachineClassList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` - // Items provides a list of CDIs + // Items provides a list of VirtualMachineClasses Items []VirtualMachineClass `json:"items"` } diff --git a/api/core/v1alpha2/vmaffinity.go b/api/core/v1alpha2/vmaffinity.go index d373e5adc0..7e558f38ba 100644 --- a/api/core/v1alpha2/vmaffinity.go +++ b/api/core/v1alpha2/vmaffinity.go @@ -21,6 +21,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// VMAffinity [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) as in the pods `spec.affinity` parameter in Kubernetes; +// +// The affinity setting is completely similar to the above documentation, the only difference is in the names of some parameters. In fact, the following analogs are used: +// * podAffinity -> virtualMachineAndPodAffinity +// * podAffinityTerm -> virtualMachineAndPodAffinityTerm type VMAffinity struct { NodeAffinity *corev1.NodeAffinity `json:"nodeAffinity,omitempty"` VirtualMachineAndPodAffinity *VirtualMachineAndPodAffinity `json:"virtualMachineAndPodAffinity,omitempty"` diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 5129163d45..019bd9913e 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -45,6 +45,8 @@ func (r Reason) String() string { } const ( + ReasonUnknown Reason = "Unknown" + ReasonAgentNotReady Reason = "AgentNotReady" ReasonClassReady Reason = "VirtualMachineClassReady" diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index d9b94b3350..89a2d42697 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -114,6 +114,24 @@ func (in *CPUSpec) DeepCopy() *CPUSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CPUStatus) DeepCopyInto(out *CPUStatus) { + *out = *in + out.RequestedCores = in.RequestedCores.DeepCopy() + out.RuntimeOverhead = in.RuntimeOverhead.DeepCopy() + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUStatus. +func (in *CPUStatus) DeepCopy() *CPUStatus { + if in == nil { + return nil + } + out := new(CPUStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Checksum) DeepCopyInto(out *Checksum) { *out = *in @@ -513,6 +531,24 @@ func (in *MemorySpec) DeepCopy() *MemorySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryStatus) DeepCopyInto(out *MemoryStatus) { + *out = *in + out.Size = in.Size.DeepCopy() + out.RuntimeOverhead = in.RuntimeOverhead.DeepCopy() + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryStatus. +func (in *MemoryStatus) DeepCopy() *MemoryStatus { + if in == nil { + return nil + } + out := new(MemoryStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeSelector) DeepCopyInto(out *NodeSelector) { *out = *in @@ -569,6 +605,24 @@ func (in *Provisioning) DeepCopy() *Provisioning { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourcesStatus) DeepCopyInto(out *ResourcesStatus) { + *out = *in + in.CPU.DeepCopyInto(&out.CPU) + in.Memory.DeepCopyInto(&out.Memory) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourcesStatus. +func (in *ResourcesStatus) DeepCopy() *ResourcesStatus { + if in == nil { + return nil + } + out := new(ResourcesStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SizingPolicy) DeepCopyInto(out *SizingPolicy) { *out = *in @@ -2059,6 +2113,22 @@ func (in *VirtualMachinePhaseTransitionTimestamp) DeepCopy() *VirtualMachinePhas return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePod) DeepCopyInto(out *VirtualMachinePod) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePod. +func (in *VirtualMachinePod) DeepCopy() *VirtualMachinePod { + if in == nil { + return nil + } + out := new(VirtualMachinePod) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { *out = *in @@ -2180,6 +2250,12 @@ func (in *VirtualMachineStatus) DeepCopyInto(out *VirtualMachineStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.VirtualMachinePods != nil { + in, out := &in.VirtualMachinePods, &out.VirtualMachinePods + *out = make([]VirtualMachinePod, len(*in)) + copy(*out, *in) + } + in.Resources.DeepCopyInto(&out.Resources) return } diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go index 4ffabbdf7c..6c2dd3e03c 100644 --- a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go +++ b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go @@ -37,6 +37,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/deckhouse/virtualization/api/core/v1alpha2.BlockDeviceStatusRef": schema_virtualization_api_core_v1alpha2_BlockDeviceStatusRef(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.CPU": schema_virtualization_api_core_v1alpha2_CPU(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.CPUSpec": schema_virtualization_api_core_v1alpha2_CPUSpec(ref), + "github.com/deckhouse/virtualization/api/core/v1alpha2.CPUStatus": schema_virtualization_api_core_v1alpha2_CPUStatus(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.Checksum": schema_virtualization_api_core_v1alpha2_Checksum(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.ClusterVirtualImage": schema_virtualization_api_core_v1alpha2_ClusterVirtualImage(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.ClusterVirtualImageDataSource": schema_virtualization_api_core_v1alpha2_ClusterVirtualImageDataSource(ref), @@ -56,8 +57,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/deckhouse/virtualization/api/core/v1alpha2.ImageStatusTarget": schema_virtualization_api_core_v1alpha2_ImageStatusTarget(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.MemoryMinMax": schema_virtualization_api_core_v1alpha2_MemoryMinMax(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.MemorySpec": schema_virtualization_api_core_v1alpha2_MemorySpec(ref), + "github.com/deckhouse/virtualization/api/core/v1alpha2.MemoryStatus": schema_virtualization_api_core_v1alpha2_MemoryStatus(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.NodeSelector": schema_virtualization_api_core_v1alpha2_NodeSelector(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.Provisioning": schema_virtualization_api_core_v1alpha2_Provisioning(ref), + "github.com/deckhouse/virtualization/api/core/v1alpha2.ResourcesStatus": schema_virtualization_api_core_v1alpha2_ResourcesStatus(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.SizingPolicy": schema_virtualization_api_core_v1alpha2_SizingPolicy(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.SizingPolicyCores": schema_virtualization_api_core_v1alpha2_SizingPolicyCores(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.SizingPolicyMemory": schema_virtualization_api_core_v1alpha2_SizingPolicyMemory(ref), @@ -117,6 +120,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineOperationSpec": schema_virtualization_api_core_v1alpha2_VirtualMachineOperationSpec(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineOperationStatus": schema_virtualization_api_core_v1alpha2_VirtualMachineOperationStatus(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachinePhaseTransitionTimestamp": schema_virtualization_api_core_v1alpha2_VirtualMachinePhaseTransitionTimestamp(ref), + "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachinePod": schema_virtualization_api_core_v1alpha2_VirtualMachinePod(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineSpec": schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStats": schema_virtualization_api_core_v1alpha2_VirtualMachineStats(ref), "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStatus": schema_virtualization_api_core_v1alpha2_VirtualMachineStatus(ref), @@ -684,9 +688,10 @@ func schema_virtualization_api_core_v1alpha2_BlockDeviceSpecRef(ref common.Refer }, "name": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The name of attached resource.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, }, @@ -711,46 +716,51 @@ func schema_virtualization_api_core_v1alpha2_BlockDeviceStatusRef(ref common.Ref }, "name": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The name of attached resource.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, "size": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The size of attached block device.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, - "target": { + "attached": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The block device is attached to the virtual machine.", + Default: false, + Type: []string{"boolean"}, + Format: "", }, }, - "attached": { + "target": { SchemaProps: spec.SchemaProps{ - Default: false, - Type: []string{"boolean"}, - Format: "", + Description: "The name of attached block device.", + Type: []string{"string"}, + Format: "", }, }, "hotplugged": { SchemaProps: spec.SchemaProps{ - Type: []string{"boolean"}, - Format: "", + Description: "Block device is attached via hot plug connection.", + Type: []string{"boolean"}, + Format: "", }, }, "virtualMachineBlockDeviceAttachmentName": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "The name of the `VirtualMachineBlockDeviceAttachment` resource that defines hot plug disk connection to the virtual machine.", + Type: []string{"string"}, + Format: "", }, }, }, - Required: []string{"kind", "name", "size", "target", "attached"}, + Required: []string{"kind", "name", "size", "attached"}, }, }, } @@ -812,29 +822,74 @@ func schema_virtualization_api_core_v1alpha2_CPUSpec(ref common.ReferenceCallbac return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "CPUSpec specifies the CPU settings for the VM.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "cores": { SchemaProps: spec.SchemaProps{ - Default: 0, - Type: []string{"integer"}, - Format: "int32", + Description: "Specifies the number of cores inside the VM. The value must be greater or equal 1.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", }, }, "coreFraction": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "Guaranteed share of CPU that will be allocated to the VM. Specified as a percentage.", + Type: []string{"string"}, + Format: "", }, }, }, - Required: []string{"cores", "coreFraction"}, + Required: []string{"cores"}, }, }, } } +func schema_virtualization_api_core_v1alpha2_CPUStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "CPUStatus defines statistics about the CPU resource usage.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "cores": { + SchemaProps: spec.SchemaProps{ + Description: "Current number of cores inside the VM.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + "coreFraction": { + SchemaProps: spec.SchemaProps{ + Description: "Current CoreFraction.", + Type: []string{"string"}, + Format: "", + }, + }, + "requestedCores": { + SchemaProps: spec.SchemaProps{ + Description: "Requested cores.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + "runtimeOverhead": { + SchemaProps: spec.SchemaProps{ + Description: "runtime overhead.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + Required: []string{"cores"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + func schema_virtualization_api_core_v1alpha2_Checksum(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -1284,18 +1339,16 @@ func schema_virtualization_api_core_v1alpha2_Disruptions(ref common.ReferenceCal return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "Disruptions describes the policy for applying changes that require rebooting the VM Changes to some VM configuration settings require a reboot of the VM to apply them. This policy allows you to specify the behavior of how the VM will respond to such changes.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "restartApprovalMode": { SchemaProps: spec.SchemaProps{ - Description: "RestartApprovalMode defines a restart approving mode: Manual or Automatic.", - Default: "", - Type: []string{"string"}, - Format: "", + Type: []string{"string"}, + Format: "", }, }, }, - Required: []string{"restartApprovalMode"}, }, }, } @@ -1488,7 +1541,8 @@ func schema_virtualization_api_core_v1alpha2_MemorySpec(ref common.ReferenceCall return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "MemorySpec specifies the memory settings for the VM.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "size": { SchemaProps: spec.SchemaProps{ @@ -1504,6 +1558,34 @@ func schema_virtualization_api_core_v1alpha2_MemorySpec(ref common.ReferenceCall } } +func schema_virtualization_api_core_v1alpha2_MemoryStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "MemoryStatus defines statistics about the Memory resource usage.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "size": { + SchemaProps: spec.SchemaProps{ + Description: "Current memory size.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + "runtimeOverhead": { + SchemaProps: spec.SchemaProps{ + Description: "Memory runtime overhead.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + Required: []string{"size"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + func schema_virtualization_api_core_v1alpha2_NodeSelector(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -1552,7 +1634,8 @@ func schema_virtualization_api_core_v1alpha2_Provisioning(ref common.ReferenceCa return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "Provisioning is a block allows you to configure the provisioning script for the VM.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "type": { SchemaProps: spec.SchemaProps{ @@ -1563,8 +1646,9 @@ func schema_virtualization_api_core_v1alpha2_Provisioning(ref common.ReferenceCa }, "userData": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "Inline cloud-init userdata script.", + Type: []string{"string"}, + Format: "", }, }, "userDataRef": { @@ -1586,6 +1670,33 @@ func schema_virtualization_api_core_v1alpha2_Provisioning(ref common.ReferenceCa } } +func schema_virtualization_api_core_v1alpha2_ResourcesStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ResourcesStatus defines resource usage statistics.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "cpu": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.CPUStatus"), + }, + }, + "memory": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.MemoryStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/deckhouse/virtualization/api/core/v1alpha2.CPUStatus", "github.com/deckhouse/virtualization/api/core/v1alpha2.MemoryStatus"}, + } +} + func schema_virtualization_api_core_v1alpha2_SizingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -1784,13 +1895,14 @@ func schema_virtualization_api_core_v1alpha2_SysprepRef(ref common.ReferenceCall return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "SysprepRef is reference to an existing Windows sysprep automation. Resource structure for the SysprepRef type: * `.data.autounattend.xml`. * `.data.unattend.xml`.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The kind of existing Windows sysprep automation resource. The following options are supported:\n - Secret", + Type: []string{"string"}, + Format: "", }, }, "name": { @@ -1801,7 +1913,7 @@ func schema_virtualization_api_core_v1alpha2_SysprepRef(ref common.ReferenceCall }, }, }, - Required: []string{"kind", "name"}, + Required: []string{"name"}, }, }, } @@ -1811,13 +1923,14 @@ func schema_virtualization_api_core_v1alpha2_UserDataRef(ref common.ReferenceCal return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "UserDataRef is reference to an existing resource with a cloud-init script. Resource structure for userDataRef type: * `.data.userData`.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The kind of existing cloud-init automation resource. The following options are supported:\n - Secret", + Type: []string{"string"}, + Format: "", }, }, "name": { @@ -1828,7 +1941,7 @@ func schema_virtualization_api_core_v1alpha2_UserDataRef(ref common.ReferenceCal }, }, }, - Required: []string{"kind", "name"}, + Required: []string{"name"}, }, }, } @@ -1838,7 +1951,8 @@ func schema_virtualization_api_core_v1alpha2_VMAffinity(ref common.ReferenceCall return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "VMAffinity [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) as in the pods `spec.affinity` parameter in Kubernetes;\n\nThe affinity setting is completely similar to the above documentation, the only difference is in the names of some parameters. In fact, the following analogs are used: * podAffinity -> virtualMachineAndPodAffinity * podAffinityTerm -> virtualMachineAndPodAffinityTerm", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "nodeAffinity": { SchemaProps: spec.SchemaProps{ @@ -2730,7 +2844,7 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachine(ref common.Reference return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "VirtualMachine specifies configuration of the virtual machine.", + Description: "VirtualMachine describes the configuration and status of a virtual machine (VM). For a running VM, parameter changes can only be applied after the VM is rebooted, except for the following parameters (they are applied on the fly): - `.metadata.labels`. - `.metadata.annotations`. - `.spec.disruptions.restartApprovalMode`. - `.spec.disruptions.runPolicy`.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -3170,7 +3284,7 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineClassList(ref common. }, "items": { SchemaProps: spec.SchemaProps{ - Description: "Items provides a list of CDIs", + Description: "Items provides a list of VirtualMachineClasses", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -3671,17 +3785,20 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineLaunchTimeDuration(re Properties: map[string]spec.Schema{ "waitingForDependencies": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "The waiting time for dependent resources. pending -> starting.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "virtualMachineStarting": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "The waiting time for the virtual machine to start. starting -> running.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "guestOSAgentStarting": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + Description: "The waiting time for the guestOsAgent to start. running -> running with guestOSAgent.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, }, @@ -3721,7 +3838,8 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineList(ref common.Refer }, "items": { SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, + Description: "Items provides a list of VirtualMachines", + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -3749,14 +3867,16 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineLocation(ref common.R Properties: map[string]spec.Schema{ "node": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "The name of the node on which the VM is currently migrating.", + Type: []string{"string"}, + Format: "", }, }, "pod": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "The name of the pod where the VM is currently being migrated.", + Type: []string{"string"}, + Format: "", }, }, }, @@ -3773,12 +3893,14 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineMigrationState(ref co Properties: map[string]spec.Schema{ "startTimestamp": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + Description: "Migration start time.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, "endTimestamp": { SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + Description: "Migration end time.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, "target": { @@ -3996,31 +4118,57 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachinePhaseTransitionTimest } } -func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_virtualization_api_core_v1alpha2_VirtualMachinePod(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ Type: []string{"object"}, Properties: map[string]spec.Schema{ - "runPolicy": { + "name": { SchemaProps: spec.SchemaProps{ - Description: "RunPolicy is a power-on behaviour of the VM.", + Description: "Name of virtual machine pod.", Default: "", Type: []string{"string"}, Format: "", }, }, + "active": { + SchemaProps: spec.SchemaProps{ + Description: "Current working pod.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"name", "active"}, + }, + }, + } +} + +func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "runPolicy": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, "virtualMachineIPAddressName": { SchemaProps: spec.SchemaProps{ - Description: "VirtualMachineIPAddress specifies a name for the associated `VirtualMachineIPAddress` resource. Defaults to `{vm name}`.", + Description: "Name for the associated `virtualMachineIPAddress` resource. Specified when it is necessary to use a previously created IP address of the VM. If not explicitly specified, by default a `virtualMachineIPAddress` resource is created for the VM with a name similar to the VM resource (`.metadata.name`).", Type: []string{"string"}, Format: "", }, }, "topologySpreadConstraints": { SchemaProps: spec.SchemaProps{ - Description: "TopologySpreadConstraints specifies how to spread matching pods among the given topology.", - Type: []string{"array"}, + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -4033,13 +4181,12 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, "affinity": { SchemaProps: spec.SchemaProps{ - Description: "Affinity is a group of affinity scheduling rules.", - Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VMAffinity"), + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VMAffinity"), }, }, "nodeSelector": { SchemaProps: spec.SchemaProps{ - Description: "NodeSelector must match a node's labels for the VM to be scheduled on that node.", + Description: "NodeSelector must match a node's labels for the VM to be scheduled on that node. [The same](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) as in the pods `spec.nodeSelector` parameter in Kubernetes.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -4055,15 +4202,14 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, "priorityClassName": { SchemaProps: spec.SchemaProps{ - Description: "PriorityClassName", - Default: "", + Description: "PriorityClassName [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) as in the pods `spec.priorityClassName` parameter in Kubernetes.", Type: []string{"string"}, Format: "", }, }, "tolerations": { SchemaProps: spec.SchemaProps{ - Description: "Tolerations define rules to tolerate node taints.", + Description: "Tolerations define rules to tolerate node taints. The same](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) as in the pods `spec.tolerations` parameter in Kubernetes.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -4077,21 +4223,19 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, "disruptions": { SchemaProps: spec.SchemaProps{ - Description: "Disruptions define an approval mode to apply disruptive (dangerous) changes.", - Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.Disruptions"), + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.Disruptions"), }, }, "terminationGracePeriodSeconds": { SchemaProps: spec.SchemaProps{ - Description: "TerminationGracePeriodSeconds", + Description: "Grace period observed after signalling a VM to stop after which the VM is force terminated.", Type: []string{"integer"}, Format: "int64", }, }, "enableParavirtualization": { SchemaProps: spec.SchemaProps{ - Description: "EnableParavirtualization flag disables virtio for virtual machine. Default value is true, so omitempty is not specified.", - Default: false, + Description: "Use the `virtio` bus to connect virtual devices of the VM. Set false to disable `virtio` for this VM. Note: To use paravirtualization mode, some operating systems require the appropriate drivers to be installed.", Type: []string{"boolean"}, Format: "", }, @@ -4110,8 +4254,10 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, "virtualMachineClassName": { SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", + Description: "Name of the `VirtualMachineClass` resource describing the requirements for a virtual CPU, memory and the resource allocation policy and node placement policies for virtual machines.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, "cpu": { @@ -4128,7 +4274,8 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, "blockDeviceRefs": { SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, + Description: "List of block devices that can be mounted by disks belonging to the virtual machine. The order of booting is determined by the order in the list.", + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -4145,7 +4292,7 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineSpec(ref common.Refer }, }, }, - Required: []string{"runPolicy", "priorityClassName", "disruptions", "enableParavirtualization", "cpu", "memory", "blockDeviceRefs", "provisioning"}, + Required: []string{"virtualMachineClassName", "cpu", "memory", "blockDeviceRefs"}, }, }, Dependencies: []string{ @@ -4161,7 +4308,8 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStats(ref common.Refe Properties: map[string]spec.Schema{ "phasesTransitions": { SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, + Description: "The history of phases.", + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -4174,8 +4322,9 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStats(ref common.Refe }, "launchTimeDuration": { SchemaProps: spec.SchemaProps{ - Default: map[string]interface{}{}, - Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineLaunchTimeDuration"), + Description: "Launch information.", + Default: map[string]interface{}{}, + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineLaunchTimeDuration"), }, }, }, @@ -4201,28 +4350,32 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStatus(ref common.Ref }, "nodeName": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "The name of the node on which the VM is currently running.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, "virtualMachineIPAddressName": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "Name of `virtualMachineIPAddressName` holding the ip address of the VirtualMachine.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, "ipAddress": { SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Description: "IP address of VM.", + Default: "", + Type: []string{"string"}, + Format: "", }, }, "blockDeviceRefs": { SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, + Description: "The list of attached block device attachments.", + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -4241,7 +4394,8 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStatus(ref common.Ref }, "conditions": { SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, + Description: "Detailed state of the virtual machine lifecycle.", + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -4254,23 +4408,26 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStatus(ref common.Ref }, "stats": { SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStats"), + Description: "VirtualMachine statistics.", + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStats"), }, }, "migrationState": { SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineMigrationState"), + Description: "Migration info.", + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineMigrationState"), }, }, "observedGeneration": { SchemaProps: spec.SchemaProps{ - Type: []string{"integer"}, - Format: "int64", + Description: "Generating a resource that was last processed by the controller.", + Type: []string{"integer"}, + Format: "int64", }, }, "restartAwaitingChanges": { SchemaProps: spec.SchemaProps{ - Description: "RestartAwaitingChanges holds operations to be manually approved before applying to the virtual machine spec.\n\nChange operation has these fields:\n\n\toperation enum(add|remove|replace)\n\tpath string\n\tcurrentValue any (bool|int|string|struct|array of structs)\n\tdesiredValue any (bool|int|string|struct|array of structs)\n\nSuch 'any' type can't be described using the OpenAPI v3 schema. The workaround is to declare a whole change operation structure using 'type: object' and 'x-kubernetes-preserve-fields: true'.", + Description: "RestartAwaitingChanges holds operations to be manually approved before applying to the virtual machine spec.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -4281,12 +4438,32 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineStatus(ref common.Ref }, }, }, + "virtualMachinePods": { + SchemaProps: spec.SchemaProps{ + Description: "List of virtual machine pods.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachinePod"), + }, + }, + }, + }, + }, + "resources": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/deckhouse/virtualization/api/core/v1alpha2.ResourcesStatus"), + }, + }, }, - Required: []string{"phase", "nodeName", "virtualMachineIPAddressName", "ipAddress", "blockDeviceRefs", "guestOSInfo"}, + Required: []string{"phase", "nodeName", "virtualMachineIPAddressName", "ipAddress"}, }, }, Dependencies: []string{ - "github.com/deckhouse/virtualization/api/core/v1alpha2.BlockDeviceStatusRef", "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineMigrationState", "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStats", "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1.JSON", "k8s.io/apimachinery/pkg/apis/meta/v1.Condition", "kubevirt.io/api/core/v1.VirtualMachineInstanceGuestOSInfo"}, + "github.com/deckhouse/virtualization/api/core/v1alpha2.BlockDeviceStatusRef", "github.com/deckhouse/virtualization/api/core/v1alpha2.ResourcesStatus", "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineMigrationState", "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachinePod", "github.com/deckhouse/virtualization/api/core/v1alpha2.VirtualMachineStats", "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1.JSON", "k8s.io/apimachinery/pkg/apis/meta/v1.Condition", "kubevirt.io/api/core/v1.VirtualMachineInstanceGuestOSInfo"}, } } diff --git a/api/scripts/update-codegen.sh b/api/scripts/update-codegen.sh index ee41a36f10..27f277c06f 100755 --- a/api/scripts/update-codegen.sh +++ b/api/scripts/update-codegen.sh @@ -32,7 +32,7 @@ function source::settings { MODULE="github.com/deckhouse/virtualization/api" PREFIX_GROUP="virtualization.deckhouse.io_" # TODO: Temporary filter until all CRDs become auto-generated. - ALLOWED_RESOURCE_GEN_CRD=("VirtualMachineClass" "ExampleKind1" "ExampleKind2") + ALLOWED_RESOURCE_GEN_CRD=("VirtualMachineClass" "VirtualMachine" "ExampleKind1" "ExampleKind2") source "${CODEGEN_PKG}/kube_codegen.sh" } diff --git a/crds/doc-ru-virtualmachineclasses.yaml b/crds/doc-ru-virtualmachineclasses.yaml index 718d0e9c45..0ee3b090b5 100644 --- a/crds/doc-ru-virtualmachineclasses.yaml +++ b/crds/doc-ru-virtualmachineclasses.yaml @@ -50,8 +50,7 @@ spec: `In` - значение ключа лейбла входит в заданный список лейблов `values`. `NotIn` - значение ключа лейбла не входит в заданный список лейблов `values`. `Exist` - ключ лейбла существует. - `DoesNotExist` - ключа лейблане существует. - + `DoesNotExist` - ключа лейбла не существует. values: description: | Строковый список значений. Если оператор `In` или `NotIn`, список значений должен быть непустым. Если оператор `Exists` или `DoesNotExist`, список значений должен быть пустым. diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 9f8e99120b..1779229d92 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -654,3 +654,33 @@ spec: observedGeneration: description: | Поколение ресурса, которое в последний раз обрабатывалось контроллером + virtualMachinePods: + description: | + Список подов виртуальной машины. + items: + properties: + active: + description: Отметка активного пода. + name: + description: Имя пода. + resources: + description: Статистика использования ресурсов. + properties: + cpu: + description: Статистика использования ресурсов ЦПУ. + properties: + coreFraction: + description: Текущее значение coreFraction. + cores: + description: Текущее количество ядер виртуальной машины. + requestedCores: + description: Запрошенные ядра. + runtimeOverhead: + description: Накладные расходы на cpu во время выполнения. + memory: + description: Статистика использования ресурсов памяти. + properties: + runtimeOverhead: + description: Накладные расходы на память во время выполнения. + size: + description: Текущий размер памяти виртуальной машины. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 667efe9b2b..97c75203b3 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1,1178 +1,1669 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: virtualmachines.virtualization.deckhouse.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 labels: heritage: deckhouse module: virtualization + name: virtualmachines.virtualization.deckhouse.io spec: group: virtualization.deckhouse.io - scope: Namespaced names: categories: - all - virtualization - plural: virtualmachines - singular: virtualmachine - listKind: VirtualMachineList kind: VirtualMachine + listKind: VirtualMachineList + plural: virtualmachines shortNames: - vm - vms - preserveUnknownFields: false + singular: virtualmachine + scope: Namespaced versions: - - name: v1alpha2 - served: true - storage: true + - additionalPrinterColumns: + - description: The phase of the virtual machine. + jsonPath: .status.phase + name: Phase + type: string + - description: The number of cores of the virtual machine. + jsonPath: .spec.cpu.cores + name: Cores + priority: 1 + type: string + - description: Virtual machine core fraction. + jsonPath: .spec.cpu.coreFraction + name: CoreFraction + priority: 1 + type: string + - description: The amount of memory of the virtual machine. + jsonPath: .spec.memory.size + name: Memory + priority: 1 + type: string + - description: A restart of the virtual machine is required. + jsonPath: .status.conditions[?(@.type=='AwaitingRestartToApplyConfiguration')].status + name: Need restart + priority: 1 + type: string + - description: Agent status. + jsonPath: .status.conditions[?(@.type=='AgentReady')].status + name: Agent + priority: 1 + type: string + - description: Is it possible to migrate a virtual machine. + jsonPath: .status.conditions[?(@.type=='Migratable')].status + name: Migratable + priority: 1 + type: string + - description: The node where the virtual machine is running. + jsonPath: .status.nodeName + name: Node + type: string + - description: The IP address of the virtual machine. + jsonPath: .status.ipAddress + name: IPAddress + type: string + - description: Time of creation resource. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 schema: openAPIV3Schema: - type: object - description: | - This resource describes the configuration and status of a virtual machine (VM). + description: |- + VirtualMachine describes the configuration and status of a virtual machine (VM). For a running VM, parameter changes can only be applied after the VM is rebooted, except for the following parameters (they are applied on the fly): - `.metadata.labels`. - `.metadata.annotations`. - `.spec.disruptions.restartApprovalMode`. - `.spec.disruptions.runPolicy`. - required: - - spec properties: - spec: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: type: object - required: - - virtualMachineClassName - - cpu - - memory - - blockDeviceRefs + spec: properties: - provisioning: - description: | - This block allows you to configure the provisioning script for the VM. - type: object - properties: - type: - description: | - This parameter defines the type of provisioning script: - - Parameters supported for using the provisioning script: - - * UserData - use the cloud-init in the .spec.provisioning.UserData section. - * UserDataRef - use a cloud-init script that resides in a different resource. - * SysprepRef - Use a Windows Automation script that resides in a different resource. - - More information: https://cloudinit.readthedocs.io/en/latest/reference/examples.html - type: string - enum: - - UserData - - UserDataRef - - SysprepRef - userData: - description: | - Inline cloud-init userdata script. - type: string - userDataRef: - description: | - A reference to an existing resource with a cloud-init script. - - Resource structure for userDataRef type: - - * `.data.userData`. - type: object - properties: - kind: - description: | - The kind of an existing cloud-init automation resource. - default: "Secret" - type: string - enum: - - Secret - name: - type: string - description: | - The name of the resource. - required: ["name", "kind"] - sysprepRef: - description: | - Reference to an existing Windows sysprep automation. - - Resource structure for the SysprepRef type: - - * `.data.autounattend.xml`. - * `.data.unattend.xml`. - type: object - properties: - kind: - description: | - The kind of an existing Windows sysprep automation resource. - default: "Secret" - type: string - enum: - - Secret - name: - type: string - description: | - Secret name. - required: ["kind", "name"] - oneOf: - - properties: - type: - enum: ["SysprepRef"] - sysprepRef: {} - required: ["sysprepRef"] - not: - anyOf: - - required: ["userData"] - - required: ["userDataRef"] - - properties: - type: - enum: ["UserData"] - required: ["userData"] - not: - anyOf: - - required: ["sysprepRef"] - - required: ["userDataRef"] - - properties: - type: - enum: ["UserDataRef"] - userDataRef: {} - required: ["userDataRef"] - not: - anyOf: - - required: ["sysprepRef"] - - required: ["userData"] - runPolicy: - type: string - enum: - - "AlwaysOn" - - "AlwaysOff" - - "Manual" - - "AlwaysOnUnlessStoppedManually" - default: "AlwaysOnUnlessStoppedManually" - description: | - This parameter defines the VM startup policy - - * `AlwaysOn` - after creation the VM is always in a running state, even in case of its shutdown by OS means. - * `AlwaysOff` - after creation the VM is always in the off state. - * `Manual` - after creation the VM is switched off, the VM state (switching on/off) is controlled via sub-resources or OS means. - * `AlwaysOnUnlessStoppedManually` - after creation the VM is always in a running state, even in case of its shutdown by means of the OS, the VM can be shut down using the corresponding subresource. - - virtualMachineIPAddressName: - type: string - description: | - Name for the associated `virtualMachineIPAddress` resource. - - Specified when it is necessary to use a previously created IP address of the VM. - - If not explicitly specified, by default a `virtualMachineIPAddress` resource is created for the VM with a name similar to the VM resource (`.metadata.name`). - - topologySpreadConstraints: - description: | - [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) as in the pods `spec.topologySpreadConstraints` parameter in Kubernetes; - - items: - properties: - labelSelector: - description: "" - properties: - matchExpressions: - description: "" - items: - properties: - key: - type: string - description: "" - operator: - type: string - description: "" - values: - items: - type: string - description: "" - type: array - description: "" - required: - - key - - operator - type: object - type: array - matchLabels: - description: "" - additionalProperties: - type: string - description: "" - type: object - type: object - maxSkew: - description: "" - format: int32 - type: integer - topologyKey: - description: "" - type: string - whenUnsatisfiable: - description: "" - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - affinity: - description: | - [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) as in the pods `spec.affinity` parameter in Kubernetes; + description: |- + VMAffinity [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) as in the pods `spec.affinity` parameter in Kubernetes; + The affinity setting is completely similar to the above documentation, the only difference is in the names of some parameters. In fact, the following analogs are used: * podAffinity -> virtualMachineAndPodAffinity * podAffinityTerm -> virtualMachineAndPodAffinityTerm - properties: nodeAffinity: + description: + Node affinity is a group of node affinity scheduling + rules. properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: + description: + A node selector term, associated with the + corresponding weight. properties: matchExpressions: + description: + A list of node selector requirements + by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: - description: "" + description: + The label key that the selector + applies to. type: string operator: - description: "" + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: - description: "" type: string type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchFields: + description: + A list of node selector requirements + by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: + The label key that the selector + applies to. type: string - description: "" operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - description: "" values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" type: object - description: "" + x-kubernetes-map-type: atomic weight: - maximum: 100 - minimum: 1 + description: + Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer - description: "" required: - preference - weight type: object - description: "" type: array - description: "" requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: + description: + Required. A list of node selector terms. + The terms are ORed. items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: + description: + A list of node selector requirements + by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: + The label key that the selector + applies to. type: string - description: "" operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - description: "" values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchFields: + description: + A list of node selector requirements + by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: + The label key that the selector + applies to. type: string - description: "" operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - description: "" values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" type: object - description: "" + x-kubernetes-map-type: atomic type: array - description: "" required: - nodeSelectorTerms type: object - description: "" + x-kubernetes-map-type: atomic type: object - description: Describes node affinity scheduling rules for the VM. virtualMachineAndPodAffinity: - description: Describes pod and VM affinity scheduling rules. properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: virtualMachineAndPodAffinityTerm: + description: + Required. A vm affinity term, associated + with the corresponding weight. properties: labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that + the selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that + the selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic namespaces: items: type: string - description: "" type: array - description: "" topologyKey: type: string - description: "" - matchLabelKeys: - items: - type: string - description: "" - type: array - description: "" - mismatchLabelKeys: - items: - type: string - description: "" - type: array - description: "" required: - topologyKey type: object - description: "" weight: - maximum: 100 - minimum: 1 + description: |- + weight associated with matching the corresponding vmAndPodAffinityTerm, + in the range 1-100. format: int32 type: integer - description: "" required: - virtualMachineAndPodAffinityTerm - weight type: object - description: "" type: array - description: "" requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that the + selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that the + selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic namespaces: items: type: string - description: "" type: array - description: "" topologyKey: type: string - description: "" - matchLabelKeys: - items: - type: string - description: "" - type: array - description: "" - mismatchLabelKeys: - items: - type: string - description: "" - type: array - description: "" required: - topologyKey type: object - description: "" type: array - description: "" type: object virtualMachineAndPodAntiAffinity: - description: Describes pod and VM anti-affinity scheduling rules. properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: virtualMachineAndPodAffinityTerm: + description: + Required. A vm affinity term, associated + with the corresponding weight. properties: labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that + the selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that + the selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic namespaces: items: type: string - description: "" type: array - description: "" topologyKey: type: string - description: "" - matchLabelKeys: - items: - type: string - description: "" - type: array - description: "" - mismatchLabelKeys: - items: - type: string - description: "" - type: array - description: "" required: - topologyKey type: object - description: "" weight: - maximum: 100 - minimum: 1 + description: |- + weight associated with matching the corresponding vmAndPodAffinityTerm, + in the range 1-100. format: int32 type: integer - description: "" required: - virtualMachineAndPodAffinityTerm - weight type: object - description: "" type: array - description: "" requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that the + selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. properties: matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: + key is the label key that the + selector applies to. type: string - description: "" operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string - description: "" values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string - description: "" type: array - description: "" required: - key - operator type: object - description: "" type: array - description: "" matchLabels: additionalProperties: type: string - description: "" + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - description: "" type: object - description: "" + x-kubernetes-map-type: atomic namespaces: items: type: string - description: "" type: array - description: "" topologyKey: type: string - description: "" - matchLabelKeys: - items: - type: string - description: "" - type: array - description: "" - mismatchLabelKeys: - items: - type: string - description: "" - type: array - description: "" required: - topologyKey type: object - description: "" type: array - description: "" type: object type: object - - nodeSelector: - additionalProperties: - type: string - description: | - [The same](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) as in the pods `spec.nodeSelector` parameter in Kubernetes; - type: object - - priorityClassName: - description: | - [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) as in the pods `spec.priorityClassName` parameter in Kubernetes; - type: string - - tolerations: - type: array - description: | - [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) as in the pods `spec.tolerations` parameter in Kubernetes; - + blockDeviceRefs: + description: |- + List of block devices that can be mounted by disks belonging to the virtual machine. + The order of booting is determined by the order in the list. items: - type: object - description: "" properties: - effect: - type: string - description: "" - key: - type: string - description: "" - operator: - type: string - description: "" - tolerationSeconds: - type: integer - description: "" - format: int64 - value: - type: string - description: "" - - disruptions: - type: object - default: - restartApprovalMode: Manual - description: | - Describing the policy for applying changes that require rebooting the VM. - - Changes to some VM configuration settings require a reboot of the VM to apply them. This policy allows you to specify the behavior of how the VM will respond to such changes. - properties: - restartApprovalMode: - type: string - description: | - Approval mode for changes that require rebooting the VM: + kind: + description: |- + The BlockDeviceKind is a type of the block device. Options are: - * `Manual` - changes will not be applied until the user reboots the VM himself. - * `Automatic` - the VM will be rebooted immediately after saving the parameters that require rebooting the VM. - enum: - - Manual - - Automatic - - terminationGracePeriodSeconds: - format: int64 - type: integer - default: 60 - description: | - Grace period observed after signalling a VM to stop after which the VM is force terminated. - - enableParavirtualization: - type: boolean - default: true - description: | - Use the `virtio` bus to connect virtual devices of the VM. Set to false to disable `virtio` for this VM. - - Note: To use paravirtualization mode, some operating systems require the appropriate drivers to be installed. - - osType: - type: string - enum: - - "Windows" - - "Generic" - default: "Generic" - description: | - The parameter allows you to select the type of used OS, for which a VM with an optimal set of required virtual devices and parameters will be created. - - * Windows - for Microsoft Windows family operating systems. - * Generic - for other types of OS. + * `ClusterVirtualImage` — Use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualImage` — Use `VirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualDisk` — Use `VirtualDisk` as the disk. This type is always mounted in RW mode. + enum: + - ClusterVirtualImage + - VirtualImage + - VirtualDisk + type: string + name: + description: The name of attached resource. + type: string + required: + - kind + - name + type: object + maxItems: 16 + minItems: 1 + type: array bootloader: - type: string - enum: - - "BIOS" - - "EFI" - - "EFIWithSecureBoot" - default: "BIOS" - description: | - Defines bootloader for VM. + default: BIOS + description: |- + The BootloaderType defines bootloader for VM. * BIOS - use legacy BIOS. * EFI - use Unified Extensible Firmware (EFI/UEFI). * EFIWithSecureBoot - use UEFI/EFI with SecureBoot support. - virtualMachineClassName: + enum: + - BIOS + - EFI + - EFIWithSecureBoot type: string - description: | - Name of the `VirtualMachineClass` resource describing the requirements for a virtual CPU, memory and the resource allocation policy and node placement policies for virtual machines. cpu: - type: object - description: Specifies the CPU settings for the VM. - required: - - cores + description: CPUSpec specifies the CPU settings for the VM. properties: + coreFraction: + default: 100% + description: + Guaranteed share of CPU that will be allocated to + the VM. Specified as a percentage. + enum: + - 5% + - 10% + - 25% + - 50% + - 100% + type: string cores: + description: + Specifies the number of cores inside the VM. The + value must be greater or equal 1. format: int32 - type: integer minimum: 1 - description: Specifies the number of cores inside the VM. The value must be greater or equal 1. - coreFraction: + type: integer + required: + - cores + type: object + disruptions: + default: + restartApprovalMode: Manual + description: |- + Disruptions describes the policy for applying changes that require rebooting the VM + Changes to some VM configuration settings require a reboot of the VM to apply them. This policy allows you to specify the behavior of how the VM will respond to such changes. + properties: + restartApprovalMode: + description: + "RestartApprovalMode defines a restart approving + mode: Manual or Automatic." + enum: + - Manual + - Automatic type: string - default: "100%" - enum: ["5%", "10%", "25%", "50%", "100%"] - description: | - Guaranteed share of CPU that will be allocated to the VM. Specified as a percentage. - - memory: type: object - description: | - Specifies the memory settings for the VM. - required: - - size + enableParavirtualization: + default: true + description: |- + Use the `virtio` bus to connect virtual devices of the VM. Set false to disable `virtio` for this VM. + Note: To use paravirtualization mode, some operating systems require the appropriate drivers to be installed. + type: boolean + memory: + description: MemorySpec specifies the memory settings for the VM. properties: size: - type: string + anyOf: + - type: integer + - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: | - Describes the maximum amount of memory resources allowed. + x-kubernetes-int-or-string: true + required: + - size + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector must match a node's labels for the VM to be scheduled on that node. + [The same](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) as in the pods `spec.nodeSelector` parameter in Kubernetes. + type: object + osType: + default: Generic + description: |- + The OsType parameter allows you to select the type of used OS, for which a VM with an optimal set of required virtual devices and parameters will be created. - blockDeviceRefs: - type: array - minItems: 1 - maxItems: 16 - description: | - List of block devices that can be mounted by disks belonging to the virtual machine. - The order of booting is determined by the order in the list. + + * Windows - for Microsoft Windows family operating systems. + * Generic - for other types of OS. + enum: + - Windows + - Generic + type: string + priorityClassName: + description: + PriorityClassName [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) as + in the pods `spec.priorityClassName` parameter in Kubernetes. + type: string + provisioning: + description: + Provisioning is a block allows you to configure the provisioning + script for the VM. + properties: + sysprepRef: + description: |- + SysprepRef is reference to an existing Windows sysprep automation. + Resource structure for the SysprepRef type: + * `.data.autounattend.xml`. + * `.data.unattend.xml`. + properties: + kind: + default: Secret + description: |- + The kind of existing Windows sysprep automation resource. + The following options are supported: + - Secret + enum: + - Secret + type: string + name: + type: string + required: + - name + type: object + type: + description: |- + ProvisioningType parameter defines the type of provisioning script: + + + Parameters supported for using the provisioning script: + * UserData - use the cloud-init in the .spec.provisioning.UserData section. + * UserDataRef - use a cloud-init script that resides in a different resource. + * SysprepRef - Use a Windows Automation script that resides in a different resource. + More information: https://cloudinit.readthedocs.io/en/latest/reference/examples.html + type: string + userData: + description: Inline cloud-init userdata script. + type: string + userDataRef: + description: |- + UserDataRef is reference to an existing resource with a cloud-init script. + Resource structure for userDataRef type: + * `.data.userData`. + properties: + kind: + default: Secret + description: |- + The kind of existing cloud-init automation resource. + The following options are supported: + - Secret + enum: + - Secret + type: string + name: + type: string + required: + - name + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: UserData cannot have userDataRef or sysprepRef. + rule: + "self.type == 'UserData' ? has(self.userData) && !has(self.userDataRef) + && !has(self.sysprepRef) : true" + - message: UserDataRef cannot have userData or sysprepRef. + rule: + "self.type == 'UserDataRef' ? has(self.userDataRef) && !has(self.userData) + && !has(self.sysprepRef) : true" + - message: SysprepRef cannot have userData or userDataRef. + rule: + "self.type == 'SysprepRef' ? has(self.sysprepRef) && !has(self.userData) + && !has(self.userDataRef) : true" + runPolicy: + default: AlwaysOnUnlessStoppedManually + description: |- + RunPolicy parameter defines the VM startup policy + * `AlwaysOn` - after creation the VM is always in a running state, even in case of its shutdown by OS means. + * `AlwaysOff` - after creation the VM is always in the off state. + * `Manual` - after creation the VM is switched off, the VM state (switching on/off) is controlled via sub-resources or OS means. + * `AlwaysOnUnlessStoppedManually` - after creation the VM is always in a running state, even in case of its shutdown by means of the OS, the VM can be shut down using the corresponding subresource. + enum: + - AlwaysOn + - AlwaysOff + - Manual + - AlwaysOnUnlessStoppedManually + type: string + terminationGracePeriodSeconds: + default: 60 + description: + Grace period observed after signalling a VM to stop after + which the VM is force terminated. + format: int64 + type: integer + tolerations: + description: |- + Tolerations define rules to tolerate node taints. + The same](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) as in the pods `spec.tolerations` parameter in Kubernetes. items: - type: object - description: | - The reference to block device. + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: - kind: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. type: string - description: | - The type of the block device. Options are: + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + items: + description: + TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. - * `ClusterVirtualImage` — Use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. - * `VirtualImage` — Use `VirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. - * `VirtualDisk` — Use `VirtualDisk` as the disk. This type is always mounted in RW mode. - enum: - - "ClusterVirtualImage" - - "VirtualImage" - - "VirtualDisk" - name: + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string - description: | - The name of attached resource. - status: - type: object - properties: - guestOSInfo: - type: object - x-kubernetes-preserve-unknown-fields: true - description: | - Guest OS information obtained from the QEMU Guest Agent. The agent must be pre-installed. + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. - More information about the agent, you can read here: https://www.qemu.org/docs/master/interop/qemu-ga.html - phase: - type: string - description: | - The current phase of the virtual machine: - * `Pending` - The process of starting the VM is in progress. - * `Running` - VM is running. - * `Degraded` - An error occurred during the startup process or while the VM is running. - * `Terminating` - The VM is currently in the process of shutting down. - * `Stopped` - The VM is stopped. - enum: - - "Pending" - - "Running" - - "Degraded" - - "Terminating" - - "Stopped" - - "Stopping" - - "Starting" - - "Migrating" - - "Pause" - nodeName: + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + virtualMachineClassName: + description: + Name of the `VirtualMachineClass` resource describing + the requirements for a virtual CPU, memory and the resource allocation + policy and node placement policies for virtual machines. type: string - description: | - The name of the node on which the VM is currently running. virtualMachineIPAddressName: + description: |- + Name for the associated `virtualMachineIPAddress` resource. + Specified when it is necessary to use a previously created IP address of the VM. + If not explicitly specified, by default a `virtualMachineIPAddress` resource is created for the VM with a name similar to the VM resource (`.metadata.name`). type: string - description: | - Name of `virtualMachineIPAddressName` holding the ip address of the VirtualMachine. - ipAddress: - type: string - description: | - IP address of VM. + required: + - blockDeviceRefs + - cpu + - memory + - virtualMachineClassName + type: object + status: + properties: blockDeviceRefs: - type: array - description: | - The list of attached block device attachments. + description: The list of attached block device attachments. items: - type: object - required: ["kind", "name", "size", "attached"] properties: attached: + description: The block device is attached to the virtual machine. type: boolean - description: | - The block device is attached to the virtual machine. hotplugged: + description: Block device is attached via hot plug connection. type: boolean - description: | - Block device is attached via hot plug connection. kind: - type: string - description: The type of block device. + description: |- + The BlockDeviceKind is a type of the block device. Options are: + + + * `ClusterVirtualImage` — Use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualImage` — Use `VirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualDisk` — Use `VirtualDisk` as the disk. This type is always mounted in RW mode. enum: - - "ClusterVirtualImage" - - "VirtualImage" - - "VirtualDisk" + - ClusterVirtualImage + - VirtualImage + - VirtualDisk + type: string name: + description: The name of attached resource. type: string - description: | - The name of attached resource. - target: + size: + description: The size of attached block device. type: string + target: description: The name of attached block device. example: sda - size: type: string - description: | - The size of attached block device. virtualMachineBlockDeviceAttachmentName: + description: + The name of the `VirtualMachineBlockDeviceAttachment` + resource that defines hot plug disk connection to the virtual + machine. type: string - description: | - The name of the `VirtualMachineBlockDeviceAttachment` resource that defines hot plug disk connection to the virtual machine. - migrationState: - type: object - description: "Migration info." + required: + - attached + - kind + - name + - size + type: object + type: array + conditions: + description: Detailed state of the virtual machine lifecycle. + items: + description: + "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + guestOSInfo: properties: - startTimestamp: - description: "Migration start time." - format: date-time - nullable: true + id: + description: Guest OS Id + type: string + kernelRelease: + description: Guest OS Kernel Release + type: string + kernelVersion: + description: Kernel version of the Guest OS type: string + machine: + description: Machine type of the Guest OS + type: string + name: + description: Name of the Guest OS + type: string + prettyName: + description: Guest OS Pretty Name + type: string + version: + description: Guest OS Version + type: string + versionId: + description: Version ID of the Guest OS + type: string + type: object + ipAddress: + description: IP address of VM. + type: string + migrationState: + description: Migration info. + properties: endTimestamp: - description: "Migration end time." + description: Migration end time. format: date-time - nullable: true type: string - target: - description: "" - type: object + result: + description: MigrationResult defines a migration result + enum: + - Succeeded + - Failed + - "" + type: string + source: properties: node: - description: "The name of the node on which the VM is currently migrating." + description: + The name of the node on which the VM is currently + migrating. type: string pod: - description: "The name of the pod where the VM is currently being migrated." + description: + The name of the pod where the VM is currently + being migrated. type: string - source: - description: "" type: object + startTimestamp: + description: Migration start time. + format: date-time + type: string + target: properties: node: - description: "The name of the node where the VM is currently running." + description: + The name of the node on which the VM is currently + migrating. type: string pod: - description: "The name of the pod on which the VM is currently running." + description: + The name of the pod where the VM is currently + being migrated. type: string - result: - description: "Migration result." - type: string - enum: - - "" - - "Succeeded" - - "Failed" - stats: + type: object + type: object + nodeName: + description: The name of the node on which the VM is currently running. + type: string + observedGeneration: + description: + Generating a resource that was last processed by the + controller. + format: int64 + type: integer + phase: + description: |- + MachinePhase defines current phase of the virtual machine: + * `Pending` - The process of starting the VM is in progress. + * `Running` - VM is running. + * `Degraded` - An error occurred during the startup process or while the VM is running. + * `Terminating` - The VM is currently in the process of shutting down. + * `Stopped` - The VM is stopped. + enum: + - Pending + - Running + - Terminating + - Stopped + - Stopping + - Starting + - Migrating + - Pause + - Degraded + type: string + resources: + description: ResourcesStatus defines resource usage statistics. + properties: + cpu: + description: + CPUStatus defines statistics about the CPU resource + usage. + properties: + coreFraction: + description: Current CoreFraction. + type: string + cores: + description: Current number of cores inside the VM. + type: integer + requestedCores: + anyOf: + - type: integer + - type: string + description: Requested cores. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + runtimeOverhead: + anyOf: + - type: integer + - type: string + description: runtime overhead. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - cores + type: object + memory: + description: + MemoryStatus defines statistics about the Memory + resource usage. + properties: + runtimeOverhead: + anyOf: + - type: integer + - type: string + description: Memory runtime overhead. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + size: + anyOf: + - type: integer + - type: string + description: Current memory size. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - size + type: object type: object - description: "VirtualMachine statistics" + restartAwaitingChanges: + description: + RestartAwaitingChanges holds operations to be manually + approved before applying to the virtual machine spec. + items: + x-kubernetes-preserve-unknown-fields: true + type: array + stats: + description: VirtualMachine statistics. properties: - phasesTransitions: - description: "The history of phases." - type: array - items: - type: object - properties: - phase: - description: "The phase of the virtual machine." - type: string - enum: - - "Pending" - - "Running" - - "Degraded" - - "Terminating" - - "Stopped" - - "Stopping" - - "Starting" - - "Migrating" - - "Pause" - timestamp: - description: "The time of transition to this phase." - format: date-time - nullable: true - type: string launchTimeDuration: - description: "Launch information." - type: object + description: Launch information. properties: - waitingForDependencies: - description: "The waiting time for dependent resources. pending -> starting." + guestOSAgentStarting: + description: + The waiting time for the guestOsAgent to start. + running -> running with guestOSAgent. nullable: true type: string virtualMachineStarting: - description: "The waiting time for the virtual machine to start. starting -> running." + description: + The waiting time for the virtual machine to start. + starting -> running. nullable: true type: string - guestOSAgentStarting: - description: "The waiting time for the guestOsAgent to start. running -> running with guestOSAgent" + waitingForDependencies: + description: + The waiting time for dependent resources. pending + -> starting. nullable: true type: string - conditions: - description: | - Specific points in VM's runtime. + type: object + phasesTransitions: + description: The history of phases. + items: + description: + VirtualMachinePhaseTransitionTimestamp gives a + timestamp in relation to when a phase is set on a vm. + properties: + phase: + description: |- + MachinePhase defines current phase of the virtual machine: + * `Pending` - The process of starting the VM is in progress. + * `Running` - VM is running. + * `Degraded` - An error occurred during the startup process or while the VM is running. + * `Terminating` - The VM is currently in the process of shutting down. + * `Stopped` - The VM is stopped. + enum: + - Pending + - Running + - Terminating + - Stopped + - Stopping + - Starting + - Migrating + - Pause + - Degraded + type: string + timestamp: + description: + PhaseTransitionTimestamp is the timestamp of + when the phase change occurred + format: date-time + nullable: true + type: string + type: object + type: array + type: object + virtualMachineIPAddressName: + description: + Name of `virtualMachineIPAddressName` holding the ip + address of the VirtualMachine. + type: string + virtualMachinePods: + description: List of virtual machine pods. items: properties: - lastProbeTime: - description: "" - format: date-time - nullable: true - type: string - lastTransitionTime: - description: "" - format: date-time - nullable: true - type: string - message: - description: "Extended message" - type: string - reason: - description: "brief reason" - type: string - status: - description: "The status of the condition. Possible values: `True`, `False`, `Unknown'." - type: string - type: - description: "The type of condition." + active: + description: Current working pod. + type: boolean + name: + description: Name of virtual machine pod. type: string - observedGeneration: - type: integer - description: "Generating a resource that was last processed by the controller" - format: int64 required: - - status - - type + - active + - name type: object type: array - restartAwaitingChanges: - description: | - A list of changes require restart for applying. - type: array - items: - type: object - x-kubernetes-preserve-unknown-fields: true - observedGeneration: - type: integer - description: "Generating a resource that was last processed by the controller" - format: int64 - additionalPrinterColumns: - - name: Phase - type: string - jsonPath: .status.phase - - name: Cores - type: string - jsonPath: .spec.cpu.cores - priority: 1 - - name: CoreFraction - type: string - jsonPath: .spec.cpu.coreFraction - priority: 1 - - name: Memory - type: string - jsonPath: .spec.memory.size - priority: 1 - - name: Need restart - type: string - jsonPath: .status.conditions[?(@.type=="AwaitingRestartToApplyConfiguration")].status - priority: 1 - - name: Agent - type: string - jsonPath: .status.conditions[?(@.type=="AgentReady")].status - priority: 1 - - name: Agent - type: string - jsonPath: .status.conditions[?(@.type=="Migratable")].status - priority: 1 - - name: Node - type: string - jsonPath: .status.nodeName - - name: IPAddress - type: string - jsonPath: .status.ipAddress - - name: Age - type: date - jsonPath: .metadata.creationTimestamp + required: + - ipAddress + - nodeName + - phase + - virtualMachineIPAddressName + type: object + required: + - spec + type: object + served: true + storage: true subresources: status: {} diff --git a/images/virtualization-artifact/pkg/controller/conditions/stringer.go b/images/virtualization-artifact/pkg/controller/conditions/stringer.go index 64ed679607..04c31fc4a0 100644 --- a/images/virtualization-artifact/pkg/controller/conditions/stringer.go +++ b/images/virtualization-artifact/pkg/controller/conditions/stringer.go @@ -25,5 +25,8 @@ type Stringer interface { type DeprecatedWrappedString string func (s DeprecatedWrappedString) String() string { + if s == "" { + return "Unknown" + } return string(s) } diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go index 97279dba1f..eef84dd0e5 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go @@ -197,7 +197,7 @@ func (b *KVVM) SetTopologySpreadConstraint(topology []corev1.TopologySpreadConst } func (b *KVVM) SetResourceRequirements(cores int, coreFraction string, memorySize resource.Quantity) error { - cpuRequest, err := b.getCPURequest(cores, coreFraction) + cpuRequest, err := GetCPURequest(cores, coreFraction) if err != nil { return err } @@ -207,16 +207,16 @@ func (b *KVVM) SetResourceRequirements(cores int, coreFraction string, memorySiz corev1.ResourceMemory: memorySize, }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: *b.getCPULimit(cores), + corev1.ResourceCPU: *GetCPULimit(cores), corev1.ResourceMemory: memorySize, }, } return nil } -func (b *KVVM) getCPURequest(cores int, coreFraction string) (*resource.Quantity, error) { +func GetCPURequest(cores int, coreFraction string) (*resource.Quantity, error) { if coreFraction == "" { - return b.getCPULimit(cores), nil + return GetCPULimit(cores), nil } fraction := intstr.FromString(coreFraction) req, err := intstr.GetScaledValueFromIntOrPercent(&fraction, cores*1000, true) @@ -224,12 +224,12 @@ func (b *KVVM) getCPURequest(cores int, coreFraction string) (*resource.Quantity return nil, fmt.Errorf("failed to calculate coreFraction. %w", err) } if req == 0 { - return b.getCPULimit(cores), nil + return GetCPULimit(cores), nil } return resource.NewMilliQuantity(int64(req), resource.DecimalSI), nil } -func (b *KVVM) getCPULimit(cores int) *resource.Quantity { +func GetCPULimit(cores int) *resource.Quantity { return resource.NewQuantity(int64(cores), resource.DecimalSI) } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/filesystem.go b/images/virtualization-artifact/pkg/controller/vm/internal/filesystem.go index 34c392862d..8458f36695 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/filesystem.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/filesystem.go @@ -70,7 +70,7 @@ func (h *FilesystemHandler) Handle(ctx context.Context, s state.VirtualMachineSt agentReady, _ := service.GetCondition(vmcondition.TypeAgentReady.String(), changed.Status.Conditions) if agentReady.Status != metav1.ConditionTrue { - cb.Status(metav1.ConditionUnknown) + cb.Status(metav1.ConditionUnknown).Reason(vmcondition.ReasonUnknown) return reconcile.Result{}, nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go b/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go index a205966a84..8891ce95e3 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go @@ -152,11 +152,10 @@ func (h *IPAMHandler) Handle(ctx context.Context, s state.VirtualMachineState) ( err = h.ipam.CheckIpAddressAvailableForBinding(current.GetName(), ipAddress) if err != nil { log.Info("Ip address is not available to be bound", "err", err, "vmipName", current.Spec.VirtualMachineIPAddress) - reason := vmcondition.ReasonIPAddressNotAvailable.String() - //nolint:staticcheck - mgr.Update(cb.Reason(conditions.DeprecatedWrappedString(reason)).Message(err.Error()).Condition()) + reason := vmcondition.ReasonIPAddressNotAvailable + mgr.Update(cb.Reason(reason).Message(err.Error()).Condition()) changed.Status.Conditions = mgr.Generate() - h.recorder.Event(changed, corev1.EventTypeWarning, reason, err.Error()) + h.recorder.Event(changed, corev1.EventTypeWarning, reason.String(), err.Error()) return reconcile.Result{}, nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/stats.go b/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go similarity index 61% rename from images/virtualization-artifact/pkg/controller/vm/internal/stats.go rename to images/virtualization-artifact/pkg/controller/vm/internal/statistic.go index 7b652199ce..017301e7a8 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/stats.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/statistic.go @@ -19,8 +19,11 @@ package internal import ( "context" "sort" + "strconv" "time" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,8 +55,20 @@ func (h *StatisticHandler) Handle(ctx context.Context, s state.VirtualMachineSta if err != nil { return reconcile.Result{}, err } - h.syncStats(current, changed, kvvmi) + + pods, err := s.Pods(ctx) + if err != nil { + return reconcile.Result{}, err + } + pod, err := s.Pod(ctx) + if err != nil { + return reconcile.Result{}, err + } + h.syncPods(changed, pod, pods) + + h.syncResources(changed, kvvmi, pod) + return reconcile.Result{}, nil } @@ -61,6 +76,123 @@ func (h *StatisticHandler) Name() string { return nameStatisticHandler } +func (h *StatisticHandler) syncResources(changed *virtv2.VirtualMachine, + kvvmi *virtv1.VirtualMachineInstance, + pod *corev1.Pod, +) { + if changed == nil { + return + } + var resources virtv2.ResourcesStatus + switch { + case pod == nil: + var ( + cpuKVVMIRequest resource.Quantity + memorySize resource.Quantity + cores int + coreFraction string + ) + if kvvmi == nil { + memorySize = changed.Spec.Memory.Size + cores = changed.Spec.CPU.Cores + coreFraction = changed.Spec.CPU.CoreFraction + } else { + cpuKVVMIRequest = kvvmi.Spec.Domain.Resources.Requests[corev1.ResourceCPU] + memorySize = kvvmi.Spec.Domain.Resources.Requests[corev1.ResourceMemory] + + cores = h.getCoresByKVVMI(kvvmi) + coreFraction = h.getCoreFractionByKVVMI(kvvmi) + } + resources = virtv2.ResourcesStatus{ + CPU: virtv2.CPUStatus{ + Cores: cores, + CoreFraction: coreFraction, + RequestedCores: cpuKVVMIRequest, + }, + Memory: virtv2.MemoryStatus{ + Size: memorySize, + }, + } + default: + if kvvmi == nil { + return + } + var ctr corev1.Container + for _, container := range pod.Spec.Containers { + if container.Name == "compute" { + ctr = container + } + } + + cpuKVVMIRequest := kvvmi.Spec.Domain.Resources.Requests[corev1.ResourceCPU] + cpuPODRequest := ctr.Resources.Requests[corev1.ResourceCPU] + + cpuOverhead := cpuPODRequest.DeepCopy() + cpuOverhead.Sub(cpuKVVMIRequest) + + cores := h.getCoresByKVVMI(kvvmi) + coreFraction := h.getCoreFractionByKVVMI(kvvmi) + + memoryKVVMIRequest := kvvmi.Spec.Domain.Resources.Requests[corev1.ResourceMemory] + memoryPodRequest := ctr.Resources.Requests[corev1.ResourceMemory] + + memoryOverhead := memoryPodRequest.DeepCopy() + memoryOverhead.Sub(memoryKVVMIRequest) + + resources = virtv2.ResourcesStatus{ + CPU: virtv2.CPUStatus{ + Cores: cores, + CoreFraction: coreFraction, + RequestedCores: cpuKVVMIRequest, + RuntimeOverhead: cpuOverhead, + }, + Memory: virtv2.MemoryStatus{ + Size: memoryKVVMIRequest, + RuntimeOverhead: memoryOverhead, + }, + } + } + changed.Status.Resources = resources +} + +func (h *StatisticHandler) getCoresByKVVMI(kvvmi *virtv1.VirtualMachineInstance) int { + if kvvmi == nil { + return -1 + } + cpuKVVMILimit := kvvmi.Spec.Domain.Resources.Limits[corev1.ResourceCPU] + return int(cpuKVVMILimit.Value()) +} + +func (h *StatisticHandler) getCoreFractionByKVVMI(kvvmi *virtv1.VirtualMachineInstance) string { + if kvvmi == nil { + return "" + } + cpuKVVMIRequest := kvvmi.Spec.Domain.Resources.Requests[corev1.ResourceCPU] + return strconv.Itoa(int(cpuKVVMIRequest.MilliValue())*100/(h.getCoresByKVVMI(kvvmi)*1000)) + "%" +} + +func (h *StatisticHandler) syncPods(changed *virtv2.VirtualMachine, pod *corev1.Pod, pods *corev1.PodList) { + if changed == nil { + return + } + if pods == nil { + changed.Status.VirtualMachinePods = nil + return + } + virtualMachinePods := make([]virtv2.VirtualMachinePod, len(pods.Items)) + for i, p := range pods.Items { + active := false + if pod != nil && p.GetUID() == pod.GetUID() { + active = true + } + virtualMachinePods[i] = virtv2.VirtualMachinePod{ + Name: p.GetName(), + Active: active, + } + } + changed.Status.VirtualMachinePods = virtualMachinePods +} + func (h *StatisticHandler) syncStats(current, changed *virtv2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) { if current == nil || changed == nil { return diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/stats_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go similarity index 77% rename from images/virtualization-artifact/pkg/controller/vm/internal/stats_test.go rename to images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go index a8898a0ac5..21a10575c7 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/stats_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/statistic_test.go @@ -23,6 +23,8 @@ import ( "time" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -47,6 +49,7 @@ func TestStatisticHandler(t *testing.T) { for _, f := range []func(*apiruntime.Scheme) error{ virtv2.AddToScheme, virtv1.AddToScheme, + corev1.AddToScheme, } { err := f(scheme) if err != nil { @@ -66,6 +69,7 @@ func TestStatisticHandler(t *testing.T) { name: "test1: virtualmachine running, if nil statistics", getObjects: func() []client.Object { return []client.Object{ + createPod(namespacedName), createVM(namespacedName, virtv2.MachinePending, nil, virtv1.VirtualMachineInstanceGuestOSInfo{}), createKVVMI(namespacedName, virtv1.Running, virtv1.VirtualMachineInstanceGuestOSInfo{Name: "test"}), } @@ -99,6 +103,7 @@ func TestStatisticHandler(t *testing.T) { getObjects: func() []client.Object { info := virtv1.VirtualMachineInstanceGuestOSInfo{Name: "test"} return []client.Object{ + createPod(namespacedName), createVM(namespacedName, virtv2.MachineRunning, createStatisticNoChange(), info), createKVVMI(namespacedName, virtv1.Running, info), } @@ -119,6 +124,7 @@ func TestStatisticHandler(t *testing.T) { getObjects: func() []client.Object { info := virtv1.VirtualMachineInstanceGuestOSInfo{} return []client.Object{ + createPod(namespacedName), createVM(namespacedName, virtv2.MachinePending, &virtv2.VirtualMachineStats{ @@ -165,6 +171,28 @@ func TestStatisticHandler(t *testing.T) { return nil }, }, + { + name: "test4: check generated .status.resources", + getObjects: func() []client.Object { + return []client.Object{ + createPod(namespacedName), + createVM(namespacedName, virtv2.MachineRunning, nil, virtv1.VirtualMachineInstanceGuestOSInfo{}), + createKVVMI(namespacedName, virtv1.Running, virtv1.VirtualMachineInstanceGuestOSInfo{Name: "test"}), + } + }, + mutateChanged: func(vm *virtv2.VirtualMachine) {}, + expect: func(vm *virtv2.VirtualMachine) error { + require.NotNil(t, vm) + res := vm.Status.Resources + require.Equal(t, res.CPU.Cores, 1) + require.Equal(t, res.CPU.CoreFraction, "50%") + require.Equal(t, res.CPU.RequestedCores.MilliValue(), int64(500)) + require.Equal(t, res.CPU.RuntimeOverhead.MilliValue(), int64(0)) + require.Equal(t, res.Memory.Size.Value(), int64(536870912)) + require.Equal(t, res.Memory.RuntimeOverhead.Value(), int64(254803968)) + return nil + }, + }, } { t.Log("Start test", test.name) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.getObjects()...).Build() @@ -231,6 +259,39 @@ func createStatisticNoChange() *virtv2.VirtualMachineStats { } } +func createPod(vmKey types.NamespacedName) *corev1.Pod { + return &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("virt-launcher-%s", vmKey.Name), + Namespace: vmKey.Namespace, + Labels: map[string]string{ + virtv1.VirtualMachineNameLabel: vmKey.Name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "compute", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("755Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("755Mi"), + }, + }, + }, + }, + }, + } +} + func createVM(key types.NamespacedName, phase virtv2.MachinePhase, stats *virtv2.VirtualMachineStats, @@ -245,6 +306,15 @@ func createVM(key types.NamespacedName, Name: key.Name, Namespace: key.Namespace, }, + Spec: virtv2.VirtualMachineSpec{ + CPU: virtv2.CPUSpec{ + Cores: 1, + CoreFraction: "50%", + }, + Memory: virtv2.MemorySpec{ + Size: resource.MustParse("512Mi"), + }, + }, Status: virtv2.VirtualMachineStatus{ Phase: phase, Stats: stats, @@ -266,6 +336,20 @@ func createKVVMI(key types.NamespacedName, Name: key.Name, Namespace: key.Namespace, }, + Spec: virtv1.VirtualMachineInstanceSpec{ + Domain: virtv1.DomainSpec{ + Resources: virtv1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + }, + }, Status: virtv1.VirtualMachineInstanceStatus{ Phase: phase, GuestOSInfo: guestOSInfo, diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/util.go b/images/virtualization-artifact/pkg/controller/vm/internal/util.go index b30b78f405..f3f870c3a2 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/util.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/util.go @@ -44,6 +44,7 @@ func addAllUnknown(vm *virtv2.VirtualMachine, conds ...vmcondition.Type) (update for _, c := range conds { if add := mgr.Add(conditions.NewConditionBuilder(c). Generation(vm.GetGeneration()). + Reason(vmcondition.ReasonUnknown). Status(metav1.ConditionUnknown). Condition()); add { update = true diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index cd0bb53791..84d7f347be 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -22,7 +22,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -42,10 +41,10 @@ const ( func SetupController( ctx context.Context, mgr manager.Manager, - log *slog.Logger, + lg *slog.Logger, dvcrSettings *dvcr.Settings, ) error { - log = log.With(logger.SlogController(controllerName)) + log := lg.With(logger.SlogController(controllerName)) recorder := mgr.GetEventRecorderFor(controllerName) mgrCache := mgr.GetCache() client := mgr.GetClient() @@ -85,21 +84,8 @@ func SetupController( return err } - vmmetrics.SetupCollector(&vmLister{vmCache: mgrCache}, metrics.Registry) + vmmetrics.SetupCollector(mgrCache, metrics.Registry, lg) log.Info("Initialized VirtualMachine controller") return nil } - -type vmLister struct { - vmCache cache.Cache -} - -func (l vmLister) List() ([]v1alpha2.VirtualMachine, error) { - vmList := v1alpha2.VirtualMachineList{} - err := l.vmCache.List(context.Background(), &vmList) - if err != nil { - return nil, err - } - return vmList.Items, nil -} diff --git a/images/virtualization-artifact/pkg/logger/attrs.go b/images/virtualization-artifact/pkg/logger/attrs.go index 8e31efbe1c..f05d817d2d 100644 --- a/images/virtualization-artifact/pkg/logger/attrs.go +++ b/images/virtualization-artifact/pkg/logger/attrs.go @@ -25,6 +25,7 @@ const ( handlerAttr = "handler" dataSourceAttr = "ds" controllerAttr = "controller" + collectorAttr = "collector" ) func SlogErr(err error) slog.Attr { @@ -50,3 +51,7 @@ func SlogHandler(handler string) slog.Attr { func SlogController(controller string) slog.Attr { return slog.String(controllerAttr, controller) } + +func SlogCollector(collector string) slog.Attr { + return slog.String(collectorAttr, collector) +} diff --git a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/collector.go b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/collector.go index f5c088f256..46281f5217 100644 --- a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/collector.go +++ b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/collector.go @@ -17,40 +17,40 @@ limitations under the License. package virtualmachine import ( + "context" + "log/slog" + "time" + "github.com/prometheus/client_golang/prometheus" - "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics" - "github.com/deckhouse/virtualization-controller/pkg/util" - virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization-controller/pkg/logger" ) -const ( - MetricVirtualMachineStatusPhase = "virtualmachine_status_phase" -) +const collectorName = "virtualmachine-collector" -var virtualMachineMetrics = map[string]*prometheus.Desc{ - MetricVirtualMachineStatusPhase: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineStatusPhase), - "The virtualmachine current phase.", - []string{"name", "namespace", "uid", "node", "phase"}, - nil), -} - -func SetupCollector(vmLister Lister, registerer prometheus.Registerer) *Collector { +func SetupCollector(reader client.Reader, + registerer prometheus.Registerer, + log *slog.Logger, +) *Collector { c := &Collector{ - lister: vmLister, + iterator: newUnsafeIterator(reader), + log: log.With(logger.SlogCollector(collectorName)), } registerer.MustRegister(c) return c } -type Lister interface { - List() ([]virtv2.VirtualMachine, error) +type handler func(m *dataMetric) (stop bool) + +type Iterator interface { + Iter(ctx context.Context, h handler) error } type Collector struct { - lister Lister + iterator Iterator + log *slog.Logger } func (c Collector) Describe(ch chan<- *prometheus.Desc) { @@ -60,63 +60,14 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { } func (c Collector) Collect(ch chan<- prometheus.Metric) { - vms, err := c.lister.List() - if err != nil { - klog.Errorf("Failed to get list of VirtualMachine: %v", err) + s := newScraper(ch, c.log) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + if err := c.iterator.Iter(ctx, func(m *dataMetric) (stop bool) { + s.Report(m) return - } - if len(vms) == 0 { + }); err != nil { + c.log.Error("Failed to iterate of VirtualMachines", logger.SlogErr(err)) return } - s := newScraper(ch) - s.Report(vms) -} - -func newScraper(ch chan<- prometheus.Metric) *scraper { - return &scraper{ch: ch} -} - -type scraper struct { - ch chan<- prometheus.Metric -} - -func (s *scraper) Report(vms []virtv2.VirtualMachine) { - for _, vm := range vms { - s.updateVMStatusPhaseMetrics(vm) - } -} - -func (s *scraper) updateVMStatusPhaseMetrics(vm virtv2.VirtualMachine) { - phase := vm.Status.Phase - if phase == "" { - phase = virtv2.MachinePending - } - phases := []struct { - value bool - name string - }{ - {phase == virtv2.MachinePending, string(virtv2.MachinePending)}, - {phase == virtv2.MachineRunning, string(virtv2.MachineRunning)}, - {phase == virtv2.MachineDegraded, string(virtv2.MachineDegraded)}, - {phase == virtv2.MachineTerminating, string(virtv2.MachineTerminating)}, - {phase == virtv2.MachineStopped, string(virtv2.MachineStopped)}, - {phase == virtv2.MachineStopping, string(virtv2.MachineStopping)}, - {phase == virtv2.MachineStarting, string(virtv2.MachineStarting)}, - {phase == virtv2.MachineMigrating, string(virtv2.MachineMigrating)}, - {phase == virtv2.MachinePause, string(virtv2.MachinePause)}, - } - desc := virtualMachineMetrics[MetricVirtualMachineStatusPhase] - for _, p := range phases { - metric, err := prometheus.NewConstMetric( - desc, - prometheus.GaugeValue, - util.BoolFloat64(p.value), - vm.GetName(), vm.GetNamespace(), string(vm.GetUID()), vm.Status.Node, p.name, - ) - if err != nil { - klog.Warningf("Error creating the new const metric for %s: %s", desc, err) - return - } - s.ch <- metric - } } diff --git a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/data_metric.go b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/data_metric.go new file mode 100644 index 0000000000..7731268419 --- /dev/null +++ b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/data_metric.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualmachine + +import ( + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/deckhouse/virtualization-controller/pkg/controller/service" + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" +) + +type dataMetric struct { + Name string + Namespace string + Node string + UID string + Phase virtv2.MachinePhase + CpuCores float64 + CpuCoreFraction float64 + CpuRequestedCores float64 + CpuRuntimeOverhead float64 + MemorySize float64 + MemoryRuntimeOverhead float64 + AwaitingRestartToApplyConfiguration bool + ConfigurationApplied bool + RunPolicy virtv2.RunPolicy + Pods []virtv2.VirtualMachinePod +} + +// DO NOT mutate VirtualMachine! +func newDataMetric(vm *virtv2.VirtualMachine) *dataMetric { + if vm == nil { + return nil + } + res := vm.Status.Resources + cf := intstr.FromString(strings.TrimSuffix(res.CPU.CoreFraction, "%")) + var ( + awaitingRestartToApplyConfiguration bool + configurationApplied bool + ) + if cond, found := service.GetCondition(vmcondition.TypeAwaitingRestartToApplyConfiguration.String(), + vm.Status.Conditions); found && cond.Status == metav1.ConditionTrue { + awaitingRestartToApplyConfiguration = true + } + if cond, found := service.GetCondition(vmcondition.TypeConfigurationApplied.String(), + vm.Status.Conditions); found && cond.Status == metav1.ConditionTrue { + configurationApplied = true + } + pods := make([]virtv2.VirtualMachinePod, len(vm.Status.VirtualMachinePods)) + for i, pod := range vm.Status.VirtualMachinePods { + pods[i] = *pod.DeepCopy() + } + return &dataMetric{ + Name: vm.Name, + Namespace: vm.Namespace, + Node: vm.Status.Node, + UID: string(vm.UID), + Phase: vm.Status.Phase, + CpuCores: float64(res.CPU.Cores), + CpuCoreFraction: float64(cf.IntValue()), + CpuRequestedCores: float64(res.CPU.RequestedCores.MilliValue()), + CpuRuntimeOverhead: float64(res.CPU.RuntimeOverhead.MilliValue()), + MemorySize: float64(res.Memory.Size.Value()), + MemoryRuntimeOverhead: float64(res.Memory.RuntimeOverhead.Value()), + AwaitingRestartToApplyConfiguration: awaitingRestartToApplyConfiguration, + ConfigurationApplied: configurationApplied, + RunPolicy: vm.Spec.RunPolicy, + Pods: pods, + } +} diff --git a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/metrics.go b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/metrics.go new file mode 100644 index 0000000000..a777c6e3e6 --- /dev/null +++ b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/metrics.go @@ -0,0 +1,123 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualmachine + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics" +) + +const ( + MetricVirtualMachineStatusPhase = "virtualmachine_status_phase" + MetricVirtualMachineConfigurationCpuCores = "virtualmachine_configuration_cpu_cores" + MetricVirtualMachineConfigurationCpuCoreFraction = "virtualmachine_configuration_cpu_core_fraction" + MetricVirtualMachineConfigurationCpuRequestedCores = "virtualmachine_configuration_cpu_requested_cores" + MetricVirtualMachineConfigurationCpuRuntimeOverhead = "virtualmachine_configuration_cpu_runtime_overhead" + MetricVirtualMachineConfigurationMemorySize = "virtualmachine_configuration_memory_size" + MetricVirtualMachineConfigurationMemoryRuntimeOverhead = "virtualmachine_configuration_memory_runtime_overhead" + MetricVirtualMachineAwaitingRestartToApplyConfiguration = "virtualmachine_awaiting_restart_to_apply_configuration" + MetricVirtualMachineConfigurationApplied = "virtualmachine_configuration_applied" + MetricVirtualMachineConfigurationRunPolicy = "virtualmachine_configuration_run_policy" + MetricVirtualMachinePod = "virtualmachine_pod" +) + +var baseLabels = []string{"name", "namespace", "uid", "node"} + +func WithBaseLabels(labels ...string) []string { + return append(baseLabels, labels...) +} + +func WithBaseLabelsByMetric(m *dataMetric, labels ...string) []string { + var base []string + if m != nil { + base = []string{ + m.Name, + m.Namespace, + m.UID, + m.Node, + } + } + return append(base, labels...) +} + +var virtualMachineMetrics = map[string]*prometheus.Desc{ + MetricVirtualMachineStatusPhase: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineStatusPhase), + "The virtualmachine current phase.", + WithBaseLabels("phase"), + nil, + ), + + MetricVirtualMachineConfigurationCpuCores: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationCpuCores), + "The virtualmachine current core count.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationCpuCoreFraction: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationCpuCoreFraction), + "The virtualmachine current coreFraction.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationCpuRequestedCores: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationCpuRequestedCores), + "The virtualmachine current requested cores.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationCpuRuntimeOverhead: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationCpuRuntimeOverhead), + "The virtualmachine current cpu runtime overhead.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationMemorySize: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationMemorySize), + "The virtualmachine current memory size.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationMemoryRuntimeOverhead: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationMemoryRuntimeOverhead), + "The virtualmachine current memory runtime overhead.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineAwaitingRestartToApplyConfiguration: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineAwaitingRestartToApplyConfiguration), + "The virtualmachine awaiting restart to apply configuration.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationApplied: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationApplied), + "The virtualmachine configuration applied.", + WithBaseLabels(), + nil, + ), + + MetricVirtualMachineConfigurationRunPolicy: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachineConfigurationRunPolicy), + "The virtualmachine current runPolicy.", + WithBaseLabels("runPolicy"), + nil, + ), + MetricVirtualMachinePod: prometheus.NewDesc(prometheus.BuildFQName(metrics.MetricNamespace, "", MetricVirtualMachinePod), + "The virtualmachine current active pod.", + WithBaseLabels("pod"), + nil, + ), +} diff --git a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/scraper.go b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/scraper.go new file mode 100644 index 0000000000..8e1d03c704 --- /dev/null +++ b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/scraper.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualmachine + +import ( + "fmt" + "log/slog" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/deckhouse/virtualization-controller/pkg/util" + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func newScraper(ch chan<- prometheus.Metric, log *slog.Logger) *scraper { + return &scraper{ch: ch, log: log} +} + +type scraper struct { + ch chan<- prometheus.Metric + log *slog.Logger +} + +func (s *scraper) Report(m *dataMetric) { + s.updateVMStatusPhaseMetrics(m) + s.updateVMCpuCoresMetrics(m) + s.updateVMCpuCoreFractionMetrics(m) + s.updateVMCpuRequestedCoresMetrics(m) + s.updateVMCpuRuntimeOverheadMetrics(m) + s.updateVMMemorySizeMetrics(m) + s.updateVMMemoryRuntimeOverheadMetrics(m) + s.updateVMAwaitingRestartToApplyConfigurationMetrics(m) + s.updateVMConfigurationAppliedMetrics(m) + s.updateVMConfigurationRunPolicyMetrics(m) + s.updateVMPodMetrics(m) +} + +func (s *scraper) updateVMStatusPhaseMetrics(m *dataMetric) { + phase := m.Phase + if phase == "" { + phase = virtv2.MachinePending + } + phases := []struct { + value bool + name string + }{ + {phase == virtv2.MachinePending, string(virtv2.MachinePending)}, + {phase == virtv2.MachineRunning, string(virtv2.MachineRunning)}, + {phase == virtv2.MachineDegraded, string(virtv2.MachineDegraded)}, + {phase == virtv2.MachineTerminating, string(virtv2.MachineTerminating)}, + {phase == virtv2.MachineStopped, string(virtv2.MachineStopped)}, + {phase == virtv2.MachineStopping, string(virtv2.MachineStopping)}, + {phase == virtv2.MachineStarting, string(virtv2.MachineStarting)}, + {phase == virtv2.MachineMigrating, string(virtv2.MachineMigrating)}, + {phase == virtv2.MachinePause, string(virtv2.MachinePause)}, + } + for _, p := range phases { + s.defaultUpdate(MetricVirtualMachineStatusPhase, + util.BoolFloat64(p.value), m, p.name) + } +} + +func (s *scraper) updateVMCpuCoresMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationCpuCores, + m.CpuCores, m) +} + +func (s *scraper) updateVMCpuCoreFractionMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationCpuCoreFraction, + m.CpuCoreFraction, m) +} + +func (s *scraper) updateVMCpuRequestedCoresMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationCpuRequestedCores, + m.CpuRequestedCores, m) +} + +func (s *scraper) updateVMCpuRuntimeOverheadMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationCpuRuntimeOverhead, + m.CpuRuntimeOverhead, m) +} + +func (s *scraper) updateVMMemorySizeMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationMemorySize, + m.MemorySize, m) +} + +func (s *scraper) updateVMMemoryRuntimeOverheadMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationMemoryRuntimeOverhead, + m.MemoryRuntimeOverhead, m) +} + +func (s *scraper) updateVMAwaitingRestartToApplyConfigurationMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineAwaitingRestartToApplyConfiguration, + util.BoolFloat64(m.AwaitingRestartToApplyConfiguration), m) +} + +func (s *scraper) updateVMConfigurationAppliedMetrics(m *dataMetric) { + s.defaultUpdate(MetricVirtualMachineConfigurationApplied, + util.BoolFloat64(m.ConfigurationApplied), m) +} + +func (s *scraper) updateVMConfigurationRunPolicyMetrics(m *dataMetric) { + policy := m.RunPolicy + policies := []struct { + value bool + name string + }{ + {policy == virtv2.AlwaysOnPolicy, string(virtv2.AlwaysOnPolicy)}, + {policy == virtv2.AlwaysOffPolicy, string(virtv2.AlwaysOffPolicy)}, + {policy == virtv2.ManualPolicy, string(virtv2.ManualPolicy)}, + {policy == virtv2.AlwaysOnUnlessStoppedManually, string(virtv2.AlwaysOnUnlessStoppedManually)}, + } + for _, p := range policies { + s.defaultUpdate(MetricVirtualMachineConfigurationRunPolicy, + util.BoolFloat64(p.value), m, p.name) + } +} + +func (s *scraper) updateVMPodMetrics(m *dataMetric) { + for _, p := range m.Pods { + s.defaultUpdate(MetricVirtualMachinePod, util.BoolFloat64(p.Active), m, p.Name) + } +} + +func (s *scraper) defaultUpdate(descName string, value float64, m *dataMetric, labels ...string) { + desc := virtualMachineMetrics[descName] + metric, err := prometheus.NewConstMetric( + desc, + prometheus.GaugeValue, + value, + WithBaseLabelsByMetric(m, labels...)..., + ) + if err != nil { + s.log.Warn(fmt.Sprintf("Error creating the new const dataMetric for %s: %s", desc, err)) + return + } + s.ch <- metric +} diff --git a/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/unsafe.go b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/unsafe.go new file mode 100644 index 0000000000..7af2f9730b --- /dev/null +++ b/images/virtualization-artifact/pkg/monitoring/metrics/virtualmachine/unsafe.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualmachine + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func newUnsafeIterator(reader client.Reader) *iterator { + return &iterator{ + reader: reader, + } +} + +type iterator struct { + reader client.Reader +} + +// Iter implements iteration on objects VirtualMachine and create new DTO. +// DO NOT mutate VirtualMachine! +func (l *iterator) Iter(ctx context.Context, h handler) error { + vms := virtv2.VirtualMachineList{} + if err := l.reader.List(ctx, &vms, client.UnsafeDisableDeepCopy); err != nil { + return err + } + for _, vm := range vms.Items { + m := newDataMetric(&vm) + if stop := h(m); stop { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + continue + } + } + return nil +}