diff --git a/src/cli/internal/clientconfig/clientconfig.go b/src/cli/internal/clientconfig/clientconfig.go index 1b7c9f3d99..2b502366ca 100644 --- a/src/cli/internal/clientconfig/clientconfig.go +++ b/src/cli/internal/clientconfig/clientconfig.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "github.com/deckhouse/virtualization/api/client/kubeclient" @@ -49,3 +50,15 @@ func ClientAndNamespaceFromContext(ctx context.Context) (client kubeclient.Clien } return client, namespace, overridden, nil } + +func GetRESTConfig(ctx context.Context) (*rest.Config, error) { + clientConfig, ok := ctx.Value(clientConfigKey).(clientcmd.ClientConfig) + if !ok { + return nil, fmt.Errorf("unable to get client config from context") + } + config, err := clientConfig.ClientConfig() + if err != nil { + return nil, err + } + return config, nil +} diff --git a/src/cli/internal/cmd/collectdebuginfo/collectdebuginfo.go b/src/cli/internal/cmd/collectdebuginfo/collectdebuginfo.go new file mode 100644 index 0000000000..861660ef66 --- /dev/null +++ b/src/cli/internal/cmd/collectdebuginfo/collectdebuginfo.go @@ -0,0 +1,130 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectdebuginfo + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/dynamic" + + "github.com/deckhouse/virtualization/api/client/kubeclient" + "github.com/deckhouse/virtualization/src/cli/internal/clientconfig" + "github.com/deckhouse/virtualization/src/cli/internal/templates" +) + +func NewCommand() *cobra.Command { + bundle := &DebugBundle{} + cmd := &cobra.Command{ + Use: "collect-debug-info (VirtualMachine)", + Short: "Collect debug information for VM: configuration, events, and logs.", + Example: usage(), + Args: templates.ExactArgs("collect-debug-info", 1), + RunE: bundle.Run, + } + + cmd.Flags().BoolVar(&bundle.saveLogs, "with-logs", false, "Include pod logs in output") + cmd.Flags().BoolVar(&bundle.debug, "debug", false, "Enable debug output for permission errors") + cmd.SetUsageTemplate(templates.UsageTemplate()) + return cmd +} + +type DebugBundle struct { + saveLogs bool + debug bool + dynamicClient dynamic.Interface + stdout io.Writer + stderr io.Writer + resourceCount int +} + +func usage() string { + return ` # Collect debug info for VirtualMachine 'myvm': + {{ProgramName}} collect-debug-info myvm + {{ProgramName}} collect-debug-info myvm.mynamespace + {{ProgramName}} collect-debug-info myvm -n mynamespace + # Include pod logs: + {{ProgramName}} collect-debug-info --with-logs myvm` +} + +func (b *DebugBundle) Run(cmd *cobra.Command, args []string) error { + client, defaultNamespace, _, err := clientconfig.ClientAndNamespaceFromContext(cmd.Context()) + if err != nil { + return err + } + + namespace, name, err := templates.ParseTarget(args[0]) + if err != nil { + return err + } + if namespace == "" { + namespace = defaultNamespace + } + + // Get dynamic client for internal resources + config, err := clientconfig.GetRESTConfig(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to get REST config: %w", err) + } + b.dynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + return fmt.Errorf("failed to create dynamic client: %w", err) + } + + // Set output writers + b.stdout = cmd.OutOrStdout() + b.stderr = cmd.ErrOrStderr() + + // Collect and output resources immediately + if err := b.collectResources(cmd.Context(), client, namespace, name); err != nil { + return err + } + + return nil +} + +func (b *DebugBundle) collectResources(ctx context.Context, client kubeclient.Client, namespace, vmName string) error { + // Collect VM and core resources + if err := b.collectVMResources(ctx, client, namespace, vmName); err != nil { + return fmt.Errorf("failed to collect VM resources: %w", err) + } + + // Collect block devices + if err := b.collectBlockDevices(ctx, client, namespace, vmName); err != nil { + return fmt.Errorf("failed to collect block devices: %w", err) + } + + // Collect pods (and logs if requested) + if err := b.collectPods(ctx, client, namespace, vmName); err != nil { + return fmt.Errorf("failed to collect pods: %w", err) + } + + return nil +} + +func (b *DebugBundle) handleError(resourceType, resourceName string, err error) bool { + if errors.IsForbidden(err) || errors.IsUnauthorized(err) { + if b.debug { + fmt.Fprintf(b.stderr, "Warning: Skipping %s/%s: permission denied\n", resourceType, resourceName) + } + return true // Skip this resource + } + return false // Don't skip, propagate error +} diff --git a/src/cli/internal/cmd/collectdebuginfo/collectors.go b/src/cli/internal/cmd/collectdebuginfo/collectors.go new file mode 100644 index 0000000000..df817c3975 --- /dev/null +++ b/src/cli/internal/cmd/collectdebuginfo/collectors.go @@ -0,0 +1,407 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectdebuginfo + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/yaml" + + "github.com/deckhouse/virtualization/api/client/kubeclient" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +var coreKinds = map[string]bool{ + "Pod": true, + "PersistentVolumeClaim": true, + "PersistentVolume": true, + "Event": true, +} + +// Resource collection functions + +func (b *DebugBundle) collectVMResources(ctx context.Context, client kubeclient.Client, namespace, vmName string) error { + // Get VM + vm, err := client.VirtualMachines(namespace).Get(ctx, vmName, metav1.GetOptions{}) + if err != nil { + if b.handleError("VirtualMachine", vmName, err) { + return nil + } + return err + } + b.outputResource("VirtualMachine", vmName, namespace, vm) + + // Get IVVM + ivvm, err := b.getInternalResource(ctx, "internalvirtualizationvirtualmachines", namespace, vmName) + if err == nil { + b.outputResource("InternalVirtualizationVirtualMachine", vmName, namespace, ivvm) + } else if !b.handleError("InternalVirtualizationVirtualMachine", vmName, err) { + return err + } + + // Get IVVMI + ivvmi, err := b.getInternalResource(ctx, "internalvirtualizationvirtualmachineinstances", namespace, vmName) + if err == nil { + b.outputResource("InternalVirtualizationVirtualMachineInstance", vmName, namespace, ivvmi) + } else if !b.handleError("InternalVirtualizationVirtualMachineInstance", vmName, err) { + return err + } + + // Get VM operations + vmUID := string(vm.UID) + vmops, err := client.VirtualMachineOperations(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("virtualization.deckhouse.io/virtual-machine-uid=%s", vmUID), + }) + if err == nil { + for _, vmop := range vmops.Items { + b.outputResource("VirtualMachineOperation", vmop.Name, namespace, &vmop) + } + } else if !b.handleError("VirtualMachineOperation", "", err) { + return err + } + + // Get migrations + migrations, err := b.getInternalResourceList(ctx, "internalvirtualizationvirtualmachineinstancemigrations", namespace) + if err == nil { + for _, item := range migrations { + vmiName, found, _ := unstructured.NestedString(item.Object, "spec", "vmiName") + if found && vmiName == vmName { + name, _, _ := unstructured.NestedString(item.Object, "metadata", "name") + b.outputResource("InternalVirtualizationVirtualMachineInstanceMigration", name, namespace, item) + } + } + } else if !b.handleError("InternalVirtualizationVirtualMachineInstanceMigration", "", err) { + return err + } + + // Get events for VM + b.collectEvents(ctx, client, namespace, "VirtualMachine", vmName) + + return nil +} + +func (b *DebugBundle) collectBlockDevices(ctx context.Context, client kubeclient.Client, namespace, vmName string) error { + vm, err := client.VirtualMachines(namespace).Get(ctx, vmName, metav1.GetOptions{}) + if err != nil { + return err + } + + // Static block devices + for _, bdRef := range vm.Spec.BlockDeviceRefs { + if err := b.collectBlockDevice(ctx, client, namespace, bdRef.Kind, bdRef.Name); err != nil { + if !b.handleError(string(bdRef.Kind), bdRef.Name, err) { + return err + } + } + } + + // Hotplug block devices + for _, bdRef := range vm.Status.BlockDeviceRefs { + if bdRef.Hotplugged { + if err := b.collectBlockDevice(ctx, client, namespace, bdRef.Kind, bdRef.Name); err != nil { + if !b.handleError(string(bdRef.Kind), bdRef.Name, err) { + return err + } + } + + // Get VMBDA + if bdRef.VirtualMachineBlockDeviceAttachmentName != "" { + vmbda, err := client.VirtualMachineBlockDeviceAttachments(namespace).Get(ctx, bdRef.VirtualMachineBlockDeviceAttachmentName, metav1.GetOptions{}) + if err == nil { + b.outputResource("VirtualMachineBlockDeviceAttachment", vmbda.Name, namespace, vmbda) + b.collectEvents(ctx, client, namespace, "VirtualMachineBlockDeviceAttachment", vmbda.Name) + } else if !b.handleError("VirtualMachineBlockDeviceAttachment", bdRef.VirtualMachineBlockDeviceAttachmentName, err) { + return err + } + } + } + } + + // Get all VMBDA that reference this VM + vmbdas, err := client.VirtualMachineBlockDeviceAttachments(namespace).List(ctx, metav1.ListOptions{}) + if err == nil { + for _, vmbda := range vmbdas.Items { + if vmbda.Spec.VirtualMachineName == vmName { + b.outputResource("VirtualMachineBlockDeviceAttachment", vmbda.Name, namespace, &vmbda) + b.collectEvents(ctx, client, namespace, "VirtualMachineBlockDeviceAttachment", vmbda.Name) + + // Get associated block device + if vmbda.Spec.BlockDeviceRef.Kind != "" && vmbda.Spec.BlockDeviceRef.Name != "" { + // Convert VMBDAObjectRefKind to BlockDeviceKind + var bdKind v1alpha2.BlockDeviceKind + switch vmbda.Spec.BlockDeviceRef.Kind { + case v1alpha2.VMBDAObjectRefKindVirtualDisk: + bdKind = v1alpha2.VirtualDiskKind + case v1alpha2.VMBDAObjectRefKindVirtualImage: + bdKind = v1alpha2.VirtualImageKind + case v1alpha2.VMBDAObjectRefKindClusterVirtualImage: + bdKind = v1alpha2.ClusterVirtualImageKind + default: + continue + } + if err := b.collectBlockDevice(ctx, client, namespace, bdKind, vmbda.Spec.BlockDeviceRef.Name); err != nil { + if !b.handleError(string(bdKind), vmbda.Spec.BlockDeviceRef.Name, err) { + return err + } + } + } + } + } + } else if !b.handleError("VirtualMachineBlockDeviceAttachment", "", err) { + return err + } + + return nil +} + +func (b *DebugBundle) collectBlockDevice(ctx context.Context, client kubeclient.Client, namespace string, kind v1alpha2.BlockDeviceKind, name string) error { + switch kind { + case v1alpha2.VirtualDiskKind: + vd, err := client.VirtualDisks(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + b.outputResource("VirtualDisk", name, namespace, vd) + b.collectEvents(ctx, client, namespace, "VirtualDisk", name) + + // Get PVC + if vd.Status.Target.PersistentVolumeClaim != "" { + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, vd.Status.Target.PersistentVolumeClaim, metav1.GetOptions{}) + if err == nil { + b.outputResource("PersistentVolumeClaim", pvc.Name, namespace, pvc) + b.collectEvents(ctx, client, namespace, "PersistentVolumeClaim", pvc.Name) + + // Get PV + if pvc.Spec.VolumeName != "" { + pv, err := client.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) + if err == nil { + b.outputResource("PersistentVolume", pv.Name, "", pv) + } else if !b.handleError("PersistentVolume", pvc.Spec.VolumeName, err) { + return err + } + } + } else if !b.handleError("PersistentVolumeClaim", vd.Status.Target.PersistentVolumeClaim, err) { + return err + } + } + + case v1alpha2.VirtualImageKind: + vi, err := client.VirtualImages(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + b.outputResource("VirtualImage", name, namespace, vi) + b.collectEvents(ctx, client, namespace, "VirtualImage", name) + + case v1alpha2.ClusterVirtualImageKind: + cvi, err := client.ClusterVirtualImages().Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + b.outputResource("ClusterVirtualImage", name, "", cvi) + + default: + return fmt.Errorf("unknown block device kind: %s", kind) + } + + return nil +} + +func (b *DebugBundle) collectPods(ctx context.Context, client kubeclient.Client, namespace, vmName string) error { + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("vm.kubevirt.internal.virtualization.deckhouse.io/name=%s", vmName), + }) + if err != nil { + if b.handleError("Pod", "", err) { + return nil + } + return err + } + + // Collect VM pods and their UIDs for finding dependent pods + vmPodUIDs := make(map[string]bool) + for _, pod := range pods.Items { + vmPodUIDs[string(pod.UID)] = true + b.outputResource("Pod", pod.Name, namespace, &pod) + b.collectEvents(ctx, client, namespace, "Pod", pod.Name) + + if b.saveLogs { + b.collectSinglePodLogs(ctx, client, namespace, pod.Name) + } + } + + // Collect pods that have ownerReference to VM pods (e.g., hotplug volume pods) + if len(vmPodUIDs) > 0 { + allPods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + // If we can't list all pods, continue without dependent pods + if !b.handleError("Pod", namespace, err) { + return err + } + } else { + for _, pod := range allPods.Items { + // Skip VM pods we already collected + if vmPodUIDs[string(pod.UID)] { + continue + } + // Check if this pod has ownerReference to any VM pod + for _, ownerRef := range pod.OwnerReferences { + if ownerRef.Kind == "Pod" && vmPodUIDs[string(ownerRef.UID)] { + b.outputResource("Pod", pod.Name, namespace, &pod) + b.collectEvents(ctx, client, namespace, "Pod", pod.Name) + if b.saveLogs { + b.collectSinglePodLogs(ctx, client, namespace, pod.Name) + } + break + } + } + } + } + } + + return nil +} + +// Event collection functions + +func (b *DebugBundle) collectEvents(ctx context.Context, client kubeclient.Client, namespace, resourceType, resourceName string) { + events, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{ + FieldSelector: fmt.Sprintf("involvedObject.name=%s", resourceName), + }) + if err != nil { + if b.handleError("Event", resourceName, err) { + return + } + return + } + + // Add each event individually to preserve TypeMeta + for i := range events.Items { + b.outputResource("Event", fmt.Sprintf("%s-%s-%d", strings.ToLower(resourceType), resourceName, i), namespace, &events.Items[i]) + } +} + +// Log collection functions + +func (b *DebugBundle) collectSinglePodLogs(ctx context.Context, client kubeclient.Client, namespace, podName string) { + logPrefix := fmt.Sprintf("logs %s/%s", namespace, podName) + + // Get current logs + req := client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{}) + if logStream, err := req.Stream(ctx); err == nil { + if logContent, err := io.ReadAll(logStream); err == nil { + fmt.Fprintf(b.stdout, "\n# %s\n", logPrefix) + fmt.Fprintf(b.stdout, "%s\n", string(logContent)) + } + logStream.Close() + } + + // Get previous logs + req = client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{Previous: true}) + if logStream, err := req.Stream(ctx); err == nil { + if logContent, err := io.ReadAll(logStream); err == nil { + fmt.Fprintf(b.stdout, "\n# %s (previous)\n", logPrefix) + fmt.Fprintf(b.stdout, "%s\n", string(logContent)) + } + logStream.Close() + } +} + +// Helper functions + +func (b *DebugBundle) getInternalResource(ctx context.Context, resource, namespace, name string) (*unstructured.Unstructured, error) { + obj, err := b.dynamicClient.Resource(schema.GroupVersionResource{ + Group: "internal.virtualization.deckhouse.io", + Version: "v1", + Resource: resource, + }).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return obj, nil +} + +func (b *DebugBundle) getInternalResourceList(ctx context.Context, resource, namespace string) ([]*unstructured.Unstructured, error) { + list, err := b.dynamicClient.Resource(schema.GroupVersionResource{ + Group: "internal.virtualization.deckhouse.io", + Version: "v1", + Resource: resource, + }).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + result := make([]*unstructured.Unstructured, len(list.Items)) + for i := range list.Items { + result[i] = &list.Items[i] + } + return result, nil +} + +func (b *DebugBundle) outputResource(kind, name, namespace string, obj runtime.Object) error { + // Output separator if not first resource + if b.resourceCount > 0 { + fmt.Fprintf(b.stdout, "\n---\n") + } + b.resourceCount++ + + // Ensure Kind is set from input if missing + gvk := obj.GetObjectKind().GroupVersionKind() + if gvk.Kind == "" { + gvk.Kind = kind + obj.GetObjectKind().SetGroupVersionKind(gvk) + } + + // If GroupVersion is missing/empty, try to get from scheme + if gvk.GroupVersion().Empty() { + gvks, _, err := kubeclient.Scheme.ObjectKinds(obj) + if err == nil && len(gvks) > 0 { + gvk = gvks[0] + obj.GetObjectKind().SetGroupVersionKind(gvk) + } else if coreKinds[kind] { + // Fallback for core Kubernetes resources if scheme doesn't know about them + gvk = schema.GroupVersionKind{Group: "", Version: "v1", Kind: kind} + obj.GetObjectKind().SetGroupVersionKind(gvk) + } + } + + // Marshal to JSON (now with TypeMeta if set) + jsonBytes, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to marshal %s/%s to JSON: %w", kind, name, err) + } + + // Convert to YAML + yamlBytes, err := yaml.JSONToYAML(jsonBytes) + if err != nil { + return fmt.Errorf("failed to convert %s/%s to YAML: %w", kind, name, err) + } + + // Output + fmt.Fprintf(b.stdout, "# %d. %s: %s\n%s", b.resourceCount, kind, name, string(yamlBytes)) + + return nil +} diff --git a/src/cli/pkg/command/virtualization.go b/src/cli/pkg/command/virtualization.go index 403c0047c2..539ae192c4 100644 --- a/src/cli/pkg/command/virtualization.go +++ b/src/cli/pkg/command/virtualization.go @@ -31,6 +31,7 @@ import ( "github.com/deckhouse/virtualization/api/client/kubeclient" "github.com/deckhouse/virtualization/src/cli/internal/clientconfig" + "github.com/deckhouse/virtualization/src/cli/internal/cmd/collectdebuginfo" "github.com/deckhouse/virtualization/src/cli/internal/cmd/console" "github.com/deckhouse/virtualization/src/cli/internal/cmd/lifecycle" "github.com/deckhouse/virtualization/src/cli/internal/cmd/portforward" @@ -84,6 +85,7 @@ func NewCommand(programName string) *cobra.Command { virtCmd.AddCommand( console.NewCommand(), + collectdebuginfo.NewCommand(), vnc.NewCommand(), portforward.NewCommand(), ssh.NewCommand(),