converter: add progress

.github/workflows/pull-request.yml

@@ -53,12 +53,12 @@ jobs:
       - name: Checkout forkliftci
         uses: actions/checkout@v4
         with:
-          repository: kubev2v/forkliftci
-          ref: v15.0
+          repository: bennyz/forkliftci
+          ref: openstack-imagebased
 
       - name: Build and setup everything with bazel
         id: forkliftci
-        uses: kubev2v/forkliftci/ci/build-and-setup@v8.0
+        uses: bennyz/forkliftci/ci/build-and-setup@openstack-imagebased
         with:
           provider_name: ${{ matrix.source_provider }}
           gh_access_token: ${{ secrets.GITHUB_TOKEN }}
@@ -72,13 +72,13 @@ jobs:
           curl -k "${{ steps.forkliftci.outputs.cluster }}/apis/forklift.konveyor.io/v1beta1/namespaces/konveyor-forklift/providers" --header "Authorization: Bearer ${{ steps.forkliftci.outputs.token }}"
 
       - name: Run e2e sanity suite
-        uses: kubev2v/forkliftci/ci/run-suite@v8.0
+        uses: bennyz/forkliftci/ci/run-suite@openstack-imagebased
         with:
           suite_name: e2e-sanity-${{ matrix.source_provider }}
 
       - name: save k8s logs and upload-artifact
         if: ${{ always() }}
-        uses: kubev2v/forkliftci/ci/save-artifacts@v8.0
+        uses: bennyz/forkliftci/ci/save-artifacts@openstack-imagebased
         with:
           source_provider: ${{ matrix.source_provider }}
 

cmd/image-converter/BUILD.bazel

@@ -1,15 +1,33 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "image-converter_lib",
     srcs = ["image-converter.go"],
     importpath = "github.com/konveyor/forklift-controller/cmd/image-converter",
     visibility = ["//visibility:private"],
-    deps = ["//vendor/k8s.io/klog/v2:klog"],
+    deps = [
+        "//pkg/metrics",
+        "//vendor/github.com/prometheus/client_golang/prometheus",
+        "//vendor/github.com/prometheus/client_model/go",
+        "//vendor/k8s.io/klog/v2:klog",
+    ],
 )
 
 go_binary(
     name = "image-converter",
     embed = [":image-converter_lib"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "image-converter_test",
+    srcs = [
+        "image-converter_test.go",
+        "image_converter_suite_test.go",
+    ],
+    embed = [":image-converter_lib"],
+    deps = [
+        "//vendor/github.com/onsi/ginkgo/v2:ginkgo",
+        "//vendor/github.com/onsi/gomega",
+    ],
+)

cmd/image-converter/image-converter.go

@@ -4,31 +4,46 @@ import (
 	"bufio"
 	"bytes"
 	"flag"
+	"os"
 	"os/exec"
+	"strconv"
+	"strings"
 
+	"github.com/konveyor/forklift-controller/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"k8s.io/klog/v2"
 )
 
 func main() {
-	var srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode string
+	var srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode, ownerUID string
 
 	flag.StringVar(&srcVolPath, "src-path", "", "Source volume path")
 	flag.StringVar(&dstVolPath, "dst-path", "", "Target volume path")
 	flag.StringVar(&srcFormat, "src-format", "", "Format of the source volume")
 	flag.StringVar(&dstFormat, "dst-format", "", "Format of the target volume")
 	flag.StringVar(&volumeMode, "volume-mode", "", "Format of the target volume")
+	flag.StringVar(&ownerUID, "owner-uid", "", "Owner UID (usually PVC UID)")
 
 	flag.Parse()
 
 	klog.Info("srcVolPath: ", srcVolPath, " dstVolPath: ", dstVolPath, " sourceFormat: ", srcFormat, " targetFormat: ", dstFormat)
-	err := convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode)
+
+	certsDirectory, err := os.MkdirTemp("", "certsdir")
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	metrics.StartPrometheusEndpoint(certsDirectory)
+
+	err = convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode, ownerUID)
 	if err != nil {
 		klog.Fatal(err)
 	}
 }
 
-func convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode string) error {
-	err := qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat)
+func convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode, ownerUID string) error {
+	err := qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat, ownerUID)
 	if err != nil {
 		return err
 	}
@@ -38,7 +53,7 @@ func convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode string) er
 	// Copy dst over src
 	switch volumeMode {
 	case "Block":
-		err = qemuimgConvert(dstVolPath, srcVolPath, dstFormat, dstFormat)
+		err = qemuimgConvert(dstVolPath, srcVolPath, dstFormat, dstFormat, ownerUID, "-W")
 		if err != nil {
 			return err
 		}
@@ -58,7 +73,7 @@ func convert(srcVolPath, dstVolPath, srcFormat, dstFormat, volumeMode string) er
 	return nil
 }
 
-func qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat string) error {
+func qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat, ownerUID string, additionalArgs ...string) error {
 	cmd := exec.Command(
 		"qemu-img",
 		"convert",
@@ -69,6 +84,10 @@ func qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat string) error {
 		dstVolPath,
 	)
 
+	if len(additionalArgs) > 0 {
+		cmd.Args = append(cmd.Args, additionalArgs...)
+	}
+
 	klog.Info("Executing command: ", cmd.String())
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
@@ -93,10 +112,20 @@ func qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat string) error {
 	}()
 
 	scanner := bufio.NewScanner(stdout)
-	for scanner.Scan() {
-		line := scanner.Text()
-		klog.Info(line)
-	}
+	go func() {
+		progressVec := createProgressCounter()
+		for scanner.Scan() {
+			line := scanner.Text()
+			klog.Info(line)
+			progress, err := parseQemuimgProgress(line)
+			if err != nil {
+				klog.Error(err)
+				continue
+			}
+
+			updateProgress(progressVec, ownerUID, progress)
+		}
+	}()
 
 	err = cmd.Wait()
 	if err != nil {
@@ -105,3 +134,54 @@ func qemuimgConvert(srcVolPath, dstVolPath, srcFormat, dstFormat string) error {
 
 	return nil
 }
+
+func updateProgress(progressVec *prometheus.CounterVec, ownerUID string, progress float64) {
+	if progress == 0 {
+		return
+	}
+	metric := &dto.Metric{}
+	if err := progressVec.WithLabelValues(ownerUID).Write(metric); err != nil {
+		klog.Errorf("updateProgress: failed to write metric; %v", err)
+	}
+
+	if progress > *metric.Counter.Value {
+		progressVec.WithLabelValues(ownerUID).Add(progress - *metric.Counter.Value)
+	}
+}
+
+func parseQemuimgProgress(line string) (float64, error) {
+	// Parse qemu-img progress
+	// Example: "(10.00/100%)"
+	trimmed := strings.Trim(line, "\r\n\t")
+	if strings.HasSuffix(trimmed, "100%)") {
+		start := strings.Index(trimmed, "(") + 1
+		end := strings.Index(trimmed, "/")
+
+		if start < end && end <= len(trimmed) {
+			progressStr := trimmed[start:end]
+			progress, err := strconv.ParseFloat(progressStr, 64)
+			if err != nil {
+				return 0, err
+			}
+			return progress, nil
+		}
+	}
+
+	return 0, nil
+}
+
+func createProgressCounter() *prometheus.CounterVec {
+	progressVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "image_converter_progress",
+			Help: "Progress of image conversion",
+		},
+		[]string{"ownerUID"},
+	)
+
+	if err := prometheus.Register(progressVec); err != nil {
+		klog.Error("Prometheus progress counter not registered:", err)
+	}
+
+	return progressVec
+}

cmd/image-converter/image-converter_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Image converter", func() {
+
+	Describe("parseQemuimgProgress function", func() {
+		It("correctly parses valid progress output", func() {
+			line := "(10.00/100%)"
+			progress, err := parseQemuimgProgress(line)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(progress).To(Equal(10.00))
+		})
+
+		It("returns an error for invalid progress output", func() {
+			line := "Invalid output"
+			progress, err := parseQemuimgProgress(line)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(progress).To(Equal(float64(0)))
+		})
+	})
+})

cmd/image-converter/image_converter_suite_test.go

@@ -0,0 +1,13 @@
+package main_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestImageConverter(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ImageConverter Suite")
+}

pkg/controller/plan/BUILD.bazel

@@ -27,6 +27,7 @@ go_library(
         "//pkg/controller/plan/context",
         "//pkg/controller/plan/handler",
         "//pkg/controller/plan/scheduler",
+        "//pkg/controller/plan/util",
         "//pkg/controller/provider/web",
         "//pkg/controller/validation",
         "//pkg/lib/client/openshift",

pkg/controller/plan/adapter/base/doc.go

@@ -89,6 +89,8 @@ type Builder interface {
 	GetPopulatorTaskName(pvc *core.PersistentVolumeClaim) (taskName string, err error)
 	// Get the virtual machine preference name
 	PreferenceName(vmRef ref.Ref, configMap *core.ConfigMap) (name string, err error)
+	// Get conversionTaskName
+	GetConversionTaskName(pvc *core.PersistentVolumeClaim) (taskName string, err error)
 }
 
 // Client API.