Merge branch 'feature/storage-data-export' of github.com:deckhouse/deckhouse-cli into feature/storage-data-export

Author: asergunov

internal/dataexport/README.md

@@ -9,3 +9,57 @@ Subcommand for the command line client for Deckhouse.
 ```shell
 d8 data create export-name pvc/test-pvc-name
 ```
+
+#### create from a snapshot
+
+1. Enable snapshot-controller module
+
+```shell
+kubectl apply -f -<<EOF
+apiVersion: deckhouse.io/v1alpha1
+kind: ModuleConfig
+metadata:
+  name: snapshot-controller
+spec:
+  enabled: true
+  version: 1
+EOF
+```
+
+2. Get a volume snapshot class name - this entity is created automatically for each type of CSI and is needed for creating a snapshot.
+For example, "sds-local-volume-snapshot-class" as shown in the example below
+
+```shell
+kubectl get volumesnapshotclass
+
+sds-local-volume-snapshot-class   local.csi.storage.deckhouse.io   Delete           22h
+```
+
+3. Create a snapshot of a needed volume 
+
+```shell
+kubectl apply -f -<<EOF
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: my-snapshot
+  namespace: <name of the namespace where the PVC is located>
+spec:
+  volumeSnapshotClassName: <volume snapshot class name>
+  source:
+    persistentVolumeClaimName: <name of the PVC to snapshot>
+EOF
+```
+
+4. Check that snapshot is created normally and ready to use (usually takes a few minutes to be ready):
+```shell
+kubectl -n <name of the namespace> get volumesnapshot my-snapshot
+
+NAMESPACE       NAME                  READYTOUSE   SOURCEPVC                        SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS                     SNAPSHOTCONTENT                                  
+<namespace>    my-snapshot          true         test-pvc-for-snapshot                           2Gi           sds-local-volume-snapshot-class   snapcontent-faf2ab1f-891d-4e5e-972c-334a490c99d8
+```
+
+5. Create data export from that snapshot using d8 command as shown in the example below 
+```shell
+d8 data create export-name snapshot/my-snapshot
+```

internal/dataexport/cmd/download/download.go

@@ -104,6 +104,54 @@ type dirResp struct {
 	Items   []dirItem `json:"items"`
 }
 
+func forRespItems(jsonStream io.ReadCloser, workFunc func(*dirItem) error) error {
+	dec := json.NewDecoder(jsonStream)
+
+	// find items list
+	for {
+		t, err := dec.Token()
+		if err != nil {
+			return err
+		}
+
+		if t == "items" {
+			t, err := dec.Token()
+			if err != nil {
+				return err
+			}
+			if t != json.Delim('[') {
+				return fmt.Errorf("JSON items is not list")
+			}
+			break
+		}
+		dec.More()
+	}
+
+	// read items
+	for dec.More() {
+		var i dirItem
+		err := dec.Decode(&i)
+		if err != nil {
+			break
+		}
+		err = workFunc(&i)
+		if err != nil {
+			return err
+		}
+	}
+
+	// check items list closed
+	t, err := dec.Token()
+	if err != nil {
+		return err
+	}
+	if t != json.Delim(']') {
+		return fmt.Errorf("items loading is not completed")
+	}
+
+	return nil
+}
+
 func recursiveDownload(ctx context.Context, sClient *safeClient.SafeClient, log *slog.Logger, sem chan struct{}, url, srcPath, dstPath string) (err error) {
 	select {
 	case <-ctx.Done():
@@ -135,22 +183,11 @@ func recursiveDownload(ctx context.Context, sClient *safeClient.SafeClient, log
 	}
 
 	if srcPath != "" && srcPath[len(srcPath)-1:] == "/" {
-		dirListBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return fmt.Errorf("Response body (%s) error: %s", srcPath, err.Error())
-		}
-
-		var dir dirResp
-		err = json.Unmarshal(dirListBody, &dir)
-		if err != nil {
-			return fmt.Errorf("Invalid dir (%s) data: %s", srcPath, err.Error())
-		}
-
 		var wg sync.WaitGroup
 		var mu sync.Mutex
 		var firstErr error
 
-		for _, item := range dir.Items {
+		err = forRespItems(resp.Body, func(item *dirItem) error {
 			subPath := item.Name
 			if item.Type == "dir" {
 				err = os.MkdirAll(filepath.Join(dstPath, subPath), os.ModePerm)
@@ -172,7 +209,13 @@ func recursiveDownload(ctx context.Context, sClient *safeClient.SafeClient, log
 					mu.Unlock()
 				}
 			}(subPath)
+
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("Response body (%s) error: %s", srcPath, err.Error())
 		}
+
 		wg.Wait()
 		return firstErr
 	} else {