feat: optimized files upload

controllers/router_v2.go

@@ -137,6 +137,11 @@ func InitRoutes(app *gin.Engine) (err error) {
 			Path:        "/:id/files/save",
 			HandlerFunc: PostSpiderSaveFile,
 		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save/batch",
+			HandlerFunc: PostSpiderSaveFiles,
+		},
 		Action{
 			Method:      http.MethodPost,
 			Path:        "/:id/files/save/dir",

controllers/spider_v2.go

@@ -22,6 +22,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"io"
 	"math"
 	"os"
 	"path/filepath"
@@ -113,12 +114,7 @@ func GetSpiderList(c *gin.Context) {
 	}
 
 	// stat list
-	query = bson.M{
-		"_id": bson.M{
-			"$in": ids,
-		},
-	}
-	stats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(query, nil)
+	spiderStats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil)
 	if err != nil {
 		HandleErrorInternalServerError(c, err)
 		return
@@ -127,7 +123,7 @@ func GetSpiderList(c *gin.Context) {
 	// cache stat list to dict
 	dict := map[primitive.ObjectID]models.SpiderStatV2{}
 	var taskIds []primitive.ObjectID
-	for _, st := range stats {
+	for _, st := range spiderStats {
 		if st.Tasks > 0 {
 			taskCount := int64(st.Tasks)
 			st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount)))
@@ -240,6 +236,18 @@ func PostSpider(c *gin.Context) {
 		return
 	}
 
+	// create folder
+	fsSvc, err := getSpiderFsSvc(c)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	err = fsSvc.CreateDir(".")
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
 	HandleSuccessWithData(c, s)
 }
 
@@ -446,7 +454,7 @@ func GetSpiderListDir(c *gin.Context) {
 
 	files, err := fsSvc.List(path)
 	if err != nil {
-		if err.Error() != "response status code: 404" {
+		if !errors.Is(err, os.ErrNotExist) {
 			HandleErrorInternalServerError(c, err)
 			return
 		}
@@ -492,26 +500,102 @@ func GetSpiderFileInfo(c *gin.Context) {
 }
 
 func PostSpiderSaveFile(c *gin.Context) {
-	var payload struct {
-		Path    string `json:"path"`
-		NewPath string `json:"new_path"`
-		Data    string `json:"data"`
-	}
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
+	fsSvc, err := getSpiderFsSvc(c)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
 		return
 	}
 
+	if c.GetHeader("Content-Type") == "application/json" {
+		var payload struct {
+			Path string `json:"path"`
+			Data string `json:"data"`
+		}
+		if err := c.ShouldBindJSON(&payload); err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	} else {
+		path, ok := c.GetPostForm("path")
+		if !ok {
+			HandleErrorBadRequest(c, errors.New("missing required field 'path'"))
+			return
+		}
+		file, err := c.FormFile("file")
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		f, err := file.Open()
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		fileData, err := io.ReadAll(f)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		if err := fsSvc.Save(path, fileData); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccess(c)
+}
+
+func PostSpiderSaveFiles(c *gin.Context) {
 	fsSvc, err := getSpiderFsSvc(c)
 	if err != nil {
-		HandleErrorBadRequest(c, err)
+		HandleErrorInternalServerError(c, err)
 		return
 	}
 
-	if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil {
-		HandleErrorInternalServerError(c, err)
+	form, err := c.MultipartForm()
+	if err != nil {
+		HandleErrorBadRequest(c, err)
 		return
 	}
+	wg := sync.WaitGroup{}
+	wg.Add(len(form.File))
+	for path := range form.File {
+		go func(path string) {
+			file, err := c.FormFile(path)
+			if err != nil {
+				log2.Warnf("invalid file header: %s", path)
+				log2.Error(err.Error())
+				wg.Done()
+				return
+			}
+			f, err := file.Open()
+			if err != nil {
+				log2.Warnf("unable to open file: %s", path)
+				log2.Error(err.Error())
+				wg.Done()
+				return
+			}
+			fileData, err := io.ReadAll(f)
+			if err != nil {
+				log2.Warnf("unable to read file: %s", path)
+				log2.Error(err.Error())
+				wg.Done()
+				return
+			}
+			if err := fsSvc.Save(path, fileData); err != nil {
+				log2.Warnf("unable to save file: %s", path)
+				log2.Error(err.Error())
+				wg.Done()
+				return
+			}
+			wg.Done()
+		}(path)
+	}
+	wg.Wait()
 
 	HandleSuccess(c)
 }
@@ -571,6 +655,9 @@ func DeleteSpiderFile(c *gin.Context) {
 		HandleErrorBadRequest(c, err)
 		return
 	}
+	if payload.Path == "~" {
+		payload.Path = "."
+	}
 
 	fsSvc, err := getSpiderFsSvc(c)
 	if err != nil {
@@ -582,6 +669,10 @@ func DeleteSpiderFile(c *gin.Context) {
 		HandleErrorInternalServerError(c, err)
 		return
 	}
+	_, err = fsSvc.GetFileInfo(".")
+	if err != nil {
+		_ = fsSvc.CreateDir("/")
+	}
 
 	HandleSuccess(c)
 }

fs/service_v2.go

@@ -73,7 +73,7 @@ func (svc *ServiceV2) List(path string) (files []interfaces.FsFileInfo, err erro
 		}
 	}
 
-	return
+	return files, err
 }
 
 func (svc *ServiceV2) GetFile(path string) (data []byte, err error) {

task/handler/runner_v2.go

@@ -591,6 +591,9 @@ func (r *RunnerV2) _updateSpiderStat(status string) {
 		}
 	case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled:
 		update = bson.M{
+			"$set": bson.M{
+				"last_task_id": r.tid, // last task id
+			},
 			"$inc": bson.M{
 				"results":          ts.ResultCount,            // results
 				"runtime_duration": ts.RuntimeDuration / 1000, // runtime duration