feat: updated controllers and router

apps/utils.go

@@ -74,7 +74,7 @@ var injectors = []interface{}{
 	user.GetUserService,
 	schedule.GetScheduleService,
 	admin.GetSpiderAdminService,
-	stats.NewStatsService,
+	stats.GetStatsService,
 	nodeconfig.NewNodeConfigService,
 	taskstats.GetTaskStatsService,
 	color.NewService,

controllers/export_v2.go

@@ -0,0 +1,87 @@
+package controllers
+
+import (
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab-core/constants"
+	"github.com/crawlab-team/crawlab-core/export"
+	"github.com/crawlab-team/crawlab-core/interfaces"
+	"github.com/gin-gonic/gin"
+)
+
+func PostExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportTarget := c.Query("target")
+	exportFilter, _ := GetFilter(c)
+
+	var exportId string
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exportId, err = export.GetCsvService().Export(exportType, exportTarget, exportFilter)
+	case constants.ExportTypeJson:
+		exportId, err = export.GetJsonService().Export(exportType, exportTarget, exportFilter)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exportId)
+}
+
+func GetExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exp)
+}
+
+func GetExportDownload(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	switch exportType {
+	case constants.ExportTypeCsv:
+		c.Header("Content-Type", "text/csv")
+	case constants.ExportTypeJson:
+		c.Header("Content-Type", "text/plain")
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
+	c.File(exp.GetDownloadPath())
+}

controllers/filter_v2.go

@@ -0,0 +1,69 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-core/entity"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetFilterColFieldOptions(c *gin.Context) {
+	colName := c.Param("col")
+	value := c.Param("value")
+	if value == "" {
+		value = "_id"
+	}
+	label := c.Param("label")
+	if label == "" {
+		label = "name"
+	}
+	query := MustGetFilterQuery(c)
+	pipelines := mongo2.Pipeline{}
+	if query != nil {
+		pipelines = append(pipelines, bson.D{{"$match", query}})
+	}
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$group",
+				bson.M{
+					"_id": bson.M{
+						"value": "$" + value,
+						"label": "$" + label,
+					},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$project",
+				bson.M{
+					"value": "$_id.value",
+					"label": bson.M{"$toString": "$_id.label"},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$sort",
+				bson.D{
+					{"value", 1},
+				},
+			},
+		},
+	)
+	var options []entity.FilterSelectOption
+	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, options)
+}

controllers/login_v2.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-core/constants"
+	"github.com/crawlab-team/crawlab-core/errors"
+	"github.com/crawlab-team/crawlab-core/user"
+	"github.com/gin-gonic/gin"
+)
+
+func PostLogin(c *gin.Context) {
+	var payload struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	userSvc, err := user.GetUserServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password)
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	c.Set(constants.UserContextKey, loggedInUser)
+	HandleSuccessWithData(c, token)
+}
+
+func PostLogout(c *gin.Context) {
+	c.Set(constants.UserContextKey, nil)
+	HandleSuccess(c)
+}

controllers/result_v2.go

@@ -0,0 +1,119 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-core/models/models"
+	"github.com/crawlab-team/crawlab-core/models/service"
+	"github.com/crawlab-team/crawlab-core/result"
+	"github.com/crawlab-team/crawlab-core/utils"
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetResultList(c *gin.Context) {
+	// data collection id
+	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// data source id
+	var dsId primitive.ObjectID
+	dsIdStr := c.Query("data_source_id")
+	if dsIdStr != "" {
+		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+	}
+
+	// data collection
+	dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// data source
+	ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId)
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			ds = &models.DataSourceV2{}
+		} else {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	// spider
+	sq := bson.M{
+		"col_id":         dc.Id,
+		"data_source_id": ds.Id,
+	}
+	s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// service
+	svc, err := result.GetResultService(s.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := getResultListQuery(c)
+
+	// get results
+	data, err := svc.List(query, &generic.ListOptions{
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate results
+	if len(data) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// total count
+	total, err := svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}