From ea1ebfafda1059c0c91c0fdf1cc1ad4449861183 Mon Sep 17 00:00:00 2001 From: yuancheng Date: Fri, 9 Jun 2023 18:19:01 +0800 Subject: [PATCH] fix some bugs. --- .gitignore | 5 +- MANUAL.md | 32 +++---- cmd/bucket.go | 38 +++++--- cmd/car.go | 60 ++----------- cmd/config.go | 65 +------------- cmd/getCmd.go | 3 + cmd/helper.go | 21 ++++- cmd/logCmd.go | 4 +- cmd/lsCmd.go | 18 +++- cmd/mbCmd.go | 6 +- cmd/object.go | 214 ++++++++++++++++++++++++++++++++++++++------- cmd/rmCmd.go | 8 +- cmd/rootCmd.go | 40 +++++++-- cmd/system.go | 26 +++++- cmd/version.go | 43 +++++++++ cmd/versionCmd.go | 2 +- config.toml.sample | 11 +-- go.mod | 5 +- go.sum | 4 +- 19 files changed, 392 insertions(+), 213 deletions(-) create mode 100644 cmd/version.go diff --git a/.gitignore b/.gitignore index c466276..8a534d5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,7 @@ testdata logs/ /uploading/* /testserver/* -/testserver \ No newline at end of file +/testserver +myfolder +myfolder_download +xyz \ No newline at end of file diff --git a/MANUAL.md b/MANUAL.md index b7976bb..1145efa 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -1,4 +1,4 @@ -# Manual +# Chainstorage-cli Manual ## Introduction - chainstorage-cli is the command line application to interact with rest api SDK of *chain strorage*. @@ -400,8 +400,7 @@ Here is an example of a configuration file (`config.toml`) with available option ``` [cli] ipfsGateway = "test-ipfs-gateway.netwarps.com/ipfs/" -ggcscmdPath = '/path/to/ggcscmd' -useHttpsProtocol = "https" +useHttpsProtocol = true bucketPrefix = 'cs://' listOffset = 20 cleanTmpData = true @@ -411,17 +410,12 @@ retryDelay = 3 [sdk] defaultRegion = 'hk-1' timeZone = 'UTC +08:00' -#链存服务API地址 chainStorageApiEndpoint = 'http://127.0.0.1:8821' -#CAR文件工作目录 +useHttpsProtocol = true carFileWorkPath = './tmp/carfile' -#CAR文件分片阈值() -carFileShardingThreshold = 1048576 -#链存服务API token +carFileShardingThreshold = 46137344 chainStorageApiToken = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...' -#HTTP request user agent (K2请求需要) httpRequestUserAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36' -#HTTP request overtime httpRequestOvertime = 30 carVersion = 1 @@ -448,17 +442,11 @@ The CLI configuration allows you to specify various settings related to the tool - Type: String - Default Value: `test-ipfs-gateway.netwarps.com/ipfs/` -#### ggcscmdPath - -- Description: The path to the `ggcscmd` executable. -- Type: String -- Default Value: `/path/to/ggcscmd` - #### useHttpsProtocol - Description: Indicates whether to use HTTPS protocol. -- Type: String -- Default Value: `https` +- Type: Boolean +- Default Value: `true` #### bucketPrefix @@ -514,6 +502,12 @@ The SDK configuration allows you to specify various settings related to the SDK' - Type: String - Default Value: `http://127.0.0.1:8821` +#### useHttpsProtocol + +- Description: Indicates whether to use HTTPS protocol. +- Type: Boolean +- Default Value: `true` + #### carFileWorkPath - Description: The working directory for CAR files. @@ -524,7 +518,7 @@ The SDK configuration allows you to specify various settings related to the SDK' - Description: The threshold (in bytes) for CAR file sharding. - Type: Integer -- Default Value: `1048576` +- Default Value: `46137344` #### chainStorageApiToken diff --git a/cmd/bucket.go b/cmd/bucket.go index 0a8bde3..8f8529e 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -1,11 +1,10 @@ package cmd import ( - "fmt" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/ulule/deepcopier" @@ -102,11 +101,18 @@ func bucketListRunOutput(cmd *cobra.Command, args []string, resp model.BucketPag // 存储网络 bucketOutput.StorageNetwork = consts.StorageNetworkCodeMapping[bucketOutput.StorageNetworkCode] - // 桶策略 - bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMapping[bucketOutput.BucketPrincipleCode] + //// 桶策略 + //bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMapping[bucketOutput.BucketPrincipleCode] + + // 桶策略（英文） + bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMappingEn[bucketOutput.BucketPrincipleCode] // 创建时间 bucketOutput.CreatedDate = bucketOutput.CreatedAt.Format("2006-01-02") + + // 已使用空间 + bucketOutput.FormatUsedSpace = convertSizeUnit(bucketOutput.UsedSpace) + bucketListOutput.List = append(bucketListOutput.List, bucketOutput) } } @@ -117,7 +123,7 @@ total {{.Count}} Status: {{.Code}} {{- else}} {{- range .List}} -{{.ObjectAmount}} {{.StorageNetwork}} {{.BucketPrinciple}} {{.UsedSpace}} {{.CreatedDate}} {{.BucketName}} +{{.StorageNetwork}} {{.BucketPrinciple}} {{.FormatUsedSpace}} {{.ObjectAmount}} {{.CreatedDate}} {{.BucketName}} {{- end}} {{- end}} ` @@ -155,6 +161,7 @@ type BucketOutput struct { StorageNetwork string `json:"storageNetwork" comment:"存储网络（10001-IPFS）"` BucketPrinciple string `json:"bucketPrinciple" comment:"桶策略（10001-公开，10000-私有）"` CreatedDate string `json:"createdDate" comment:"创建日期"` + FormatUsedSpace string `json:"formatUsedSpace" comment:"格式化已使用空间"` } // endregion Bucket List @@ -245,8 +252,11 @@ func bucketCreateRunOutput(cmd *cobra.Command, args []string, resp model.BucketC // 存储网络 bucketOutput.StorageNetwork = consts.StorageNetworkCodeMapping[bucketOutput.StorageNetworkCode] - // 桶策略 - bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMapping[bucketOutput.BucketPrincipleCode] + //// 桶策略 + //bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMapping[bucketOutput.BucketPrincipleCode] + + // 桶策略（英文） + bucketOutput.BucketPrinciple = consts.BucketPrincipleCodeMappingEn[bucketOutput.BucketPrincipleCode] // 创建时间 todo: timezone //bucketOutput.CreatedDate = bucketOutput.CreatedAt.Format("2006-01-02") @@ -406,10 +416,10 @@ func bucketEmptyRun(cmd *cobra.Command, args []string) { } // todo: remove it - fmt.Sprint(force) - //if !force { - // Error(cmd, args, errors.New("empty bucket operation, add --force to confirm emptying")) - //} + //fmt.Sprint(force) + if !force { + Error(cmd, args, errors.New("empty bucket operation, add --force to confirm emptying")) + } sdk, err := chainstoragesdk.New(&appConfig) if err != nil { diff --git a/cmd/car.go b/cmd/car.go index 7c77f1c..f06c3c4 100644 --- a/cmd/car.go +++ b/cmd/car.go @@ -4,11 +4,11 @@ import ( "fmt" "github.com/alanshaw/go-carbites" "github.com/cheggaaa/pb/v3" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/consts" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" - "github.com/paradeum-team/chainstorage-sdk/sdk/utils" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" + "github.com/paradeum-team/chainstorage-sdk/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -117,49 +117,6 @@ func carUploadRunOutput(cmd *cobra.Command, args []string, resp model.ObjectCrea Error(cmd, args, err) } - //对象上传 - //通过命令向固定桶内上传对象，包括文件、目录 - // - //模版 - // - //gcscmd put FILE[/DIR...] cs://BUCKET - //BUCKET - // - //桶名称 - // - //命令行例子 - // - //上传文件 - // - //当前目录 - // - //gcscmd put ./aaa.mp4 cs://bbb - //绝对路径 - // - //gcscmd put /home/pz/aaa.mp4 cs://bbb - //相对路径 - // - //gcscmd put ../pz/aaa.mp4 cs://bbb - //上传目录 - // - //gcscmd put ./aaaa cs://bbb - //上传 carfile - // - //gcscmd put ./aaa.car cs://bbb --carfile - //响应 - // - //过程 - // - //################ 15% - //Tarkov.mp4 - //完成 - // - //CID: QmWgnG7pPjG31w328hZyALQ2BgW5aQrZyKpT47jVpn8CNo - //Name:Tarkov.mp4 - //报错 - // - //Error: This file is a car file, add --carfile to confirm uploading car - templateContent := ` CID: {{.Data.ObjectCid}} Name: {{.Data.ObjectName}} @@ -507,7 +464,7 @@ func UploadBigCarFile(sdk *chainstoragesdk.CssClient, req *model.CarFileUploadRe "index": i, "retry": j, }).Info("upload sharding car file") - fmt.Printf("UploadBigCarFile => UploadShardingCarFileExt, index:%d, parameter uploadingReq:%+v\n", i, uploadingReq) + //fmt.Printf("UploadBigCarFile => UploadShardingCarFileExt, index:%d, parameter uploadingReq:%+v\n", i, uploadingReq) uploadResp, err := sdk.Car.UploadShardingCarFileExt(&uploadingReq, extReader) if err == nil && uploadResp.Code == http.StatusOK { uploadRespList = append(uploadRespList, uploadResp) @@ -597,8 +554,9 @@ func carImportRun(cmd *cobra.Command, args []string) { // CAR文件类型检查 if !strings.HasSuffix(strings.ToLower(dataPath), ".car") { - err := sdkcode.ErrCarUploadFileInvalidDataPath - Error(cmd, args, err) + //err := sdkcode.ErrCarUploadFileInvalidDataPath + //Error(cmd, args, err) + Error(cmd, args, errors.New("please specify car format file with .car suffix")) } //// CAR文件标识 diff --git a/cmd/config.go b/cmd/config.go index b8d688e..735879f 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -10,7 +10,7 @@ import ( type CliConfig struct { IpfsGateway string `toml:"ipfsGateway"` GgcscmdPath string `toml:"ggcscmdPath"` - UseHTTPSProtocol string `toml:"useHttpsProtocol"` + UseHTTPSProtocol bool `toml:"useHttpsProtocol"` BucketPrefix string `toml:"bucketPrefix"` ListOffset int `toml:"listOffset"` CleanTmpData bool `toml:"cleanTmpData"` @@ -28,6 +28,7 @@ type SdkConfig struct { HTTPRequestUserAgent string `toml:"httpRequestUserAgent"` HTTPRequestOvertime int `toml:"httpRequestOvertime"` CarVersion int `toml:"carVersion"` + UseHTTPSProtocol bool `toml:"useHttpsProtocol"` } type LoggerConfig struct { @@ -49,19 +50,6 @@ type CscConfig struct { // region Config show -//var bucketListCmd = &cobra.Command{ -// Use: "ls", -// Short: "ls", -// Long: "List links from object or bucket", -// Example: "gcscmd ls [--Offset=]", -// -// Run: func(cmd *cobra.Command, args []string) { -// //cmd.Help() -// //fmt.Printf("%s %s\n", cmd.Name(), strconv.Itoa(offset)) -// bucketListRun(cmd, args) -// }, -//} - func configShowRun(cmd *cobra.Command, args []string) { configFileUsed := viper.ConfigFileUsed() fmt.Fprintln(os.Stderr, "Using config file:", configFileUsed) @@ -69,55 +57,6 @@ func configShowRun(cmd *cobra.Command, args []string) { if err != nil { Error(cmd, args, err) } - - configShowRunOutput(cmd, args) } -func configShowRunOutput(cmd *cobra.Command, args []string) { - // templateContent := ` - //total {{.Count}} - //{{- if eq (len .List) 0}} - //Status: {{.Code}} - //{{- else}} - //{{- range .List}} - //{{.ObjectAmount}} {{.StorageNetwork}} {{.BucketPrinciple}} {{.UsedSpace}} {{.CreatedDate}} {{.BucketName}} - //{{- end}} - //{{- end}} - //` - // - // t, err := template.New("configShowTemplate").Parse(templateContent) - // if err != nil { - // Error(cmd, args, err) - // } - // - // err = t.Execute(os.Stdout, bucketListOutput) - // if err != nil { - // Error(cmd, args, err) - // } -} - -//type BucketListOutput struct { -// RequestId string `json:"requestId,omitempty"` -// Code int32 `json:"code,omitempty"` -// Msg string `json:"msg,omitempty"` -// Status string `json:"status,omitempty"` -// Count int `json:"count,omitempty"` -// PageIndex int `json:"pageIndex,omitempty"` -// PageSize int `json:"pageSize,omitempty"` -// List []BucketOutput `json:"list,omitempty"` -//} -// -//type BucketOutput struct { -// Id int `json:"id" comment:"桶ID"` -// BucketName string `json:"bucketName" comment:"桶名称（3-63字长度限制）"` -// StorageNetworkCode int `json:"storageNetworkCode" comment:"存储网络编码（10001-IPFS）"` -// BucketPrincipleCode int `json:"bucketPrincipleCode" comment:"桶策略编码（10001-公开，10000-私有）"` -// UsedSpace int64 `json:"usedSpace" comment:"已使用空间（字节）"` -// ObjectAmount int `json:"objectAmount" comment:"对象数量"` -// CreatedAt time.Time `json:"createdAt" comment:"创建时间"` -// StorageNetwork string `json:"storageNetwork" comment:"存储网络（10001-IPFS）"` -// BucketPrinciple string `json:"bucketPrinciple" comment:"桶策略（10001-公开，10000-私有）"` -// CreatedDate string `json:"createdDate" comment:"创建日期"` -//} - // endregion Config show diff --git a/cmd/getCmd.go b/cmd/getCmd.go index 0370a6e..18e4707 100644 --- a/cmd/getCmd.go +++ b/cmd/getCmd.go @@ -54,4 +54,7 @@ func init() { // 对象CID getCmd.Flags().StringP("cid", "c", "", "cid of object") + + // 用户自定义目录 + getCmd.Flags().StringP("downloadFolder", "d", "", "download folder") } diff --git a/cmd/helper.go b/cmd/helper.go index e081963..de51b27 100644 --- a/cmd/helper.go +++ b/cmd/helper.go @@ -3,7 +3,7 @@ package cmd import ( "bufio" "fmt" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" "github.com/spf13/cobra" "io" "os" @@ -189,3 +189,22 @@ func printFileContent(filename string) error { return nil } + +func convertSizeUnit(size int64) string { + // 2^10 = 1024 + const unit = 1024 + if size < unit { + return fmt.Sprintf("%dB", size) + } + + div, exp := int64(unit), 0 + for size >= div && exp < 8 { + div *= unit + exp++ + } + + convertedSize := float64(size) / float64(div/unit) + //fmt.Printf("size:%d\n", size) + //fmt.Printf("div:%d\n", div) + return fmt.Sprintf("%.1f%cB", convertedSize, "KMGTPEZY"[exp-1]) +} diff --git a/cmd/logCmd.go b/cmd/logCmd.go index aafc01e..5309359 100644 --- a/cmd/logCmd.go +++ b/cmd/logCmd.go @@ -31,8 +31,8 @@ import ( // logCmd represents the log command var logCmd = &cobra.Command{ Use: "log ", - Short: "Manage and show logs of running daemon", - Long: `Manage and show logs of running daemon`, + Short: "Manage and show logs. Supported level of log are: trace, debug, info, warn and error their lower-case forms", + Long: `Manage and show logs. Supported level of log are: trace, debug, info, warn and error their lower-case forms`, Run: func(cmd *cobra.Command, args []string) { //fmt.Println("log called") if len(args) == 0 { diff --git a/cmd/lsCmd.go b/cmd/lsCmd.go index 48f9fc9..6a577b5 100644 --- a/cmd/lsCmd.go +++ b/cmd/lsCmd.go @@ -28,9 +28,23 @@ import ( // lsCmd represents the ls command var lsCmd = &cobra.Command{ - Use: "ls [--Offset=]", + Use: "ls [cs://] [--name=] [--cid=] [--Offset=]", Short: "List links from object or bucket", - Long: `List links from object or bucket`, + Long: `List links from object or bucket +Example + +To list all buckets, use the following command: +gcscmd ls --offset +Replace with the desired list offset value. + +To list links from an object or bucket, use the following command: +gcscmd ls cs:// --cid --offset +Replace with the name of the bucket, with the CID of the object, and with the desired list offset value. + +Alternatively, you can specify the object name using the --name flag: +gcscmd ls cs:// --name --offset +Replace with the name of the object and with the desired list offset value. +`, Run: func(cmd *cobra.Command, args []string) { //fmt.Println("ls called") diff --git a/cmd/mbCmd.go b/cmd/mbCmd.go index 2f6f2ef..1f54772 100644 --- a/cmd/mbCmd.go +++ b/cmd/mbCmd.go @@ -27,7 +27,7 @@ import ( // mbCmd represents the mb command var mbCmd = &cobra.Command{ - Use: "mb cs://[BUCKET] [--storageNetworkCode=] [--bucketPrincipleCode=]", + Use: "mb cs:// [--storageNetworkCode=] [--bucketPrincipleCode=]", Short: "create bucket", Long: `create bucket`, Run: func(cmd *cobra.Command, args []string) { @@ -48,6 +48,6 @@ func init() { // Cobra supports local flags which will only run when this command // is called directly, e.g.: // mbCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - mbCmd.Flags().IntP("storageNetworkCode", "s", 10001, "storage network code") - mbCmd.Flags().IntP("bucketPrincipleCode", "b", 10001, "bucket principle code") + mbCmd.Flags().IntP("storageNetworkCode", "s", 10001, "storage network code. option value: 10001-IPFS") + mbCmd.Flags().IntP("bucketPrincipleCode", "b", 10001, "bucket principle code. option value: 10001-public,10000-private") } diff --git a/cmd/object.go b/cmd/object.go index 12e47cf..a41b6f3 100644 --- a/cmd/object.go +++ b/cmd/object.go @@ -5,14 +5,19 @@ import ( "fmt" "github.com/Code-Hex/pget" "github.com/ipfs/go-cid" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - sdkcode "github.com/paradeum-team/chainstorage-sdk/sdk/code" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + sdkcode "github.com/paradeum-team/chainstorage-sdk/code" + "github.com/paradeum-team/chainstorage-sdk/consts" + "github.com/paradeum-team/chainstorage-sdk/model" + "github.com/paradeum-team/chainstorage-sdk/utils" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/ulule/deepcopier" "net/http" + "net/url" "os" + "path/filepath" "text/template" "time" ) @@ -164,6 +169,10 @@ func objectListRunOutput(cmd *cobra.Command, args []string, resp model.ObjectPag // 创建时间 objectOutput.CreatedDate = objectOutput.CreatedAt.Format("2006-01-02") + + // 对象大小 + objectOutput.FormatObjectSize = convertSizeUnit(objectOutput.ObjectSize) + objectListOutput.List = append(objectListOutput.List, objectOutput) } } @@ -174,7 +183,7 @@ total {{.Count}} Status: {{.Code}} {{else}} {{- range .List}} -{{.ObjectCid}} {{.ObjectSize}} {{.CreatedDate}} {{.ObjectName}} +{{.ObjectCid}} {{.FormatObjectSize}} {{.CreatedDate}} {{.ObjectName}} {{- end}} {{end}}` @@ -201,16 +210,17 @@ type ObjectListOutput struct { } type ObjectOutput struct { - Id int `json:"id" comment:"对象ID"` - BucketId int `json:"bucketId" comment:"桶主键"` - ObjectName string `json:"objectName" comment:"对象名称（255字限制）"` - ObjectTypeCode int `json:"objectTypeCode" comment:"对象类型编码"` - ObjectSize int64 `json:"objectSize" comment:"对象大小（字节）"` - IsMarked int `json:"isMarked" comment:"星标（1-已标记，0-未标记）"` - ObjectCid string `json:"objectCid" comment:"对象CID"` - CreatedAt time.Time `json:"createdAt" comment:"创建时间"` - UpdatedAt time.Time `json:"updatedAt" comment:"最后更新时间"` - CreatedDate string `json:"createdDate" comment:"创建日期"` + Id int `json:"id" comment:"对象ID"` + BucketId int `json:"bucketId" comment:"桶主键"` + ObjectName string `json:"objectName" comment:"对象名称（255字限制）"` + ObjectTypeCode int `json:"objectTypeCode" comment:"对象类型编码"` + ObjectSize int64 `json:"objectSize" comment:"对象大小（字节）"` + IsMarked int `json:"isMarked" comment:"星标（1-已标记，0-未标记）"` + ObjectCid string `json:"objectCid" comment:"对象CID"` + CreatedAt time.Time `json:"createdAt" comment:"创建时间"` + UpdatedAt time.Time `json:"updatedAt" comment:"最后更新时间"` + CreatedDate string `json:"createdDate" comment:"创建日期"` + FormatObjectSize string `json:"formatObjectSize" comment:"格式化对象大小"` } // endregion Object List @@ -274,10 +284,10 @@ func objectRenameRun(cmd *cobra.Command, args []string) { Error(cmd, args, err) } - // todo: return succeed? - if rename == objectName { - Error(cmd, args, errors.New("the new name of object can't be equal to the raw name of object")) - } + //// todo: return succeed? + //if rename == objectName { + // Error(cmd, args, errors.New("the new name of object can't be equal to the raw name of object")) + //} // 强制覆盖 force, err := cmd.Flags().GetBool("force") @@ -322,12 +332,15 @@ func objectRenameRun(cmd *cobra.Command, args []string) { count := respObject.Data.Count if count == 1 { - objectId = respObject.Data.List[0].Id + objectData := respObject.Data.List[0] + objectId = objectData.Id + objectName = objectData.ObjectName } else if count == 0 { Error(cmd, args, sdkcode.ErrObjectNotFound) } else if count > 1 { // todo: please use name query? - Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use cid query\n")) + //Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use cid query\n")) + Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use name query\n")) } } else { respObject, err := sdk.Object.GetObjectByName(bucketId, objectName) @@ -344,10 +357,17 @@ func objectRenameRun(cmd *cobra.Command, args []string) { objectId = respObject.Data.Id } - // 重命名对象 - response, err := sdk.Object.RenameObject(objectId, rename, force) - if err != nil { - Error(cmd, args, err) + response := model.ObjectRenameResponse{} + // 重名名与对象名称相同，直接返回成功 + if rename == objectName { + response.Code = http.StatusOK + //Error(cmd, args, errors.New("the new name of object can't be equal to the raw name of object")) + } else { + // 重命名对象 + response, err = sdk.Object.RenameObject(objectId, rename, force) + if err != nil { + Error(cmd, args, err) + } } objectRenameRunOutput(cmd, args, response) @@ -658,11 +678,23 @@ func objectDownloadRun(cmd *cobra.Command, args []string) { Error(cmd, args, errors.New("please specify the name or cid")) } - //// 输出路径 - //target, err := cmd.Flags().GetString("Target") - //if err != nil { - // Error(cmd, args, err) - //} + // 用户自定义目录 + downloadFolder, err := cmd.Flags().GetString("downloadFolder") + if err != nil { + Error(cmd, args, err) + } + + // 用户自定义目录无效 + if len(downloadFolder) != 0 { + fileInfo, err := os.Stat(downloadFolder) + if err != nil { + Error(cmd, args, errors.New("please specify the valid download folder")) + } + + if !fileInfo.IsDir() { + Error(cmd, args, errors.New("please specify the valid download folder")) + } + } sdk, err := chainstoragesdk.New(&appConfig) if err != nil { @@ -704,7 +736,8 @@ func objectDownloadRun(cmd *cobra.Command, args []string) { Error(cmd, args, sdkcode.ErrObjectNotFound) } else if count > 1 { // todo: please use name query? - Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use cid query\n")) + //Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use cid query\n")) + Error(cmd, args, errors.New("Error: Multiple objects match this query, cannot perform this operation, please use name query\n")) } objectData := respObjectList.Data.List[0] @@ -729,14 +762,27 @@ func objectDownloadRun(cmd *cobra.Command, args []string) { objectCid = respObject.Data.ObjectCid } + isFolder := respObject.Data.ObjectTypeCode == consts.ObjectTypeCodeDir + if isFolder { + downloadFolderData(cmd, args, &respObject) + return + } + //ipfsGateway := viper.GetString("cli.ipfsGateway") - ipfsGateway := cliConfig.IpfsGateway - downloadUrl := fmt.Sprintf("https://%s%s", ipfsGateway, objectCid) + //downloadUrl := fmt.Sprintf("https://%s%s", ipfsGateway, objectCid) + //downloadUrl := fmt.Sprintf(ipfsGateway, objectCid) + //ipfsGateway := cliConfig.IpfsGateway + //downloadUrl := ipfsGateway + objectCid + + downloadUrl := generateDownloadUrl(objectCid, false) + outputPath := objectName + if len(downloadFolder) != 0 { + outputPath = filepath.Join(downloadFolder, objectName) + } cli := pget.New() cli.URLs = []string{downloadUrl} - cli.Output = objectName - + cli.Output = outputPath version := "" downloadArgs := []string{"-t", "10"} if err := cli.Run(context.Background(), version, downloadArgs); err != nil { @@ -794,4 +840,104 @@ type ObjectDownloadOutput struct { Data ObjectOutput `json:"objectOutput,omitempty"` } +func downloadFolderData(cmd *cobra.Command, args []string, respObject *model.ObjectCreateResponse) { + objectCid := respObject.Data.ObjectCid + objectName := respObject.Data.ObjectName + downloadUrl := generateDownloadUrl(objectCid, true) + + sdk, err := chainstoragesdk.New(&appConfig) + if err != nil { + Error(cmd, args, err) + } + + outputPath := sdk.Car.GenerateTempFileName(utils.CurrentDate()+"_", ".tmp") + cli := pget.New() + cli.URLs = []string{downloadUrl} + cli.Output = outputPath + version := "" + downloadArgs := []string{"-t", "10"} + if err := cli.Run(context.Background(), version, downloadArgs); err != nil { + //if cli.Trace { + // fmt.Fprintf(os.Stderr, "Error:\n%+v\n", err) + //} else { + // fmt.Fprintf(os.Stderr, "Error:\n %v\n", err) + //} + Error(cmd, args, err) + } + + // 用户自定义目录 + downloadFolder, err := cmd.Flags().GetString("downloadfolder") + if err != nil { + Error(cmd, args, err) + } + + // make data folder + folderDestination := objectName + if len(downloadFolder) != 0 { + folderDestination = filepath.Join(downloadFolder, objectName) + } + + // Check if the folder exists + if _, err := os.Stat(folderDestination); os.IsNotExist(err) { + // Folder does not exist, create a new folder + err := os.MkdirAll(folderDestination, 0755) + if err != nil { + fmt.Println("Failed to create folder:", err) + Error(cmd, args, err) + } + + fmt.Println("Folder created:", folderDestination) + } + + // extract car, delete temp car, override file + err = sdk.Car.ExtractCarFile(outputPath, folderDestination) + if err != nil { + Error(cmd, args, err) + } + defer func(fileDestination string) { + //if !viper.GetBool("cli.cleanTmpData") { + if !cliConfig.CleanTmpData { + return + } + + err := os.Remove(fileDestination) + if err != nil { + log.WithError(err). + WithFields(logrus.Fields{ + "fileDestination": fileDestination, + }).Error("Fail to remove car file") + //fmt.Printf("Error:%+v\n", err) + } + }(outputPath) + + objectDownloadRunOutput(cmd, args, *respObject) +} + // endregion Object Download + +func generateDownloadUrl(objectCid string, isFolder bool) string { + downloadUrl := "" + + ipfsGateway := cliConfig.IpfsGateway + base, err := url.Parse(ipfsGateway) + if err != nil { + log.Fatal(err) + } + + ref, err := url.Parse(objectCid) + if err != nil { + log.Fatal(err) + } + + if isFolder { + queryString := ref.Query() + queryString.Set("format", "car") + ref.RawQuery = queryString.Encode() + } + + u := base.ResolveReference(ref) + downloadUrl = u.String() + //fmt.Println(downloadUrl) + + return downloadUrl +} diff --git a/cmd/rmCmd.go b/cmd/rmCmd.go index d618d68..91788d0 100644 --- a/cmd/rmCmd.go +++ b/cmd/rmCmd.go @@ -39,7 +39,13 @@ var rmCmd = &cobra.Command{ Error(cmd, args, err) } - if len(objectName) == 0 { + // 对象CID + objectCid, err := cmd.Flags().GetString("cid") + if err != nil { + Error(cmd, args, err) + } + + if len(objectName) == 0 && len(objectCid) == 0 { bucketEmptyRun(cmd, args) } else { objectRemoveRun(cmd, args) diff --git a/cmd/rootCmd.go b/cmd/rootCmd.go index 38536e2..5262fcd 100644 --- a/cmd/rootCmd.go +++ b/cmd/rootCmd.go @@ -23,9 +23,10 @@ package cmd import ( "fmt" - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" "github.com/ulule/deepcopier" "os" + "strings" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -135,19 +136,48 @@ func initConfig() { err := viper.Unmarshal(&cscConfig) if err != nil { fmt.Fprintln(os.Stderr, "config Unmarshal fail, error:%+v\n", err) + os.Exit(1) } - cliConfig = cscConfig.Cli - sdkConfig = cscConfig.Sdk - loggerConfig = cscConfig.Logger - //fmt.Printf("Config struct: %#v\n", cscConfig) //fmt.Printf("All configuration: %+v\n", viper.AllSettings()) appConfig = chainstoragesdk.ApplicationConfig{} + checkConfig(&cscConfig) + + cliConfig = cscConfig.Cli + sdkConfig = cscConfig.Sdk + loggerConfig = cscConfig.Logger + // 设置SDK配置 deepcopier.Copy(&sdkConfig).To(&appConfig.Server) deepcopier.Copy(&loggerConfig).To(&appConfig.Logger) initLogger() } + +func checkConfig(config *CscConfig) { + cliConfig := &config.Cli + + //check chain-storage-api base address + if len(cliConfig.IpfsGateway) > 0 { + ipfsGateway := cliConfig.IpfsGateway + if !strings.HasPrefix(ipfsGateway, "http://") && + !strings.HasPrefix(ipfsGateway, "https://") { + + if cliConfig.UseHTTPSProtocol { + cliConfig.IpfsGateway = "https://" + ipfsGateway + } else { + cliConfig.IpfsGateway = "http://" + ipfsGateway + } + } + + if !strings.HasSuffix(ipfsGateway, "/") { + cliConfig.IpfsGateway += "/" + } + } else { + fmt.Println("ERROR: no ipfs gateway provided in Configuration, at least 1 valid http/https ipfs gateway must be given, exiting") + os.Exit(1) + } + +} diff --git a/cmd/system.go b/cmd/system.go index b22f03a..45becab 100644 --- a/cmd/system.go +++ b/cmd/system.go @@ -1,8 +1,8 @@ package cmd import ( - chainstoragesdk "github.com/paradeum-team/chainstorage-sdk/sdk" - "github.com/paradeum-team/chainstorage-sdk/sdk/model" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + "github.com/paradeum-team/chainstorage-sdk/model" "github.com/spf13/cobra" "os" "text/template" @@ -44,3 +44,25 @@ IPFS Version: {{.Version}} Error(cmd, args, err) } } + +func versionRun(cmd *cobra.Command, args []string) { + versionInfo := GetVersionInfo() + versionRunOutput(cmd, args, versionInfo) +} + +func versionRunOutput(cmd *cobra.Command, args []string, versionInfo *VersionInfo) { + templateContent := ` +Client Version: version.Info{Version:"v{{.Version}}"} +Server Version: version.Info{Version:"v{{.ApiVersion}}"} +` + + t, err := template.New("versionTemplate").Parse(templateContent) + if err != nil { + Error(cmd, args, err) + } + + err = t.Execute(os.Stdout, versionInfo) + if err != nil { + Error(cmd, args, err) + } +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..f0f70ab --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "fmt" + chainstoragesdk "github.com/paradeum-team/chainstorage-sdk" + "os" + "runtime" +) + +// CurrentVersionNumber is the current application's version literal +const CurrentVersionNumber = "0.0.4" + +func GetApiVersion() string { + sdk, err := chainstoragesdk.New(&appConfig) + if err != nil { + fmt.Fprintln(os.Stderr, "get api version fail, error:%+v\n", err) + os.Exit(1) + } + + response, err := sdk.GetApiVersion() + if err != nil { + fmt.Fprintln(os.Stderr, "get api version fail, error:%+v\n", err) + os.Exit(1) + } + + return response.Data.Version +} + +type VersionInfo struct { + Version string + ApiVersion string + System string + Golang string +} + +func GetVersionInfo() *VersionInfo { + return &VersionInfo{ + ApiVersion: GetApiVersion(), + Version: CurrentVersionNumber, + System: runtime.GOARCH + "/" + runtime.GOOS, + Golang: runtime.Version(), + } +} diff --git a/cmd/versionCmd.go b/cmd/versionCmd.go index 2de4c08..ae00d20 100644 --- a/cmd/versionCmd.go +++ b/cmd/versionCmd.go @@ -32,7 +32,7 @@ var versionCmd = &cobra.Command{ Long: `Show IPFS version information`, Run: func(cmd *cobra.Command, args []string) { //fmt.Println("version called") - ipfsVersionRun(cmd, args) + versionRun(cmd, args) }, } diff --git a/config.toml.sample b/config.toml.sample index 279b339..65c22c6 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -1,7 +1,6 @@ [cli] ipfsGateway = "chainstorage-gateway.solarfs.io/ipfs/" -ggcscmdPath = '' -useHttpsProtocol = "https" +useHttpsProtocol = true bucketPrefix = 'cs://' listOffset = 20 cleanTmpData = true @@ -11,17 +10,11 @@ retryDelay = 3 [sdk] defaultRegion = 'hk-1' timeZone = 'UTC +08:00' -#链存服务API地址 chainStorageApiEndpoint = 'chainstorage-api.solarfs.io' -#CAR文件工作目录 +useHttpsProtocol = true carFileWorkPath = './tmp/carfile' -#CAR文件分片阈值 -carFileShardingThreshold = 1048576 -#链存服务API token chainStorageApiToken = '' -#HTTP request user agent (K2请求需要) httpRequestUserAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36' -#HTTP request overtime httpRequestOvertime = 30 carVersion = 1 diff --git a/go.mod b/go.mod index 4a3ecda..cc448ea 100644 --- a/go.mod +++ b/go.mod @@ -115,8 +115,7 @@ replace github.com/go-resty/resty => gopkg.in/resty.v1 v1.11.0 require ( github.com/Code-Hex/pget v0.1.1 github.com/cheggaaa/pb/v3 v3.0.8 - //github.com/paradeum-team/chainstorage-sdk/sdk v0.0.0-20230522083708-3022df5c4d3c - github.com/paradeum-team/chainstorage-sdk/sdk v0.0.0-20230606085429-eeaa8c4e8d0a + github.com/paradeum-team/chainstorage-sdk v0.0.6 ) -//replace github.com/paradeum-team/chainstorage-sdk/sdk => /Users/yuan/code/chainstorage-sdk/sdk +//replace github.com/paradeum-team/chainstorage-sdk => /Users/yuan/code/chainstorage-sdk diff --git a/go.sum b/go.sum index 9f72a21..38fecb6 100644 --- a/go.sum +++ b/go.sum @@ -900,8 +900,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/paradeum-team/chainstorage-sdk/sdk v0.0.0-20230606085429-eeaa8c4e8d0a h1:uXd+oe9O3j4CEWaO28RpbCLWJ08MZS2H/GwcyRQuSVM= -github.com/paradeum-team/chainstorage-sdk/sdk v0.0.0-20230606085429-eeaa8c4e8d0a/go.mod h1:kulW+ws77DF2ZTw6X+zVtA74FWzLIkQtKZsAYYv6RZM= +github.com/paradeum-team/chainstorage-sdk v0.0.6 h1:Vaj5PhXqS5TpTzRngu/aUFZOfxrSCVVNDLqhs9FsMfQ= +github.com/paradeum-team/chainstorage-sdk v0.0.6/go.mod h1:xauTpwK3qyNv1wfmoVp7IDilwdSPtfcwMWrlMlkHY40= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=