Merge branch 'feat/m12' into 'main' (merge request !73)

averiewang · averiewang · commit 1d340f078a76 · 2025-01-20T14:51:05.000Z
feat: failIndexedDocumentSets and collection/collectionView count
* feat: support the user interface to grant or revoke privileges
* feat: support to sort the data by the sortRules firstly, when quering documents
* feat: return [FailIndexedDocumentSets] in describing collectionView's response
* feat: return the number of collections/collectionViews in listing database's response
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,14 @@
 # Changelog
 
+## v1.6.0
+* feat: support the user interface to grant or revoke privileges
+* feat: support to sort the data by the sortRules firstly, when quering documents
+* feat: return [FailIndexedDocumentSets] in describing collectionView's response
+* feat: return the number of collections/collectionViews in listing database's response 
+
 ## v1.5.5
 * refactor: update default stopwords file
+
 ## v1.5.4
 * feat: support search radius for searching in collection
  
diff --git a/example/demo/main.go b/example/demo/main.go
@@ -233,8 +233,8 @@ func (d *Demo) QueryData(ctx context.Context, database, collection string) error
 		Filter:         filter,
 		RetrieveVector: true,
 		OutputFields:   outputField,
-		Limit:          2,
-		Offset:         1,
+		Limit:          5,
+		Offset:         0,
 		Sort: []document.SortRule{
 			{
 				FieldName: "page",
diff --git a/tcvectordb/ai_collection_view.go b/tcvectordb/ai_collection_view.go
@@ -65,7 +65,8 @@ type AICollectionViewInterface interface {
 //     for splitting document chunks. See [SplitterPreprocess] for more information.
 //   - ParsingProcess: A pointer to a [ParsingProcess] object, which includes the parameters
 //     for parsing files. See [ParsingProcess] for more information.
-//   - IndexedDocumentSets: The number of documentSets that have been processed.
+//   - FailIndexedDocumentSets: The number of documentSets that failed to be processed.
+//   - IndexedDocumentSets: The number of documentSets that have been processed successfully.
 //   - TotalDocumentSets: The total number of documentSets in this collectionView.
 //   - UnIndexedDocumentSets: The number of documentSets that haven't been processed.
 //   - FilterIndexes: A [Indexes] object that includes a list of the scalar filter index properties for the documentSets in a collectionView.
@@ -79,6 +80,7 @@ type AICollectionView struct {
 	Embedding               *collection_view.DocumentEmbedding  `json:"embedding"`
 	SplitterPreprocess      *collection_view.SplitterPreprocess `json:"splitterPreprocess"`
 	ParsingProcess          *api.ParsingProcess                 `json:"parsingProcess"`
+	FailIndexedDocumentSets uint64                              `json:"failIndexedDocumentSets"`
 	IndexedDocumentSets     uint64                              `json:"indexedDocumentSets"`
 	TotalDocumentSets       uint64                              `json:"totalDocumentSets"`
 	UnIndexedDocumentSets   uint64                              `json:"unIndexedDocumentSets"`
@@ -368,6 +370,7 @@ func (i *implementerCollectionView) toCollectionView(item *collection_view.Descr
 	}
 
 	if item.Status != nil {
+		coll.FailIndexedDocumentSets = item.Status.FailIndexedDocumentSets
 		coll.IndexedDocumentSets = item.Status.IndexedDocumentSets
 		coll.TotalDocumentSets = item.Status.TotalDocumentSets
 		coll.UnIndexedDocumentSets = item.Status.UnIndexedDocumentSets
diff --git a/tcvectordb/api/collection_view/api.go b/tcvectordb/api/collection_view/api.go
@@ -87,9 +87,10 @@ type DescribeCollectionViewItem struct {
 }
 
 type Status struct {
-	IndexedDocumentSets   uint64 `json:"indexedDocumentSets"`
-	TotalDocumentSets     uint64 `json:"totalDocumentSets"`
-	UnIndexedDocumentSets uint64 `json:"unIndexedDocumentSets"`
+	FailIndexedDocumentSets uint64 `json:"failIndexedDocumentSets"`
+	IndexedDocumentSets     uint64 `json:"indexedDocumentSets"`
+	TotalDocumentSets       uint64 `json:"totalDocumentSets"`
+	UnIndexedDocumentSets   uint64 `json:"unIndexedDocumentSets"`
 }
 
 // DropReq delete collectionView request
diff --git a/tcvectordb/api/database/api.go b/tcvectordb/api/database/api.go
@@ -68,4 +68,5 @@ type ListRes struct {
 type DatabaseInfo struct {
 	CreateTime string `json:"createTime,omitempty"`
 	DbType     string `json:"dbType,omitempty"`
+	Count      int64  `json:"count"`
 }
diff --git a/tcvectordb/base_database.go b/tcvectordb/base_database.go
@@ -263,11 +263,13 @@ func (i *implementerDatabase) ListDatabase(ctx context.Context) (result *ListDat
 		if res.Info[v].DbType == AIDOCDbType || res.Info[v].DbType == DbTypeAI {
 			db := i.AIDatabase(v)
 			db.Info.CreateTime = res.Info[v].CreateTime
+			db.Info.Count = res.Info[v].Count
 			result.AIDatabases = append(result.AIDatabases, *db)
 		} else {
 			db := i.Database(v)
 			db.Info.CreateTime = res.Info[v].CreateTime
 			db.Info.DbType = res.Info[v].DbType
+			db.Info.Count = res.Info[v].Count
 			result.Databases = append(result.Databases, *db)
 		}
 	}
@@ -353,6 +355,7 @@ func (d Database) IsAIDatabase() bool {
 type DatabaseItem struct {
 	CreateTime string `json:"createTime,omitempty"`
 	DbType     string `json:"dbType,omitempty"`
+	Count      int64  `json:"count"`
 }
 
 // Debug sets the debug mode for the SdkClient, which prints the request and response of a network call.
diff --git a/tcvectordb/olama/olama.pb.go b/tcvectordb/olama/olama.pb.go
diff --git a/tcvectordb/olama/olama.proto b/tcvectordb/olama/olama.proto
@@ -79,8 +79,9 @@ enum DataType {
 }
 
 message DatabaseItem {
-    int64 create_time   = 1;
+    int64 create_time = 1;
     DataType db_type  = 2;
+    int64 count       = 3;
 }
 
 enum IndexMetricType {
diff --git a/tcvectordb/olama/olama_grpc.pb.go b/tcvectordb/olama/olama_grpc.pb.go
diff --git a/tcvectordb/rpc_base_database.go b/tcvectordb/rpc_base_database.go
@@ -133,11 +133,13 @@ func (r *rpcImplementerDatabase) ListDatabase(ctx context.Context) (result *List
 		if res.Info[v].DbType == olama.DataType_AI_DOC {
 			db := r.AIDatabase(v)
 			db.Info.CreateTime = strconv.FormatInt(res.Info[v].CreateTime, 10)
+			db.Info.Count = int64(res.Info[v].Count)
 			result.AIDatabases = append(result.AIDatabases, *db)
 		} else {
 			db := r.Database(v)
 			db.Info.CreateTime = strconv.FormatInt(res.Info[v].CreateTime, 10)
 			db.Info.DbType = ConvertDbType(res.Info[v].DbType)
+			db.Info.Count = int64(res.Info[v].Count)
 			result.Databases = append(result.Databases, *db)
 		}
 	}
diff --git a/tcvectordb/version.go b/tcvectordb/version.go
@@ -18,4 +18,4 @@
 
 package tcvectordb
 
-const SDKVersion = "v1.5.5"
+const SDKVersion = "v1.6.0"
diff --git a/test/normal_test.go b/test/normal_test.go
@@ -58,12 +58,14 @@ func TestListDatabase(t *testing.T) {
 	printErr(err)
 	log.Printf("base database ======================")
 	for _, db := range dbList.Databases {
-		log.Printf("database: %s, createTime: %s, dbType: %s", db.DatabaseName, db.Info.CreateTime, db.Info.DbType)
+		log.Printf("database: %s, createTime: %s, dbType: %s, collectionNum: %v", db.DatabaseName,
+			db.Info.CreateTime, db.Info.DbType, db.Info.Count)
 	}
 
 	log.Printf("AI database ======================")
 	for _, db := range dbList.AIDatabases {
-		log.Printf("database: %s, createTime: %s, dbType: %s", db.DatabaseName, db.Info.CreateTime, db.Info.DbType)
+		log.Printf("database: %s, createTime: %s, dbType: %s, collectionViewNum: %v", db.DatabaseName,
+			db.Info.CreateTime, db.Info.DbType, db.Info.Count)
 	}
 }
 
diff --git a/test/user_test.go b/test/user_test.go
@@ -110,17 +110,18 @@ func Test_User(t *testing.T) {
 	printErr(err)
 	println(ToJson(res))
 
-	cli, err := tcvectordb.NewRpcClient("http://gz-vdb-jco3bdic.sql.tencentcdb.com:8100", "test_user_1",
+	newCli, err := tcvectordb.NewRpcClient("http://xx",
+		"test_user_1",
 		"123..A.!", &tcvectordb.ClientOption{Timeout: 10 * time.Second,
 			ReadConsistency: tcvectordb.StrongConsistency})
 	printErr(err)
-	cli.Debug(true)
+	newCli.Debug(true)
 
-	collRes, err := cli.Database(database).ListCollection(ctx)
+	collRes, err := newCli.Database(database).ListCollection(ctx)
 	printErr(err)
 	println(ToJson(collRes))
 
-	collRes, err = cli.Database(database + "1").ListCollection(ctx)
+	collRes, err = newCli.Database(database + "1").ListCollection(ctx)
 	printErr(err)
 	println(ToJson(collRes))
 

Original file line number	Diff line number	Diff line change
`@@ -87,9 +87,10 @@ type DescribeCollectionViewItem struct {`
`87`	`87`	`}`
`88`	`88`
`89`	`89`	`type Status struct {`
`90`		- IndexedDocumentSets uint64 `json:"indexedDocumentSets"`
`91`		- TotalDocumentSets uint64 `json:"totalDocumentSets"`
`92`		- UnIndexedDocumentSets uint64 `json:"unIndexedDocumentSets"`
	`90`	+ FailIndexedDocumentSets uint64 `json:"failIndexedDocumentSets"`
	`91`	+ IndexedDocumentSets uint64 `json:"indexedDocumentSets"`
	`92`	+ TotalDocumentSets uint64 `json:"totalDocumentSets"`
	`93`	+ UnIndexedDocumentSets uint64 `json:"unIndexedDocumentSets"`
`93`	`94`	`}`
`94`	`95`
`95`	`96`	`// DropReq delete collectionView request`
Original file line number	Diff line number	Diff line change
`@@ -68,4 +68,5 @@ type ListRes struct {`
`68`	`68`	`type DatabaseInfo struct {`
`69`	`69`	CreateTime string `json:"createTime,omitempty"`
`70`	`70`	DbType string `json:"dbType,omitempty"`
	`71`	+ Count int64 `json:"count"`
`71`	`72`	`}`
Original file line number	Diff line number	Diff line change
`@@ -79,8 +79,9 @@ enum DataType {`
`79`	`79`	`}`
`80`	`80`
`81`	`81`	`message DatabaseItem {`
`82`		`- int64 create_time = 1;`
	`82`	`+ int64 create_time = 1;`
`83`	`83`	`DataType db_type = 2;`
	`84`	`+ int64 count = 3;`
`84`	`85`	`}`
`85`	`86`
`86`	`87`	`enum IndexMetricType {`
Original file line number	Diff line number	Diff line change
`@@ -133,11 +133,13 @@ func (r rpcImplementerDatabase) ListDatabase(ctx context.Context) (result List`
`133`	`133`	`if res.Info[v].DbType == olama.DataType_AI_DOC {`
`134`	`134`	`db := r.AIDatabase(v)`
`135`	`135`	`db.Info.CreateTime = strconv.FormatInt(res.Info[v].CreateTime, 10)`
	`136`	`+ db.Info.Count = int64(res.Info[v].Count)`
`136`	`137`	`result.AIDatabases = append(result.AIDatabases, *db)`
`137`	`138`	`} else {`
`138`	`139`	`db := r.Database(v)`
`139`	`140`	`db.Info.CreateTime = strconv.FormatInt(res.Info[v].CreateTime, 10)`
`140`	`141`	`db.Info.DbType = ConvertDbType(res.Info[v].DbType)`
	`142`	`+ db.Info.Count = int64(res.Info[v].Count)`
`141`	`143`	`result.Databases = append(result.Databases, *db)`
`142`	`144`	`}`
`143`	`145`	`}`
Original file line number	Diff line number	Diff line change
`@@ -18,4 +18,4 @@`
`18`	`18`
`19`	`19`	`package tcvectordb`
`20`	`20`
`21`		`-const SDKVersion = "v1.5.5"`
	`21`	`+const SDKVersion = "v1.6.0"`
Original file line number	Diff line number	Diff line change
`@@ -58,12 +58,14 @@ func TestListDatabase(t *testing.T) {`
`58`	`58`	`printErr(err)`
`59`	`59`	`log.Printf("base database ======================")`
`60`	`60`	`for _, db := range dbList.Databases {`
`61`		`- log.Printf("database: %s, createTime: %s, dbType: %s", db.DatabaseName, db.Info.CreateTime, db.Info.DbType)`
	`61`	`+ log.Printf("database: %s, createTime: %s, dbType: %s, collectionNum: %v", db.DatabaseName,`
	`62`	`+ db.Info.CreateTime, db.Info.DbType, db.Info.Count)`
`62`	`63`	`}`
`63`	`64`
`64`	`65`	`log.Printf("AI database ======================")`
`65`	`66`	`for _, db := range dbList.AIDatabases {`
`66`		`- log.Printf("database: %s, createTime: %s, dbType: %s", db.DatabaseName, db.Info.CreateTime, db.Info.DbType)`
	`67`	`+ log.Printf("database: %s, createTime: %s, dbType: %s, collectionViewNum: %v", db.DatabaseName,`
	`68`	`+ db.Info.CreateTime, db.Info.DbType, db.Info.Count)`
`67`	`69`	`}`
`68`	`70`	`}`
`69`	`71`