Merge branch 'feat/averiewang/index-interface-new' into 'main' (merge request !80)

feat/averiewang/index-interface-new
feat: get stopwords from cos

Author: averiewang

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.7.0
+* feat: support /index/drop interface
+* feat: /document/count support config the ReadConsistency param
+
 ## v1.6.5
 * feat: get stopwords from cos
 

example/add_and_drop_index/main.go

@@ -0,0 +1,283 @@
+package main
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
+)
+
+type Demo struct {
+	//client *tcvectordb.Client
+	client *tcvectordb.RpcClient
+}
+
+func NewDemo(url, username, key string) (*Demo, error) {
+	// cli, err := tcvectordb.NewClient(url, username, key, &tcvectordb.ClientOption{
+	// 	ReadConsistency: tcvectordb.EventualConsistency})
+	cli, err := tcvectordb.NewRpcClient(url, username, key, &tcvectordb.ClientOption{
+		ReadConsistency: tcvectordb.StrongConsistency})
+	if err != nil {
+		return nil, err
+	}
+	// disable/enable http request log print
+	// cli.Debug(false)
+	return &Demo{client: cli}, nil
+}
+
+func (d *Demo) Clear(ctx context.Context, database string) error {
+	log.Println("--------------------------- DropDatabase ---------------------------")
+	result, err := d.client.DropDatabase(ctx, database)
+	if err != nil {
+		return err
+	}
+	log.Printf("drop database result: %+v", result)
+	return nil
+}
+
+func (d *Demo) DeleteAndDrop(ctx context.Context, database, collection string) error {
+	// 删除collection，删除collection的同时，其中的数据也将被全部删除
+	log.Println("-------------------------- DropCollection --------------------------")
+	colDropResult, err := d.client.Database(database).DropCollection(ctx, collection)
+	if err != nil {
+		return err
+	}
+	log.Printf("drop collection result: %+v", colDropResult)
+
+	log.Println("--------------------------- DropDatabase ---------------------------")
+	// 删除db，db下的所有collection都将被删除
+	dbDropResult, err := d.client.DropDatabase(ctx, database)
+	if err != nil {
+		return err
+	}
+	log.Printf("drop database result: %+v", dbDropResult)
+	return nil
+}
+
+func (d *Demo) CreateDBAndCollection(ctx context.Context, database, collection, alias string) error {
+	// 创建DB--'book'
+	log.Println("-------------------------- CreateDatabase --------------------------")
+	db, err := d.client.CreateDatabaseIfNotExists(ctx, database)
+	if err != nil {
+		return err
+	}
+
+	log.Println("------------------------- CreateCollection -------------------------")
+	// 创建 Collection
+
+	// 第一步，设计索引（不是设计 Collection 的结构）
+	// 1. 【重要的事】向量对应的文本字段不要建立索引，会浪费较大的内存，并且没有任何作用。
+	// 2. 【必须的索引】：主键id、向量字段 vector 这两个字段目前是固定且必须的，参考下面的例子；
+	// 3. 【其他索引】：检索时需作为条件查询的字段，比如要按书籍的作者进行过滤，这个时候 author 字段就需要建立索引，
+	//     否则无法在查询的时候对 author 字段进行过滤，不需要过滤的字段无需加索引，会浪费内存；
+	// 4.  向量数据库支持动态 Schema，写入数据时可以写入任何字段，无需提前定义，类似 MongoDB.
+	// 5.  例子中创建一个书籍片段的索引，例如书籍片段的信息包括 {id, vector, segment, bookName, author, page},
+	//     id 为主键需要全局唯一，segment 为文本片段, vector 字段需要建立向量索引，假如我们在查询的时候要查询指定书籍
+	//     名称的内容，这个时候需要对 bookName 建立索引，其他字段没有条件查询的需要，无需建立索引。
+	index := tcvectordb.Indexes{}
+	index.VectorIndex = append(index.VectorIndex, tcvectordb.VectorIndex{
+		FilterIndex: tcvectordb.FilterIndex{
+			FieldName: "vector",
+			FieldType: tcvectordb.Vector,
+			IndexType: tcvectordb.HNSW,
+		},
+		Dimension:  3,
+		MetricType: tcvectordb.IP,
+		Params: &tcvectordb.HNSWParam{
+			M:              16,
+			EfConstruction: 200,
+		},
+	})
+	index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "id", FieldType: tcvectordb.String, IndexType: tcvectordb.PRIMARY})
+	index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "bookName", FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER})
+	index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "page", FieldType: tcvectordb.Uint64, IndexType: tcvectordb.FILTER})
+
+	// 第二步：创建 Collection
+	// 创建collection耗时较长，需要调整客户端的timeout
+	// 这里以三可用区实例作为参考，具体实例不同的规格所支持的shard和replicas区间不同，需要参考官方文档
+	db.WithTimeout(time.Second * 30)
+	_, err = db.CreateCollection(ctx, collection, 3, 1, "test collection", index)
+	if err != nil {
+		return err
+	}
+
+	log.Println("------------------------ DescribeCollection ------------------------")
+	// 查看 Collection 信息
+	colRes, err := db.DescribeCollection(ctx, collection)
+	if err != nil {
+		return err
+	}
+	log.Printf("DescribeCollection: %+v", colRes)
+	return nil
+}
+
+func (d *Demo) UpsertData(ctx context.Context, database, collection string) error {
+	// 获取 Collection 对象
+	coll := d.client.Database(database).Collection(collection)
+
+	log.Println("------------------------------ Upsert ------------------------------")
+	// upsert 写入数据，可能会有一定延迟
+	// 1. 支持动态 Schema，除了 id、vector 字段必须写入，可以写入其他任意字段；
+	// 2. upsert 会执行覆盖写，若文档id已存在，则新数据会直接覆盖原有数据(删除原有数据，再插入新数据)
+
+	documentList := []tcvectordb.Document{
+		{
+			Id:     "0001",
+			Vector: []float32{0.2123, 0.21, 0.213},
+			Fields: map[string]tcvectordb.Field{
+				"bookName": {Val: "西游记"},
+				"author":   {Val: "吴承恩"},
+				"page":     {Val: 21},
+				"segment":  {Val: "富贵功名，前缘分定，为人切莫欺心。"},
+			},
+		},
+		{
+			Id:     "0002",
+			Vector: []float32{0.2123, 0.22, 0.213},
+			Fields: map[string]tcvectordb.Field{
+				"bookName": {Val: "西游记"},
+				"author":   {Val: "吴承恩"},
+				"page":     {Val: 22},
+				"segment":  {Val: "正大光明，忠良善果弥深。些些狂妄天加谴，眼前不遇待时临。"},
+			},
+		},
+		{
+			Id:     "0003",
+			Vector: []float32{0.2123, 0.23, 0.213},
+			Fields: map[string]tcvectordb.Field{
+				"bookName": {Val: "三国演义"},
+				"author":   {Val: "罗贯中"},
+				"page":     {Val: 23},
+				"segment":  {Val: "细作探知这个消息，飞报吕布。"},
+			},
+		},
+		{
+			Id:     "0004",
+			Vector: []float32{0.2123, 0.24, 0.213},
+			Fields: map[string]tcvectordb.Field{
+				"bookName": {Val: "三国演义"},
+				"author":   {Val: "罗贯中"},
+				"page":     {Val: 24},
+				"segment":  {Val: "布大惊，与陈宫商议。宫曰：“闻刘玄德新领徐州，可往投之。”布从其言，竟投徐州来。有人报知玄德。"},
+			},
+		},
+		{
+			Id:     "0005",
+			Vector: []float32{0.2123, 0.25, 0.213},
+			Fields: map[string]tcvectordb.Field{
+				"bookName": {Val: "三国演义"},
+				"author":   {Val: "罗贯中"},
+				"page":     {Val: 25},
+				"segment":  {Val: "玄德曰：“布乃当今英勇之士，可出迎之。”糜竺曰：“吕布乃虎狼之徒，不可收留；收则伤人矣。"},
+			},
+		},
+	}
+	result, err := coll.Upsert(ctx, documentList)
+	if err != nil {
+		return err
+	}
+	log.Printf("UpsertResult: %+v", result)
+	return nil
+}
+
+func (d *Demo) AddIndex(ctx context.Context, database, collection string) error {
+	log.Println("------------------------------ AddIndex ------------------------------")
+
+	addFilterIndexs := []tcvectordb.FilterIndex{
+		{FieldName: "author", FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER}}
+	buildExistedData := true
+	err := d.client.AddIndex(ctx, database, collection, &tcvectordb.AddIndexParams{
+		FilterIndexs:     addFilterIndexs,
+		BuildExistedData: &buildExistedData})
+	if err != nil {
+		return err
+	}
+
+	db := d.client.Database(database)
+	for {
+		time.Sleep(10 * time.Second)
+		colRes, err := db.DescribeCollection(ctx, collection)
+		if err != nil {
+			return err
+		}
+		if colRes.IndexStatus.Status == "ready" {
+			break
+		}
+	}
+
+	queryResult, queryErr := d.client.Query(ctx, database, collection, nil, &tcvectordb.QueryDocumentParams{
+		Filter: tcvectordb.NewFilter("author=\"罗贯中\""),
+		Limit:  100,
+	})
+	printErr(queryErr)
+	log.Printf("query by new added filter, total doc: %d, should be 3", queryResult.Total)
+	for _, doc := range queryResult.Documents {
+		log.Printf("document: %+v", doc)
+	}
+	return nil
+}
+
+func (d *Demo) DropIndex(ctx context.Context, database, collection string) error {
+	log.Println("------------------------------ DropIndex ------------------------------")
+
+	err := d.client.DropIndex(ctx, database, collection, tcvectordb.DropIndexParams{
+		FieldNames: []string{"author"}})
+	if err != nil {
+		return err
+	}
+
+	_, queryErr := d.client.Query(ctx, database, collection, nil, &tcvectordb.QueryDocumentParams{
+		Filter: tcvectordb.NewFilter("author=\"罗贯中\""),
+		Limit:  100,
+	})
+	if queryErr != nil {
+		log.Printf("query by dropped filter, queryErr: %+v", queryErr.Error())
+	}
+	return nil
+}
+
+func (d *Demo) DescribeCollection(ctx context.Context, database, collection string) error {
+	db := d.client.Database(database)
+	log.Println("------------------------ DescribeCollection ------------------------")
+	// 查看 Collection 信息
+	colRes, err := db.DescribeCollection(ctx, collection)
+	if err != nil {
+		return err
+	}
+	log.Printf("DescribeCollection: %+v", colRes)
+	log.Printf("DescribeCollection: %+v", colRes.Indexes.VectorIndex[0].Params)
+	return nil
+}
+
+func printErr(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func main() {
+	database := "go-sdk-demo-db"
+	collectionName := "go-sdk-demo-col"
+	collectionAlias := "go-sdk-demo-alias"
+
+	ctx := context.Background()
+	testVdb, err := NewDemo("vdb http url or ip and port", "vdb username", "key get from web console")
+	printErr(err)
+	err = testVdb.Clear(ctx, database)
+	printErr(err)
+	err = testVdb.CreateDBAndCollection(ctx, database, collectionName, collectionAlias)
+	printErr(err)
+
+	err = testVdb.UpsertData(ctx, database, collectionName)
+	printErr(err)
+	time.Sleep(2 * time.Second)
+
+	err = testVdb.AddIndex(ctx, database, collectionName)
+	printErr(err)
+
+	err = testVdb.DropIndex(ctx, database, collectionName)
+	printErr(err)
+	err = testVdb.DeleteAndDrop(ctx, database, collectionName)
+	printErr(err)
+}

tcvectordb/api/index/api.go

@@ -47,6 +47,17 @@ type AddRes struct {
 	api.CommonRes
 }
 
+type DropReq struct {
+	api.Meta   `path:"/index/drop" tags:"Index" method:"Post" summary:"删除collection的索引"`
+	Database   string   `json:"database,omitempty"`
+	Collection string   `json:"collection,omitempty"`
+	FieldNames []string `json:"fieldNames,omitempty"`
+}
+
+type DropRes struct {
+	api.CommonRes
+}
+
 type ModifyVectorIndexReq struct {
 	api.Meta      `path:"/index/modifyVectorIndex" tags:"Index" method:"Post" summary:"调整collection的向量索引参数"`
 	Database      string             `json:"database,omitempty"`

tcvectordb/base_document.go

@@ -970,6 +970,7 @@ func (i *implementerFlatDocument) Count(ctx context.Context, databaseName, colle
 	req := new(document.CountReq)
 	req.Database = databaseName
 	req.Collection = collectionName
+	req.ReadConsistency = string(i.SdkClient.Options().ReadConsistency)
 	req.Query = new(document.CountQueryCond)
 
 	if len(params) != 0 {

tcvectordb/base_index.go

@@ -33,6 +33,9 @@ type IndexInterface interface {
 	// [AddIndex] adds scalar field index to an existing collection.
 	AddIndex(ctx context.Context, params ...*AddIndexParams) (err error)
 
+	// [DropIndex] drops scalar field index to an existing collection.
+	DropIndex(ctx context.Context, params DropIndexParams) (err error)
+
 	// [ModifyVectorIndex] modifies vector indexes to an existing collection.
 	ModifyVectorIndex(ctx context.Context, param ModifyVectorIndexParam) (err error)
 }
@@ -78,6 +81,10 @@ func (i *implementerIndex) AddIndex(ctx context.Context, params ...*AddIndexPara
 	return i.flat.AddIndex(ctx, i.database.DatabaseName, i.collection.CollectionName, params...)
 }
 
+func (i *implementerIndex) DropIndex(ctx context.Context, params DropIndexParams) error {
+	return i.flat.DropIndex(ctx, i.database.DatabaseName, i.collection.CollectionName, params)
+}
+
 // [ModifyVectorIndex] modifies vector indexes to an existing collection.
 func (i *implementerIndex) ModifyVectorIndex(ctx context.Context, param ModifyVectorIndexParam) error {
 	return i.flat.ModifyVectorIndex(ctx, i.database.DatabaseName, i.collection.CollectionName, param)

tcvectordb/base_index_flat.go

@@ -19,6 +19,9 @@ type FlatIndexInterface interface {
 	// [AddIndex] adds scalar field index to an existing collection.
 	AddIndex(ctx context.Context, databaseName, collectionName string, params ...*AddIndexParams) (err error)
 
+	// [DropIndex] drops scalar field index to an existing collection.
+	DropIndex(ctx context.Context, databaseName, collectionName string, params DropIndexParams) (err error)
+
 	// [ModifyVectorIndex] modifies vector indexes to an existing collection.
 	ModifyVectorIndex(ctx context.Context, databaseName, collectionName string, param ModifyVectorIndexParam) (err error)
 }
@@ -139,6 +142,20 @@ func (i *implementerFlatIndex) AddIndex(ctx context.Context, databaseName, colle
 	return nil
 }
 
+type DropIndexParams struct {
+	FieldNames []string
+}
+
+func (i *implementerFlatIndex) DropIndex(ctx context.Context, databaseName, collectionName string, params DropIndexParams) error {
+	req := new(index.DropReq)
+	req.Database = databaseName
+	req.Collection = collectionName
+	req.FieldNames = params.FieldNames
+	res := new(index.DropRes)
+	err := i.Request(ctx, req, res)
+	return err
+}
+
 // [ModifyVectorIndex] modifies vector indexes to an existing collection.
 func (i *implementerFlatIndex) ModifyVectorIndex(ctx context.Context, databaseName, collectionName string, param ModifyVectorIndexParam) error {
 	req := new(index.ModifyVectorIndexReq)