Merge branch 'feat/averiewang/encoder-init' into 'main' (merge request !77)

feat: tcvdbtext add [NewBM25EncoderByFiles] functions to init encoder

Author: averiewang

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## v1.6.3
+* feat: tcvdbtext add [NewBM25EncoderByFiles] functions to init encoder  in an offline environment without internet access
+
 ## v1.6.2
 * fix: upload file by io.Reader
 

tcvdbtext/README.md

@@ -0,0 +1,24 @@
+# Tencent VectorDB Sparse Encoder SDK
+
+Go SDK for [Tencent VectorDB Sparse Encoder](https://cloud.tencent.com/document/product/1709/111372).
+
+## Getting started
+
+### Prerequisites
+1. Go 1.17 or higher
+
+### Install TencentCloud VectorDB Go SDK
+
+1. Use `go get` to install the latest version of the TencentCloud VectorDB Sparse Encoder SDK dependencies: 
+```sh
+go get -u github.com/tencent/vectordatabase-sdk-go/tcvdbtext
+```
+
+2. Try [sparse_vector_demo](examples/sparse_vector_demo/main.go) in an online environment with internet access.
+
+3. Try [sparse_vector_offline_demo](examples/sparse_vector_offline_demo/main.go) in an offline environment without internet access. 
+Before running the code, please download files which you need.
+
+    - [Chinese Words Frequency File](https://vectordb-public-1310738255.cos.ap-guangzhou.myqcloud.com/sparsevector/bm25_zh_default.json)
+    - [English Words Frequency File](https://vectordb-public-1310738255.cos.ap-guangzhou.myqcloud.com/sparsevector/bm25_en_default.json)
+    - [Default Stopwords File](https://vectordb-public-1310738255.cos.ap-guangzhou.myqcloud.com/sparsevector/default_stopwords.txt)

tcvdbtext/encoder/bm25_encoder.go

@@ -37,6 +37,18 @@ type BM25EncoderParams struct {
 	Bm25Language string
 }
 
+// [BM25EncoderFileParams] holds the parameters for initing bm25 encoder by local files.
+//
+// Fields:
+//   - WordsFreqFile: The local file path of the words frequency.
+//   - StopWordsFile: The local file path of the stopwords.
+//   - UserDictFile: The local file path of the user define dictionary.
+type BM25EncoderFileParams struct {
+	WordsFreqFile string
+	StopWordsFile string
+	UserDictFile  string
+}
+
 type BM25LearnedParams struct {
 	TokenFreq        map[string]float64 `json:"token_freq,omitempty"`
 	DocCount         int64              `json:"doc_count,omitempty"`
@@ -84,6 +96,63 @@ func NewBM25Encoder(params *BM25EncoderParams) (SparseEncoder, error) {
 	return bm25, nil
 }
 
+func NewBM25EncoderByFiles(params *BM25EncoderFileParams) (SparseEncoder, error) {
+	bm25 := new(BM25Encoder)
+	var stopWords interface{}
+	if params.StopWordsFile == "" {
+		stopWords = false
+	} else {
+		stopWords = params.StopWordsFile
+	}
+	JiebaTokenizer, err := tokenizer.NewJiebaTokenizer(&tokenizer.TokenizerParams{
+		StopWords:        stopWords,
+		UserDictFilePath: params.UserDictFile,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	bm25.Tokenizer = JiebaTokenizer
+
+	if params.WordsFreqFile == "" {
+		return bm25, nil
+	}
+
+	var data []byte
+	if !tcvdbtext.FileExists(params.WordsFreqFile) {
+		return nil, fmt.Errorf("the filepath %v doesn't exist", params.WordsFreqFile)
+	} else {
+		data, err = os.ReadFile(params.WordsFreqFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read file: %v", err)
+		}
+	}
+
+	bm25ParamsByFile := new(BM25Params)
+	err = json.Unmarshal(data, bm25ParamsByFile)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse file %v to JSON, err: %v", params.WordsFreqFile, err.Error())
+	}
+
+	bm25.B = *bm25ParamsByFile.B
+	bm25.K1 = *bm25ParamsByFile.K1
+	bm25.BM25LearnedParams = bm25ParamsByFile.BM25LearnedParams
+
+	err = bm25.Tokenizer.UpdateParameters(tokenizer.TokenizerParams{
+		ForSearch: bm25ParamsByFile.ForSearch,
+		CutAll:    bm25ParamsByFile.CutAll,
+		Hmm:       bm25ParamsByFile.Hmm,
+
+		HashFunction: bm25ParamsByFile.HashFunction,
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("update parameters by file %v failed, err: %v", params.WordsFreqFile, err.Error())
+	}
+
+	return bm25, nil
+}
+
 func (bm25 *BM25Encoder) GetTokenizer() tokenizer.Tokenizer {
 	return bm25.Tokenizer
 }

tcvdbtext/examples/sparse_vector_demo/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
+)
+
+func main() {
+	bm25, err := encoder.NewBM25Encoder(&encoder.BM25EncoderParams{Bm25Language: "zh"})
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	text := "什么是腾讯云向量数据库。"
+
+	// 如需了解分词的情况，可参考下一行代码获取
+	tokens := bm25.GetTokenizer().Tokenize(text)
+	fmt.Println("tokens: ", tokens)
+
+	sparse_vectors, err := bm25.EncodeText(text)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+	fmt.Println("sparse vectors: ", sparse_vectors)
+}

tcvdbtext/examples/sparse_vector_offline_demo/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
+)
+
+func main() {
+	bm25, err := encoder.NewBM25EncoderByFiles(&encoder.BM25EncoderFileParams{
+		StopWordsFile: "./stopwords.txt",
+		//WordsFreqFile: "./bm25_zh_default.json",
+		UserDictFile: "",
+	})
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	text := "什么是腾讯云向量数据库。"
+
+	// 如需了解分词的情况，可参考下一行代码获取
+	tokens := bm25.GetTokenizer().Tokenize(text)
+	fmt.Println("tokens: ", tokens)
+
+	// [EncodeText] can be used after set WordsFreqFile in [NewBM25EncoderByFiles]
+	// sparse_vectors, err := bm25.EncodeText(text)
+	// if err != nil {
+	// 	log.Fatalf(err.Error())
+	// }
+	// fmt.Println("sparse vectors: ", sparse_vectors)
+}