[Fix](analyzer) add ownership flag to Field's TokenStream value and A…

…nalyzer's Reader
apache · Jun 11, 2024 · 5422ac1 · 5422ac1
1 parent 1936207
commit 5422ac1
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 9 deletions.
diff --git a/src/core/CLucene/analysis/AnalysisHeader.h b/src/core/CLucene/analysis/AnalysisHeader.h
@@ -10,6 +10,7 @@
 #include "CLucene/index/Payload.h"
 #include "CLucene/util/VoidList.h"
 #include "CLucene/LuceneThreads.h"
+#include "CLucene/util/CLStreams.h"
 
 #include <unordered_set>
 
@@ -304,6 +305,10 @@ class CLUCENE_EXPORT Analyzer{
         _stopwords = stopwords;
     }
 
+    virtual void set_ownReader(bool ownReader) {
+        _ownReader = ownReader;
+    }
+
 private:
 
     DEFINE_MUTEX(THIS_LOCK)
@@ -322,6 +327,7 @@ class CLUCENE_EXPORT Analyzer{
     virtual void setPreviousTokenStream(TokenStream* obj);
 
     bool _lowercase = false;
+    bool _ownReader = false;
     std::unordered_set<std::string_view>* _stopwords = nullptr;
 
 public:
@@ -359,19 +365,23 @@ class CLUCENE_EXPORT Tokenizer:public TokenStream {
     /** The text source for this Tokenizer. */
     CL_NS(util)::Reader* input;
     bool lowercase = false;
+    bool ownReader = false;
     std::unordered_set<std::string_view>* stopwords = nullptr;
 
 public:
     /** Construct a tokenizer with null input. */
     Tokenizer():input(nullptr){}
     /** Construct a token stream processing the given input. */
-    explicit Tokenizer(CL_NS(util)::Reader* _input):input(_input){}
+    explicit Tokenizer(CL_NS(util)::Reader* _input, bool _ownReader = false):input(_input), ownReader(_ownReader){}
 
     /** By default, closes the input Reader. */
     virtual void close() {
         if (input != NULL) {
-            // ? delete input;
-            input = NULL;
+            if (ownReader) {
+                _CLDELETE(input);
+            } else {
+                input = NULL;
+            }
         }
     };
 

diff --git a/src/core/CLucene/analysis/Analyzers.cpp b/src/core/CLucene/analysis/Analyzers.cpp
@@ -51,11 +51,13 @@ template class LowerCaseTokenizer<TCHAR>;
 template<typename T>
 SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in) : LowerCaseTokenizer<T>(in) {
     Tokenizer::lowercase = true;
+    Tokenizer::ownReader = false;
 }
 
 template<typename T>
-SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in, bool lowercase) : LowerCaseTokenizer<T>(in) {
+SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in, bool lowercase, bool ownReader) : LowerCaseTokenizer<T>(in) {
     Tokenizer::lowercase = lowercase;
+    Tokenizer::ownReader = ownReader;
 }
 
 template<typename T>

diff --git a/src/core/CLucene/analysis/Analyzers.h b/src/core/CLucene/analysis/Analyzers.h
@@ -140,7 +140,7 @@ class CLUCENE_EXPORT SimpleTokenizer:public LowerCaseTokenizer<T> {
 public:
     /** Construct a new SimpleTokenizer. */
     explicit SimpleTokenizer(CL_NS(util)::Reader* in);
-    SimpleTokenizer(CL_NS(util)::Reader* in, bool lowercase);
+    SimpleTokenizer(CL_NS(util)::Reader* in, bool lowercase, bool ownReader = false);
     virtual ~SimpleTokenizer();
 
     Token* next(Token* token) override {
@@ -182,12 +182,13 @@ class CLUCENE_EXPORT SimpleAnalyzer: public Analyzer {
 public:
     SimpleAnalyzer(){
         _lowercase = true;
+        _ownReader = false;
     }
 
     bool isSDocOpt() override { return true; }
 
     TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) override{
-        return _CLNEW SimpleTokenizer<T>(reader, _lowercase);
+        return _CLNEW SimpleTokenizer<T>(reader, _lowercase, _ownReader);
     }
     TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) override{
         if (tokenizer_ == nullptr) {

diff --git a/src/core/CLucene/document/Field.cpp b/src/core/CLucene/document/Field.cpp
@@ -176,7 +176,8 @@ void Field::setValue(ValueArray<uint8_t>* value) {
 }
 
 /** Expert: change the value of this field.  See <a href="#setValue(java.lang.String)">setValue(String)</a>. */
-void Field::setValue(CL_NS(analysis)::TokenStream* value) {
+void Field::setValue(CL_NS(analysis)::TokenStream* value, bool own_stream) {
+    ownStream = own_stream;
 	_resetValue();
     fieldsData = value;
 	valueType = VALUE_TOKENSTREAM;
@@ -340,7 +341,10 @@ void Field::_resetValue() {
 	} else if (valueType & VALUE_BINARY) {
 		ValueArray<uint8_t>* v = static_cast<ValueArray<uint8_t>*>(fieldsData);
 		_CLDELETE(v);
-	}
+	} else if (valueType & VALUE_TOKENSTREAM && ownStream) {
+        auto* v = static_cast<CL_NS(analysis)::TokenStream*>(fieldsData);
+        _CLDELETE(v);
+    }
 	valueType=VALUE_NONE;
 }
 const char* Field::getObjectName() const{

diff --git a/src/core/CLucene/document/Field.h b/src/core/CLucene/document/Field.h
@@ -305,7 +305,7 @@ class CLUCENE_EXPORT Field : public CL_NS(util)::NamedObject{
 	void setValue(CL_NS(util)::ValueArray<uint8_t>* value) ;
 
 	/** Expert: change the value of this field.  See <a href="#setValue(TCHAR*)">setValue(TCHAR*)</a>. */
-	void setValue(CL_NS(analysis)::TokenStream* value);
+	void setValue(CL_NS(analysis)::TokenStream* value, bool own_stream = false);
 
     //void setValue(CL_NS(analysis)::STokenStream* value);
 
@@ -334,6 +334,7 @@ class CLUCENE_EXPORT Field : public CL_NS(util)::NamedObject{
 	float_t boost;
 
 	IndexVersion indexVersion_ = IndexVersion::kV1;
+    bool ownStream = false;
 };
 CL_NS_END
 #endif