[Pick 2.0](analyzer) add reader ownership for chinese and standard analyzer

src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp

@@ -121,7 +121,7 @@ TokenStream *LanguageBasedAnalyzer::tokenStream(const TCHAR *fieldName, Reader *
     if (_tcscmp(lang, _T("cjk")) == 0) {
         ret = _CLNEW CL_NS2(analysis, cjk)::CJKTokenizer(reader);
     } else if (_tcscmp(lang, _T("chinese")) == 0) {
-        ret = _CLNEW CL_NS2(analysis, jieba)::ChineseTokenizer(reader, mode, Analyzer::_lowercase);
+        ret = _CLNEW CL_NS2(analysis, jieba)::ChineseTokenizer(reader, mode, Analyzer::_lowercase, Analyzer::_ownReader);
     } else {
         CL_NS(util)::BufferedReader* bufferedReader = reader->__asBufferedReader();
 

src/contribs-lib/CLucene/analysis/jieba/ChineseTokenizer.cpp

@@ -10,11 +10,13 @@ CL_NS_USE(util)
 ChineseTokenizer::ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode m) : Tokenizer(reader), mode(m) {
     reset(reader);
     Tokenizer::lowercase = false;
+    Tokenizer::ownReader = false;
 }
 
-ChineseTokenizer::ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode m, bool lowercase) : Tokenizer(reader), mode(m) {
+ChineseTokenizer::ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode m, bool lowercase, bool ownReader) : Tokenizer(reader), mode(m) {
     reset(reader);
     Tokenizer::lowercase = lowercase;
+    Tokenizer::ownReader = ownReader;
 }
 
 void ChineseTokenizer::init(const ChineseDict* chineseDict) {

src/contribs-lib/CLucene/analysis/jieba/ChineseTokenizer.h

@@ -56,7 +56,7 @@ class ChineseTokenizer : public lucene::analysis::Tokenizer {
 public:
     // Constructor
     explicit ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode mode);
-    explicit ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode mode, bool lowercase);
+    explicit ChineseTokenizer(lucene::util::Reader *reader, AnalyzerMode mode, bool lowercase, bool ownReader=false);
     static void init(const ChineseDict* chineseDict);
 
     // Destructor

src/core/CLucene/analysis/AnalysisHeader.h

@@ -10,6 +10,7 @@
 #include "CLucene/index/Payload.h"
 #include "CLucene/util/VoidList.h"
 #include "CLucene/LuceneThreads.h"
+#include "CLucene/util/CLStreams.h"
 
 #include <unordered_set>
 
@@ -304,6 +305,10 @@ class CLUCENE_EXPORT Analyzer{
         _stopwords = stopwords;
     }
 
+    virtual void set_ownReader(bool ownReader) {
+        _ownReader = ownReader;
+    }
+
 private:
 
     DEFINE_MUTEX(THIS_LOCK)
@@ -322,6 +327,7 @@ class CLUCENE_EXPORT Analyzer{
     virtual void setPreviousTokenStream(TokenStream* obj);
 
     bool _lowercase = false;
+    bool _ownReader = false;
     std::unordered_set<std::string_view>* _stopwords = nullptr;
 
 public:
@@ -359,19 +365,23 @@ class CLUCENE_EXPORT Tokenizer:public TokenStream {
     /** The text source for this Tokenizer. */
     CL_NS(util)::Reader* input;
     bool lowercase = false;
+    bool ownReader = false;
     std::unordered_set<std::string_view>* stopwords = nullptr;
 
 public:
     /** Construct a tokenizer with null input. */
     Tokenizer():input(nullptr){}
     /** Construct a token stream processing the given input. */
-    explicit Tokenizer(CL_NS(util)::Reader* _input):input(_input){}
+    explicit Tokenizer(CL_NS(util)::Reader* _input, bool _ownReader = false):input(_input), ownReader(_ownReader){}
 
     /** By default, closes the input Reader. */
     virtual void close() {
         if (input != NULL) {
-            // ? delete input;
-            input = NULL;
+            if (ownReader) {
+                _CLDELETE(input);
+            } else {
+                input = NULL;
+            }
         }
     };
 

src/core/CLucene/analysis/Analyzers.cpp

@@ -51,11 +51,13 @@ template class LowerCaseTokenizer<TCHAR>;
 template<typename T>
 SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in) : LowerCaseTokenizer<T>(in) {
     Tokenizer::lowercase = true;
+    Tokenizer::ownReader = false;
 }
 
 template<typename T>
-SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in, bool lowercase) : LowerCaseTokenizer<T>(in) {
+SimpleTokenizer<T>::SimpleTokenizer(CL_NS(util)::Reader *in, bool lowercase, bool ownReader) : LowerCaseTokenizer<T>(in) {
     Tokenizer::lowercase = lowercase;
+    Tokenizer::ownReader = ownReader;
 }
 
 template<typename T>

src/core/CLucene/analysis/Analyzers.h

@@ -140,7 +140,7 @@ class CLUCENE_EXPORT SimpleTokenizer:public LowerCaseTokenizer<T> {
 public:
     /** Construct a new SimpleTokenizer. */
     explicit SimpleTokenizer(CL_NS(util)::Reader* in);
-    SimpleTokenizer(CL_NS(util)::Reader* in, bool lowercase);
+    SimpleTokenizer(CL_NS(util)::Reader* in, bool lowercase, bool ownReader = false);
     virtual ~SimpleTokenizer();
 
     Token* next(Token* token) override {
@@ -182,12 +182,13 @@ class CLUCENE_EXPORT SimpleAnalyzer: public Analyzer {
 public:
     SimpleAnalyzer(){
         _lowercase = true;
+        _ownReader = false;
     }
 
     bool isSDocOpt() override { return true; }
 
     TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) override{
-        return _CLNEW SimpleTokenizer<T>(reader, _lowercase);
+        return _CLNEW SimpleTokenizer<T>(reader, _lowercase, _ownReader);
     }
     TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) override{
         if (tokenizer_ == nullptr) {

src/core/CLucene/analysis/standard95/StandardAnalyzer.h

@@ -8,14 +8,15 @@ class StandardAnalyzer : public Analyzer {
  public:
   StandardAnalyzer() : Analyzer() { 
     _lowercase = true;
+    _ownReader = false;
     _stopwords = nullptr;
   }
 
   bool isSDocOpt() override { return true; }
 
   TokenStream* tokenStream(const TCHAR* fieldName,
                            lucene::util::Reader* reader) override {
-    return _CLNEW StandardTokenizer(reader, _lowercase, _stopwords);
+    return _CLNEW StandardTokenizer(reader, _lowercase, _stopwords, _ownReader);
   }
 
   TokenStream* reusableTokenStream(const TCHAR* fieldName,

src/core/CLucene/analysis/standard95/StandardTokenizer.h

@@ -23,13 +23,15 @@ class StandardTokenizer : public Tokenizer {
       : Tokenizer(in) {
     scanner_ = std::make_unique<StandardTokenizerImpl>(in);
     Tokenizer::lowercase = true;
+    Tokenizer::lowercase = false;
     Tokenizer::stopwords = nullptr;
   }
-  StandardTokenizer(lucene::util::Reader* in, bool lowercase, std::unordered_set<std::string_view>* stopwords)
+  StandardTokenizer(lucene::util::Reader* in, bool lowercase, std::unordered_set<std::string_view>* stopwords, bool ownReader=false)
           : Tokenizer(in) {
       scanner_ = std::make_unique<StandardTokenizerImpl>(in);
       Tokenizer::lowercase = lowercase;
       Tokenizer::stopwords = stopwords;
+      Tokenizer::ownReader = ownReader;
   }
 
   Token* next(Token* token) override {

src/core/CLucene/document/Field.cpp

@@ -176,7 +176,8 @@ void Field::setValue(ValueArray<uint8_t>* value) {
 }
 
 /** Expert: change the value of this field.  See <a href="#setValue(java.lang.String)">setValue(String)</a>. */
-void Field::setValue(CL_NS(analysis)::TokenStream* value) {
+void Field::setValue(CL_NS(analysis)::TokenStream* value, bool own_stream) {
+    ownStream = own_stream;
 	_resetValue();
     fieldsData = value;
 	valueType = VALUE_TOKENSTREAM;
@@ -340,7 +341,10 @@ void Field::_resetValue() {
 	} else if (valueType & VALUE_BINARY) {
 		ValueArray<uint8_t>* v = static_cast<ValueArray<uint8_t>*>(fieldsData);
 		_CLDELETE(v);
-	}
+	} else if (valueType & VALUE_TOKENSTREAM && ownStream) {
+        auto* v = static_cast<CL_NS(analysis)::TokenStream*>(fieldsData);
+        _CLDELETE(v);
+    }
 	valueType=VALUE_NONE;
 }
 const char* Field::getObjectName() const{

src/core/CLucene/document/Field.h

@@ -305,7 +305,7 @@ class CLUCENE_EXPORT Field : public CL_NS(util)::NamedObject{
 	void setValue(CL_NS(util)::ValueArray<uint8_t>* value) ;
 
 	/** Expert: change the value of this field.  See <a href="#setValue(TCHAR*)">setValue(TCHAR*)</a>. */
-	void setValue(CL_NS(analysis)::TokenStream* value);
+	void setValue(CL_NS(analysis)::TokenStream* value, bool own_stream = false);
 
     //void setValue(CL_NS(analysis)::STokenStream* value);
 
@@ -334,6 +334,7 @@ class CLUCENE_EXPORT Field : public CL_NS(util)::NamedObject{
 	float_t boost;
 
 	IndexVersion indexVersion_ = IndexVersion::kV1;
+    bool ownStream = false;
 };
 CL_NS_END
 #endif