Atualização para Lucene 8.11.1 issue #4

criar arquivo de teste.txt

@@ -0,0 +1 @@
+ mx /bases/lilG4/lil.lil/lilacs.mst "pft=if p(v12) and p(v83) then  v2'~^'v12[1]^*'~^'v83[1]^*/ fi" now lw=0 > ~/lilacs_2_12_83.txt

nbproject/build-impl.xml

@@ -46,8 +46,8 @@ is divided into following sections:
         <property file="${user.properties.file}"/>
         <!-- The two properties below are usually overridden -->
         <!-- by the active platform. Just a fallback. -->
-        <property name="default.javac.source" value="1.6"/>
-        <property name="default.javac.target" value="1.6"/>
+        <property name="default.javac.source" value="1.8"/>
+        <property name="default.javac.target" value="1.8"/>
     </target>
     <target depends="-pre-init,-init-private,-init-user" name="-init-project">
         <property file="nbproject/configs/${config}.properties"/>

nbproject/configs/IndexDecs.properties

@@ -0,0 +1 @@
+main.class=org.bireme.dia.util.IndexDecs

nbproject/configs/Search.properties

@@ -0,0 +1 @@
+main.class=org.bireme.dia.util.Search

nbproject/configs/ShowPerformance.properties

@@ -0,0 +1 @@
+main.class=org.bireme.dia.util.ShowPerformance

nbproject/configs/ShowTermDocs.properties

@@ -0,0 +1 @@
+main.class=org.bireme.dia.util.ShowTermDocs

nbproject/genfiles.properties

@@ -1,6 +1,6 @@
 build.xml.data.CRC32=6dce84d2
 build.xml.script.CRC32=e371a08e
-build.xml.stylesheet.CRC32=8064a381@1.75.2.48
+build.xml.stylesheet.CRC32=f85dc8f2@1.98.0.48
 # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
 # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
 nbproject/build-impl.xml.data.CRC32=6dce84d2

nbproject/private/config.properties

@@ -0,0 +1 @@
+config=ShowTermDocs

nbproject/private/configs/ShowTermDocs.properties

@@ -0,0 +1 @@
+application.args=-index=index -idFieldName=id  -outFile=./reportBug8.txt

nbproject/private/private.properties

@@ -1,6 +1,8 @@
 compile.on.save=true
 do.depend=false
 do.jar=true
+do.jlink=false
 javac.debug=true
 javadoc.preview=true
-user.properties.file=/home/heitor/.netbeans/8.2/build.properties
+jlink.strip=false
+user.properties.file=/home/heitor/.netbeans/12.3/build.properties

nbproject/private/private.xml

@@ -2,6 +2,11 @@
 <project-private xmlns="http://www.netbeans.org/ns/project-private/1">
     <editor-bookmarks xmlns="http://www.netbeans.org/ns/editor-bookmarks/2" lastBookmarkId="0"/>
     <open-files xmlns="http://www.netbeans.org/ns/projectui-open-files/2">
-        <group/>
+        <group>
+            <file>file:/home/javaapps/iahx-analyzer-8.11.1/src/org/bireme/dia/util/ShowTermDocs.java</file>
+            <file>file:/home/javaapps/iahx-analyzer-8.11.1/src/org/bireme/dia/analysis/RemoveInvalidCharFilter.java</file>
+            <file>file:/home/javaapps/iahx-analyzer-8.11.1/src/org/bireme/dia/analysis/RemoveSlashFilter.java</file>
+            <file>file:/home/javaapps/iahx-analyzer-8.11.1/src/org/bireme/dia/util/ShowPerformance.java</file>
+        </group>
     </open-files>
 </project-private>

nbproject/project.properties

@@ -5,6 +5,7 @@ annotation.processing.run.all.processors=true
 annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output
 application.title=iahx-analyzer
 application.vendor=heitor
+auxiliary.org-netbeans-spi-editor-hints-projects.perProjectHintSettingsFile=nbproject/cfg_hints.xml
 build.classes.dir=${build.dir}/classes
 build.classes.excludes=**/*.java,**/*.form
 # This directory is removed when the project is cleaned:
@@ -19,8 +20,12 @@ build.test.results.dir=${build.dir}/test/results
 #debug.transport=dt_socket
 debug.classpath=\
     ${run.classpath}
+debug.modulepath=\
+    ${run.modulepath}
 debug.test.classpath=\
     ${run.test.classpath}
+debug.test.modulepath=\
+    ${run.test.modulepath}
 # Os arquivos em build.classes.dir que devem ser exclu\u00eddos do jar de distribui\u00e7\u00e3o
 dist.archive.excludes=
 # This directory is removed when the project is cleaned:
@@ -31,34 +36,43 @@ endorsed.classpath=
 excludes=
 file.reference.hamcrest-core-1.3.jar=lib/hamcrest-core-1.3.jar
 file.reference.junit-4.12.jar=lib/junit-4.12.jar
-file.reference.lucene-analyzers-common-5.4.1.jar=lib/lucene-analyzers-common-5.4.1.jar
-file.reference.lucene-core-5.4.1.jar=lib/lucene-core-5.4.1.jar
-file.reference.lucene-queryparser-5.4.1.jar=lib/lucene-queryparser-5.4.1.jar
+file.reference.lucene-analyzers-common-8.11.1.jar=lib/lucene-analyzers-common-8.11.1.jar
+file.reference.lucene-backward-codecs-8.11.1.jar=lib/lucene-backward-codecs-8.11.1.jar
+file.reference.lucene-core-8.11.1.jar=lib/lucene-core-8.11.1.jar
+file.reference.lucene-queryparser-8.11.1.jar=lib/lucene-queryparser-8.11.1.jar
 includes=**
+jar.archive.disabled=${jnlp.enabled}
 jar.compress=false
+jar.index=${jnlp.enabled}
 javac.classpath=\
     ${file.reference.hamcrest-core-1.3.jar}:\
     ${file.reference.junit-4.12.jar}:\
-    ${file.reference.lucene-analyzers-common-5.4.1.jar}:\
-    ${file.reference.lucene-core-5.4.1.jar}:\
-    ${file.reference.lucene-queryparser-5.4.1.jar}
+    ${file.reference.lucene-analyzers-common-8.11.1.jar}:\
+    ${file.reference.lucene-backward-codecs-8.11.1.jar}:\
+    ${file.reference.lucene-core-8.11.1.jar}:\
+    ${file.reference.lucene-queryparser-8.11.1.jar}
 # Space-separated list of extra javac options
-javac.compilerargs=
+javac.compilerargs=-Xlint:deprecation
 javac.deprecation=false
 javac.external.vm=false
+javac.modulepath=
+javac.processormodulepath=
 javac.processorpath=\
     ${javac.classpath}
-javac.source=1.6
-javac.target=1.6
+javac.source=1.8
+javac.target=1.8
 javac.test.classpath=\
     ${javac.classpath}:\
     ${build.classes.dir}:\
     ${libs.junit_4.classpath}
+javac.test.modulepath=\
+    ${javac.modulepath}
 javac.test.processorpath=\
     ${javac.test.classpath}
 javadoc.additionalparam=
 javadoc.author=false
 javadoc.encoding=${source.encoding}
+javadoc.html5=false
 javadoc.noindex=false
 javadoc.nonavbar=false
 javadoc.notree=false
@@ -67,7 +81,26 @@ javadoc.splitindex=true
 javadoc.use=true
 javadoc.version=false
 javadoc.windowtitle=
+jlink.launcher=false
+jlink.launcher.name=iahx-analyzer
+jnlp.codebase.type=no.codebase
+jnlp.descriptor=application
+jnlp.enabled=false
+jnlp.mixed.code=default
+jnlp.offline-allowed=false
+jnlp.signed=false
+jnlp.signing=
+jnlp.signing.alias=
+jnlp.signing.keystore=
 main.class=org.bireme.dia.util.TestAnalyzers
+# Substitui\u00e7\u00e3o opcional do atributo padr\u00e3o Application-Library-Allowable-Codebase que identifica os locais onde se espera que o RIA assinado seja encontrado.
+manifest.custom.application.library.allowable.codebase=
+# Substitui\u00e7\u00e3o opcional do atributo padr\u00e3o Caller-Allowable-Codebase que identifica os dom\u00ednios dos quais o c\u00f3digo JavaScript pode fazer chamadas para o RIA sem avisos de seguran\u00e7a.
+manifest.custom.caller.allowable.codebase=
+# Substitui\u00e7\u00e3o opcional do atributo de manifesto da Base de c\u00f3digo; use para impedir que RIAs sejam adaptados
+manifest.custom.codebase=
+# Substitui\u00e7\u00e3o opcional do atributo de manifesto de Permiss\u00f5es (valores suportados: sandbox, all-permissions)
+manifest.custom.permissions=
 manifest.file=manifest.mf
 meta.inf.dir=${src.dir}/META-INF
 mkdist.disabled=false
@@ -79,9 +112,13 @@ run.classpath=\
 # You may also define separate properties like run-sys-prop.name=value instead of -Dname=value.
 # To set system properties for unit tests define test-sys-prop.name=value:
 run.jvmargs=
+run.modulepath=\
+    ${javac.modulepath}
 run.test.classpath=\
     ${javac.test.classpath}:\
     ${build.test.classes.dir}
+run.test.modulepath=\
+    ${javac.test.modulepath}
 source.encoding=UTF-8
 src.dir=src
 test.src.dir=test

src/org/bireme/dia/analysis/BVSStandardAnalyzer.java

@@ -8,7 +8,7 @@
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
-import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator;
 
 public class BVSStandardAnalyzer extends Analyzer {
@@ -42,6 +42,7 @@ public BVSStandardAnalyzer() throws IOException {
         }
         engine = new DeCSEngine("resources/decs/main", CATEGORY, SYN, KEYQLF,
                                                                        ONLYQLF);
+        /*
         wordDelimiterConfig = WordDelimiterFilter.GENERATE_WORD_PARTS +
                                   WordDelimiterFilter.GENERATE_NUMBER_PARTS +
                                 //WordDelimiterFilter.CATENATE_WORDS +
@@ -50,8 +51,23 @@ public BVSStandardAnalyzer() throws IOException {
                                   WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE +
                                   WordDelimiterFilter.CATENATE_ALL +        
                                   WordDelimiterFilter.PRESERVE_ORIGINAL; // VA/HB 20161128
+        */
+
+        wordDelimiterConfig = WordDelimiterGraphFilter.GENERATE_WORD_PARTS +
+                              WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS +
+                              WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE +
+                              WordDelimiterGraphFilter.CATENATE_ALL +        
+                              WordDelimiterGraphFilter.PRESERVE_ORIGINAL; // VA/HB 20161128
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new WhitespaceTokenizer();
@@ -61,8 +77,11 @@ protected TokenStreamComponents createComponents(final String fieldName) {
                                                                  WORDS, PRECOD);
         //final TokenStream filter4 =  new WordDelimiterFilter(filter3, 
         //                                         wordDelimiterConfig, null);        
-        final TokenStream filter4 =  new WordDelimiterFilter(filter3, table, 
-                                                     wordDelimiterConfig, null);
+        //final TokenStream filter4 =  new WordDelimiterFilter(filter3, table, 
+        //                                             wordDelimiterConfig, null);
+        final TokenStream filter4 =  new WordDelimiterGraphFilter(filter3, false,
+                                           table, wordDelimiterConfig, null);
+
         final TokenStream filter5 = new TrimFilter(filter4);
         final TokenStream filter6 = new LowerCaseFilter(filter5);
         final TokenStream filter7 = new ASCIIFoldingFilter(filter6);

src/org/bireme/dia/analysis/DeCSAnalyzer.java

@@ -33,6 +33,14 @@ public DeCSAnalyzer() throws IOException {
         }
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new KeywordTokenizer();

src/org/bireme/dia/analysis/DeCSAuthorizedTermAndCategoryAnalyzer.java

@@ -27,6 +27,14 @@ public DeCSAuthorizedTermAndCategoryAnalyzer() throws IOException {
                                                                        ONLYQLF);
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new KeywordTokenizer();

src/org/bireme/dia/analysis/DeCSCode.java

@@ -13,25 +13,26 @@
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MMapDirectory;
 
 public class DeCSCode {
+    private final DirectoryReader reader;
     private final IndexSearcher decs;
 
     public DeCSCode() throws IOException {
         final ClassLoader loader = this.getClass().getClassLoader();
         final URL dirUrl = loader.getResource("./"); // get current directory of classes
         final File indexDir = new File("resources/decs/code");
-        final RAMDirectory ramDir = new RAMDirectory(
-                                    FSDirectory.open(indexDir.toPath()), 
-                                                     IOContext.READ);
-        final DirectoryReader reader = DirectoryReader.open(ramDir);
-
+        final MMapDirectory ramDir = new MMapDirectory(indexDir.toPath());
+
+        reader = DirectoryReader.open(ramDir);
         decs = new IndexSearcher(reader);
     }
 
+    public void close() throws IOException {
+        reader.close();
+    }
+
     public String getDescritorCode(final String descriptor) throws IOException,
                                                                 ParseException {
         String descriptorCode = null;
@@ -44,7 +45,7 @@ public String getDescritorCode(final String descriptor) throws IOException,
             final Query query = qParser.parse(descriptorPhrase);
             final TopDocs hits = decs.search(query, 1);
 
-            if (hits.totalHits > 0){
+            if (hits.totalHits.value > 0){
                 final int docID = hits.scoreDocs[0].doc;
                 final Document doc = decs.doc(docID);
                 descriptorCode = doc.get("id");
@@ -61,7 +62,7 @@ public String getDescriptorTerm(final String code,
         final TopDocs hits = decs.search(query, 1);
         String ret = null;
 
-        if (hits.totalHits > 0) {
+        if (hits.totalHits.value > 0) {
             final int docID = hits.scoreDocs[0].doc;
             final Document doc = decs.doc(docID);
             final String[] descriptorTerm = doc.getValues("descriptor");

src/org/bireme/dia/analysis/DeCSEngine.java

@@ -14,9 +14,10 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MMapDirectory;
 
 public class DeCSEngine implements SynonymEngine {
+    private final DirectoryReader reader;
     private final IndexSearcher searcher;
 
     // flag para informar se sera gerado chaves com as categorias e sinomimos dos descritores
@@ -37,14 +38,17 @@ public DeCSEngine(final String indexPath,
         this.onlyQualifiers = onlyqlf;
 
         final File indexDir = new File(indexPath);
-        final RAMDirectory ramDir = new RAMDirectory(
-                                    FSDirectory.open(indexDir.toPath()), 
-                                                     IOContext.READ);
-        final DirectoryReader reader = DirectoryReader.open(ramDir);
-
+        final MMapDirectory ramDir = new MMapDirectory(indexDir.toPath());
+
+        reader = DirectoryReader.open(ramDir);
         searcher = new IndexSearcher(reader);
     }
 
+    @Override
+    public void close() throws IOException {
+        reader.close();
+    }
+
     @Override
     public String[] getSynonyms(final String term) throws IOException {
         final ArrayList<String> synList = new ArrayList<String>();
@@ -148,7 +152,7 @@ private Document decsKey(final String code,
         Document key = null;
 
         final TopDocs hits = searcher.search(query, 1);
-        if (hits.totalHits > 0) {
+        if (hits.totalHits.value > 0) {
             final int docID = hits.scoreDocs[0].doc;
             key = searcher.doc(docID);
         }

src/org/bireme/dia/analysis/DeCSKeywordAnalyzer.java

@@ -27,6 +27,14 @@ public DeCSKeywordAnalyzer() throws IOException {
                                                                        ONLYQLF);
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new KeywordTokenizer();

src/org/bireme/dia/analysis/DeCSQualifierAnalyzer.java

@@ -28,6 +28,14 @@ public DeCSQualifierAnalyzer() throws IOException {
                                                                        ONLYQLF);
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new KeywordTokenizer();

src/org/bireme/dia/analysis/DeCSSimpleAnalyzer.java

@@ -27,6 +27,14 @@ public DeCSSimpleAnalyzer() throws IOException {
                                                                        ONLYQLF);
     }
 
+    @Override
+    public void close() {
+        try {
+            engine.close();
+            super.close();
+        } catch(IOException ioe) {}
+    }
+
     @Override
     protected TokenStreamComponents createComponents(final String fieldName) {
         final Tokenizer source = new KeywordTokenizer();