Merge pull request #1221 from Apollon77/entityfeat2

More strict entity matching and better handle duplicate enum entity cases

packages/ner/src/extractor-builtin.js

@@ -45,7 +45,12 @@ class ExtractorBuiltin {
     input.edges = input.edges || [];
     if (newInput.edges) {
       for (let i = 0; i < newInput.edges.length; i += 1) {
-        input.edges.push(newInput.edges[i]);
+        if (
+          !input.nerLimitToEntities ||
+          input.intentEntities.includes(newInput.edges[i].entity)
+        ) {
+          input.edges.push(newInput.edges[i]);
+        }
       }
     }
     input.edges = reduceEdges(input.edges, false);

packages/ner/src/extractor-enum.js

@@ -342,7 +342,7 @@ class ExtractorEnum {
       }
     }
     edges.sort((a, b) => a.start - b.start);
-    input.edges = reduceEdges(edges, false);
+    input.edges = reduceEdges(edges, false, input.intentEntities);
     return input;
   }
 

packages/ner/src/ner.js

@@ -154,9 +154,28 @@ class Ner extends Clonable {
     return result;
   }
 
-  decideRules(srcInput) {
+  decideRules(srcInput, intentEntities) {
     const input = srcInput;
-    input.nerRules = this.getRules(input.locale || 'en');
+    let nerRules = this.getRules(input.locale || 'en');
+    if (intentEntities && this.settings.considerOnlyIntentEntities) {
+      nerRules = nerRules.filter((rule) => intentEntities.includes(rule.name));
+    } else if (intentEntities) {
+      // entities in the current intent get a higher priority when
+      // sorting out overlapping matches
+      const intentRelevantRule = [];
+      const nonIntentRelevantRule = [];
+      nerRules.forEach((rule) => {
+        if (intentEntities.includes(rule.name)) {
+          intentRelevantRule.push(rule);
+        } else {
+          nonIntentRelevantRule.push(rule);
+        }
+      });
+      nerRules = intentRelevantRule.concat(nonIntentRelevantRule);
+    }
+    input.nerRules = nerRules;
+    input.nerLimitToEntities = this.settings.considerOnlyIntentEntities;
+    input.intentEntities = intentEntities;
     return input;
   }
 
@@ -343,10 +362,12 @@ class Ner extends Clonable {
     input.entities = input.edges;
     delete input.edges;
     delete input.nerRules;
+    delete input.nerLimitToEntities;
+    delete input.intentEntities;
     return input;
   }
 
-  async defaultPipelineProcess(input) {
+  async defaultPipelineProcess(input, intentEntities) {
     if (!this.cache) {
       this.cache = {
         extractEnum: this.container.get('extract-enum'),
@@ -371,7 +392,7 @@ class Ner extends Clonable {
         this.cache.extractBuiltin = this.container.get('extract-builtin');
       }
     }
-    let output = await this.decideRules(input);
+    let output = await this.decideRules(input, intentEntities);
     if (this.cache.extractEnum) {
       output = await this.cache.extractEnum.run(output);
     }
@@ -388,8 +409,11 @@ class Ner extends Clonable {
     return output;
   }
 
-  async process(srcInput) {
-    const input = { threshold: this.settings.threshold || 0.8, ...srcInput };
+  async process(srcInput, consideredEntities) {
+    const input = {
+      threshold: this.settings.threshold || 0.8,
+      ...srcInput,
+    };
     let result;
     if (input.locale) {
       const pipeline = this.container.getPipeline(
@@ -402,7 +426,12 @@ class Ner extends Clonable {
       result = await this.runPipeline(input, this.pipelineProcess);
     }
     if (!result) {
-      result = await this.defaultPipelineProcess(input);
+      result = await this.defaultPipelineProcess(input, consideredEntities);
+    } else if (consideredEntities) {
+      // when custom pipeline is used then we can not be sure it is handled correctly
+      result.entities = result.entities.filter((entity) =>
+        consideredEntities.includes(entity.entity)
+      );
     }
     delete result.threshold;
     return result;

packages/ner/src/reduce-edges.js

@@ -22,7 +22,7 @@
  */
 const { TrimTypesList } = require('./trim-types');
 
-function runDiscard(srcEdge, srcOther, useMaxLength) {
+function runDiscard(srcEdge, srcOther, useMaxLength, intentEntities = []) {
   let edge;
   let other;
   if (
@@ -44,15 +44,29 @@ function runDiscard(srcEdge, srcOther, useMaxLength) {
         other.entity === 'number') &&
       other.len <= edge.len
     ) {
-      // Do nothing! entities have same priority
+      // Entities have same priority
       if (
         other.start === edge.start &&
         other.end === edge.end &&
         other.type === edge.type &&
         other.entity === edge.entity &&
         other.option === edge.option
       ) {
+        // same type and none of them is an enum or both are an enum
         other.discarded = true;
+      } else if (
+        other.start === edge.start &&
+        other.end === edge.end &&
+        other.entity === edge.entity &&
+        other.type !== edge.type
+      ) {
+        if (edge.type === 'trim' && other.type !== 'trim') {
+          edge.discarded = true;
+        } else if (edge.type !== 'trim' && other.type === 'trim') {
+          other.discarded = true;
+        } else {
+          other.discarded = true;
+        }
       }
     } else if (
       (useMaxLength ||
@@ -62,7 +76,18 @@ function runDiscard(srcEdge, srcOther, useMaxLength) {
     ) {
       edge.discarded = true;
     } else if (edge.type === 'enum' && other.type === 'enum') {
-      if (
+      const edgeIncludedInIntentEntities = intentEntities.includes(edge.entity);
+      const otherIncludedInIntentEntities = intentEntities.includes(
+        other.entity
+      );
+      if (edgeIncludedInIntentEntities && !otherIncludedInIntentEntities) {
+        other.discarded = true;
+      } else if (
+        !edgeIncludedInIntentEntities &&
+        otherIncludedInIntentEntities
+      ) {
+        edge.discarded = true;
+      } else if (
         edge.len <= other.len &&
         other.utteranceText.includes(edge.utteranceText)
       ) {
@@ -121,7 +146,7 @@ function splitEdges(edges) {
   return edges;
 }
 
-function reduceEdges(edges, useMaxLength = true) {
+function reduceEdges(edges, useMaxLength = true, intentEntities = []) {
   edges = splitEdges(edges);
   const edgeslen = edges.length;
   for (let i = 0; i < edgeslen; i += 1) {
@@ -133,8 +158,17 @@ function reduceEdges(edges, useMaxLength = true) {
       for (let j = i + 1; j < edgeslen; j += 1) {
         const other = edges[j];
         if (!other.discarded) {
-          runDiscard(edge, other, useMaxLength);
+          runDiscard(edge, other, useMaxLength, intentEntities);
         }
+        if (edge.discarded) {
+          break;
+        }
+      }
+    }
+    if (!edge.discarded) {
+      const knownEntityPos = intentEntities.indexOf(edge.entity);
+      if (knownEntityPos !== -1) {
+        intentEntities.splice(knownEntityPos, 1);
       }
     }
   }

packages/nlp/src/nlp.js

@@ -723,7 +723,10 @@ class Nlp extends Clonable {
     }
     output.context = context;
     if (forceNER || !this.slotManager.isEmpty) {
-      output = await this.ner.process({ ...output });
+      const intentEntities = this.slotManager.getIntentEntityNames(
+        output.intent
+      );
+      output = await this.ner.process({ ...output }, intentEntities);
     } else {
       output.entities = [];
       output.sourceEntities = [];