Added inclusive and exclusive range tokenization

netgen · Apr 5, 2018 · 285543d · 285543d
1 parent 6af5e8b
commit 285543d
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 6 deletions.
diff --git a/lib/Languages/Galach/TokenExtractor/Full.php b/lib/Languages/Galach/TokenExtractor/Full.php
@@ -36,7 +36,7 @@ final class Full extends TokenExtractor
         '/(?<lexeme>(?:(?<marker>(?<!\\\\)\#)(?<tag>[a-zA-Z0-9_][a-zA-Z0-9_\-.]*)))(?:[\s"()+!]|$)/Au' => Tokenizer::TOKEN_TERM,
         '/(?<lexeme>(?:(?<marker>(?<!\\\\)@)(?<user>[a-zA-Z0-9_][a-zA-Z0-9_\-.]*)))(?:[\s"()+!]|$)/Au' => Tokenizer::TOKEN_TERM,
         '/(?<lexeme>(?:(?<domain>[a-zA-Z_][a-zA-Z0-9_\-.]*):)?(?<quote>(?<!\\\\)["])(?<phrase>.*?)(?:(?<!\\\\)(?P=quote)))/Aus' => Tokenizer::TOKEN_TERM,
-        '/(?<lexeme>(?:(?<domain>[a-zA-Z_][a-zA-Z0-9_\-.]*):)?\[(?<rangeFrom>[a-zA-Z0-9]+) TO (?<rangeTo>[a-zA-Z0-9]+)\])/Aus' => Tokenizer::TOKEN_TERM,
+        '/(?<lexeme>(?:(?<domain>[a-zA-Z_][a-zA-Z0-9_\-.]*):)?(?<rangeStartSymbol>[\[\{])(?<rangeFrom>[a-zA-Z0-9]+) TO (?<rangeTo>[a-zA-Z0-9]+)[\]\}])/Aus' => Tokenizer::TOKEN_TERM,
         '/(?<lexeme>(?:(?<domain>[a-zA-Z_][a-zA-Z0-9_\-.]*):)?(?<word>(?:\\\\\\\\|\\\\ |\\\\\(|\\\\\)|\\\\"|[^"()\s])+?))(?:(?<!\\\\)["]|\(|\)|$|\s)/Au' => Tokenizer::TOKEN_TERM,
     ];
 
@@ -50,12 +50,13 @@ protected function createTermToken($position, array $data)
         $lexeme = $data['lexeme'];
 
         switch (true) {
-            case isset($data['rangeFrom']) && isset($data['rangeTo']):
+            case isset($data['rangeStartSymbol']):
                 return new Range(
                     $lexeme,
                     $position,
                     $data['domain'],
-                    $data['rangeFrom'], $data['rangeTo']
+                    $data['rangeFrom'], $data['rangeTo'],
+                    Range::getTypeByStart($data['rangeStartSymbol'])
                 );
             case isset($data['word']):
                 return new Word(

diff --git a/lib/Languages/Galach/Values/Token/Range.php b/lib/Languages/Galach/Values/Token/Range.php
@@ -12,6 +12,9 @@
  */
 final class Range extends Token
 {
+    const TYPE_INCLUSIVE = 'inclusive';
+    const TYPE_EXCLUSIVE = 'exclusive';
+
     /**
      * Holds domain string.
      *
@@ -29,19 +32,50 @@ final class Range extends Token
      */
     public $rangeTo;
 
+    /**
+     * @var string
+     */
+    public $type;
+
     /**
      * @param string $lexeme
-     * @param int $position
+     * @param int    $position
      * @param string $domain
      * @param string $rangeFrom
      * @param string $rangeTo
+     * @param string $type
      */
-    public function __construct($lexeme, $position, $domain, $rangeFrom, $rangeTo)
+    public function __construct($lexeme, $position, $domain, $rangeFrom, $rangeTo, $type)
     {
+        if (!in_array($type, [self::TYPE_EXCLUSIVE, self::TYPE_INCLUSIVE])) {
+            throw new \InvalidArgumentException(sprintf('Invalid range type: %s', $type));
+        }
+
         parent::__construct(Tokenizer::TOKEN_TERM, $lexeme, $position);
 
         $this->domain = $domain;
         $this->rangeFrom = $rangeFrom;
         $this->rangeTo = $rangeTo;
+        $this->type = $type;
+    }
+
+    /**
+     * Returns the range type, given the starting symbol.
+     *
+     * @param string $startSymbol the start symbol, either '[' or '{'
+     *
+     * @return string
+     */
+    public static function getTypeByStart($startSymbol)
+    {
+        if ('[' === $startSymbol) {
+            return self::TYPE_INCLUSIVE;
+        }
+
+        if ('{' === $startSymbol) {
+            return self::TYPE_EXCLUSIVE;
+        }
+
+        throw new \InvalidArgumentException(sprintf('Invalid range start symbol: %s', $startSymbol));
     }
 }
diff --git a/tests/Galach/Tokenizer/FullTokenizerTest.php b/tests/Galach/Tokenizer/FullTokenizerTest.php
@@ -116,7 +116,13 @@ public function providerForTestTokenize()
             [
                 '[a TO b]',
                 [
-                    new RangeToken('[a TO b]', 0, '', 'a', 'b'),
+                    new RangeToken('[a TO b]', 0, '', 'a', 'b', 'inclusive'),
+                ],
+            ],
+            [
+                '{a TO b}',
+                [
+                    new RangeToken('{a TO b}', 0, '', 'a', 'b', 'exclusive'),
                 ],
             ],
             [

diff --git a/tests/Galach/Tokenizer/TextTokenizerTest.php b/tests/Galach/Tokenizer/TextTokenizerTest.php
@@ -102,6 +102,13 @@ public static function setUpBeforeClass()
                 new Token(Tokenizer::TOKEN_WHITESPACE, ' ', 5),
                 new WordToken('b]', 6, '', 'b]'),
             ],
+            '{a TO b}' => [
+                new WordToken('{a', 0, '', '{a'),
+                new Token(Tokenizer::TOKEN_WHITESPACE, ' ', 2),
+                new WordToken('TO', 3, '', 'TO'),
+                new Token(Tokenizer::TOKEN_WHITESPACE, ' ', 5),
+                new WordToken('b}', 6, '', 'b}'),
+            ],
             'domain:domain:' => [
                 new WordToken('domain:domain:', 0, '', 'domain:domain:'),
             ],

diff --git a/tests/Galach/Values/Token/RangeTest.php b/tests/Galach/Values/Token/RangeTest.php
@@ -0,0 +1,65 @@
+<?php
+
+namespace QueryTranslator\Tests\Galach\Values\Token;
+
+use PHPUnit\Framework\TestCase;
+use QueryTranslator\Languages\Galach\Values\Token\Range;
+
+class RangeTest extends TestCase
+{
+    public function failingStartSymbolDataprovider()
+    {
+        return [
+            [''],
+            ['/'],
+            ['('],
+        ];
+    }
+
+    /**
+     * @dataProvider failingStartSymbolDataprovider
+     * @param string $startSymbol
+     */
+    public function testGetTypeByStartFails($startSymbol)
+    {
+        $this->expectException(\InvalidArgumentException::class);
+        Range::getTypeByStart($startSymbol);
+    }
+
+    public function successfulStartSymbolDataprovider()
+    {
+        return [
+            ['inclusive', '['],
+            ['exclusive', '{'],
+        ];
+    }
+
+    /**
+     * @dataProvider successfulStartSymbolDataprovider
+     * @param string $expectedType
+     * @param string $startSymbol
+     */
+    public function testGetTypeByStartSucceeds($expectedType, $startSymbol)
+    {
+        $this->assertSame($expectedType, Range::getTypeByStart($startSymbol));
+    }
+
+    public function failingTypeDataprovider()
+    {
+        return [
+            [''],
+            [null],
+            ['other'],
+        ];
+    }
+
+    /**
+     * @dataProvider failingTypeDataprovider
+     * @param string $type
+     */
+    public function testConstructorFailsWrongType($type)
+    {
+        $this->expectException(\InvalidArgumentException::class);
+        new Range('[a TO b]', 0, '', 'a', 'b', $type);
+    }
+}