Merge pull request #89 from talis/remove-incorrect-char-encoding

90% - Remove incorrect char encoding
techfromsage · Aug 6, 2015 · 7cd9c7f · 7cd9c7f
2 parents 91bd384 + f26ebb5
commit 7cd9c7f
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 35 deletions.
diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php
@@ -262,15 +262,15 @@ protected function addFields(Array $source, Array $fieldsOrIndices, Array &$targ
 
                     if(isset($source[$p])){
                         if(isset($source[$p]['u'])){
-                            $values[] = ($isIndex) ? strtolower(trim($source[$p]['u'])) : trim($source[$p]['u']);
+                            $values[] = ($isIndex) ? mb_strtolower(trim($source[$p]['u']), 'UTF-8') : trim($source[$p]['u']);
                         } else if (isset($source[$p]['l'])){
-                            $values[] = ($isIndex) ? strtolower(trim($source[$p]['l'])) : trim($source[$p]['l']);
+                            $values[] = ($isIndex) ? mb_strtolower(trim($source[$p]['l']), 'UTF-8') : trim($source[$p]['l']);
                         } else {
                             foreach($source[$p] as $v){
                                 if(isset($v['u'])){
-                                    $values[] = ($isIndex) ? strtolower(trim($v['u'])) : trim($v['u']);
+                                    $values[] = ($isIndex) ? mb_strtolower(trim($v['u']), 'UTF-8') : trim($v['u']);
                                 } else {
-                                    $values[] = ($isIndex) ? strtolower(trim($v['l'])) : trim($v['l']);
+                                    $values[] = ($isIndex) ? mb_strtolower(trim($v['l']), 'UTF-8') : trim($v['l']);
                                 }
                             }
                         }
@@ -330,7 +330,7 @@ private function addValuesToTarget($values, $field, &$target)
 
 		if(count($values) > 0){						
 	        for ($i=0; $i<$limit; $i++) {	        	
-	            $v = utf8_encode($values[$i]); //todo: added by CJC
+	            $v = $values[$i];
 	            if (empty($objName)) {
 	                if (!isset($target[$name])) {
 	                    $target[$name] = $v;

diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php
@@ -58,7 +58,7 @@ public function testSearchIndexing() {
         // assert that there are only 12 based on the data we loaded into tripod
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-        $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
+        $this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 13 search documents based on searchData.json");
 
         // define the expected search documents, this is what each of them should look like
         $expectedSearchDocs = array(
@@ -194,7 +194,7 @@ public function testSearchIndexingRemovesDocWhenTypeHasNoCorrespondingSearchdocS
 
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-        $this->assertEquals(11, $actualSearchDocumentCount, "Should only be 11 search documents now that one of them has had its type changed with no corresponding search doc spec");
+        $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents now that one of them has had its type changed with no corresponding search doc spec");
 
 
         foreach(\Tripod\Mongo\Config::getInstance()->getCollectionsForSearch('tripod_php_testing') as $collection)
@@ -225,7 +225,7 @@ public function testSearchIndexingGeneratesNewDocForChangedTypeThatHasACorrespon
 
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-        $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents");
+        $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents");
 
         $result = array();
         foreach(\Tripod\Mongo\Config::getInstance()->getCollectionsForSearch('tripod_php_testing') as $collection)
@@ -260,7 +260,7 @@ public function testSearchIndexingGeneratesTwoDocumentsForGivenResourceTheDelete
 
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-        $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents");
+        $this->assertEquals(14, $actualSearchDocumentCount, "Should only be 14 search documents");
 
         $results = array();
         // We don't know where exactly these might have stored
@@ -307,7 +307,7 @@ public function testSearchIndexingGeneratesTwoDocumentsForGivenResourceTheDelete
 
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-        $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents");
+        $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents");
 
         $results = array();
         // We don't know where exactly these might have stored
@@ -393,10 +393,10 @@ public function testSearchLimitAndOffset()
     public function testSearchSingleIndex()
     {
         // simple search
-        $results = $this->searchProvider->search("john locke poetry", "i_search_resource", array("search_terms"), array("result"), 3, 0);
+        $results = $this->searchProvider->search("john locke poetry", "i_search_resource", array("search_terms"), array("result"), 4, 0);
         $this->assertEquals(6, $results['head']['count']);
-        $this->assertEquals(3, $results['head']['limit']);
-        $this->assertEquals(3, count($results['results']));
+        $this->assertEquals(4, $results['head']['limit']);
+        $this->assertEquals(4, count($results['results']));
         $this->assertEquals(0, $results['head']['offset']);
         $this->assertEquals("john locke poetry",$results['head']["query"]);
         $this->assertEquals(array("john","locke","poetry"), $results['head']["query_terms_used"]);
@@ -406,6 +406,7 @@ public function testSearchSingleIndex()
             array("title"=>"Paradise Lost" , "link"=>"http://talisaspire.com/resources/doc4" , "author"=>"John Locke"),
             array("title"=>"The Divine Comedy" , "link"=>"http://talisaspire.com/resources/doc5" , "author"=>"John Locke"),
             array("title"=>"A Light in the Attic" , "link"=>"http://talisaspire.com/resources/doc6" , "author"=>"John Locke"),
+            array("title"=>"Revolting Rhymes" , "link"=>"http://talisaspire.com/resources/doc7" , "author"=>"John Locke"),
         );
 
         $this->assertEquals($expectedResults, $results['results']);
@@ -453,7 +454,7 @@ public function testSearchSingleIndex()
     public function testSearchMultipleIndices()
     {
         $results = $this->searchProvider->search("bibo:Book", "i_search_resource", array("search_terms","other_terms"), array("result","rdftype"), 3, 0);
-        $this->assertEquals(12, $results['head']['count']);
+        $this->assertEquals(13, $results['head']['count']);
         $this->assertEquals(3, $results['head']['limit']);
         $this->assertEquals(3, count($results['results']));
         $this->assertEquals(0, $results['head']['offset']);
@@ -498,25 +499,25 @@ public function testSearchWorksDirectlyFromTripod()
         $this->assertEquals($expectedResults, $results['results']);
 
     }
-    
+
     public function testDeleteSearchDocumentsByTypeIdThrowsExceptionForInvalidType()
     {
     	$mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod));
     	$mockSearchProvider->expects($this->once())
     						->method('getSearchDocumentSpecification')
     						->with('i_some_type')
     						->will($this->returnValue(null));
-    	
+
     	$this->setExpectedException("Exception","Could not find a search specification for i_some_type");
     	$mockSearchProvider->deleteSearchDocumentsByTypeId('i_some_type');
     }
-    
+
     public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound()
     {
     	// first, assert that there are only 12 based on the data we loaded into tripod
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-    	$this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
+    	$this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
 
         /** @var \Tripod\Mongo\MongoSearchProvider|PHPUnit_Framework_MockObject_MockObject $mockSearchProvider */
     	$mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod));
@@ -532,66 +533,66 @@ public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound(
         {
             $this->assertEquals("Search document id 'i_some_type' not in configuration for store 'tripod_php_testing'", $e->getMessage());
         }
-    	
-    	//search document count should remain same, because we expect that there was nothing to delete 
+
+    	//search document count should remain same, because we expect that there was nothing to delete
         $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-    	$this->assertEquals(12, $newSearchDocumentCount, "Should have generated 12 search documents, because there was no match to remove");
+    	$this->assertEquals(13, $newSearchDocumentCount, "Should have generated 12 search documents, because there was no match to remove");
     }
-    
+
     public function testDeleteSearchDocumentsByTypeIdDeleteAllMatchingDocuments()
     {
     	// first, assert that there are only 12 based on the data we loaded into tripod
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-    	$this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
-    	 
+    	$this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
+
     	$mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod));
     	$mockSearchProvider->expects($this->once())
     					->method('getSearchDocumentSpecification')
 				    	->with('i_search_resource')
 				    	->will($this->returnValue(array('i_search_resource' => array())));
-    
+
     	$mockSearchProvider->deleteSearchDocumentsByTypeId('i_search_resource');
-    	 
+
     	//search document count should be 0, because we expect that everything should be deleted
         $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
     	$this->assertEquals(0, $newSearchDocumentCount, "Should have 0 search documents after removing all matching documents");
     }
-    
+
     public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments()
     {
     	// first, assert that there are only 12 based on the data we loaded into tripod
         $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-    	$this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json");
+    	$this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 13 search documents based on searchData.json");
 
     	$id = array('_id.r'=>'http://talisaspire.com/resources/doc1');
-    	
+
     	$newData = array(
     			"rdf:type"=>array(array("u"=>"resourcelist:List"),array("u"=>"bibo:Book")),
     			"spec:name"=>array("l"=>"my list title"),
     			"resourcelist:description"=>array("l"=>"foo bar baz"),
     	);
     	$this->getTripodCollection($this->tripod)->update($id, array('$set'=> $newData));
-    	
+
     	// reindex
     	$this->indexer->generateAndIndexSearchDocuments('http://talisaspire.com/resources/doc1', 'http://talisaspire.com/', $this->tripod->getPodName());
-    	 
+
     	//assert that there are now 13 documents after adding new document to collection
         $updatedSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 
-    	$this->assertEquals(13, $updatedSearchDocumentCount, "Should have generated 13 search documents after adding a new document to collection");
-    
+    	$this->assertEquals(14, $updatedSearchDocumentCount, "Should have generated 14 search documents after adding a new document to collection");
+
     	$mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod));
     	$mockSearchProvider->expects($this->once())
 				    	->method('getSearchDocumentSpecification')
 				    	->with('i_search_resource')
 				    	->will($this->returnValue(array('i_search_resource' => array())));
-    
+
     	$mockSearchProvider->deleteSearchDocumentsByTypeId('i_search_resource');
-    
+
     	//search document count should be 1, since there is one document not matching the type id provided for delete
         $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod);
 

diff --git a/test/unit/mongo/MongoTripodSearchDocumentsTest.php b/test/unit/mongo/MongoTripodSearchDocumentsTest.php
@@ -81,6 +81,15 @@ public function testGenerateSearchDocumentBasedOnSpecId()
     }
 
 
+    public function testGenerateSearchDocumentPreservesDiacritics()
+    {
+        $searchDocuments = $this->getSearchDocuments($this->tripod);
+        $generatedDocuments = $searchDocuments->generateSearchDocumentBasedOnSpecId('i_search_resource', 'http://talisaspire.com/resources/doc13', 'http://talisaspire.com/');
+        $this->assertEquals('René Chapus', $generatedDocuments['result']['author']);
+        $this->assertContains('rené chapus', $generatedDocuments['search_terms']);
+        $this->assertEquals('http://talisaspire.com/resources/doc13', $generatedDocuments['_id']['r']);
+    }
+
     public function testGenerateSearchDocumentBasedOnSpecIdWithFieldNamePredicatesHavingNoValueInCollection()
     {
         $searchSpecs = json_decode('{"_id":"i_search_resource","type":["bibo:Book"],"from":"CBD_testing","filter":[{"condition":{"dct:title.l":{"$exists":true}}}],"indices":[{"fieldName":"search_terms","predicates":["dct:title","dct:subject"]},{"fieldName":"other_terms","predicates":["rdf:type"]}],"fields":[{"fieldName":"result.title","predicates":["dct:title"],"limit":1},{"fieldName":"result.link","value":"link"},{"fieldName":"rdftype","predicates":["rdf:type"],"limit":1}],"joins":{"dct:creator":{"indices":[{"fieldName":"search_terms","predicates":["foaf:name"]}],"fields":[{"fieldName":"result.author","predicates":["foaf:name"],"limit":1}, {"fieldName":"result.role","predicates":["siocAccess:Role"], "limit":1}] } }}', true);

diff --git a/test/unit/mongo/data/searchData.json b/test/unit/mongo/data/searchData.json
@@ -343,6 +343,32 @@
             }
         ]
     },
+    {
+        "_id": {
+            "r":"http://talisaspire.com/resources/doc13",
+            "c":"http://talisaspire.com/"
+        },
+        "_version": 0,
+        "dct:title": {
+            "l": "Droit du contentieux administratif"
+        },
+        "rdf:type": [
+            {
+                "u": "http://purl.org/ontology/bibo/Book"
+            },
+            {
+                "u": "http://talisaspire.com/schema#Resource"
+            }
+        ],
+        "dct:creator":{
+            "u": "http://talisaspire.com/authors/4"
+        },
+        "dct:subject":[
+            {
+                "l": "legal"
+            }
+        ]
+    },
     {
         "_id": {
             "r":"http://talisaspire.com/authors/1",
@@ -381,5 +407,18 @@
         "rdf:type": {
                 "u": "http://xmlns.com/foaf/0.1/Person"
         }
+    },
+    {
+        "_id": {
+            "r":"http://talisaspire.com/authors/4",
+            "c":"http://talisaspire.com/"
+        },
+        "_version": 0,
+        "foaf:name": {
+            "l": "René Chapus"
+        },
+        "rdf:type": {
+                "u": "http://xmlns.com/foaf/0.1/Person"
+        }
     }
 ]