From faa91db4be653d39017ba3beb7844b1c75d9a918 Mon Sep 17 00:00:00 2001 From: Richard Tattersall Date: Thu, 6 Aug 2015 12:52:09 +0100 Subject: [PATCH 1/4] Removed incorrect char encoding --- src/mongo/delegates/SearchDocuments.class.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php index aabf1ddb..5e443c35 100644 --- a/src/mongo/delegates/SearchDocuments.class.php +++ b/src/mongo/delegates/SearchDocuments.class.php @@ -330,7 +330,7 @@ private function addValuesToTarget($values, $field, &$target) if(count($values) > 0){ for ($i=0; $i<$limit; $i++) { - $v = utf8_encode($values[$i]); //todo: added by CJC + $v = $values[$i]; if (empty($objName)) { if (!isset($target[$name])) { $target[$name] = $v; From c5534c7df183e19111e38538b0e171c8479b794c Mon Sep 17 00:00:00 2001 From: Richard Tattersall Date: Thu, 6 Aug 2015 17:02:25 +0100 Subject: [PATCH 2/4] Added tests, fixed a strtolower utf8 issue --- src/mongo/delegates/SearchDocuments.class.php | 8 ++-- test/unit/mongo/MongoSearchProviderTest.php | 32 +++++++-------- .../mongo/MongoTripodSearchDocumentsTest.php | 9 +++++ test/unit/mongo/data/searchData.json | 39 +++++++++++++++++++ 4 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php index 5e443c35..058de0c6 100644 --- a/src/mongo/delegates/SearchDocuments.class.php +++ b/src/mongo/delegates/SearchDocuments.class.php @@ -262,15 +262,15 @@ protected function addFields(Array $source, Array $fieldsOrIndices, Array &$targ if(isset($source[$p])){ if(isset($source[$p]['u'])){ - $values[] = ($isIndex) ? strtolower(trim($source[$p]['u'])) : trim($source[$p]['u']); + $values[] = ($isIndex) ? mb_strtolower(trim($source[$p]['u']), 'UTF-8') : trim($source[$p]['u']); } else if (isset($source[$p]['l'])){ - $values[] = ($isIndex) ? strtolower(trim($source[$p]['l'])) : trim($source[$p]['l']); + $values[] = ($isIndex) ? mb_strtolower(trim($source[$p]['l']), 'UTF-8') : trim($source[$p]['l']); } else { foreach($source[$p] as $v){ if(isset($v['u'])){ - $values[] = ($isIndex) ? strtolower(trim($v['u'])) : trim($v['u']); + $values[] = ($isIndex) ? mb_strtolower(trim($v['u']), 'UTF-8') : trim($v['u']); } else { - $values[] = ($isIndex) ? strtolower(trim($v['l'])) : trim($v['l']); + $values[] = ($isIndex) ? mb_strtolower(trim($v['l']), 'UTF-8') : trim($v['l']); } } } diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php index fd423953..b78a9f9e 100644 --- a/test/unit/mongo/MongoSearchProviderTest.php +++ b/test/unit/mongo/MongoSearchProviderTest.php @@ -498,7 +498,7 @@ public function testSearchWorksDirectlyFromTripod() $this->assertEquals($expectedResults, $results['results']); } - + public function testDeleteSearchDocumentsByTypeIdThrowsExceptionForInvalidType() { $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); @@ -506,11 +506,11 @@ public function testDeleteSearchDocumentsByTypeIdThrowsExceptionForInvalidType() ->method('getSearchDocumentSpecification') ->with('i_some_type') ->will($this->returnValue(null)); - + $this->setExpectedException("Exception","Could not find a search specification for i_some_type"); $mockSearchProvider->deleteSearchDocumentsByTypeId('i_some_type'); } - + public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound() { // first, assert that there are only 12 based on the data we loaded into tripod @@ -532,34 +532,34 @@ public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound( { $this->assertEquals("Search document id 'i_some_type' not in configuration for store 'tripod_php_testing'", $e->getMessage()); } - - //search document count should remain same, because we expect that there was nothing to delete + + //search document count should remain same, because we expect that there was nothing to delete $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); $this->assertEquals(12, $newSearchDocumentCount, "Should have generated 12 search documents, because there was no match to remove"); } - + public function testDeleteSearchDocumentsByTypeIdDeleteAllMatchingDocuments() { // first, assert that there are only 12 based on the data we loaded into tripod $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); - + $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); $mockSearchProvider->expects($this->once()) ->method('getSearchDocumentSpecification') ->with('i_search_resource') ->will($this->returnValue(array('i_search_resource' => array()))); - + $mockSearchProvider->deleteSearchDocumentsByTypeId('i_search_resource'); - + //search document count should be 0, because we expect that everything should be deleted $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); $this->assertEquals(0, $newSearchDocumentCount, "Should have 0 search documents after removing all matching documents"); } - + public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments() { // first, assert that there are only 12 based on the data we loaded into tripod @@ -568,30 +568,30 @@ public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); $id = array('_id.r'=>'http://talisaspire.com/resources/doc1'); - + $newData = array( "rdf:type"=>array(array("u"=>"resourcelist:List"),array("u"=>"bibo:Book")), "spec:name"=>array("l"=>"my list title"), "resourcelist:description"=>array("l"=>"foo bar baz"), ); $this->getTripodCollection($this->tripod)->update($id, array('$set'=> $newData)); - + // reindex $this->indexer->generateAndIndexSearchDocuments('http://talisaspire.com/resources/doc1', 'http://talisaspire.com/', $this->tripod->getPodName()); - + //assert that there are now 13 documents after adding new document to collection $updatedSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); $this->assertEquals(13, $updatedSearchDocumentCount, "Should have generated 13 search documents after adding a new document to collection"); - + $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); $mockSearchProvider->expects($this->once()) ->method('getSearchDocumentSpecification') ->with('i_search_resource') ->will($this->returnValue(array('i_search_resource' => array()))); - + $mockSearchProvider->deleteSearchDocumentsByTypeId('i_search_resource'); - + //search document count should be 1, since there is one document not matching the type id provided for delete $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); diff --git a/test/unit/mongo/MongoTripodSearchDocumentsTest.php b/test/unit/mongo/MongoTripodSearchDocumentsTest.php index 4d4548e8..9bf937f5 100644 --- a/test/unit/mongo/MongoTripodSearchDocumentsTest.php +++ b/test/unit/mongo/MongoTripodSearchDocumentsTest.php @@ -81,6 +81,15 @@ public function testGenerateSearchDocumentBasedOnSpecId() } + public function testGenerateSearchDocumentPreservesDiacritics() + { + $searchDocuments = $this->getSearchDocuments($this->tripod); + $generatedDocuments = $searchDocuments->generateSearchDocumentBasedOnSpecId('i_search_resource', 'http://talisaspire.com/resources/doc13', 'http://talisaspire.com/'); + $this->assertEquals('René Chapus', $generatedDocuments['result']['author']); + $this->assertContains('rené chapus', $generatedDocuments['search_terms']); + $this->assertEquals('http://talisaspire.com/resources/doc13', $generatedDocuments['_id']['r']); + } + public function testGenerateSearchDocumentBasedOnSpecIdWithFieldNamePredicatesHavingNoValueInCollection() { $searchSpecs = json_decode('{"_id":"i_search_resource","type":["bibo:Book"],"from":"CBD_testing","filter":[{"condition":{"dct:title.l":{"$exists":true}}}],"indices":[{"fieldName":"search_terms","predicates":["dct:title","dct:subject"]},{"fieldName":"other_terms","predicates":["rdf:type"]}],"fields":[{"fieldName":"result.title","predicates":["dct:title"],"limit":1},{"fieldName":"result.link","value":"link"},{"fieldName":"rdftype","predicates":["rdf:type"],"limit":1}],"joins":{"dct:creator":{"indices":[{"fieldName":"search_terms","predicates":["foaf:name"]}],"fields":[{"fieldName":"result.author","predicates":["foaf:name"],"limit":1}, {"fieldName":"result.role","predicates":["siocAccess:Role"], "limit":1}] } }}', true); diff --git a/test/unit/mongo/data/searchData.json b/test/unit/mongo/data/searchData.json index fa39f604..1a13cb95 100644 --- a/test/unit/mongo/data/searchData.json +++ b/test/unit/mongo/data/searchData.json @@ -343,6 +343,32 @@ } ] }, + { + "_id": { + "r":"http://talisaspire.com/resources/doc13", + "c":"http://talisaspire.com/" + }, + "_version": 0, + "dct:title": { + "l": "Droit du contentieux administratif" + }, + "rdf:type": [ + { + "u": "http://purl.org/ontology/bibo/Book" + }, + { + "u": "http://talisaspire.com/schema#Resource" + } + ], + "dct:creator":{ + "u": "http://talisaspire.com/authors/4" + }, + "dct:subject":[ + { + "l": "legal" + } + ] + }, { "_id": { "r":"http://talisaspire.com/authors/1", @@ -381,5 +407,18 @@ "rdf:type": { "u": "http://xmlns.com/foaf/0.1/Person" } + }, + { + "_id": { + "r":"http://talisaspire.com/authors/4", + "c":"http://talisaspire.com/" + }, + "_version": 0, + "foaf:name": { + "l": "René Chapus" + }, + "rdf:type": { + "u": "http://xmlns.com/foaf/0.1/Person" + } } ] \ No newline at end of file From 382eb99dae69d7bd6a1de22f3e98717398c01a6c Mon Sep 17 00:00:00 2001 From: Richard Tattersall Date: Thu, 6 Aug 2015 18:06:09 +0100 Subject: [PATCH 3/4] Fixed up tests which have changed now we have more search data --- test/unit/mongo/MongoSearchProviderTest.php | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php index b78a9f9e..7668446b 100644 --- a/test/unit/mongo/MongoSearchProviderTest.php +++ b/test/unit/mongo/MongoSearchProviderTest.php @@ -58,7 +58,7 @@ public function testSearchIndexing() { // assert that there are only 12 based on the data we loaded into tripod $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 13 search documents based on searchData.json"); // define the expected search documents, this is what each of them should look like $expectedSearchDocs = array( @@ -194,7 +194,7 @@ public function testSearchIndexingRemovesDocWhenTypeHasNoCorrespondingSearchdocS $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(11, $actualSearchDocumentCount, "Should only be 11 search documents now that one of them has had its type changed with no corresponding search doc spec"); + $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents now that one of them has had its type changed with no corresponding search doc spec"); foreach(\Tripod\Mongo\Config::getInstance()->getCollectionsForSearch('tripod_php_testing') as $collection) @@ -225,7 +225,7 @@ public function testSearchIndexingGeneratesNewDocForChangedTypeThatHasACorrespon $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents"); $result = array(); foreach(\Tripod\Mongo\Config::getInstance()->getCollectionsForSearch('tripod_php_testing') as $collection) @@ -260,7 +260,7 @@ public function testSearchIndexingGeneratesTwoDocumentsForGivenResourceTheDelete $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents"); + $this->assertEquals(14, $actualSearchDocumentCount, "Should only be 14 search documents"); $results = array(); // We don't know where exactly these might have stored @@ -307,7 +307,7 @@ public function testSearchIndexingGeneratesTwoDocumentsForGivenResourceTheDelete $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should only be 12 search documents"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should only be 13 search documents"); $results = array(); // We don't know where exactly these might have stored @@ -393,10 +393,10 @@ public function testSearchLimitAndOffset() public function testSearchSingleIndex() { // simple search - $results = $this->searchProvider->search("john locke poetry", "i_search_resource", array("search_terms"), array("result"), 3, 0); + $results = $this->searchProvider->search("john locke poetry", "i_search_resource", array("search_terms"), array("result"), 4, 0); $this->assertEquals(6, $results['head']['count']); - $this->assertEquals(3, $results['head']['limit']); - $this->assertEquals(3, count($results['results'])); + $this->assertEquals(4, $results['head']['limit']); + $this->assertEquals(4, count($results['results'])); $this->assertEquals(0, $results['head']['offset']); $this->assertEquals("john locke poetry",$results['head']["query"]); $this->assertEquals(array("john","locke","poetry"), $results['head']["query_terms_used"]); @@ -406,6 +406,7 @@ public function testSearchSingleIndex() array("title"=>"Paradise Lost" , "link"=>"http://talisaspire.com/resources/doc4" , "author"=>"John Locke"), array("title"=>"The Divine Comedy" , "link"=>"http://talisaspire.com/resources/doc5" , "author"=>"John Locke"), array("title"=>"A Light in the Attic" , "link"=>"http://talisaspire.com/resources/doc6" , "author"=>"John Locke"), + array("title"=>"Revolting Rhymes" , "link"=>"http://talisaspire.com/resources/doc7" , "author"=>"John Locke"), ); $this->assertEquals($expectedResults, $results['results']); @@ -452,16 +453,17 @@ public function testSearchSingleIndex() public function testSearchMultipleIndices() { - $results = $this->searchProvider->search("bibo:Book", "i_search_resource", array("search_terms","other_terms"), array("result","rdftype"), 3, 0); - $this->assertEquals(12, $results['head']['count']); - $this->assertEquals(3, $results['head']['limit']); - $this->assertEquals(3, count($results['results'])); + $results = $this->searchProvider->search("bibo:Book", "i_search_resource", array("search_terms","other_terms"), array("result","rdftype"), 4, 0); + $this->assertEquals(13, $results['head']['count']); + $this->assertEquals(4, $results['head']['limit']); + $this->assertEquals(4, count($results['results'])); $this->assertEquals(0, $results['head']['offset']); $this->assertEquals("bibo:Book",$results['head']["query"]); $this->assertEquals(array("bibo:book"), $results['head']["query_terms_used"]); $this->assertArrayHasKey("duration", $results['head']); $expectedResults = array( + array("result"=>array("title"=>"Droit du contentieux administratif" , "link"=>"http://talisaspire.com/resources/doc13" , "author"=>"René Chapus"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Physics for Engineers and Scientists" , "link"=>"http://talisaspire.com/resources/doc1" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Astrophysics for Beginners" , "link"=>"http://talisaspire.com/resources/doc2" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Fundementals of Heat and Mass Transfers" , "link"=>"http://talisaspire.com/resources/doc3" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book") @@ -516,7 +518,7 @@ public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound( // first, assert that there are only 12 based on the data we loaded into tripod $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); /** @var \Tripod\Mongo\MongoSearchProvider|PHPUnit_Framework_MockObject_MockObject $mockSearchProvider */ $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); @@ -536,7 +538,7 @@ public function testDeleteSearchDocumentsByTypeIdDeletesNothingWhenNoMatchFound( //search document count should remain same, because we expect that there was nothing to delete $newSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $newSearchDocumentCount, "Should have generated 12 search documents, because there was no match to remove"); + $this->assertEquals(13, $newSearchDocumentCount, "Should have generated 12 search documents, because there was no match to remove"); } public function testDeleteSearchDocumentsByTypeIdDeleteAllMatchingDocuments() @@ -544,7 +546,7 @@ public function testDeleteSearchDocumentsByTypeIdDeleteAllMatchingDocuments() // first, assert that there are only 12 based on the data we loaded into tripod $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); $mockSearchProvider->expects($this->once()) @@ -565,7 +567,7 @@ public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments // first, assert that there are only 12 based on the data we loaded into tripod $actualSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(12, $actualSearchDocumentCount, "Should have generated 12 search documents based on searchData.json"); + $this->assertEquals(13, $actualSearchDocumentCount, "Should have generated 13 search documents based on searchData.json"); $id = array('_id.r'=>'http://talisaspire.com/resources/doc1'); @@ -582,7 +584,7 @@ public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments //assert that there are now 13 documents after adding new document to collection $updatedSearchDocumentCount = $this->getCountForSearchSpecs($this->tripod); - $this->assertEquals(13, $updatedSearchDocumentCount, "Should have generated 13 search documents after adding a new document to collection"); + $this->assertEquals(14, $updatedSearchDocumentCount, "Should have generated 14 search documents after adding a new document to collection"); $mockSearchProvider = $this->getMock("\Tripod\Mongo\MongoSearchProvider", array('getSearchDocumentSpecification'), array($this->tripod)); $mockSearchProvider->expects($this->once()) From f26ebb586f65d4e64f3cab7253fe2a31932efba3 Mon Sep 17 00:00:00 2001 From: Richard Tattersall Date: Thu, 6 Aug 2015 18:12:09 +0100 Subject: [PATCH 4/4] Fixed up another test --- test/unit/mongo/MongoSearchProviderTest.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php index 7668446b..5d7943c5 100644 --- a/test/unit/mongo/MongoSearchProviderTest.php +++ b/test/unit/mongo/MongoSearchProviderTest.php @@ -453,17 +453,16 @@ public function testSearchSingleIndex() public function testSearchMultipleIndices() { - $results = $this->searchProvider->search("bibo:Book", "i_search_resource", array("search_terms","other_terms"), array("result","rdftype"), 4, 0); + $results = $this->searchProvider->search("bibo:Book", "i_search_resource", array("search_terms","other_terms"), array("result","rdftype"), 3, 0); $this->assertEquals(13, $results['head']['count']); - $this->assertEquals(4, $results['head']['limit']); - $this->assertEquals(4, count($results['results'])); + $this->assertEquals(3, $results['head']['limit']); + $this->assertEquals(3, count($results['results'])); $this->assertEquals(0, $results['head']['offset']); $this->assertEquals("bibo:Book",$results['head']["query"]); $this->assertEquals(array("bibo:book"), $results['head']["query_terms_used"]); $this->assertArrayHasKey("duration", $results['head']); $expectedResults = array( - array("result"=>array("title"=>"Droit du contentieux administratif" , "link"=>"http://talisaspire.com/resources/doc13" , "author"=>"René Chapus"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Physics for Engineers and Scientists" , "link"=>"http://talisaspire.com/resources/doc1" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Astrophysics for Beginners" , "link"=>"http://talisaspire.com/resources/doc2" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book"), array("result"=>array("title"=>"Fundementals of Heat and Mass Transfers" , "link"=>"http://talisaspire.com/resources/doc3" , "author"=>"Sayid Jarrah"), "rdftype"=>"bibo:Book")