unit tests for SURT keys from non-utf-8 encoded URLs, tests #6

commoncrawl · Jan 24, 2017 · e393643 · e393643
1 parent eb66fc4
commit e393643
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java b/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java
@@ -143,6 +143,9 @@ public void testUnescapeRepeatedly() {
 		assertEquals("%",guc.unescapeRepeatedly("%25%32%35"));
 
 		assertEquals("168.188.99.26",guc.unescapeRepeatedly("%31%36%38%2e%31%38%38%2e%39%39%2e%32%36"));
+
+		assertEquals("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5",
+				guc.unescapeRepeatedly("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5"));
 	}
 
 	public void testAttemptIPFormats() throws URIException {

diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java
@@ -23,6 +23,10 @@ public void testMakeKey() throws URISyntaxException {
 		assertEquals("org,archive)/goo?a&b", km.makeKey("http://archive.org/goo/?b&a"));
 		assertEquals("org,archive)/goo?a=1&a=2&b", km.makeKey("http://archive.org/goo/?a=2&b&a=1"));
 		assertEquals("org,archive)/", km.makeKey("http://archive.org:/"));
+		assertEquals("ua,1kr)/newslist.html?tag=%e4%ee%f8%ea%ee%eb%fc%ed%ee%e5",
+				km.makeKey("http://1kr.ua/newslist.html?tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5"));
+		assertEquals("com,aluroba)/tags/%c3%ce%ca%c7%d1%e5%c7.htm",
+				km.makeKey("http://www.aluroba.com/tags/%C3%CE%CA%C7%D1%E5%C7.htm"));
 	}
 
 }