Merge pull request #2 from xiffy/master

python 3 compatible

Author: Willem Jan Faber

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 __pycache__
+build

example.py

@@ -11,8 +11,8 @@
 oai_handler = oai
 oai_handler.current_set = "ANP"
 
-response = sru.search("beatrix AND juliana AND Bernhard AND telegram", "ANP")
-print "Number of records: %i" % response.sru.nr_of_records
+response = sru.search("karel AND reve AND hooftprijs", "ANP")
+print ("Number of records: %i" % response.sru.nr_of_records)
 
 record_nr = 0
 
@@ -26,10 +26,9 @@
 
 
     oai_handler.DEBUG = True
-    record = oai_handler.get(record.identifiers[0])
-    print(record.alto)
-    for alto in record.alto:
-        #print("Fulltext: %s" % alto_to_text(alto))
-        print(alto)
+    r = oai_handler.get(record.identifiers[0])
+    for alto in r.alto:
+        print("Fulltext: %s" % alto_to_text(alto))
+        #print(alto)
 
     print("*************************************\n\n")

kb/nl/api/oai.py

@@ -7,7 +7,7 @@
 OAI_BASEURL = 'http://services.kb.nl/mdo/oai'
 
 
-class oai():
+class oai:
 
     """
         OAI interface to the National Library of the Netherlands.

kb/nl/api/sru.py

@@ -1,6 +1,7 @@
 import sys
 import requests
 import urllib
+import pprint
 
 try:
     from urllib import quote  # Python 2.X
@@ -10,7 +11,7 @@
 from kb.nl.collections import SETS
 from kb.nl.helpers import etree
 
-SRU_BASEURL = 'http://jsru.kb.nl/sru/sru'
+SRU_BASEURL = 'https://jsru.kb.nl/sru/sru'
 SRU_BASEURL += '?version=1.2&maximumRecords=%i'
 SRU_BASEURL += '&operation=searchRetrieve'
 SRU_BASEURL += '&startRecord=%i'
@@ -36,7 +37,7 @@ def records(self):
     # TODO: distinguish by xsi:type
     @property
     def identifiers(self):
-        baseurl = 'http://resolver.kb.nl/resolve?urn='
+        baseurl = 'https://resolver.kb.nl/resolve?urn='
         result = [r.text.replace(baseurl, '') for r in self.record_data.iter() if
                   r.tag.endswith('identifier') and r.text.find(':') > -1]
         return result
@@ -108,7 +109,7 @@ def __init__(self, record_data, sru):
     def __iter__(self):
         return self
 
-    def next(self):
+    def __next__(self):
         if self.sru.nr_of_records == 0:
             raise StopIteration
         if self.sru.startrecord < self.sru.nr_of_records + 1:
@@ -118,9 +119,12 @@ def next(self):
         else:
             raise StopIteration
 
+    def next(self):
+        return self.__next__()
+
 
 class sru():
-    DEBUG = False
+    DEBUG = True
 
     collection = False
     maximumrecords = 50
@@ -166,7 +170,7 @@ def run_query(self):
         url = SRU_BASEURL % (self.maximumrecords, self.startrecord,
                              self.recordschema, self.collection, self.query)
         if self.DEBUG:
-                sys.stdout.write(url)
+            print("run_query: %s" % url)
 
         r = requests.get(url)
 

kb/nl/collections/__init__.py

@@ -12,14 +12,19 @@
                 'recordschema': 'ddd',
                 'setname': 'DPO',
                 'time_period': [1781, 1800]},
-        'BYVANCK': {'description_en': 'Medieval Illuminated Manuscripts',
+        'BYVANCK': {'collection': 'MISC',
+                    'description_en': 'Medieval Illuminated Manuscripts',
                     'description_nl': 'Middeleeuwse Verluchte Handschriften',
                     'metadataPrefix': 'dcx',
+                    'recordschema': 'dcx',
                     'setname': 'BYVANCK',
-                    'time_period': [500, 1500]},
-        'SGD': {'description_en': 'States General Digital',
+                    'time_period': [500, 1500],
+                    'extra_query' : '(dcterms:isPartOf="ByvanckB" OR dcterms:isPartOf="BYVANCK")'},
+        'SGD': {'collection': 'SGD',
+        	      'description_en': 'States General Digital',
                 'description_nl': 'Staten-Generaal Digitaal',
                 'metadataPrefix': 'dcx',
+                'recordschema': 'dcx',
                 'setname': 'sgd:register',
                 'time_period': [1962, 1994]},
         'GGC': {'collection': 'GGC',
@@ -28,4 +33,5 @@
                 'metadataPrefix': 'dcx',
                 'recordschema': 'dcx',
                 'setname': 'ggc',
-                'time_period': [1937, 2016]}}
+                'time_period': [1937, 2016]}
+       }

kb/nl/helpers/__init__.py

@@ -18,11 +18,19 @@
                     import elementtree.ElementTree as etree
                 except ImportError:
                     raise("Failed to import ElementTree from any known place")
-
+import codecs
 
 def alto_to_text(alto_data):
     ''' Grab the selected text blocks and write them to disk '''
-    alto_data = etree.fromstring(alto_data.encode('utf-8'))
+    try:
+        # some files have a BOM available which does not get stripped on windows
+        if not alto_data[0] == '<':
+            alto_data = alto_data[3:]
+        alto_data = etree.fromstring(alto_data.encode('utf-8'))
+
+    except etree.XMLSyntaxError as e:
+        print ("error! %s" % alto_str[:40])
+        return None
 
     alto_text = u""
     prev_was_hyp = False