Add enricher tests

Author: andersju

__init__.py

@@ -27,7 +27,7 @@ def recover_doi(body, field):
     if translated != doi:
         doi = translated
         update_at_path(body, path, doi)
-        field.events.append(make_event(type="enrichment", code="unicode", initial_value=initial, value=doi, result="enriched"))
+        field.events.append(make_event(type="enrichment", code="recovery", initial_value=initial, value=doi, result="enriched"))
         field.value = doi
 
     if doi.startswith(VALID_STARTS):

pipeline/enrichers/doi.py

@@ -33,7 +33,7 @@ def recover_isbn(body, field):
         update_at_path(body, field.path, res[0])
         field.value = res[0]
         field.enrichment_status = "enriched"
-        field.events.append(make_event(type="enrichment", code="split", value=res[0], initial_value=isbn, result="enriched"))
+        field.events.append(make_event(type="enrichment", code="recovery", value=res[0], initial_value=isbn, result="enriched"))
 
     if field.enrichment_status != 'enriched':
         field.enrichment_status = 'unsuccessful'

pipeline/enrichers/isbn.py

@@ -37,7 +37,7 @@ def recover_orcid(body, field):
     orcid_list = [h.group() for h in hit]
     if len(orcid_list) != 0:
         if orcid_list[0] != orcid:
-            field.events.append(make_event(type="enrichment", code="split", value=orcid_list[0], initial_value=orcid, result="enriched"))
+            field.events.append(make_event(type="enrichment", code="recovery", value=orcid_list[0], initial_value=orcid, result="enriched"))
             update_at_path(body, path, orcid_list[0])
             field.enrichment_status = 'enriched'
             field.value = orcid_list[0]

pipeline/enrichers/orcid.py

@@ -0,0 +1,111 @@
+from pipeline.validate import FieldMeta
+from pipeline.enrichers.doi import recover_doi
+
+
+def test_doi_enrich_remove_whitespace():
+    test_data = "10.5468 /ogs.2010.59.10.1"
+    expected_result = "10.5468/ogs.2010.59.10.1"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_unicode_cat_cc():
+    # Test removal of tab from unicode category Cc.
+    test_data = "10.10/invalid\t"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_unicode_cat_cf():
+    # Test removal of zero width space from unicode category Cf.
+    test_data = "10.10/invalid\u200b"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_unicode_cat_zl():
+    # Test removal of line separator from unicode category Zl.
+    test_data = "10.10/invalid\u2028"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_unicode_cat_zp():
+    # Test removal of paragraph separator from unicode category Zp.
+    test_data = "10.10/invalid\u2029"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_unicode_cat_zs():
+    # Test removal of white space (no-break space) from unicode category Zs.
+    # Regular space also belongs in this category.
+    test_data = "10.10/invalid\xa0"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_replace_fraction_slash():
+    # Test replacement of fractional slash with regular slash.
+    test_data = "10.10\u2044invalid"
+    expected_result = "10.10/invalid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_remove_prefix():
+    # Test removal of invalid prefixes.
+    test_data = "Doi:10.10/valid"
+    expected_result = "10.10/valid"
+    _test_doi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_doi_enrich_valid_prefixes():
+    # Test valid prefixes left untouched.
+    test_data = "https://doi.org/10.10/valid"
+    expected_result = "https://doi.org/10.10/valid"
+    body = {"partOf": [{"identifiedBy": [{"value": test_data}]}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        path="partOf.[0].identifiedBy.[0].value",
+    )
+
+    recover_doi(body, field)
+    assert field.value == expected_result
+    assert field.events == []
+
+    test_data = "http://doi.org/10.10/valid"
+    expected_result = "http://doi.org/10.10/valid"
+    body = {"partOf": [{"identifiedBy": [{"value": test_data}]}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        path="partOf.[0].identifiedBy.[0].value",
+    )
+
+    recover_doi(body, field)
+    assert field.value == expected_result
+    assert field.events == []
+
+
+def _test_doi_enrichment(test_data, expected_result, expected_codes):
+    body = {"partOf": [{"identifiedBy": [{"value": test_data}]}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        path="partOf.[0].identifiedBy.[0].value",
+    )
+
+    recover_doi(body, field)
+
+    assert field.value == expected_result
+    if len(field.events) > 0:
+        assert field.events[0]["type"] == "enrichment"
+        for event in field.events:
+            assert event["initial_value"] == test_data
+            assert event["code"] in expected_codes

pipeline/tests/enrichers/__init__.py

@@ -0,0 +1,136 @@
+from pipeline.validate import FieldMeta
+from pipeline.enrichers.isbn import recover_isbn
+
+
+def test_isbn_with_prefix():
+    test_data = "ISBN 978-91-86857-16-5"
+    expected_result = "978-91-86857-16-5"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn_with_prefix_and_suffix():
+    test_data = "ISBN: 978-91-7740-107-0 (pdf)"
+    expected_result = "978-91-7740-107-0"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn10_with_prefix_and_numeric_control():
+    test_data = "ISBN 0415579961"
+    expected_result = "0415579961"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn10_with_prefix_and_lowercase_x_control():
+    test_data = "ISBN 041557996x"
+    expected_result = "041557996x"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn10_with_prefix_and_capital_X_control():
+    test_data = "ISBN: 917291341X"
+    expected_result = "917291341X"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn13_with_prefix():
+    test_data = "ISBN: 9172-91-341-X"
+    expected_result = "9172-91-341-X"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isbn13_with_numerical_prefix():
+    test_data = "13 9780853583080"
+    expected_result = "9780853583080"
+    _test_isbn_enrichment(test_data, expected_result, ["recovery"])
+
+
+# TODO: Fix split in enricher
+# def test_dual_isbn():
+#    test_data = '0415579961, 9780415579964'
+#    original_field_data = {'value': test_data, 'enrichment_status': 'pending', 'path': '<path>',
+#                           'validation_status': 'true', 'events': []}
+#    field, actions = isbn_enricher(FieldMetadata(original_field_data))
+#    assert field.value == '0415579961'
+#    action1 = actions[0]
+#    event1 = field.events[0]
+#    assert isinstance(action1, DuplicateFieldAction)
+#    assert action1.field_type == 'isbn'
+#    assert event1['new_value'] == '0415579961'
+#    assert event1['old_value'] == '0415579961, 9780415579964'
+#    assert event1['code'] == 'recovery'
+#    action2 = actions[1]
+#    event2 = field.events[1]
+#    assert isinstance(action2, ChangeAction)
+#    assert action2.field_type == 'isbn'
+#    assert event2['created_value'] == '9780415579964'
+#    assert event2['old_value'] == '0415579961, 9780415579964'
+#    assert event2['code'] == 'split'
+
+
+def test_isbn_non_recoverable_wrong_length_and_strange_prefix():
+    test_data = "2-s2.0-84863732124"
+    _test_none_isbn_enrichment(test_data)
+
+
+def test_isbn_non_recoverable_issn():
+    test_data = "0022-2380"
+    _test_none_isbn_enrichment(test_data)
+
+
+def test_isbn_non_recoverable_doi():
+    test_data = "10.1046/j.1365-2834.1999.00138.x"
+    _test_none_isbn_enrichment(test_data)
+
+
+def test_isbn_non_recoverable_to_many_digits():
+    test_data = "12345678901234567890"
+    _test_none_isbn_enrichment(test_data)
+
+
+def test_isbn_non_recoverable_doi_containing_isbn():
+    test_data = "10.1046/j.978-91-86857-16-5"
+    _test_none_isbn_enrichment(test_data)
+
+
+def test_isbn_non_recoverable_numerical_prefix_without_delimiter():
+    test_data = "13978-91-975576-6-5"
+    _test_none_isbn_enrichment(test_data)
+
+
+def _test_isbn_enrichment(test_data, expected_result, expected_codes):
+    body = {"identifiedBy": [{"value": test_data}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        enrichment_status="pending",
+        path="identifiedBy.[0].value",
+    )
+
+    recover_isbn(body, field)
+
+    assert field.value == expected_result
+    if len(field.events) > 0:
+        assert field.events[0]["type"] == "enrichment"
+        for event in field.events:
+            assert event["initial_value"] == test_data
+            assert event["code"] in expected_codes
+
+
+def _test_none_isbn_enrichment(test_data):
+    body = {"identifiedBy": [{"value": test_data}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        enrichment_status="pending",
+        path="identifiedBy.[0].value",
+    )
+
+    recover_isbn(body, field)
+
+    assert field.value == test_data
+    assert not field.is_enriched()
+    assert len(field.events) == 0

pipeline/tests/enrichers/test_doi_enricher.py

@@ -0,0 +1,93 @@
+from pipeline.validate import FieldMeta
+from pipeline.enrichers.isi import recover_isi
+
+
+def test_correct_isi():
+    test_data = "000123456789123"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_correct_isi_with_prefix():
+    test_data = "ISI: 000321123456789"
+    expected_result = "000321123456789"
+    _test_isi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_correct_isi_with_suffix():
+    test_data = "A19123456789XYZ checked"
+    expected_result = "A19123456789XYZ"
+    _test_isi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_correct_isi_with_incorrect_first_sign():
+    test_data = ":000321123456789"
+    expected_result = "000321123456789"
+    _test_isi_enrichment(test_data, expected_result, ["recovery"])
+
+
+def test_isi_non_recoverable_values_incorrect_prefix_001_instead_of_000():
+    test_data = "13978-91-975576-6-5"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_isi_non_recoverable_values_incorrect_prefix_8_instead_of_9():
+    test_data = "A18123456789123"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_isi_non_recoverable_values_to_short():
+    test_data = "00012345678912"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_isi_non_recoverable_values_to_long():
+    test_data = "00012345678912345"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_isi_non_recoverable_values_invalid_characters():
+    test_data = "A1912A456!@#$%^"
+    _test_none_isi_enrichment(test_data)
+
+
+def test_isi_non_recoverable_values_incorrect_prefix_000_with_non_numeric_characters():
+    test_data = "000A23456789123"
+    _test_none_isi_enrichment(test_data)
+
+
+def _test_isi_enrichment(test_data, expected_result, expected_codes):
+    body = {"identifiedBy": [{"value": test_data}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        enrichment_status="pending",
+        path="identifiedBy.[0].value",
+    )
+
+    recover_isi(body, field)
+
+    assert field.value == expected_result
+    if len(field.events) > 0:
+        assert field.events[0]["type"] == "enrichment"
+        for event in field.events:
+            assert event["initial_value"] == test_data
+            assert event["code"] in expected_codes
+
+
+def _test_none_isi_enrichment(test_data):
+    body = {"identifiedBy": [{"value": test_data}]}
+
+    field = FieldMeta(
+        value=test_data,
+        normalization_status="unchanged",
+        validation_status="valid",
+        enrichment_status="pending",
+        path="identifiedBy.[0].value",
+    )
+
+    recover_isi(body, field)
+    assert field.value == test_data
+    assert not field.is_enriched()
+    assert len(field.events) == 0