Skip to content

Commit

Permalink
fixed test parser
Browse files Browse the repository at this point in the history 
Signed-off-by: minff <[email protected]>
  • Loading branch information
@minff
minff committed Oct 13, 2023
1 parent 6d86f10 commit 8fdb658
Showing 1 changed file with 101 additions and 29 deletions.
130 changes: 101 additions & 29 deletions test/test_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
format_map_fields,
)

from qgis.core import QgsFields, QgsVectorLayer
from qgis.core import QgsFields, QgsVectorLayer, QgsWkbTypes
from qgis.testing import unittest
from XYZHubConnector.xyz_qgis.layer import parser

Expand All @@ -31,7 +31,23 @@
class TestParser(BaseTestAsync):
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self.similarity_threshold = 80
self.similarity_threshold = 0
self.mixed_case_duplicate = False

# util for debug
def assertEqual(self, first, second, msg=None):
if first != second:
msg = self._log_error(msg)
super().assertEqual(first, second, msg)

def assertTrue(self, expr, msg=None):
if not expr:
msg = self._log_error(msg)
super().assertTrue(expr, msg)

def fail(self, msg):
msg = self._log_error(msg)
return super().fail(msg)

# ######## Parse xyz geojson -> QgsFeature
def test_parse_xyzjson(self):
Expand All @@ -50,9 +66,14 @@ def subtest_parse_xyzjson(self, folder, fname):
obj_feat = obj["features"]
fields = QgsFields()
feat = [parser.xyz_json_to_feature(ft, fields) for ft in obj_feat]
o1 = obj_feat[0] if len(obj_feat) else None
geom_str = o1 and o1["geometry"] and o1["geometry"]["type"]

self._assert_parsed_id(obj_feat, feat, fields)
self._assert_parsed_fields_unorder(obj_feat, feat, fields)
self._assert_parsed_fields(obj_feat, feat, fields)
self._assert_parsed_geom(obj_feat, feat, fields)
self._assert_parsed_geom_unorder(obj_feat, feat, fields, geom_str)
self._assert_parsed_geom(obj_feat, feat, fields, geom_str)
return feat

def _assert_parsed_fields_unorder(self, obj_feat, feat, fields):
Expand All @@ -66,22 +87,21 @@ def _assert_parsed_fields_unorder(self, obj_feat, feat, fields):
self.assertEqual(len(obj_feat), len(feat))

def _assert_parsed_fields(self, obj_feat, feat, fields):
self._assert_parsed_fields_unorder(obj_feat, feat, fields)

def msg_fields(obj):
return (
"{sep}{0}{sep}{1}"
"{sep}fields-props {2}"
"{sep}props-fields {3}"
"{sep}props {0}"
"{sep}fields {1}"
"{sep}props-fields {2} (should be 0)"
"{sep}fields-props {3}"
"{sep}json {4}".format(
*tuple(
map(
lambda x: "%s %s" % (len(x), x),
[
obj_props,
fields.names(),
set(fields.names()).difference(obj_props),
set(obj_props).difference(fields.names()),
set(fields.names()).difference(obj_props),
],
)
),
Expand All @@ -92,31 +112,56 @@ def msg_fields(obj):

for o in obj_feat:
obj_props = list(o["properties"].keys())
obj_props_non_null = [k for k, v in o["properties"].items() if v is not None]
self.assertLessEqual(len(obj_props), fields.size(), msg_fields(o))
self.assertTrue(set(obj_props) < set(fields.names()), msg_fields(o))
# self._log_debug(msg_fields(o).replace(">>", "++"))

obj_props_is_subset_of_fields = any(
[
set(obj_props) < set(fields.names()),
set(obj_props_non_null) < set(fields.names()),
]
)
self.assertTrue(obj_props_is_subset_of_fields, msg_fields(o))
# self.assertEqual( obj_props, fields.names(), msg_fields(o)) # strict assert

def wkb_type_to_wkt_str(self, typ):
return QgsWkbTypes.displayString(typ)

def wkb_type_to_geom_str(self, typ):
return QgsWkbTypes.displayString(typ % 1000) if typ else None

def wkb_type_to_geom_display_str(self, typ):
return (
QgsWkbTypes.geometryDisplayString(QgsWkbTypes.geometryType(typ))
if typ
else "No geometry"
)

def _assert_parsed_geom_unorder(self, obj_feat, feat, fields, geom_str):
wkt_ref = self.wkb_type_to_wkt_str(QgsWkbTypes.parseType(geom_str))
for ft in feat:
geom = json.loads(
ft.geometry().asJson()
) # limited to 13 or 14 precison (ogr.CreateGeometryFromJson)
self.assertEqual(geom and geom["type"], geom_str)

def _assert_parsed_geom(self, obj_feat, feat, fields):
geom_str_ft = self.wkb_type_to_geom_str(ft.geometry().wkbType())
wkt_ft = self.wkb_type_to_wkt_str(ft.geometry().wkbType())
msg = "wkt string {} != {}".format(wkt_ft, wkt_ref)
self.assertEqual(geom_str_ft, geom_str, msg)

def _assert_parsed_geom(self, obj_feat, feat, fields, geom_str):
# both crs is WGS84
for o, ft in zip(obj_feat, feat):
geom = json.loads(
ft.geometry().asJson()
) # limited to 13 or 14 precison (ogr.CreateGeometryFromJson)
obj_geom = o["geometry"]

self.assertEqual(geom["type"], obj_geom["type"])

id_ = ft.attribute(parser.QGS_XYZ_ID)
obj_id_ = o["id"]
self.assertEqual(id_, obj_id_)
self.assertEqual(geom and geom["type"], obj_geom and obj_geom["type"])
if not geom or not obj_geom:
continue

# self._log_debug(geom)
# self._log_debug(obj_geom)
Expand All @@ -130,17 +175,32 @@ def _assert_parsed_geom(self, obj_feat, feat, fields):

c1 = np.array(obj_geom["coordinates"])
c2 = np.array(geom["coordinates"])
c1_flatten = np.array(flatten(obj_geom["coordinates"]))
c2_flatten = np.array(flatten(geom["coordinates"]))
if c1.shape != c2.shape:
self._log_debug(
"\nWARNING: Geometry has mismatch shape",
c1.shape,
c2.shape,
"\nOriginal geom has problem. Testing parsed geom..",
)
self.assertEqual(c2.shape[-1], 2, "parsed geom has wrong shape of coord")
msg = (
"parsed geom has wrong shape of coord for geom {geom}. {} != {}".format(
c1.shape, c2.shape, geom=geom["type"]
),
)
self.assertEqual(c2.shape[-1], 2, msg)
continue
else:
self.assertLess(np.max(np.abs(c1 - c2)), 1e-13, "parsed geometry error > 1e-13")
self.assertLess(
np.max(np.abs(c1_flatten - c2_flatten)), 1e-13, "parsed geometry error > 1e-13"
)

def _assert_parsed_id(self, obj_feat, feat, fields):
for o, ft in zip(obj_feat, feat):
id_ = ft.attribute(parser.QGS_XYZ_ID)
obj_id_ = o["id"]
self.assertEqual(id_, obj_id_)

# @unittest.skip("large")
def test_parse_xyzjson_large(self):
Expand Down Expand Up @@ -187,6 +247,7 @@ def test_parse_xyzjson_map_similarity_0(self):
self.similarity_threshold = s

def test_parse_xyzjson_map_dupe_case(self):
self.mixed_case_duplicate = True
folder = "xyzjson-small"
fnames = [
"airport-xyz.geojson",
Expand Down Expand Up @@ -302,7 +363,7 @@ def subtest_parse_xyzjson_mix(self, folder, fnames):
def subtest_parse_xyzjson_map_multi_chunk(self, obj, lst_chunk_size=None):
if not lst_chunk_size:
p10 = 1 + len(str(len(obj["features"])))
lst_chunk_size = [10 ** i for i in range(p10)]
lst_chunk_size = [10**i for i in range(p10)]
with self.subTest(lst_chunk_size=lst_chunk_size):
ref_map_feat, ref_map_fields = self.do_test_parse_xyzjson_map(obj)
lst_map_fields = list()
Expand All @@ -323,13 +384,14 @@ def subtest_parse_xyzjson_map_shuffle(self, obj, n_shuffle=5, chunk_size=10):
lst_map_fields = list()
random.seed(0.5)
for i in range(n_shuffle):
random.shuffle(o["features"])
map_fields = self.subtest_parse_xyzjson_map_chunk(o, chunk_size)
if map_fields is None:
continue
lst_map_fields.append(map_fields)
with self.subTest(shuffle=i):
random.shuffle(o["features"])
map_fields = self.subtest_parse_xyzjson_map_chunk(o, chunk_size)
if map_fields is None:
continue
lst_map_fields.append(map_fields)

# self._log_debug("parsed fields shuffle", len_of_struct(map_fields))
# self._log_debug("parsed fields shuffle", len_of_struct(map_fields))

for i, map_fields in enumerate(lst_map_fields):
with self.subTest(shuffle=i):
Expand All @@ -339,7 +401,7 @@ def subtest_parse_xyzjson_map_chunk(self, obj, chunk_size=100):
similarity_threshold = self.similarity_threshold
with self.subTest(chunk_size=chunk_size, similarity_threshold=similarity_threshold):
o = dict(obj)
obj_feat = obj["features"]
obj_feat = list(obj["features"])
lst_map_feat = list()
map_fields = dict()
for i0 in range(0, len(obj_feat), chunk_size):
Expand Down Expand Up @@ -397,12 +459,22 @@ def _assert_parsed_map(self, obj_feat, map_feat, map_fields):

# NOTE: obj_feat order does not corresponds to that of map_feat
# -> use unorder assert
# NOTE: group obj_feat by geom_str for element-wise assert
for geom_str in map_feat:
for feat, fields in zip(map_feat[geom_str], map_fields[geom_str]):
o = obj_feat[: len(feat)]
self._assert_parsed_fields_unorder(o, feat, fields)
o = [
o1
for o1 in obj_feat
if (o1["geometry"] and o1["geometry"]["type"]) == geom_str
]
self._assert_parsed_id(o, feat, fields)
self._assert_parsed_geom_unorder(o, feat, fields, geom_str)
obj_feat = obj_feat[len(feat) :]
self._assert_parsed_fields_unorder(o, feat, fields)

if not self.mixed_case_duplicate:
# element-wise assert
self._assert_parsed_geom(o, feat, fields, geom_str)
self._assert_parsed_fields(o, feat, fields)

def _assert_len_map_feat_fields(self, map_feat, map_fields):
self.assertEqual(map_feat.keys(), map_fields.keys())
Expand Down

0 comments on commit 8fdb658

Please sign in to comment.