Skip to content

Commit

Permalink
Prevent some error when open geojson with non unique entries in id field
Browse files Browse the repository at this point in the history
  • Loading branch information
mthh committed Apr 18, 2018
1 parent 9ffaebc commit 4df5d31
Show file tree
Hide file tree
Showing 11 changed files with 55 additions and 13 deletions.
6 changes: 6 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
Changes
=======

0.7.4 (2018-04-18)
------------------

- Prevent some error when opening layer with non unique entries in field named 'id' (internally caused by the fact we use geojson and fiona is failing on opening geojson with duplicates in that field).


0.7.3 (2018-03-21)
------------------

Expand Down
2 changes: 1 addition & 1 deletion magrit_app/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# -*- coding: utf-8 -*-

__version__ = '0.7.3'
__version__ = '0.7.4'
33 changes: 32 additions & 1 deletion magrit_app/helpers/geo.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import ujson as json
import cchardet as chardet
import tempfile
from fiona import _err as fiona_err
from osgeo.ogr import GetDriverByName, Feature as OgrFeature
from osgeo.osr import SpatialReference, CoordinateTransformation
from osgeo.gdal import VectorTranslate, OpenEx, UseExceptions as gdal_UseExceptions
Expand Down Expand Up @@ -351,7 +352,9 @@ def make_geojson_links(ref_layer_geojson, csv_table, field_i, field_j, field_fij


def make_carto_doug(file_path, field_name, iterations):
gdf = GeoDataFrame.from_file(file_path)
gdf, replaced_id_field = try_open_geojson(file_path)
if replaced_id_field and field_name == 'id':
field_name = '_id'
if not gdf[field_name].dtype in (int, float):
gdf.loc[:, field_name] = gdf[field_name].replace('', np.NaN)
gdf.loc[:, field_name] = gdf[field_name].astype(float)
Expand Down Expand Up @@ -572,3 +575,31 @@ def multi_to_single(gdf, columns=None):
geometry=geoms,
columns=columns or [i for i in gdf.columns if i != 'geometry']
)


def replace_geojson_id_field(input_file, fb=False):
with open(input_file, 'rb') as f:
data = f.read()
nb_ft = data.count(b'"Feature"')
_nb_ft = data.count(b'"geometry"')
if not fb and data.count(b'"id"') == nb_ft and nb_ft == _nb_ft:
data = data.replace(b'"id"', b'"_id"')
with open(input_file, 'wb') as f:
f.write(data)
else:
data = json.loads(data)
for ix, ft in enumerate(data['features']):
ft['properties']['_id'] = ft['properties']['id']
del ft['properties']['id']
with open(input_file, 'wb') as f:
f.write(json.dumps(data).encode())

def try_open_geojson(file_path):
replaced = False
try:
gdf = GeoDataFrame.from_file(file_path)
except fiona_err.CPLE_AppDefinedError as err:
replace_geojson_id_field(file_path)
replaced = True
gdf = GeoDataFrame.from_file(file_path)
return gdf, replaced
12 changes: 6 additions & 6 deletions magrit_app/helpers/grid_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@
"""
from geopandas import GeoDataFrame, GeoSeries
from shapely.geometry import Polygon
from math import ceil
from shapely.ops import cascaded_union
from shapely import speedups
import rtree
import numpy as np
import ujson as json
from .geo import repairCoordsPole, TopologicalError, multi_to_single
from .geo import (
repairCoordsPole, TopologicalError,
multi_to_single, try_open_geojson)
from .grid_helpers import (
square_grid_gen, diams_grid_gen, hex_grid_gen, to_float, make_index)

Expand All @@ -21,8 +20,9 @@ def get_grid_layer(input_file, height, field_name, grid_shape="square"):
proj_robinson = (
"+proj=robin +lon_0=0 +x_0=0 +y_0=0 "
"+ellps=WGS84 +datum=WGS84 +units=m +no_defs")
gdf = GeoDataFrame.from_file(input_file)

gdf, replaced_id_field = try_open_geojson(input_file)
if replaced_id_field and field_name == 'id':
field_name = '_id'
if not gdf[field_name].dtype in (int, float):
# gdf.loc[:, field_name] = gdf[field_name].replace('', np.NaN)
# gdf.loc[:, field_name] = gdf[field_name].astype(float)
Expand Down
9 changes: 7 additions & 2 deletions magrit_app/helpers/grid_layer_pt.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
from shapely import speedups
import numpy as np
import ujson as json
from .geo import repairCoordsPole, TopologicalError, multi_to_single
from .geo import (
repairCoordsPole, TopologicalError,
multi_to_single, try_open_geojson)
from .grid_helpers import (
square_grid_gen, diams_grid_gen, hex_grid_gen, to_float, make_index)

Expand Down Expand Up @@ -39,7 +41,10 @@ def get_grid_layer_pt(input_file, height, field_name,
proj_robinson = (
"+proj=robin +lon_0=0 +x_0=0 +y_0=0 "
"+ellps=WGS84 +datum=WGS84 +units=m +no_defs")
gdf = GeoDataFrame.from_file(input_file)

gdf, replaced_id_field = try_open_geojson(input_file)
if replaced_id_field and field_name == 'id':
field_name = '_id'

if field_name:
if not gdf[field_name].dtype in (int, float):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1261,7 +1261,7 @@ function parseQuery(search) {
lng: lang,
fallbackLng: _app.existing_lang[0],
backend: {
loadPath: 'static/locales/{{lng}}/translation.9949d18a0537.json'
loadPath: 'static/locales/{{lng}}/translation.0f4c519d1d51.json'
}
}, function (err, tr) {
if (err) {
Expand Down
4 changes: 2 additions & 2 deletions magrit_app/templates/modules.html
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<title> {{ app_name }} </title>
<link rel="icon" type="image/png" href="static/img/favicon.png" />
<link href="static/css/scoped-bootstrap.min.css" rel="stylesheet" type="text/css">
<link href="static/css/style.9949d18a0537.min.css" rel="stylesheet" type="text/css">
<link href="static/css/style.0f4c519d1d51.min.css" rel="stylesheet" type="text/css">
<link href="static/css/_vanilla-dataTables.min.css" rel="stylesheet" type="text/css">
<link href="static/css/alertify.min.css" rel="stylesheet" type="text/css">
<link href="static/css/semantic.min.css" rel="stylesheet" type="text/css">
Expand Down Expand Up @@ -52,6 +52,6 @@
</div>
</noscript>
<script src="static/js/lib/bootstrap-native3.mod.min.js"></script>
<script src="static/js/app.9949d18a0537.js"></script>
<script src="static/js/app.0f4c519d1d51.js"></script>
</body>
</html>

0 comments on commit 4df5d31

Please sign in to comment.