Merge pull request #1012 from sickelap/refactor_geocode

Make use of Geopy for reverse geolocation

api/api_util.py

@@ -1,7 +1,6 @@
 import os
 import random
 import stat
-from collections import Counter
 from datetime import datetime
 
 import numpy as np
@@ -84,7 +83,6 @@ def jump_by_month(start_date, end_date, month_step=1):
 
 
 def get_location_timeline(user):
-    city_start_end_duration = []
     with connection.cursor() as cursor:
         raw_sql = """
             WITH data AS (
@@ -741,33 +739,11 @@ def get_location_clusters(user):
         """
         cursor.execute(raw_sql, [user.id])
         res = [[float(row[2]), float(row[1]), row[0]] for row in cursor.fetchall()]
-
         elapsed = (datetime.now() - start).total_seconds()
         logger.info("location clustering took %.2f seconds" % elapsed)
         return res
 
 
-def get_photo_country_counts(user):
-    with connection.cursor() as cursor:
-        raw_sql = """
-            SELECT
-                DISTINCT ON(jsonb_extract_path("feature", 'place_name')) jsonb_extract_path("feature", 'place_name') "place_name"
-                , COUNT(jsonb_extract_path("feature", 'place_name'))
-            FROM
-                "api_photo"
-                , jsonb_array_elements(jsonb_extract_path("api_photo"."geolocation_json", 'features')) "feature"
-            WHERE
-                (
-                    "api_photo"."owner_id" = %s
-                    AND jsonb_extract_path_text("feature", 'place_type', '0') = 'country'
-                )
-            GROUP BY
-                "place_name";
-        """
-        cursor.execute(raw_sql, [user.id])
-        return Counter({row[0]: row[1] for row in cursor.fetchall()})
-
-
 def get_location_sunburst(user):
     levels = []
     with connection.cursor() as cursor:

api/background_tasks.py

@@ -25,7 +25,7 @@ def geolocate(overwrite=False):
     for photo in photos:
         try:
             logger.info("geolocating %s" % photo.main_file.path)
-            photo._geolocate_mapbox()
+            photo._geolocate()
             photo._add_location_to_album_dates()
         except Exception:
             logger.exception("could not geolocate photo: " + photo)

api/directory_watcher.py

@@ -161,7 +161,7 @@ def handle_new_image(user, path, job_id):
                     job_id, path, elapsed
                 )
             )
-            photo._geolocate_mapbox(True)
+            photo._geolocate(True)
             elapsed = (datetime.datetime.now() - start).total_seconds()
             util.logger.info(
                 "job {}: geolocate: {}, elapsed: {}".format(job_id, path, elapsed)
@@ -244,7 +244,7 @@ def rescan_image(user, path, job_id):
             photo = Photo.objects.filter(Q(files__path=path)).get()
             photo._generate_thumbnail(True)
             photo._calculate_aspect_ratio(False)
-            photo._geolocate_mapbox(True)
+            photo._geolocate(True)
             photo._extract_exif_data(True)
             photo._extract_date_time_from_exif(True)
             photo._add_location_to_album_dates()

api/geocode/__init__.py

@@ -0,0 +1 @@
+GEOCODE_VERSION = "1"

api/geocode/config.py

@@ -6,45 +6,49 @@
 from .parsers.photon import parse as parse_photon
 from .parsers.tomtom import parse as parse_tomtom
 
-config = {
-    "mapbox": {
-        "geocode_args": {"api_key": settings.MAP_API_KEY},
-        "parser": parse_mapbox,
-    },
-    "maptiler": {
-        "geocode_args": {"api_key": settings.MAP_API_KEY},
-        "parser": parse_mapbox,
-    },
-    "tomtom": {
-        "geocode_args": {"api_key": settings.MAP_API_KEY},
-        "parser": parse_tomtom,
-    },
-    "photon": {
-        "geocode_args": {
-            "domain": "photon.komoot.io",
+
+def _get_config():
+    return {
+        "mapbox": {
+            "geocode_args": {"api_key": settings.MAP_API_KEY},
+            "parser": parse_mapbox,
+        },
+        "maptiler": {
+            "geocode_args": {"api_key": settings.MAP_API_KEY},
+            "parser": parse_mapbox,
+        },
+        "tomtom": {
+            "geocode_args": {"api_key": settings.MAP_API_KEY},
+            "parser": parse_tomtom,
+        },
+        "photon": {
+            "geocode_args": {
+                "domain": "photon.komoot.io",
+            },
+            "parser": parse_photon,
+        },
+        "nominatim": {
+            "geocode_args": {"user_agent": "librephotos"},
+            "parser": parse_nominatim,
         },
-        "parser": parse_photon,
-    },
-    "nominatim": {
-        "geocode_args": {"user_agent": "librephotos"},
-        "parser": parse_nominatim,
-    },
-    "opencage": {
-        "geocode_args": {
-            "api_key": settings.MAP_API_KEY,
+        "opencage": {
+            "geocode_args": {
+                "api_key": settings.MAP_API_KEY,
+            },
+            "parser": parse_opencage,
         },
-        "parser": parse_opencage,
-    },
-}
+    }
 
 
 def get_provider_config(provider) -> dict:
+    config = _get_config()
     if provider not in config:
         raise Exception(f"Map provider not found: {provider}.")
     return config[provider]["geocode_args"]
 
 
 def get_provider_parser(provider) -> callable:
+    config = _get_config()
     if provider not in config:
         raise Exception(f"Map provider not found: {provider}.")
     return config[provider]["parser"]

api/geocode/parsers/mapbox.py

@@ -1,13 +1,17 @@
+from api.geocode import GEOCODE_VERSION
+
+
 def parse(location):
     context = location.raw["context"]
-    center = location.raw["center"]
+    center = [location.raw["center"][0], location.raw["center"][1]]
     local_name = location.raw["text"]
     places = [local_name] + [
         i["text"] for i in context if not i["id"].startswith("post")
     ]
     return {
-        "features": [{"text": place} for place in places],
+        "features": [{"text": place, "center": center} for place in places],
         "places": places,
         "address": location.address,
-        "center": [center[1], center[0]],
+        "center": center,
+        "_v": GEOCODE_VERSION,
     }

api/geocode/parsers/nominatim.py

@@ -1,3 +1,6 @@
+from api.geocode import GEOCODE_VERSION
+
+
 def parse(location):
     data = location.raw["address"]
     props = [
@@ -13,9 +16,11 @@ def parse(location):
         "country",
     ]
     places = [data[prop] for prop in props if prop in data]
+    center = [float(location.raw["lat"]), float(location.raw["lon"])]
     return {
-        "features": [{"text": place} for place in places],
+        "features": [{"text": place, "center": center} for place in places],
         "places": places,
         "address": location.address,
-        "center": [float(location.raw["lon"]), float(location.raw["lat"])],
+        "center": center,
+        "_v": GEOCODE_VERSION,
     }

api/geocode/parsers/opencage.py

@@ -1,6 +1,9 @@
+from api.geocode import GEOCODE_VERSION
+
+
 def parse(location):
     data = location.raw["components"]
-    center = location.raw["geometry"]
+    center = [location.raw["geometry"]["lat"], location.raw["geometry"]["lng"]]
     props = [
         data["_type"],
         "road",
@@ -15,8 +18,9 @@ def parse(location):
     ]
     places = [data[prop] for prop in props if prop in data]
     return {
-        "features": [{"text": place} for place in places],
+        "features": [{"text": place, "center": center} for place in places],
         "places": places,
         "address": location.address,
-        "center": [center["lat"], center["lng"]],
+        "center": center,
+        "_v": GEOCODE_VERSION,
     }

api/geocode/parsers/photon.py

@@ -1,3 +1,6 @@
+from api.geocode import GEOCODE_VERSION
+
+
 def parse(location):
     data = location.raw["properties"]
     props = [
@@ -9,9 +12,14 @@ def parse(location):
         "country",
     ]
     places = [data[prop] for prop in props if prop in data]
+    center = [
+        float(location.raw["geometry"]["coordinates"][0]),
+        float(location.raw["geometry"]["coordinates"][1]),
+    ]
     return {
-        "features": [{"text": place} for place in places],
+        "features": [{"text": place, "center": center} for place in places],
         "places": places,
         "address": location.address,
-        "center": location.raw["geometry"]["coordinates"],
+        "center": center,
+        "_v": GEOCODE_VERSION,
     }

api/geocode/parsers/tomtom.py

@@ -1,5 +1,7 @@
 from functools import reduce
 
+from api.geocode import GEOCODE_VERSION
+
 
 def _dedup(iterable):
     unique_items = set()
@@ -31,8 +33,9 @@ def parse(location):
         [data[prop] for prop in props if prop in data and len(data[prop]) > 2]
     )
     return {
-        "features": [{"text": place} for place in places],
+        "features": [{"text": place, "center": center} for place in places],
         "places": places,
         "address": address,
         "center": center,
+        "_v": GEOCODE_VERSION,
     }

api/models/photo.py

@@ -16,6 +16,8 @@
 import api.models
 import api.util as util
 from api.exif_tags import Tags
+from api.geocode import GEOCODE_VERSION
+from api.geocode.geocode import reverse_geocode
 from api.im2txt.sample import im2txt
 from api.models.file import File
 from api.models.user import User, get_deleted_user
@@ -425,7 +427,7 @@ def exif_getter(tags):
             self.save()
         album_date.save()
 
-    def _geolocate_mapbox(self, commit=True):
+    def _geolocate(self, commit=True):
         old_gps_lat = self.exif_gps_lat
         old_gps_lon = self.exif_gps_lon
         new_gps_lat, new_gps_lon = get_metadata(
@@ -442,36 +444,31 @@ def _geolocate_mapbox(self, commit=True):
             and old_gps_lon == float(new_gps_lon)
             and old_album_places.count() != 0
             and self.geolocation_json
+            and "_v" in self.geolocation_json
+            and self.geolocation_json["_v"] == GEOCODE_VERSION
         ):
             return
         self.exif_gps_lon = float(new_gps_lon)
         self.exif_gps_lat = float(new_gps_lat)
         if commit:
             self.save()
-        # Skip if the request fails or is empty
-        res = None
         try:
-            res = util.mapbox_reverse_geocode(new_gps_lat, new_gps_lon)
-            if not res or len(res) == 0:
-                return
-            if "features" not in res.keys():
+            res = reverse_geocode(new_gps_lat, new_gps_lon)
+            if not res:
                 return
         except Exception as e:
             util.logger.exception("something went wrong with geolocating")
             raise e
 
         self.geolocation_json = res
-        if "search_text" in res.keys():
-            if self.search_location:
-                self.search_location = self.search_location + " " + res["search_text"]
-            else:
-                self.search_location = res["search_text"]
-        # Delete photo from album places if location has changed
+        self.search_location = res["address"]
 
+        # Delete photo from album places if location has changed
         if old_album_places is not None:
             for old_album_place in old_album_places:
                 old_album_place.photos.remove(self)
                 old_album_place.save()
+
         # Add photo to new album places
         for geolocation_level, feature in enumerate(self.geolocation_json["features"]):
             if "text" not in feature.keys() or feature["text"].isnumeric():
@@ -493,11 +490,7 @@ def _add_location_to_album_dates(self):
         if not self.geolocation_json:
             return
         album_date = self._find_album_date()
-        city_name = [
-            f["text"]
-            for f in self.geolocation_json["features"][1:-1]
-            if not f["text"].isdigit()
-        ][0]
+        city_name = self.geolocation_json["places"][-2]
         if album_date.location and len(album_date.location) > 0:
             prev_value = album_date.location
             new_value = prev_value