Initial commit

Author: juyrjola

data/placeholder.txt

@@ -0,0 +1,86 @@
+from django.contrib.gis.geos import Point
+from tastypie.resources import ModelResource
+from tastypie.exceptions import InvalidFilterError
+from tastypie.constants import ALL, ALL_WITH_RELATIONS
+from tastypie.contrib.gis.resources import ModelResource as GeometryModelResource
+from tastypie import fields
+from geo.models import *
+import json
+
+class MunicipalityResource(ModelResource):
+    def _convert_to_geojson(self, bundle):
+        muni = bundle.obj
+        data = {'type': 'Feature'}
+        data['properties'] = bundle.data
+        data['id'] = bundle.obj.pk
+        borders = muni.municipalityboundary.borders
+        data['geometry'] = json.loads(borders.geojson)
+        bundle.data = data
+        return bundle
+    def alter_detail_data_to_serialize(self, request, bundle):
+        if request.GET.get('format') == 'geojson':
+            return self._convert_to_geojson(bundle)
+        return bundle
+    def alter_list_data_to_serialize(self, request, bundles):
+        if request.GET.get('format') != 'geojson':
+            return bundles
+        data = {'type': 'FeatureCollection'}
+        data['meta'] = bundles['meta']
+        data['features'] = [self._convert_to_geojson(bundle) for bundle in bundles['objects']]
+        return data
+    def apply_filters(self, request, filters):
+        obj_list = super(MunicipalityResource, self).apply_filters(request, filters)
+        if request.GET.get('format') == 'geojson':
+            obj_list = obj_list.select_related('municipalityboundary')
+        return obj_list
+    def dehydrate(self, bundle):
+        alt_names = bundle.obj.municipalityname_set.all()
+        for an in alt_names:
+            bundle.data['name_%s' % an.language] = an.name
+        return bundle
+    def determine_format(self, request):
+        if request.GET.get('format') == 'geojson':
+            return 'application/json'
+        return super(MunicipalityResource, self).determine_format(request)
+    class Meta:
+        queryset = Municipality.objects.all().order_by('name').select_related('municipalityboundary')
+        resource_name = 'municipality'
+
+class MunicipalityBoundaryResource(GeometryModelResource):
+    municipality = fields.ToOneField('geo.api.MunicipalityResource', 'municipality')
+    class Meta:
+        queryset = MunicipalityBoundary.objects.all()
+        resource_name = 'municipality_boundary'
+        filtering = {
+            'municipality': ALL
+        }
+
+class AddressResource(GeometryModelResource):
+    municipality = fields.ToOneField('geo.api.MunicipalityResource', 'municipality')
+
+    def apply_sorting(self, objects, options=None):
+        if options and 'lon' in options and 'lat' in options:
+            try:
+                lat = float(options['lat'])
+                lon = float(options['lon'])
+            except TypeError:
+                raise InvalidFilterError("'lon' and 'lat' need to be floats")
+            objects = objects.distance(Point(lon, lat, srid=4326)).order_by('distance')
+        return super(AddressResource, self).apply_sorting(objects, options)
+    
+    def dehydrate(self, bundle):
+        distance = getattr(bundle.obj, 'distance', None)
+        if distance is not None:
+            bundle.data['distance'] = distance
+        print bundle
+        return bundle
+
+    class Meta:
+        queryset = Address.objects.all()
+        filtering = {
+            'municipality': ALL,
+            'street': ALL,
+            'number': ALL,
+            'letter': ALL,
+            'location': ALL,
+        }

geo/__init__.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+import os
+import csv
+
+from django.core.management.base import BaseCommand
+from geo.models import *
+from utils.http import HttpFetcher
+from django.conf import settings
+from django import db
+from django.contrib.gis.gdal import DataSource, SpatialReference, CoordTransform
+from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point
+
+MUNI_URL = "http://tilastokeskus.fi/meta/luokitukset/kunta/001-2013/tekstitiedosto.txt"
+
+class Command(BaseCommand):
+    help = "Manage stats app"
+
+    def import_municipalities(self):
+        s = self.http.open_url(MUNI_URL, "muni")
+        # strip first 4 lines of header and any blank/empty lines at EOF
+        count = 0
+        for line in s.rstrip().split('\n')[4:]:
+            dec_line = line.decode('iso8859-1').rstrip().split('\t')
+            (muni_id, muni_name) = dec_line
+            muni_id = int(muni_id)
+            muni_name = muni_name.split(' - ')[0]
+
+            try:
+                muni = Municipality.objects.get(id=muni_id)
+            except:
+                muni = Municipality(id=muni_id, name=muni_name)
+                muni.save()
+                count += 1
+        print "%d municipalities added." % count
+
+    def import_addresses(self):
+        f = open(os.path.join(self.data_path, 'pks_osoite.csv'))
+        reader = csv.reader(f, delimiter=',')
+        reader.next()
+        muni_list = Municipality.objects.all()
+        muni_dict = {}
+        for muni in muni_list:
+            muni_dict[muni.name] = muni
+        for idx, row in enumerate(reader):
+            street = row[0]
+            num = int(row[1])
+            num2 = row[2]
+            letter = row[3]
+            coord_n = int(row[8])
+            coord_e = int(row[9])
+            muni_name = row[10]
+            row_type = int(row[-1])
+            if row_type != 1:
+                continue
+            id_s = "%s %d" % (street, num)
+            if id_s == 'Eliel Saarisen tie 4':
+                muni_name = 'Helsinki'
+            muni = muni_dict[muni_name]
+            args = dict(municipality=muni, street=street, number=num, letter=letter)
+            try:
+                addr = Address.objects.get(**args)
+            except Address.DoesNotExist:
+                addr = Address(**args)
+            # 3879
+            to_wgs84 = CoordTransform(SpatialReference('3879'), SpatialReference('WGS84'))
+            pnt = Point(coord_e, coord_n, srid=3879)
+            pnt.transform(to_wgs84)
+            #print "%s: %s %d%s N%d E%d (%f,%f)" % (muni_name, street, num, letter, coord_n, coord_e, pnt.y, pnt.x)
+            if idx > 0 and idx % 1000 == 0:
+                print "%d addresses processed" % idx
+                # Reset DB query store to free up memory
+                db.reset_queries()
+            addr.location = pnt
+            addr.save()
+
+    def handle(self, **options):
+        http = HttpFetcher()
+        http.set_cache_dir(os.path.join(settings.PROJECT_ROOT, ".cache"))
+        self.data_path = os.path.join(settings.PROJECT_ROOT, '..', 'data')
+        self.http = http
+        print "Importing municipalities"
+        self.import_municipalities()
+        print "Importing addresses"
+        self.import_addresses()

geo/api.py

@@ -0,0 +1,32 @@
+from django.contrib.gis.db import models
+
+class Municipality(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.CharField(max_length=50)
+
+    def __unicode__(self):
+        return self.name
+
+# For municipality names in other languages
+class MunicipalityName(models.Model):
+    municipality = models.ForeignKey(Municipality)
+    language = models.CharField(max_length=8)
+    name = models.CharField(max_length=50)
+
+class MunicipalityBoundary(models.Model):
+    municipality = models.OneToOneField(Municipality)
+    borders = models.MultiPolygonField()
+
+    objects = models.GeoManager()
+
+class Address(models.Model):
+    street = models.CharField(max_length=50, db_index=True)
+    number = models.PositiveIntegerField()
+    letter = models.CharField(max_length=2, blank=True, null=True)
+    location = models.PointField()
+    municipality = models.ForeignKey(Municipality, db_index=True)
+
+    objects = models.GeoManager()
+
+    class Meta:
+        unique_together = (('municipality', 'street', 'number', 'letter'),)

geo/management/__init__.py

@@ -0,0 +1,16 @@
+"""
+This file demonstrates writing tests using the unittest module. These will pass
+when you run "manage.py test".
+
+Replace this with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+
+class SimpleTest(TestCase):
+    def test_basic_addition(self):
+        """
+        Tests that 1 + 1 always equals 2.
+        """
+        self.assertEqual(1 + 1, 2)

geo/management/commands/__init__.py

@@ -0,0 +1 @@
+# Create your views here.

geo/management/commands/geo_import.py

@@ -0,0 +1,159 @@
+import os
+
+# Django settings for geocoder project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', '[email protected]'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.contrib.gis.db.backends.postgis', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'osoite',                      # Or path to database file if using sqlite3.
+        # The following settings are not used with sqlite3:
+        'USER': 'osoite',
+        'PASSWORD': 'osoite',
+        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
+        'PORT': '',                      # Set to empty string for default.
+    }
+}
+
+PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# In a Windows environment this must be set to your system time zone.
+TIME_ZONE = 'Europe/Helsinki'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'fi'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale.
+USE_L10N = True
+
+# If you set this to False, Django will not use timezone-aware datetimes.
+USE_TZ = True
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/var/www/example.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://example.com/media/", "http://media.example.com/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/var/www/example.com/static/"
+STATIC_ROOT = ''
+
+# URL prefix for static files.
+# Example: "http://example.com/static/", "http://static.example.com/"
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = 'gp_*jv(bspmp5rv-=hg9lue-$%pf!a))ch#x&(uf%a_xsj*#sy'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    # Uncomment the next line for simple clickjacking protection:
+    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+)
+
+ROOT_URLCONF = 'geocoder.urls'
+
+# Python dotted path to the WSGI application used by Django's runserver.
+WSGI_APPLICATION = 'geocoder.wsgi.application'
+
+TEMPLATE_DIRS = (
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.gis',
+    # Uncomment the next line to enable the admin:
+    # 'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'tastypie',
+    'geo',
+)
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        }
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+    }
+}