Merge pull request #584 from datamade/2.5

Upgrade LA Metro to Councilmatic 2.5

.gitignore

@@ -33,3 +33,5 @@ debug.log
 /scripts/after_install.sh
 /configs/settings_deployment.production.py
 /configs/settings_deployment.staging.py
+
+*.csv

.travis.yml

@@ -10,20 +10,24 @@ install:
 - pip install --upgrade pip
 - pip install --upgrade -r requirements.txt
 sudo: required
-dist: trusty
-group: deprecated-2017Q4
+
 addons:
-  postgresql: '9.4'
+  postgresql: '10'
+  apt:
+    packages:
+    - postgresql-10-postgis-2.4
+
 before_script:
-- mv configs/test_settings_deployment.py councilmatic/settings_deployment.py
+  - psql -U postgres -c "create extension postgis"
+  - mv configs/test_settings_deployment.py councilmatic/settings_deployment.py
 script: python manage.py migrate && pytest
 deploy:
   - provider: codedeploy
     access_key_id: AKIAJNYGDLOJKHOUWCAA
     secret_access_key:
       secure: HkMlaMSE/qhzk5QvTccId2/eoEx8iPrS5GFRr95YXA5/r0l1kUnBCcv+yTPOjDSg969P/V3vs0bRMKVDU0ibFpc2FgQQdAKrbc8l7ZBlnMEMSId/LyyTxszVcfOUPqqSqK0bFjjqud/k9rmL3Du3sUrQSkkkRWwrSRdbu6+vKKYTduhQ++bzgjTyVab49eCoRCh0JeswxONBx7wWsXxZxSDBEVaRipg2tsZMzfxbVbCQK5yL5xeoeQj0JMV6mTK+hwwRzFZKi5tNiLQQ/26xTUVdKU2FZOp+0cXMFNweYHo0Ca3WGbEVZ54ur9OGm3nwJhLII7vv5Z66qjkiyVTpdqIkZRWYzA8xO1w2tK6TdfpJTvs/8jyG1QKK6FOBf9qfcLt6Z5Kc+h6XMTfd8QS/g/uIemoO1oFPGi+2keHUX8H8qUlLcpc5JPKDbYsXGlsGIdDxJbcCI9J/O34tG5hOZz4hH9SYQtkFdV+oGxvBf4KEUMoWd/d9sggtr3ezX4ltrOpCk7VjH5wqYkzzWdO8Oo3aecaOsPJsBa0//WRdAcxxI8SlaPUBgauFx5wfDHQVwgx3BqP4eQGYMkM9+0MBlnYi1Ub8AZTEpjUqkM7VH8v56DA3jD/yo7VwhfFLhjir9CYp7tK4HOR7ZoHOpqyi97HVQXzOh+Cy8wmH9hA9A18=
     revision_type: github
-    application: lametro-councilmatic
+    application: la-metro-councilmatic
     deployment_group: production
     on:
       tags: true
@@ -32,7 +36,7 @@ deploy:
     secret_access_key:
       secure: HkMlaMSE/qhzk5QvTccId2/eoEx8iPrS5GFRr95YXA5/r0l1kUnBCcv+yTPOjDSg969P/V3vs0bRMKVDU0ibFpc2FgQQdAKrbc8l7ZBlnMEMSId/LyyTxszVcfOUPqqSqK0bFjjqud/k9rmL3Du3sUrQSkkkRWwrSRdbu6+vKKYTduhQ++bzgjTyVab49eCoRCh0JeswxONBx7wWsXxZxSDBEVaRipg2tsZMzfxbVbCQK5yL5xeoeQj0JMV6mTK+hwwRzFZKi5tNiLQQ/26xTUVdKU2FZOp+0cXMFNweYHo0Ca3WGbEVZ54ur9OGm3nwJhLII7vv5Z66qjkiyVTpdqIkZRWYzA8xO1w2tK6TdfpJTvs/8jyG1QKK6FOBf9qfcLt6Z5Kc+h6XMTfd8QS/g/uIemoO1oFPGi+2keHUX8H8qUlLcpc5JPKDbYsXGlsGIdDxJbcCI9J/O34tG5hOZz4hH9SYQtkFdV+oGxvBf4KEUMoWd/d9sggtr3ezX4ltrOpCk7VjH5wqYkzzWdO8Oo3aecaOsPJsBa0//WRdAcxxI8SlaPUBgauFx5wfDHQVwgx3BqP4eQGYMkM9+0MBlnYi1Ub8AZTEpjUqkM7VH8v56DA3jD/yo7VwhfFLhjir9CYp7tK4HOR7ZoHOpqyi97HVQXzOh+Cy8wmH9hA9A18=
     revision_type: github
-    application: lametro-councilmatic
+    application: la-metro-councilmatic
     deployment_group: staging
     on:
       branch: master

Dockerfile

@@ -6,14 +6,17 @@ ENV PYTHONUNBUFFERED=1
 RUN apt-get update && \
     apt-get install -y libxml2-dev libxslt1-dev antiword unrtf poppler-utils \
                        pstotext tesseract-ocr flac ffmpeg lame libmad0 \
-                       libsox-fmt-mp3 sox libjpeg-dev swig
+                       libsox-fmt-mp3 sox libjpeg-dev swig gdal-bin
 
 RUN mkdir /app
 WORKDIR /app
 
 COPY ./requirements.txt /app/requirements.txt
-RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install textract==1.6.3
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
 
 COPY . /app
+
+RUN DATABASE_URL='' python manage.py collectstatic --noinput
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

Makefile

@@ -0,0 +1,22 @@
+GENERATED_FILES=councilmatic_core_event.csv councilmatic_core_person.csv
+
+all : $(GENERATED_FILES)
+
+clean :
+	rm $(GENERATED_FILES) || :
+
+councilmatic_core_event.csv :
+	ssh [email protected] " \
+	    psql -U postgres -d lametro -c \" \
+		    COPY ( \
+			    SELECT name, start_time, slug \
+			    FROM councilmatic_core_event \
+		    ) TO STDOUT WITH CSV HEADER\"" > $@
+
+councilmatic_core_person.csv :
+	ssh [email protected] " \
+	    psql -U postgres -d lametro -c \" \
+		    COPY ( \
+			    SELECT name, slug \
+			    FROM councilmatic_core_person \
+		    ) TO STDOUT WITH CSV HEADER\"" > $@

configs/test_settings_deployment.py

@@ -2,6 +2,14 @@
 
 import os
 
+import dj_database_url
+
+ALLOWED_HOSTS = [
+    'localhost',
+    '127.0.0.1',
+    '0.0.0.0',
+]
+
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = 'testing secrets'
 
@@ -12,23 +20,14 @@
 # Database
 # https://docs.djangoproject.com/en/1.8/ref/settings/#databases
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql',
-        'NAME': 'travis',
-        'USER': 'travis',
-        'PASSWORD': '',
-        'HOST': 'localhost',
-        'PORT': '5432',
-    }
-}
+DATABASES = {'default': dj_database_url.config(default='postgis://travis:/travis')}
 
 HAYSTACK_CONNECTIONS = {
     'default': {
         'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
         #'URL': 'http://127.0.0.1:8983/solr'
         # ...or for multicore...
-        'URL': 'http://127.0.0.1:8983/solr/lametro',
+        'URL': os.environ.get('SOLR_URL') or 'http://127.0.0.1:8983/solr/lametro',
     },
 }
 
@@ -53,12 +52,6 @@
 HEADSHOT_PATH = os.path.join(os.path.dirname(__file__), '..'
                              '/lametro/static/images/')
 
-EXTRA_APPS = ('raven.contrib.django.raven_compat',)
-
-RAVEN_CONFIG = {
-    'dsn': '',
-}
-
 SHOW_TEST_EVENTS = True
 
 SMART_LOGIC_KEY = 'smartlogic api key'

councilmatic/settings.py

@@ -22,12 +22,12 @@
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
 
-
 try:
     from .settings_deployment import ALLOWED_HOSTS
 except ImportError:
     ALLOWED_HOSTS = [
-        'localhost', '127.0.0.1',
+        'localhost',
+        '127.0.0.1',
         '.datamade.us',
         '.councilmatic.org'
     ]
@@ -42,7 +42,10 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.contrib.gis',
     'haystack',
+    'opencivicdata.core',
+    'opencivicdata.legislative',
     'lametro',
     'councilmatic_core',
     'adv_cache_tag',
@@ -54,26 +57,23 @@
 except NameError:
     pass
 
-MIDDLEWARE_CLASSES = (
+MIDDLEWARE = (
+    'debug_toolbar.middleware.DebugToolbarMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
-    # 'django.middleware.cache.UpdateCacheMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
-    'debug_toolbar.middleware.DebugToolbarMiddleware',
-    # 'django.middleware.cache.FetchFromCacheMiddleware',
 )
 
 ROOT_URLCONF = 'councilmatic.urls'
 
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [],
+        'DIRS': [os.path.join(BASE_DIR, 'lametro/templates')],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -87,6 +87,7 @@
     },
 ]
 
+
 WSGI_APPLICATION = 'councilmatic.wsgi.application'
 
 
@@ -113,7 +114,7 @@
 MERGER_BASE_URL = 'https://metro-pdf-merger.datamade.us'
 # MERGER_BASE_URL = 'http://0.0.0.0:5000'
 
-PIC_BASE_URL = 'https://pic.datamade.us/lametro/document/'    
+PIC_BASE_URL = 'https://pic.datamade.us/lametro/document/'
 # PIC_BASE_URL = 'http://127.0.0.1:5000/lametro/document/'
 
 HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

councilmatic/settings_deployment.py.example

@@ -20,7 +20,7 @@ DEBUG = True
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
         'NAME': 'lametro',
         'USER': 'postgres',
         'HOST': 'postgres',
@@ -86,5 +86,9 @@ LOGGING = {
 # Set to False in production!
 SHOW_TEST_EVENTS = True
 
+import socket
+hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+INTERNAL_IPS = [ip[:-1] + '1' for ip in ips] + ['127.0.0.1', '10.0.2.2']
+
 SMART_LOGIC_KEY = 'smartlogic api key'
 SMART_LOGIC_ENVIRONMENT = '0ef5d755-1f43-4a7e-8b06-7591bed8d453'

councilmatic/settings_jurisdiction.py

@@ -4,7 +4,7 @@
 # These settings are required #
 ###############################
 
-OCD_CITY_COUNCIL_ID = 'ocd-organization/42e23f04-de78-436a-bec5-ab240c1b977c'
+OCD_CITY_COUNCIL_NAME = 'Board of Directors'
 CITY_COUNCIL_NAME = 'Metro'
 OCD_JURISDICTION_IDS = ['ocd-jurisdiction/country:us/state:ca/county:los_angeles/transit_authority']
 LEGISLATIVE_SESSIONS = ['2014', '2015', '2016', '2017', '2018', '2019'] # the last one in this list should be the current legislative session
@@ -39,14 +39,6 @@
 LEGISTAR_URL = ''           # e.g. 'https://chicago.legistar.com/Legislation.aspx'
 
 
-# this is for the boundaries of municipal districts, to add
-# shapes to posts & ultimately display a map with the council
-# member listing. the boundary set should be the relevant
-# slug from the ocd api's boundary service
-# available boundary sets here: http://ocd.datamade.us/boundary-sets/
-
-BOUNDARY_SET = ['la-metro-supervisory-districts', 'la-metro-committee-districts', 'city-of-la']
-
 MAP_CONFIG = {
     'center': [34.0522, -118.2437],
     'zoom': 10,

councilmatic/urls.py

@@ -1,6 +1,6 @@
 from django.conf.urls import include, url
 from django.contrib import admin
-from django.contrib.staticfiles import views as staticviews
+from django.views.static import serve
 from django.views.generic.base import RedirectView
 from django.conf import settings
 from django.views.decorators.cache import never_cache
@@ -39,7 +39,7 @@
 
 urlpatterns = [
     url(r'', include(patterns)),
-    url(r'^admin/', include(admin.site.urls)),
+    url(r'^admin/', admin.site.urls),
     url(r'^metro-login/$', metro_login, name='metro_login'),
     url(r'^metro-logout/$', metro_logout, name='metro_logout'),
     url(r'^ses-token/$', SmartLogicAPI.as_view(), name='ses_token'),
@@ -51,6 +51,7 @@
 if settings.DEBUG:
     import debug_toolbar
     urlpatterns += [
-      url(r'^static/(?P<path>.*)/$', staticviews.serve),
-      url(r'^__debug__/', include(debug_toolbar.urls)),
+        url(r'^static/(?P<path>.*)/$', serve),
+        url(r'^images/(?P<path>.*)/$', serve, {'document_root': settings.STATIC_ROOT + '/images/'}),
+        url(r'^__debug__/', include(debug_toolbar.urls)),
     ]

data/Makefile

@@ -0,0 +1,34 @@
+all : boundary.geojson
+
+boundary.geojson : boundary_full_precision.geojson
+	ogr2ogr -lco COORDINATE_PRECISION=5 $@ $<
+
+.INTERMEDIATE: boundary_full_precision.geojson
+boundary_full_precision.geojson : supervisory_districts.geojson		\
+                   city_of_la.geojson long_beach.geojson		\
+                   southwest_corridor.geojson san_fernando.geojson	\
+                   san_gabriel.geojson
+	python scripts/combine.py $^ | \
+            python scripts/add_districts.py > $@
+
+%.geojson : %.json
+	cat $< | python scripts/ocd2geojson.py $* > $@
+
+supervisory_districts.json :
+	wget -O $@ https://ocd.datamade.us/boundaries/la-metro-supervisory-districts/shape
+
+city_of_la.json :
+	wget -O $@ wget -O $@ https://ocd.datamade.us/boundaries/city-of-la/shape
+
+long_beach.json :
+	wget -O $@ https://ocd.datamade.us/boundaries/la-metro-committee-districts/la-metro-committee-district-southeast_long_beach/shape
+
+southwest_corridor.json :
+	wget -O $@ https://ocd.datamade.us/boundaries/la-metro-committee-districts/la-metro-committee-district-southwest_corridor/shape
+
+san_fernando.json :
+	wget -O $@ https://ocd.datamade.us/boundaries/la-metro-committee-districts/la-metro-committee-district-north_county_san_fernando_valley/shape
+
+san_gabriel.json : 
+	wget -O $@ https://ocd.datamade.us/boundaries/la-metro-committee-districts/la-metro-committee-district-san_gabriel_valley/shape
+

data/README.md

@@ -0,0 +1,17 @@
+This directory contains a `boundary.geojson` file and the Makefile and associated scripts used to created it.
+
+You'll use the geojson file to load in shapes of districts with
+
+`python manage.py import_shapes data/boundary.geojson`
+
+
+You shouldn't need to rerun the Makefile, but keep it for historical
+reference.
+
+This Makefile currently depends on pulling data from DataMade's legacy
+OCD API. In the future, if boundary changes, we won't put the data
+into the that API at all, but will more create an updated
+`boundary.geojson` file much more directly.
+
+
+

data/scripts/add_districts.py

@@ -0,0 +1,22 @@
+import sys
+import json
+
+DIVISION = {'la-metro-supervisory-district-1': 'ocd-division/country:us/state:ca/county:los_angeles/council_district:1',
+            'la-metro-supervisory-district-2': 'ocd-division/country:us/state:ca/county:los_angeles/council_district:2',
+            'la-metro-supervisory-district-3': 'ocd-division/country:us/state:ca/county:los_angeles/council_district:3',
+            'la-metro-supervisory-district-4': 'ocd-division/country:us/state:ca/county:los_angeles/council_district:4',
+            'la-metro-supervisory-district-5': 'ocd-division/country:us/state:ca/county:los_angeles/council_district:5',
+            'Los Angeles': 'ocd-division/country:us/state:ca/place:los_angeles',
+             'long_beach': 'ocd-division/country:us/state:ca/county:los_angeles/la_metro_sector:southeast_long_beach',
+            'southwest_corridor': 'ocd-division/country:us/state:ca/county:los_angeles/la_metro_sector:southwest_corridor',
+            'san_fernando': 'ocd-division/country:us/state:ca/county:los_angeles/la_metro_sector:north_county_san_fernando_valley',
+            'san_gabriel': 'ocd-division/country:us/state:ca/county:los_angeles/la_metro_sector:san_gabriel_valley',
+             }
+
+geojson = json.load(sys.stdin)
+
+for feature in geojson['features']:
+    name = feature['properties']['name']
+    feature['properties']['division_id'] = DIVISION[name]
+
+json.dump(geojson, sys.stdout)

data/scripts/combine.py

@@ -0,0 +1,12 @@
+import sys
+import json
+
+geojson = {'type': "FeatureCollection",
+            'features': []}
+
+for file_name in sys.argv[1:]:
+    with open(file_name) as f:
+        sub_geojson = json.load(f)
+    geojson['features'].extend(sub_geojson['features'])
+
+json.dump(geojson, sys.stdout)

data/scripts/ocd2geojson.py

@@ -0,0 +1,22 @@
+import sys
+import json
+
+ocd_map = json.load(sys.stdin)
+geojson = {'type': "FeatureCollection",
+            'features': []}
+
+name = sys.argv[1]
+
+try:
+    for obj in ocd_map['objects']:
+        feature = {'type': "Feature"}
+        feature['geometry'] = obj['shape']
+        feature['properties'] = {'name': obj['name']}
+        geojson['features'].append(feature)
+except KeyError:
+        feature = {'type': "Feature"}
+        feature['geometry'] = ocd_map
+        feature['properties'] = {'name': name}
+        geojson['features'].append(feature)
+
+json.dump(geojson, sys.stdout)