From 0de5b1e90f2d5a9adb2b241a6256e07cd284dad0 Mon Sep 17 00:00:00 2001 From: kevinearldenny Date: Wed, 7 Oct 2020 09:55:43 -0400 Subject: [PATCH 1/3] Import default speed for a neighborhood into AnalysisJob model - Display default speed for a neighborhood on frontend - Pass defaultSpeedLimit to PlaceMap component - Make defaultSpeedLimit available on PlaceMap scope - Create initial MapSpeedLimit component - Add MapSpeedLimit component to map - Pass data to MapSpeedLimitLegend - Add working and semi-styled SpeedLimitLegend - Add changelog entry - Fix positioning of speed limit legend - Convert 'speed_limit_src' choices to an enumeration class - Remove analysis-side code for storing speed limit Reverts the parts of the upstream changes that modify the analysis, which leaves just the model and front-end changes. --- CHANGELOG.md | 1 + .../src/app/components/map/map.constants.js | 3 ++ .../src/app/components/map/map.directive.js | 4 +++ .../src/app/components/places.service.js | 6 ++-- .../leaflet/map-speed-limit-legend.control.js | 36 +++++++++++++++++++ .../app/places/detail/place-map.directive.js | 11 ++++-- .../src/app/places/detail/place-map.html | 1 + .../src/app/places/detail/places-detail.html | 6 ++-- src/angularjs/src/styles/components/_map.scss | 32 +++++++++++++++++ .../migrations/0041_auto_20201006_2320.py | 23 ++++++++++++ src/django/pfb_analysis/models.py | 20 +++++++++-- src/django/pfb_analysis/serializers.py | 1 + src/django/pfb_analysis/views.py | 3 +- 13 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js create mode 100644 src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa45f132..3e556d6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. #### Added - S3 caching of Census data files - Upgrade Django (2.2.10 -> 2.2.16) and psycopg2 (2.8.4 -> 2.8.5) +- Add default speed limit for neighborhood to AnalysisJob model and display it on frontend ## [0.13.0] - 2020-02-14 diff --git a/src/angularjs/src/app/components/map/map.constants.js b/src/angularjs/src/app/components/map/map.constants.js index 2a373935..a764ceb9 100644 --- a/src/angularjs/src/app/components/map/map.constants.js +++ b/src/angularjs/src/app/components/map/map.constants.js @@ -47,6 +47,9 @@ position: 'bottomright', colors: ['#009fdf', '#ff3300'], labels: ['Low Stress', 'High Stress'] + }, + speedLimit: { + position: 'bottomright' } } }; diff --git a/src/angularjs/src/app/components/map/map.directive.js b/src/angularjs/src/app/components/map/map.directive.js index ea5f99b7..14edb405 100644 --- a/src/angularjs/src/app/components/map/map.directive.js +++ b/src/angularjs/src/app/components/map/map.directive.js @@ -4,6 +4,7 @@ function MapController($element) { var ctl = this; ctl.map = null; + ctl.speedLimit = null; ctl.mapHTMLElement = $element[0]; ctl.pfbMapOptions = ctl.pfbMapOptions || {}; @@ -43,6 +44,9 @@ ctl.map.addLayer(ctl.baseLayer); } } + if (changes.speedLimit) { + ctl.speedLimit = changes.speedLimit + } }; } diff --git a/src/angularjs/src/app/components/places.service.js b/src/angularjs/src/app/components/places.service.js index a4b9a99c..613f757e 100644 --- a/src/angularjs/src/app/components/places.service.js +++ b/src/angularjs/src/app/components/places.service.js @@ -45,14 +45,16 @@ dfd.resolve({}); return dfd.promise; } - place.results = results; place.scores = results.overall_scores; + _.each(results.overall_scores, function (scores, key) { scores.score_normalized = (key === 'population_total' ? scores.score_original : scores.score_normalized) }); - + place.scores.default_speed_limit = { + score_normalized: results.residential_speed_limit + }; dfd.resolve(place); }); }); diff --git a/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js b/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js new file mode 100644 index 00000000..ccb6e556 --- /dev/null +++ b/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js @@ -0,0 +1,36 @@ +(function() { + + // Custom private Leaflet control implements a simple legend for all map layers + L.Control.SpeedLegend = L.Control.extend({ + initialize: function (options) { + L.Util.setOptions(this, options); + this.speedLimit = options.speedLimit; + if (this.speedLimit <= 25) { + this.stressLevel = 'low' + } else if (this.speedLimit <= 30) { + this.stressLevel = 'med.' + } else { + this.stressLevel = 'high' + } + }, + onAdd: function () { + var div = L.DomUtil.create('div', 'leaflet-control-layers pfb-speed-limit-legend'); + var kphLimit = Math.round(this.speedLimit * 1.609); + div.innerHTML += '
Residential Speed Limit
'; + div.innerHTML += '
' + + '
' + this.speedLimit + '
mph
' + + '
' + kphLimit + '
km/h
' + + '
'; + div.innerHTML += '
Stress impact: ' + this.stressLevel + '
'; + return div; + }, + stressLevel: function () { + + } + }); + if (!L.control.speedLegend) { + L.control.speedLegend = function (opts) { + return new L.Control.SpeedLegend(opts); + } + } +})(); diff --git a/src/angularjs/src/app/places/detail/place-map.directive.js b/src/angularjs/src/app/places/detail/place-map.directive.js index 0870adb2..5a1712ff 100644 --- a/src/angularjs/src/app/places/detail/place-map.directive.js +++ b/src/angularjs/src/app/places/detail/place-map.directive.js @@ -72,7 +72,6 @@ if (!layers) { return; } - var satelliteLayer = L.tileLayer(MapConfig.baseLayers.Satellite.url, { attribution: MapConfig.baseLayers.Satellite.attribution, maxZoom: MapConfig.conusMaxZoom @@ -97,6 +96,11 @@ $window.print(); }).addTo(ctl.map); } + if (ctl.pfbPlaceMapSpeedLimit && !ctl.speedLimitLegend) { + var speedLimitLegendOptions = MapConfig.legends["speedLimit"]; + speedLimitLegendOptions.speedLimit = ctl.pfbPlaceMapSpeedLimit; + ctl.speedLimitLegend = L.control.speedLegend(speedLimitLegendOptions).addTo(ctl.map); + } _.map(layers.tileLayers, function(layerObj) { // Get desired label @@ -149,6 +153,8 @@ }); }); + + function onEachFeature(feature, layer) { // TODO: Style marker and popup layer.on({ @@ -202,7 +208,8 @@ restrict: 'E', scope: { pfbPlaceMapLayers: '<', - pfbPlaceMapUuid: '<' + pfbPlaceMapUuid: '<', + pfbPlaceMapSpeedLimit: '<' }, controller: 'PlaceMapController', controllerAs: 'ctl', diff --git a/src/angularjs/src/app/places/detail/place-map.html b/src/angularjs/src/app/places/detail/place-map.html index c6f92da9..d33d0cf1 100644 --- a/src/angularjs/src/app/places/detail/place-map.html +++ b/src/angularjs/src/app/places/detail/place-map.html @@ -1,6 +1,7 @@
{{placeDetail.place.label_suffix}} ng-class="m.subscoreClass"> {{ ::m.label }}
- {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} + {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} + {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} mph
@@ -78,7 +79,8 @@

{{placeDetail.place.label_suffix}}

+ pfb-place-map-uuid="placeDetail.place.uuid" + pfb-place-map-speed-limit="placeDetail.scores.default_speed_limit.score_normalized">
diff --git a/src/angularjs/src/styles/components/_map.scss b/src/angularjs/src/styles/components/_map.scss index 6217e9a0..79d191cd 100644 --- a/src/angularjs/src/styles/components/_map.scss +++ b/src/angularjs/src/styles/components/_map.scss @@ -54,3 +54,35 @@ opacity: 0.7; } } + +.pfb-speed-limit-legend { + width: 210px; + color: #555; + text-align: center; + + h6 { + margin-top: 0; + padding-top: 0.3rem; + background: lightgray; + } + .speed-limit-inner { + margin: auto; + width: 140px; + } + + .speed-limit-block { + display: inline-block; + margin: 0 15px 0 15px; + } + + .speed-limit { + font-weight: 700; + font-size: 24pt; + } + .speed-stress { + font-weight: 500; + font-size: 1.4rem; + color: white; + } + +} diff --git a/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py b/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py new file mode 100644 index 00000000..99dea60b --- /dev/null +++ b/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.10 on 2020-10-06 23:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pfb_analysis', '0040_neighborhood_analysis_job_ondelete'), + ] + + operations = [ + migrations.AddField( + model_name='analysisjob', + name='default_speed_limit', + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='analysisjob', + name='speed_limit_src', + field=models.CharField(blank=True, choices=[('State', 'State'), ('City', 'City')], max_length=20, null=True), + ), + ] diff --git a/src/django/pfb_analysis/models.py b/src/django/pfb_analysis/models.py index 247efa44..60aa5e1f 100644 --- a/src/django/pfb_analysis/models.py +++ b/src/django/pfb_analysis/models.py @@ -487,7 +487,8 @@ def get_queryset(self): ('overall_score', 'score_normalized'))) .annotate(population_total=ObjectAtPath('overall_scores', ('population_total', 'score_original'), - output_field=models.PositiveIntegerField()))) + output_field=models.PositiveIntegerField())) + ) return qs @@ -531,6 +532,15 @@ class Status: (ERROR, 'Error',), ) + class SpeedLimitSource: + STATE = 'State' + CITY = 'City' + + CHOICES = ( + (STATE, 'State'), + (CITY, 'City') + ) + batch_job_id = models.CharField(max_length=256, blank=True, null=True) neighborhood = models.ForeignKey(Neighborhood, related_name='analysis_jobs', @@ -549,7 +559,13 @@ class Status: start_time = models.DateTimeField(null=True, blank=True) final_runtime = models.PositiveIntegerField(default=0) status = models.CharField(choices=Status.CHOICES, max_length=12, default=Status.CREATED) - + default_speed_limit = models.PositiveIntegerField(blank=True, null=True) + speed_limit_src = models.CharField( + choices=SpeedLimitSource.CHOICES, + max_length=20, + blank=True, + null=True + ) objects = AnalysisJobManager() @property diff --git a/src/django/pfb_analysis/serializers.py b/src/django/pfb_analysis/serializers.py index 9a6d7177..b277a5c9 100644 --- a/src/django/pfb_analysis/serializers.py +++ b/src/django/pfb_analysis/serializers.py @@ -210,6 +210,7 @@ class Meta: 'analysis_job_definition', '_analysis_job_name',) read_only_fields = ('uuid', 'createdAt', 'modifiedAt', 'createdBy', 'modifiedBy', 'batch_job_id', 'batch', 'census_block_count', 'final_runtime', + 'default_speed_limit', 'local_upload_task') diff --git a/src/django/pfb_analysis/views.py b/src/django/pfb_analysis/views.py index 931097ee..d5d275a8 100644 --- a/src/django/pfb_analysis/views.py +++ b/src/django/pfb_analysis/views.py @@ -62,7 +62,7 @@ def get_queryset(self): filter_class = AnalysisJobFilterSet filter_backends = (DjangoFilterBackend, OrderingFilter, OrgAutoFilterBackend) ordering_fields = ('created_at', 'modified_at', 'overall_score', 'neighborhood__label', - 'neighborhood__country', 'neighborhood__state_abbrev', 'population_total') + 'neighborhood__country', 'neighborhood__state_abbrev', 'population_total', 'default_speed_limit') ordering = ('-created_at',) def perform_create(self, serializer): @@ -86,6 +86,7 @@ def results(self, request, pk=None): results = OrderedDict([ ('census_block_count', job.census_block_count), ('census_blocks_url', job.census_blocks_url), + ('residential_speed_limit', job.default_speed_limit), ('connected_census_blocks_url', job.connected_census_blocks_url), ('destinations_urls', job.destinations_urls), ('tile_urls', job.tile_urls), From a7730d885835bc29679a8d4048714a4dc9a5e868 Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Thu, 12 Nov 2020 08:44:28 -0500 Subject: [PATCH 2/3] Write default speed limit to file and load it on the job Adds - a step to 'import_osm.sh' after the default speed limit is loaded to write the answer to a new table - a step in 'export_connectivity.sh' to export the table to a CSV file - a command, also called in 'export_connectivity.sh', to pass the CSV file path to a Django management command that loads it onto the job instance, like we've been doing for overall scores Still to do: load the value onto the instance during import --- src/analysis/import/import_osm.sh | 21 +++++++++++++++++++++ src/analysis/scripts/export_connectivity.sh | 5 +++++ src/analysis/scripts/utils.sh | 9 +++++++++ 3 files changed, 35 insertions(+) diff --git a/src/analysis/import/import_osm.sh b/src/analysis/import/import_osm.sh index 9e8d91bb..7b7484d4 100755 --- a/src/analysis/import/import_osm.sh +++ b/src/analysis/import/import_osm.sh @@ -207,6 +207,27 @@ else echo "The city residential default speed is ${CITY_DEFAULT}." fi +# Save default speed limit to a table for export later +psql -h $NB_POSTGRESQL_HOST -U $NB_POSTGRESQL_USER -d $NB_POSTGRESQL_DB \ + -c "CREATE TABLE IF NOT EXISTS \"residential_speed_limit\" ( + fips_code_state char(2), + fips_code_city char(7), + state_speed smallint, + city_speed smallint + );" +psql -h $NB_POSTGRESQL_HOST -U $NB_POSTGRESQL_USER -d $NB_POSTGRESQL_DB \ + -c "INSERT INTO \"residential_speed_limit\" ( + state_fips_code, + city_fips_code, + state_speed, + city_speed + ) VALUES ( + ${PFB_STATE_FIPS}, + ${PFB_CITY_FIPS}, + ${STATE_DEFAULT}, + ${CITY_DEFAULT} + );" + rm -rf "${SPEED_TEMPDIR}" echo "DONE: Importing city default residential speed" diff --git a/src/analysis/scripts/export_connectivity.sh b/src/analysis/scripts/export_connectivity.sh index 66b28e9e..90ab32ba 100755 --- a/src/analysis/scripts/export_connectivity.sh +++ b/src/analysis/scripts/export_connectivity.sh @@ -176,6 +176,11 @@ then # Send overall_scores to Django app update_overall_scores "${OUTPUT_DIR}/neighborhood_overall_scores.csv" + # Export residential_speed_limit as CSV + ec_export_table_csv "${OUTPUT_DIR}" "residential_speed_limit" + # Send residential_speed_limit to Django app + update_residential_speed_limit "${OUTPUT_DIR}/residential_speed_limit.csv" + if [ -n "${AWS_STORAGE_BUCKET_NAME}" ] && [ -n "${PFB_S3_RESULTS_PATH}" ] then sync # Probably superfluous, but the s3 command said "file changed while reading" once diff --git a/src/analysis/scripts/utils.sh b/src/analysis/scripts/utils.sh index a5041390..905cdc4f 100644 --- a/src/analysis/scripts/utils.sh +++ b/src/analysis/scripts/utils.sh @@ -28,6 +28,15 @@ function update_overall_scores() { fi } +function update_residential_speed_limit() { + # Usage: + # update_residential_speed_limit RESIDENTIAL_SPEED_LIMIT_CSV + if [ -n "${PFB_JOB_ID}" ]; + then + /opt/pfb/django/manage.py load_residential_speed_limit "${PFB_JOB_ID}" "$@" + fi +} + function set_job_attr() { # Usage: # update_job_attr ATTRIBUTE VALUE From 21fe1b982e74c4a6c49356d45506de636ce1f14b Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Thu, 12 Nov 2020 10:19:01 -0500 Subject: [PATCH 3/3] Load default speed limit when importing analysis results Changes the 'upload_local_analysis' task to load the default residential speed limit from the file where it gets exported by the analysis. Unlike the other files that are part of the export, it won't crash if this one is missing, so that exports of older analysis runs will still be importable. The front-end only shows the speed limit box if there's a default speed limit set on the job, so leaving it blank when we don't have the answer is fine. --- src/analysis/import/import_osm.sh | 4 +- .../commands/load_residential_speed_limit.py | 52 +++++++++++++++++++ src/django/pfb_analysis/tasks.py | 16 +++++- 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 src/django/pfb_analysis/management/commands/load_residential_speed_limit.py diff --git a/src/analysis/import/import_osm.sh b/src/analysis/import/import_osm.sh index 7b7484d4..ebcfb618 100755 --- a/src/analysis/import/import_osm.sh +++ b/src/analysis/import/import_osm.sh @@ -210,8 +210,8 @@ fi # Save default speed limit to a table for export later psql -h $NB_POSTGRESQL_HOST -U $NB_POSTGRESQL_USER -d $NB_POSTGRESQL_DB \ -c "CREATE TABLE IF NOT EXISTS \"residential_speed_limit\" ( - fips_code_state char(2), - fips_code_city char(7), + state_fips_code char(2), + city_fips_code char(7), state_speed smallint, city_speed smallint );" diff --git a/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py b/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py new file mode 100644 index 00000000..c7346b23 --- /dev/null +++ b/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py @@ -0,0 +1,52 @@ +import csv + +from django.core.management.base import BaseCommand + +from pfb_analysis.models import AnalysisJob + + +def load_speed_limit(job, csv_filename): + with open(csv_filename, 'r') as csv_file: + reader = csv.DictReader(csv_file) + row = next(reader) + # Take the city value if there is one, or fall back to the state value. + # In both cases check for truthiness, since a missing value will come back as zero + if row.get("city_speed"): + job.default_speed_limit = row["city_speed"] + job.speed_limit_src = AnalysisJob.SpeedLimitSource.CITY + elif row.get("state_speed"): + job.default_speed_limit = row["state_speed"] + job.speed_limit_src = AnalysisJob.SpeedLimitSource.STATE + job.save() + + +class Command(BaseCommand): + help = """ Load 'residential_speed_limit.csv' produced by the analysis into + AnalysisJob.default_speed_limit + + Expected CSV format: one line, with state and city FIPS codes and speeds. + The city code and speed can be blank. + + state_fips_code,city_fips_code,state_speed,city_speed + 24,2404000,30,25 + + Saves the city speed limit if present, otherwise the state limit. + """ + + def add_arguments(self, parser): + parser.add_argument('job_uuid', type=str) + parser.add_argument('csv_file', type=str, + help='Absolute path to residential speed limit csv to load') + + def handle(self, *args, **options): + job_uuid = options['job_uuid'] + csv_filename = options['csv_file'] + + try: + job = AnalysisJob.objects.get(pk=job_uuid) + except (AnalysisJob.DoesNotExist): + print('WARNING: Tried to set default_speed_limit for invalid job {} ' + 'from file {}'.format(job_uuid, csv_filename)) + raise + load_speed_limit(job, csv_filename) + self.stdout.write('{}: Loaded default_speed_limit from {}'.format(job, csv_filename)) diff --git a/src/django/pfb_analysis/tasks.py b/src/django/pfb_analysis/tasks.py index 2e5efa75..24d7fcdd 100644 --- a/src/django/pfb_analysis/tasks.py +++ b/src/django/pfb_analysis/tasks.py @@ -11,6 +11,7 @@ from pfb_analysis.management.commands.import_results_shapefiles import add_results_geoms from pfb_analysis.management.commands.load_overall_scores import load_scores +from pfb_analysis.management.commands.load_residential_speed_limit import load_speed_limit from pfb_analysis.models import AnalysisBatch, AnalysisJob, AnalysisLocalUploadTask from pfb_network_connectivity.utils import download_file from users.models import PFBUser @@ -21,13 +22,15 @@ for destination in settings.PFB_ANALYSIS_DESTINATIONS]) OVERALL_SCORES_FILE = 'neighborhood_overall_scores.csv' +SPEED_LIMIT_FILE = 'residential_speed_limit.csv' OTHER_RESULTS_FILES = set([ 'neighborhood_score_inputs.csv', 'neighborhood_ways.zip', 'neighborhood_census_blocks.zip', OVERALL_SCORES_FILE, - 'neighborhood_connected_census_blocks.csv.zip']) + 'neighborhood_connected_census_blocks.csv.zip', +]) # The set of all the results files from a local analysis run to upload on import LOCAL_ANALYSIS_FILES = DESTINATION_ANALYSIS_FILES.union(OTHER_RESULTS_FILES) @@ -85,7 +88,11 @@ def download_and_extract_local_results(tmpdir, task): # Verify all expected results files are in the upload missing = LOCAL_ANALYSIS_FILES.difference(set(results_files)) - if missing: + if missing == set([SPEED_LIMIT_FILE]): + # Warn but don't fail if there's no speed limit file, to support older exported + # analysis files + logging.warning('Analysis upload file has no {} file.'.format(SPEED_LIMIT_FILE)) + elif missing: raise LocalAnalysisFetchException('Missing expected results files: {files}'.format( files=', '.join(missing))) logging.info('Results files extracted for upload task {uuid}'.format( @@ -114,6 +121,11 @@ def download_and_extract_local_results(tmpdir, task): # set the overall scores on the job local_scores_file = os.path.join(tmpdir, OVERALL_SCORES_FILE) load_scores(task.job, local_scores_file, 'score_id', None) + # set the default residential speed limit on the job, but don't crash if the file is missing + try: + load_speed_limit(task.job, os.path.join(tmpdir, SPEED_LIMIT_FILE)) + except FileNotFoundError: + pass # Mark this upload task and its associated analysis job as completed. task.status = AnalysisLocalUploadTask.Status.COMPLETE