diff --git a/CHANGELOG.md b/CHANGELOG.md index fa45f132..3e556d6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. #### Added - S3 caching of Census data files - Upgrade Django (2.2.10 -> 2.2.16) and psycopg2 (2.8.4 -> 2.8.5) +- Add default speed limit for neighborhood to AnalysisJob model and display it on frontend ## [0.13.0] - 2020-02-14 diff --git a/src/analysis/import/import_osm.sh b/src/analysis/import/import_osm.sh index 9e8d91bb..ebcfb618 100755 --- a/src/analysis/import/import_osm.sh +++ b/src/analysis/import/import_osm.sh @@ -207,6 +207,27 @@ else echo "The city residential default speed is ${CITY_DEFAULT}." fi +# Save default speed limit to a table for export later +psql -h $NB_POSTGRESQL_HOST -U $NB_POSTGRESQL_USER -d $NB_POSTGRESQL_DB \ + -c "CREATE TABLE IF NOT EXISTS \"residential_speed_limit\" ( + state_fips_code char(2), + city_fips_code char(7), + state_speed smallint, + city_speed smallint + );" +psql -h $NB_POSTGRESQL_HOST -U $NB_POSTGRESQL_USER -d $NB_POSTGRESQL_DB \ + -c "INSERT INTO \"residential_speed_limit\" ( + state_fips_code, + city_fips_code, + state_speed, + city_speed + ) VALUES ( + ${PFB_STATE_FIPS}, + ${PFB_CITY_FIPS}, + ${STATE_DEFAULT}, + ${CITY_DEFAULT} + );" + rm -rf "${SPEED_TEMPDIR}" echo "DONE: Importing city default residential speed" diff --git a/src/analysis/scripts/export_connectivity.sh b/src/analysis/scripts/export_connectivity.sh index 66b28e9e..90ab32ba 100755 --- a/src/analysis/scripts/export_connectivity.sh +++ b/src/analysis/scripts/export_connectivity.sh @@ -176,6 +176,11 @@ then # Send overall_scores to Django app update_overall_scores "${OUTPUT_DIR}/neighborhood_overall_scores.csv" + # Export residential_speed_limit as CSV + ec_export_table_csv "${OUTPUT_DIR}" "residential_speed_limit" + # Send residential_speed_limit to Django app + update_residential_speed_limit "${OUTPUT_DIR}/residential_speed_limit.csv" + if [ -n "${AWS_STORAGE_BUCKET_NAME}" ] && [ -n "${PFB_S3_RESULTS_PATH}" ] then sync # Probably superfluous, but the s3 command said "file changed while reading" once diff --git a/src/analysis/scripts/utils.sh b/src/analysis/scripts/utils.sh index a5041390..905cdc4f 100644 --- a/src/analysis/scripts/utils.sh +++ b/src/analysis/scripts/utils.sh @@ -28,6 +28,15 @@ function update_overall_scores() { fi } +function update_residential_speed_limit() { + # Usage: + # update_residential_speed_limit RESIDENTIAL_SPEED_LIMIT_CSV + if [ -n "${PFB_JOB_ID}" ]; + then + /opt/pfb/django/manage.py load_residential_speed_limit "${PFB_JOB_ID}" "$@" + fi +} + function set_job_attr() { # Usage: # update_job_attr ATTRIBUTE VALUE diff --git a/src/angularjs/src/app/components/map/map.constants.js b/src/angularjs/src/app/components/map/map.constants.js index 2a373935..a764ceb9 100644 --- a/src/angularjs/src/app/components/map/map.constants.js +++ b/src/angularjs/src/app/components/map/map.constants.js @@ -47,6 +47,9 @@ position: 'bottomright', colors: ['#009fdf', '#ff3300'], labels: ['Low Stress', 'High Stress'] + }, + speedLimit: { + position: 'bottomright' } } }; diff --git a/src/angularjs/src/app/components/map/map.directive.js b/src/angularjs/src/app/components/map/map.directive.js index ea5f99b7..14edb405 100644 --- a/src/angularjs/src/app/components/map/map.directive.js +++ b/src/angularjs/src/app/components/map/map.directive.js @@ -4,6 +4,7 @@ function MapController($element) { var ctl = this; ctl.map = null; + ctl.speedLimit = null; ctl.mapHTMLElement = $element[0]; ctl.pfbMapOptions = ctl.pfbMapOptions || {}; @@ -43,6 +44,9 @@ ctl.map.addLayer(ctl.baseLayer); } } + if (changes.speedLimit) { + ctl.speedLimit = changes.speedLimit + } }; } diff --git a/src/angularjs/src/app/components/places.service.js b/src/angularjs/src/app/components/places.service.js index a4b9a99c..613f757e 100644 --- a/src/angularjs/src/app/components/places.service.js +++ b/src/angularjs/src/app/components/places.service.js @@ -45,14 +45,16 @@ dfd.resolve({}); return dfd.promise; } - place.results = results; place.scores = results.overall_scores; + _.each(results.overall_scores, function (scores, key) { scores.score_normalized = (key === 'population_total' ? scores.score_original : scores.score_normalized) }); - + place.scores.default_speed_limit = { + score_normalized: results.residential_speed_limit + }; dfd.resolve(place); }); }); diff --git a/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js b/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js new file mode 100644 index 00000000..ccb6e556 --- /dev/null +++ b/src/angularjs/src/app/leaflet/map-speed-limit-legend.control.js @@ -0,0 +1,36 @@ +(function() { + + // Custom private Leaflet control implements a simple legend for all map layers + L.Control.SpeedLegend = L.Control.extend({ + initialize: function (options) { + L.Util.setOptions(this, options); + this.speedLimit = options.speedLimit; + if (this.speedLimit <= 25) { + this.stressLevel = 'low' + } else if (this.speedLimit <= 30) { + this.stressLevel = 'med.' + } else { + this.stressLevel = 'high' + } + }, + onAdd: function () { + var div = L.DomUtil.create('div', 'leaflet-control-layers pfb-speed-limit-legend'); + var kphLimit = Math.round(this.speedLimit * 1.609); + div.innerHTML += '
Residential Speed Limit
'; + div.innerHTML += '
' + + '
' + this.speedLimit + '
mph
' + + '
' + kphLimit + '
km/h
' + + '
'; + div.innerHTML += '
Stress impact: ' + this.stressLevel + '
'; + return div; + }, + stressLevel: function () { + + } + }); + if (!L.control.speedLegend) { + L.control.speedLegend = function (opts) { + return new L.Control.SpeedLegend(opts); + } + } +})(); diff --git a/src/angularjs/src/app/places/detail/place-map.directive.js b/src/angularjs/src/app/places/detail/place-map.directive.js index 0870adb2..5a1712ff 100644 --- a/src/angularjs/src/app/places/detail/place-map.directive.js +++ b/src/angularjs/src/app/places/detail/place-map.directive.js @@ -72,7 +72,6 @@ if (!layers) { return; } - var satelliteLayer = L.tileLayer(MapConfig.baseLayers.Satellite.url, { attribution: MapConfig.baseLayers.Satellite.attribution, maxZoom: MapConfig.conusMaxZoom @@ -97,6 +96,11 @@ $window.print(); }).addTo(ctl.map); } + if (ctl.pfbPlaceMapSpeedLimit && !ctl.speedLimitLegend) { + var speedLimitLegendOptions = MapConfig.legends["speedLimit"]; + speedLimitLegendOptions.speedLimit = ctl.pfbPlaceMapSpeedLimit; + ctl.speedLimitLegend = L.control.speedLegend(speedLimitLegendOptions).addTo(ctl.map); + } _.map(layers.tileLayers, function(layerObj) { // Get desired label @@ -149,6 +153,8 @@ }); }); + + function onEachFeature(feature, layer) { // TODO: Style marker and popup layer.on({ @@ -202,7 +208,8 @@ restrict: 'E', scope: { pfbPlaceMapLayers: '<', - pfbPlaceMapUuid: '<' + pfbPlaceMapUuid: '<', + pfbPlaceMapSpeedLimit: '<' }, controller: 'PlaceMapController', controllerAs: 'ctl', diff --git a/src/angularjs/src/app/places/detail/place-map.html b/src/angularjs/src/app/places/detail/place-map.html index c6f92da9..d33d0cf1 100644 --- a/src/angularjs/src/app/places/detail/place-map.html +++ b/src/angularjs/src/app/places/detail/place-map.html @@ -1,6 +1,7 @@
{{placeDetail.place.label_suffix}} ng-class="m.subscoreClass"> {{ ::m.label }}
- {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} + {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} + {{ ::placeDetail.scores[m.name].score_normalized | number:0 }} mph
@@ -78,7 +79,8 @@

{{placeDetail.place.label_suffix}}

+ pfb-place-map-uuid="placeDetail.place.uuid" + pfb-place-map-speed-limit="placeDetail.scores.default_speed_limit.score_normalized">
diff --git a/src/angularjs/src/styles/components/_map.scss b/src/angularjs/src/styles/components/_map.scss index 6217e9a0..79d191cd 100644 --- a/src/angularjs/src/styles/components/_map.scss +++ b/src/angularjs/src/styles/components/_map.scss @@ -54,3 +54,35 @@ opacity: 0.7; } } + +.pfb-speed-limit-legend { + width: 210px; + color: #555; + text-align: center; + + h6 { + margin-top: 0; + padding-top: 0.3rem; + background: lightgray; + } + .speed-limit-inner { + margin: auto; + width: 140px; + } + + .speed-limit-block { + display: inline-block; + margin: 0 15px 0 15px; + } + + .speed-limit { + font-weight: 700; + font-size: 24pt; + } + .speed-stress { + font-weight: 500; + font-size: 1.4rem; + color: white; + } + +} diff --git a/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py b/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py new file mode 100644 index 00000000..c7346b23 --- /dev/null +++ b/src/django/pfb_analysis/management/commands/load_residential_speed_limit.py @@ -0,0 +1,52 @@ +import csv + +from django.core.management.base import BaseCommand + +from pfb_analysis.models import AnalysisJob + + +def load_speed_limit(job, csv_filename): + with open(csv_filename, 'r') as csv_file: + reader = csv.DictReader(csv_file) + row = next(reader) + # Take the city value if there is one, or fall back to the state value. + # In both cases check for truthiness, since a missing value will come back as zero + if row.get("city_speed"): + job.default_speed_limit = row["city_speed"] + job.speed_limit_src = AnalysisJob.SpeedLimitSource.CITY + elif row.get("state_speed"): + job.default_speed_limit = row["state_speed"] + job.speed_limit_src = AnalysisJob.SpeedLimitSource.STATE + job.save() + + +class Command(BaseCommand): + help = """ Load 'residential_speed_limit.csv' produced by the analysis into + AnalysisJob.default_speed_limit + + Expected CSV format: one line, with state and city FIPS codes and speeds. + The city code and speed can be blank. + + state_fips_code,city_fips_code,state_speed,city_speed + 24,2404000,30,25 + + Saves the city speed limit if present, otherwise the state limit. + """ + + def add_arguments(self, parser): + parser.add_argument('job_uuid', type=str) + parser.add_argument('csv_file', type=str, + help='Absolute path to residential speed limit csv to load') + + def handle(self, *args, **options): + job_uuid = options['job_uuid'] + csv_filename = options['csv_file'] + + try: + job = AnalysisJob.objects.get(pk=job_uuid) + except (AnalysisJob.DoesNotExist): + print('WARNING: Tried to set default_speed_limit for invalid job {} ' + 'from file {}'.format(job_uuid, csv_filename)) + raise + load_speed_limit(job, csv_filename) + self.stdout.write('{}: Loaded default_speed_limit from {}'.format(job, csv_filename)) diff --git a/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py b/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py new file mode 100644 index 00000000..99dea60b --- /dev/null +++ b/src/django/pfb_analysis/migrations/0041_auto_20201006_2320.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.10 on 2020-10-06 23:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pfb_analysis', '0040_neighborhood_analysis_job_ondelete'), + ] + + operations = [ + migrations.AddField( + model_name='analysisjob', + name='default_speed_limit', + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='analysisjob', + name='speed_limit_src', + field=models.CharField(blank=True, choices=[('State', 'State'), ('City', 'City')], max_length=20, null=True), + ), + ] diff --git a/src/django/pfb_analysis/models.py b/src/django/pfb_analysis/models.py index 247efa44..60aa5e1f 100644 --- a/src/django/pfb_analysis/models.py +++ b/src/django/pfb_analysis/models.py @@ -487,7 +487,8 @@ def get_queryset(self): ('overall_score', 'score_normalized'))) .annotate(population_total=ObjectAtPath('overall_scores', ('population_total', 'score_original'), - output_field=models.PositiveIntegerField()))) + output_field=models.PositiveIntegerField())) + ) return qs @@ -531,6 +532,15 @@ class Status: (ERROR, 'Error',), ) + class SpeedLimitSource: + STATE = 'State' + CITY = 'City' + + CHOICES = ( + (STATE, 'State'), + (CITY, 'City') + ) + batch_job_id = models.CharField(max_length=256, blank=True, null=True) neighborhood = models.ForeignKey(Neighborhood, related_name='analysis_jobs', @@ -549,7 +559,13 @@ class Status: start_time = models.DateTimeField(null=True, blank=True) final_runtime = models.PositiveIntegerField(default=0) status = models.CharField(choices=Status.CHOICES, max_length=12, default=Status.CREATED) - + default_speed_limit = models.PositiveIntegerField(blank=True, null=True) + speed_limit_src = models.CharField( + choices=SpeedLimitSource.CHOICES, + max_length=20, + blank=True, + null=True + ) objects = AnalysisJobManager() @property diff --git a/src/django/pfb_analysis/serializers.py b/src/django/pfb_analysis/serializers.py index 9a6d7177..b277a5c9 100644 --- a/src/django/pfb_analysis/serializers.py +++ b/src/django/pfb_analysis/serializers.py @@ -210,6 +210,7 @@ class Meta: 'analysis_job_definition', '_analysis_job_name',) read_only_fields = ('uuid', 'createdAt', 'modifiedAt', 'createdBy', 'modifiedBy', 'batch_job_id', 'batch', 'census_block_count', 'final_runtime', + 'default_speed_limit', 'local_upload_task') diff --git a/src/django/pfb_analysis/tasks.py b/src/django/pfb_analysis/tasks.py index 2e5efa75..24d7fcdd 100644 --- a/src/django/pfb_analysis/tasks.py +++ b/src/django/pfb_analysis/tasks.py @@ -11,6 +11,7 @@ from pfb_analysis.management.commands.import_results_shapefiles import add_results_geoms from pfb_analysis.management.commands.load_overall_scores import load_scores +from pfb_analysis.management.commands.load_residential_speed_limit import load_speed_limit from pfb_analysis.models import AnalysisBatch, AnalysisJob, AnalysisLocalUploadTask from pfb_network_connectivity.utils import download_file from users.models import PFBUser @@ -21,13 +22,15 @@ for destination in settings.PFB_ANALYSIS_DESTINATIONS]) OVERALL_SCORES_FILE = 'neighborhood_overall_scores.csv' +SPEED_LIMIT_FILE = 'residential_speed_limit.csv' OTHER_RESULTS_FILES = set([ 'neighborhood_score_inputs.csv', 'neighborhood_ways.zip', 'neighborhood_census_blocks.zip', OVERALL_SCORES_FILE, - 'neighborhood_connected_census_blocks.csv.zip']) + 'neighborhood_connected_census_blocks.csv.zip', +]) # The set of all the results files from a local analysis run to upload on import LOCAL_ANALYSIS_FILES = DESTINATION_ANALYSIS_FILES.union(OTHER_RESULTS_FILES) @@ -85,7 +88,11 @@ def download_and_extract_local_results(tmpdir, task): # Verify all expected results files are in the upload missing = LOCAL_ANALYSIS_FILES.difference(set(results_files)) - if missing: + if missing == set([SPEED_LIMIT_FILE]): + # Warn but don't fail if there's no speed limit file, to support older exported + # analysis files + logging.warning('Analysis upload file has no {} file.'.format(SPEED_LIMIT_FILE)) + elif missing: raise LocalAnalysisFetchException('Missing expected results files: {files}'.format( files=', '.join(missing))) logging.info('Results files extracted for upload task {uuid}'.format( @@ -114,6 +121,11 @@ def download_and_extract_local_results(tmpdir, task): # set the overall scores on the job local_scores_file = os.path.join(tmpdir, OVERALL_SCORES_FILE) load_scores(task.job, local_scores_file, 'score_id', None) + # set the default residential speed limit on the job, but don't crash if the file is missing + try: + load_speed_limit(task.job, os.path.join(tmpdir, SPEED_LIMIT_FILE)) + except FileNotFoundError: + pass # Mark this upload task and its associated analysis job as completed. task.status = AnalysisLocalUploadTask.Status.COMPLETE diff --git a/src/django/pfb_analysis/views.py b/src/django/pfb_analysis/views.py index 931097ee..d5d275a8 100644 --- a/src/django/pfb_analysis/views.py +++ b/src/django/pfb_analysis/views.py @@ -62,7 +62,7 @@ def get_queryset(self): filter_class = AnalysisJobFilterSet filter_backends = (DjangoFilterBackend, OrderingFilter, OrgAutoFilterBackend) ordering_fields = ('created_at', 'modified_at', 'overall_score', 'neighborhood__label', - 'neighborhood__country', 'neighborhood__state_abbrev', 'population_total') + 'neighborhood__country', 'neighborhood__state_abbrev', 'population_total', 'default_speed_limit') ordering = ('-created_at',) def perform_create(self, serializer): @@ -86,6 +86,7 @@ def results(self, request, pk=None): results = OrderedDict([ ('census_block_count', job.census_block_count), ('census_blocks_url', job.census_blocks_url), + ('residential_speed_limit', job.default_speed_limit), ('connected_census_blocks_url', job.connected_census_blocks_url), ('destinations_urls', job.destinations_urls), ('tile_urls', job.tile_urls),