WIP make formal import process

codefordurham · Oct 6, 2015 · 5797a45 · 5797a45
1 parent 969749c
commit 5797a45
Show file tree

Hide file tree

Showing 8 changed files with 194 additions and 47 deletions.
diff --git a/README.md b/README.md
@@ -6,8 +6,7 @@ The goal of the application is to create color-coded bike routes based on relati
 
 A snapshot of the master branch is [on heroku](http://bikesafetee.herokuapp.com/). This has been continuous integration server setup with Heroku - ping @dsummersl if its not working for you!
 
-TODO
-====
+# TODO
 
 [![Stories in Ready](https://badge.waffle.io/bikesafety/bikesafety.svg?label=ready&title=Ready)](http://waffle.io/bikesafety/bikesafety)
 
@@ -48,17 +47,17 @@ You only need [Node.js](http://nodejs.org) to develop this platform. Install tha
 
 `npm start` will open a server on port 3000 of your computer. Go there to view the app.
 
-Test
-====
+# Test
 
     npm test
 
 Under development you might want to rerun tests when files change:
 
     npm run-script start-test
 
-TopoJSON
-========
+# Data
+
+## Map Data
 
 The frontend application uses [TopoJSON](https://github.com/mbostock/topojson)
 to render paths. If the source geojson is updated then the topojson needs to be
@@ -68,3 +67,16 @@ To convert the geojson file to topojson:
 
     ./node_modules/.bin/topojson -p BIKE_FACIL --id-property OBJECTID_12 html/src/data/durham-bike-lanes.geojson > html/src/data/durham-bike-lanes.topojson
 
+## Crash Data
+
+Data is read from [Code for America Socrata](https://brigades.opendatanetwork.com/TRANSPORTATION/North-Carolina-Bicycle-Crash-Data/5tve-iede), supplied by the NCDOT bi-annually. We migrate and clean
+the data. If the data changes or the cleanup process changes, one should
+download the new CSV data and import it:
+
+To do so:
+
+1. Data is stored in the data/ directory.
+1. Start the server with Firebase credentials:
+
+    FIREBASE_SECRET=SECRET npm start
+    curl localhost:3000/api/refreshCrashMapping
diff --git a/data/North_Carolina_Bicycle_Crash_Data.csv.gz b/data/North_Carolina_Bicycle_Crash_Data.csv.gz
diff --git a/lib/createFirebase.js b/lib/createFirebase.js
@@ -0,0 +1,68 @@
+var csv = require('csv');
+var h = require('highland');
+var _ = require('lodash');
+var sanitizeCrashes = require('./sanitizeCrashes');
+
+// Read in a CSV stream, and create a JSON stream of the format expected by
+// createFirebaseTable().
+//
+// Returns a stream of objects created by sanitizeCrashes.js
+var createFirebaseFormat = function(csvStream) {
+    return h.pipeline(
+        h(csvStream),
+        csv.parse({columns: true, trim: true}),
+        h.map(function(d) {
+            // lowercase the headers for the sanitize code.
+            _(_.keys(d)).forEach(function(k) {
+                d[k.toLowerCase()] = d[k];
+            }).value();
+            return sanitizeCrashes(d);
+        })
+    );
+};
+
+// Create a new table 'tableName' in 'crashDB' from a JSON stream 'dataStream'
+// (created by createFirebaseFormat).
+var createFirebaseTable = function(crashDB, tableName, dataStream) {
+    var Firebase = require('firebase');
+    var bikeSafetyDB = new Firebase(crashDB);
+    bikeSafetyDB.authWithCustomToken(process.env.FIREBASE_SECRET, function(error, authData) {
+
+        var crashDB = bikeSafetyDB.child(oldCrashTable);
+
+        crashDB.once('value', function(data) {
+            var crashes = data.val();
+            var values = {};
+            var newCrashes = [];
+
+            for (var i = 0; i < crashes.length; i++) {
+                var crash = crashes[i];
+
+                //Get all the current values for each attribute
+                /*
+                for (var key in crash) {
+                    if (!(key in values)) {
+                        values[key] = [];
+                    }
+                    var value = crash[key];
+                    if (values[key].indexOf(value) == -1) {
+                        values[key].push(value);
+                    }
+                }
+                */
+
+                newCrashes.push(createNewCrash(crash));
+            }
+
+            var newCrashDB = bikeSafetyDB.child(newCrashTable);
+            newCrashDB.set(newCrashes);
+        });
+
+    });
+};
+
+module.exports.createFormat = createFirebaseFormat;
+module.exports.createTable = function(crashDB, tableName) {
+    var formattedData = createFirebaseFormat(fs.createReadStream(__dirname +'/testdata.csv'));
+    return createFirebaseTable(crashDB, tableName, formattedData);
+};
diff --git a/lib/sanatizeCrashes.js → lib/sanitizeCrashes.js b/lib/sanatizeCrashes.js → lib/sanitizeCrashes.js
@@ -102,45 +102,6 @@ function createNewCrash(oldCrash) {
     };
 }
 
-function sanitize(crashDB, oldCrashTable, newCrashTable) {
-    var Firebase = require('firebase');
-    var bikeSafetyDB = new Firebase(crashDB);
-    console.log(process.env.FIREBASE_SECRET);
-    bikeSafetyDB.authWithCustomToken(process.env.FIREBASE_SECRET, function(error, authData) {
-
-        var crashDB = bikeSafetyDB.child(oldCrashTable);
-
-        crashDB.once('value', function(data) {
-            var crashes = data.val();
-            var values = {};
-            var newCrashes = [];
-
-            for (var i = 0; i < crashes.length; i++) {
-                var crash = crashes[i];
-
-                //Get all the current values for each attribute 
-                /*
-                for (var key in crash) {
-                    if (!(key in values)) {
-                        values[key] = [];
-                    }
-                    var value = crash[key];
-                    if (values[key].indexOf(value) == -1) {
-                        values[key].push(value);
-                    }
-                }
-                */
-
-                newCrashes.push(createNewCrash(crash));
-            }
-
-            var newCrashDB = bikeSafetyDB.child(newCrashTable);
-            newCrashDB.set(newCrashes);
-        });
-
-    });
-}
-
-module.exports = sanitize;
+module.exports = createNewCrash;
 
-sanitize('https://bikesafety.firebaseio.com', 'Crashes', 'Crashes_Sanitzed');
+// sanitize('https://bikesafety.firebaseio.com', 'Crashes', 'Crashes_Sanitzed');
diff --git a/package.json b/package.json
@@ -43,10 +43,13 @@
   },
   "dependencies": {
     "bower": "^1.3.12",
+    "csv": "^0.4.2",
     "ejs": "~2.2.4",
     "express": "~3.19.2",
     "firebase": "~2.2.0",
+    "highland": "^2.5.0",
     "jade": "~1.9.2",
+    "lodash": "^3.8.0",
     "nconf": "~0.7.1"
   },
   "devDependencies": {

diff --git a/spec/lib/createFirebaseSpec.js b/spec/lib/createFirebaseSpec.js
@@ -0,0 +1,94 @@
+var createFirebase = require('../../lib/createFirebase');
+var fs = require('fs');
+var h = require('highland');
+
+describe('createFirebase', function() {
+    describe('createFormat()', function() {
+        it('returns an empty stream if there is no input', function(done) {
+            h(createFirebase.createFormat(h([])))
+                .errors(function(err,push) {
+                    expect(err).toEqual(null);
+                })
+                .toArray(function(data) {
+                    expect(data).toEqual([]);
+                    done();
+                });
+        });
+
+        it('converts a single row', function(done) {
+            h(createFirebase.createFormat(fs.createReadStream(__dirname +'/testrow.csv')))
+                .errors(function(err,push) {
+                    expect(err).toEqual(null);
+                })
+                .toArray(function(data) {
+                    expect(data).toEqual([{
+                      biker: {
+                          age: '11',
+                          alcohol: 'No',
+                          injury: 'Evident Injury',
+                          race: 'Unknown',
+                          sex: 'Male',
+                          direction: 'With Traffic',
+                          position: 'Sidewalk / Crosswalk / Driveway Crossing'
+                      },
+                      driver: {
+                          age: '35',
+                          alcohol: 'No',
+                          injury: 'No Injury',
+                          race: 'Unknown',
+                          sex: 'Male',
+                          vehicle_type: 'Passenger Car',
+                          estimated_speed: '31-35 mph'
+                      },
+                      location: {
+                          city: 'Durham',
+                          county: 'Durham',
+                          region: 'Piedmont',
+                          development: 'Residential',
+                          latitude: '36.03949',
+                          longitude: '-78.883896',
+                          locality: 'Urban (>70% Developed)',
+                          lanes: '1 lane',
+                          characteristics: 'Straight - Level',
+                          class: 'Local Street',
+                          configuration: 'Two-Way, Divided, Unprotected Median',
+                          feature: 'No Special Feature',
+                          surface: 'Smooth Asphalt',
+                          rural_urban: 'Urban',
+                          speed_limit: '30 - 35  MPH',
+                          traffic_control: 'No Control Present'
+                      },
+                      crash: {
+                          ambulance: 'Yes',
+                          group: 'Parallel Paths - Other Circumstances',
+                          location: 'Non-Intersection',
+                          type: 'Bicyclist Ride Out - Parallel Path',
+                          hit_and_run: 'No',
+                          timestamp: '01/02/2007 12:00:00 AMT16:41:00',
+                          light_conditions: 'Daylight',
+                          road_conditions: 'Dry',
+                          road_defects: 'None',
+                          weather: 'Clear',
+                          workzone: 'No'
+                      }
+                    }]);
+                    done();
+                });
+        });
+
+        it("converts a standard dataset", function(done) {
+            h(createFirebase.createFormat(fs.createReadStream(__dirname +'/testdata.csv')))
+                .toArray(function(data) {
+                    expect(data.length).toEqual(6);
+                    done();
+                });
+        });
+    });
+
+    describe('createTable()', function() {
+        // TODO mock firebase
+        it('does nothing if there is no data', function(done) {
+          done();
+        });
+    });
+});
diff --git a/spec/lib/testdata.csv b/spec/lib/testdata.csv
@@ -0,0 +1,7 @@
+FID,OBJECTID,AmbulanceR,BikeAge_Gr,Bike_Age,Bike_Alc_D,Bike_Dir,Bike_Injur,Bike_Pos,Bike_Race,Bike_Sex,City,County,CrashAlcoh,CrashDay,Crash_Date,Crash_Grp,Crash_Hour,Crash_Loc,Crash_Mont,Crash_Time,Crash_Type,Crash_Ty_1,Crash_Year,Crsh_Sevri,Developmen,DrvrAge_Gr,Drvr_Age,Drvr_Alc_D,Drvr_EstSp,Drvr_Injur,Drvr_Race,Drvr_Sex,Drvr_VehTy,ExcsSpdInd,Hit_Run,Latitude,Light_Cond,Locality,Longitude,Num_Lanes,Num_Units,Rd_Charact,Rd_Class,Rd_Conditi,Rd_Config,Rd_Defects,Rd_Feature,Rd_Surface,Region,Rural_Urba,Speed_Limi,Traff_Cntr,Weather,Workzone_I,Location
+0,1,Yes,,11,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,Black,Male,Durham,Durham,No,Tuesday,01/02/2007 12:00:00 AM,Parallel Paths - Other Circumstances,16,Non-Intersection,January,16:41,Bicyclist Ride Out - Parallel Path,331225,2007,B: Evident Injury,Residential,30-39,35,No,31-35 mph,O: No Injury,White,Male,Passenger Car,No,No,36.03949,Daylight,Urban (>70% Developed),-78.883896,1 lane,2,Straight - Level,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,30 - 35  MPH,No Control Present,Clear,No,"(36.03949, -78.883896)"
+1,2,Yes,20-24,20,No,Facing Traffic,C: Possible Injury,Sidewalk / Crosswalk / Driveway Crossing,Hispanic,Male,Cary,Wake,No,Friday,01/12/2007 12:00:00 AM,Motorist Failed to Yield - Signalized Intersection,9,Intersection,January,9:34,Motorist Drive Out - Right Turn on Red,132151,2007,C: Possible Injury,Residential,60-69,64,No,0-5 mph,O: No Injury,White,Male,Passenger Car,No,No,35.751118,Daylight,Urban (>70% Developed),-78.7828,3 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35  MPH,Stop And Go Signal,Clear,No,"(35.751118, -78.7828)"
+2,3,Yes,30-39,37,No,,B: Evident Injury,Non-Roadway,Black,Male,Stallings,Union,No,Monday,01/15/2007 12:00:00 AM,Non-Roadway,17,Non-Roadway,January,17:55,Non-Roadway,460910,2007,B: Evident Injury,Commercial,30-39,39,No,6-10 mph,O: No Injury,White,Female,Passenger Car,No,No,35.084732,Dusk,Urban (>70% Developed),-80.69782,2 lanes,2,Straight - Level,Public Vehicular Area,Dry,"Two-Way, Not Divided",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,20 - 25  MPH,No Control Present,Cloudy,No,"(35.084732, -80.69782)"
+3,4,Yes,30-39,30,No,With Traffic,C: Possible Injury,Travel Lane,White,Male,Salisbury,Rowan,No,Friday,01/19/2007 12:00:00 AM,Motorist Left Turn / Merge,17,Intersection,January,17:25,Motorist Left Turn - Opposite Direction,111212,2007,C: Possible Injury,Commercial,,,No,31-35 mph, Injury,/Missing,,Sport Utility,No,Yes,35.6844,Daylight,Urban (>70% Developed),-80.47932,2 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Not Divided",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35  MPH,No Control Present,Cloudy,No,"(35.6844, -80.47932)"
+4,5,Yes,40-49,45,No,With Traffic,B: Evident Injury,Travel Lane,Black,Male,Fayetteville,Cumberland,No,Friday,01/12/2007 12:00:00 AM,Motorist Overtaking Bicyclist,12,Non-Intersection,January,12:41,Motorist Overtaking - Bicyclist Swerved,311235,2007,B: Evident Injury,Commercial,50-59,51,No,31-35 mph,O: No Injury,Black,Female,Van,No,No,34.999428,Daylight,Urban (>70% Developed),-78.90445,2 lanes,2,Straight - Level,Local Street,Dry,"Two-Way, Not Divided",None,No Special Feature,Coarse Asphalt,Coastal,Urban,30 - 35  MPH,"Double Yellow Line, No Passing Zone",Clear,No,"(34.999428, -78.90445)"
+5,6,Yes,50-59,58,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,White,Male,Salisbury,Rowan,No,Wednesday,01/31/2007 12:00:00 AM,Bicyclist Failed to Yield - Signalized Intersection,9,Intersection,January,9:25,Bicyclist Ride Out - Signalized  Intersection,131153,2007,B: Evident Injury,Residential,,,No,21-25 mph, Injury,/Missing,,Passenger Car,No,Yes,35.666668,Daylight,Urban (>70% Developed),-80.47759,2 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Not Divided",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35  MPH,Stop And Go Signal,Clear,No,"(35.666668, -80.47759)"
diff --git a/spec/lib/testrow.csv b/spec/lib/testrow.csv
@@ -0,0 +1,2 @@
+FID,OBJECTID,AmbulanceR,BikeAge_Gr,Bike_Age,Bike_Alc_D,Bike_Dir,Bike_Injur,Bike_Pos,Bike_Race,Bike_Sex,City,County,CrashAlcoh,CrashDay,Crash_Date,Crash_Grp,Crash_Hour,Crash_Loc,Crash_Mont,Crash_Time,Crash_Type,Crash_Ty_1,Crash_Year,Crsh_Sevri,Developmen,DrvrAge_Gr,Drvr_Age,Drvr_Alc_D,Drvr_EstSp,Drvr_Injur,Drvr_Race,Drvr_Sex,Drvr_VehTy,ExcsSpdInd,Hit_Run,Latitude,Light_Cond,Locality,Longitude,Num_Lanes,Num_Units,Rd_Charact,Rd_Class,Rd_Conditi,Rd_Config,Rd_Defects,Rd_Feature,Rd_Surface,Region,Rural_Urba,Speed_Limi,Traff_Cntr,Weather,Workzone_I,Location
+0,1,Yes,,11,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,Black,Male,Durham,Durham,No,Tuesday,01/02/2007 12:00:00 AM,Parallel Paths - Other Circumstances,16,Non-Intersection,January,16:41,Bicyclist Ride Out - Parallel Path,331225,2007,B: Evident Injury,Residential,30-39,35,No,31-35 mph,O: No Injury,White,Male,Passenger Car,No,No,36.03949,Daylight,Urban (>70% Developed),-78.883896,1 lane,2,Straight - Level,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,30 - 35  MPH,No Control Present,Clear,No,"(36.03949, -78.883896)"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		FID,OBJECTID,AmbulanceR,BikeAge_Gr,Bike_Age,Bike_Alc_D,Bike_Dir,Bike_Injur,Bike_Pos,Bike_Race,Bike_Sex,City,County,CrashAlcoh,CrashDay,Crash_Date,Crash_Grp,Crash_Hour,Crash_Loc,Crash_Mont,Crash_Time,Crash_Type,Crash_Ty_1,Crash_Year,Crsh_Sevri,Developmen,DrvrAge_Gr,Drvr_Age,Drvr_Alc_D,Drvr_EstSp,Drvr_Injur,Drvr_Race,Drvr_Sex,Drvr_VehTy,ExcsSpdInd,Hit_Run,Latitude,Light_Cond,Locality,Longitude,Num_Lanes,Num_Units,Rd_Charact,Rd_Class,Rd_Conditi,Rd_Config,Rd_Defects,Rd_Feature,Rd_Surface,Region,Rural_Urba,Speed_Limi,Traff_Cntr,Weather,Workzone_I,Location
		0,1,Yes,,11,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,Black,Male,Durham,Durham,No,Tuesday,01/02/2007 12:00:00 AM,Parallel Paths - Other Circumstances,16,Non-Intersection,January,16:41,Bicyclist Ride Out - Parallel Path,331225,2007,B: Evident Injury,Residential,30-39,35,No,31-35 mph,O: No Injury,White,Male,Passenger Car,No,No,36.03949,Daylight,Urban (>70% Developed),-78.883896,1 lane,2,Straight - Level,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,No Control Present,Clear,No,"(36.03949, -78.883896)"