diff --git a/README.md b/README.md index cf9f01b..f4084dd 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,7 @@ The goal of the application is to create color-coded bike routes based on relati A snapshot of the master branch is [on heroku](http://bikesafetee.herokuapp.com/). This has been continuous integration server setup with Heroku - ping @dsummersl if its not working for you! -TODO -==== +# TODO [![Stories in Ready](https://badge.waffle.io/bikesafety/bikesafety.svg?label=ready&title=Ready)](http://waffle.io/bikesafety/bikesafety) @@ -48,8 +47,7 @@ You only need [Node.js](http://nodejs.org) to develop this platform. Install tha `npm start` will open a server on port 3000 of your computer. Go there to view the app. -Test -==== +# Test npm test @@ -57,8 +55,9 @@ Under development you might want to rerun tests when files change: npm run-script start-test -TopoJSON -======== +# Data + +## Map Data The frontend application uses [TopoJSON](https://github.com/mbostock/topojson) to render paths. If the source geojson is updated then the topojson needs to be @@ -68,3 +67,16 @@ To convert the geojson file to topojson: ./node_modules/.bin/topojson -p BIKE_FACIL --id-property OBJECTID_12 html/src/data/durham-bike-lanes.geojson > html/src/data/durham-bike-lanes.topojson +## Crash Data + +Data is read from [Code for America Socrata](https://brigades.opendatanetwork.com/TRANSPORTATION/North-Carolina-Bicycle-Crash-Data/5tve-iede), supplied by the NCDOT bi-annually. We migrate and clean +the data. If the data changes or the cleanup process changes, one should +download the new CSV data and import it: + +To do so: + +1. Data is stored in the data/ directory. +1. Start the server with Firebase credentials: + + FIREBASE_SECRET=SECRET npm start + curl localhost:3000/api/refreshCrashMapping diff --git a/data/North_Carolina_Bicycle_Crash_Data.csv.gz b/data/North_Carolina_Bicycle_Crash_Data.csv.gz new file mode 100644 index 0000000..1e0005c Binary files /dev/null and b/data/North_Carolina_Bicycle_Crash_Data.csv.gz differ diff --git a/lib/createFirebase.js b/lib/createFirebase.js new file mode 100644 index 0000000..72b77de --- /dev/null +++ b/lib/createFirebase.js @@ -0,0 +1,68 @@ +var csv = require('csv'); +var h = require('highland'); +var _ = require('lodash'); +var sanitizeCrashes = require('./sanitizeCrashes'); + +// Read in a CSV stream, and create a JSON stream of the format expected by +// createFirebaseTable(). +// +// Returns a stream of objects created by sanitizeCrashes.js +var createFirebaseFormat = function(csvStream) { + return h.pipeline( + h(csvStream), + csv.parse({columns: true, trim: true}), + h.map(function(d) { + // lowercase the headers for the sanitize code. + _(_.keys(d)).forEach(function(k) { + d[k.toLowerCase()] = d[k]; + }).value(); + return sanitizeCrashes(d); + }) + ); +}; + +// Create a new table 'tableName' in 'crashDB' from a JSON stream 'dataStream' +// (created by createFirebaseFormat). +var createFirebaseTable = function(crashDB, tableName, dataStream) { + var Firebase = require('firebase'); + var bikeSafetyDB = new Firebase(crashDB); + bikeSafetyDB.authWithCustomToken(process.env.FIREBASE_SECRET, function(error, authData) { + + var crashDB = bikeSafetyDB.child(oldCrashTable); + + crashDB.once('value', function(data) { + var crashes = data.val(); + var values = {}; + var newCrashes = []; + + for (var i = 0; i < crashes.length; i++) { + var crash = crashes[i]; + + //Get all the current values for each attribute + /* + for (var key in crash) { + if (!(key in values)) { + values[key] = []; + } + var value = crash[key]; + if (values[key].indexOf(value) == -1) { + values[key].push(value); + } + } + */ + + newCrashes.push(createNewCrash(crash)); + } + + var newCrashDB = bikeSafetyDB.child(newCrashTable); + newCrashDB.set(newCrashes); + }); + + }); +}; + +module.exports.createFormat = createFirebaseFormat; +module.exports.createTable = function(crashDB, tableName) { + var formattedData = createFirebaseFormat(fs.createReadStream(__dirname +'/testdata.csv')); + return createFirebaseTable(crashDB, tableName, formattedData); +}; diff --git a/lib/sanatizeCrashes.js b/lib/sanitizeCrashes.js similarity index 70% rename from lib/sanatizeCrashes.js rename to lib/sanitizeCrashes.js index 0348318..6ee5247 100644 --- a/lib/sanatizeCrashes.js +++ b/lib/sanitizeCrashes.js @@ -102,45 +102,6 @@ function createNewCrash(oldCrash) { }; } -function sanitize(crashDB, oldCrashTable, newCrashTable) { - var Firebase = require('firebase'); - var bikeSafetyDB = new Firebase(crashDB); - console.log(process.env.FIREBASE_SECRET); - bikeSafetyDB.authWithCustomToken(process.env.FIREBASE_SECRET, function(error, authData) { - - var crashDB = bikeSafetyDB.child(oldCrashTable); - - crashDB.once('value', function(data) { - var crashes = data.val(); - var values = {}; - var newCrashes = []; - - for (var i = 0; i < crashes.length; i++) { - var crash = crashes[i]; - - //Get all the current values for each attribute - /* - for (var key in crash) { - if (!(key in values)) { - values[key] = []; - } - var value = crash[key]; - if (values[key].indexOf(value) == -1) { - values[key].push(value); - } - } - */ - - newCrashes.push(createNewCrash(crash)); - } - - var newCrashDB = bikeSafetyDB.child(newCrashTable); - newCrashDB.set(newCrashes); - }); - - }); -} - -module.exports = sanitize; +module.exports = createNewCrash; -sanitize('https://bikesafety.firebaseio.com', 'Crashes', 'Crashes_Sanitzed'); +// sanitize('https://bikesafety.firebaseio.com', 'Crashes', 'Crashes_Sanitzed'); diff --git a/package.json b/package.json index 53b2bea..ca36f9a 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,13 @@ }, "dependencies": { "bower": "^1.3.12", + "csv": "^0.4.2", "ejs": "~2.2.4", "express": "~3.19.2", "firebase": "~2.2.0", + "highland": "^2.5.0", "jade": "~1.9.2", + "lodash": "^3.8.0", "nconf": "~0.7.1" }, "devDependencies": { diff --git a/spec/lib/createFirebaseSpec.js b/spec/lib/createFirebaseSpec.js new file mode 100644 index 0000000..ab5f4ef --- /dev/null +++ b/spec/lib/createFirebaseSpec.js @@ -0,0 +1,94 @@ +var createFirebase = require('../../lib/createFirebase'); +var fs = require('fs'); +var h = require('highland'); + +describe('createFirebase', function() { + describe('createFormat()', function() { + it('returns an empty stream if there is no input', function(done) { + h(createFirebase.createFormat(h([]))) + .errors(function(err,push) { + expect(err).toEqual(null); + }) + .toArray(function(data) { + expect(data).toEqual([]); + done(); + }); + }); + + it('converts a single row', function(done) { + h(createFirebase.createFormat(fs.createReadStream(__dirname +'/testrow.csv'))) + .errors(function(err,push) { + expect(err).toEqual(null); + }) + .toArray(function(data) { + expect(data).toEqual([{ + biker: { + age: '11', + alcohol: 'No', + injury: 'Evident Injury', + race: 'Unknown', + sex: 'Male', + direction: 'With Traffic', + position: 'Sidewalk / Crosswalk / Driveway Crossing' + }, + driver: { + age: '35', + alcohol: 'No', + injury: 'No Injury', + race: 'Unknown', + sex: 'Male', + vehicle_type: 'Passenger Car', + estimated_speed: '31-35 mph' + }, + location: { + city: 'Durham', + county: 'Durham', + region: 'Piedmont', + development: 'Residential', + latitude: '36.03949', + longitude: '-78.883896', + locality: 'Urban (>70% Developed)', + lanes: '1 lane', + characteristics: 'Straight - Level', + class: 'Local Street', + configuration: 'Two-Way, Divided, Unprotected Median', + feature: 'No Special Feature', + surface: 'Smooth Asphalt', + rural_urban: 'Urban', + speed_limit: '30 - 35 MPH', + traffic_control: 'No Control Present' + }, + crash: { + ambulance: 'Yes', + group: 'Parallel Paths - Other Circumstances', + location: 'Non-Intersection', + type: 'Bicyclist Ride Out - Parallel Path', + hit_and_run: 'No', + timestamp: '01/02/2007 12:00:00 AMT16:41:00', + light_conditions: 'Daylight', + road_conditions: 'Dry', + road_defects: 'None', + weather: 'Clear', + workzone: 'No' + } + }]); + done(); + }); + }); + + it("converts a standard dataset", function(done) { + h(createFirebase.createFormat(fs.createReadStream(__dirname +'/testdata.csv'))) + .toArray(function(data) { + expect(data.length).toEqual(6); + done(); + }); + }); + }); + + describe('createTable()', function() { + // TODO mock firebase + it('does nothing if there is no data', function(done) { + done(); + }); + }); +}); diff --git a/spec/lib/testdata.csv b/spec/lib/testdata.csv new file mode 100644 index 0000000..2e2f0b8 --- /dev/null +++ b/spec/lib/testdata.csv @@ -0,0 +1,7 @@ +FID,OBJECTID,AmbulanceR,BikeAge_Gr,Bike_Age,Bike_Alc_D,Bike_Dir,Bike_Injur,Bike_Pos,Bike_Race,Bike_Sex,City,County,CrashAlcoh,CrashDay,Crash_Date,Crash_Grp,Crash_Hour,Crash_Loc,Crash_Mont,Crash_Time,Crash_Type,Crash_Ty_1,Crash_Year,Crsh_Sevri,Developmen,DrvrAge_Gr,Drvr_Age,Drvr_Alc_D,Drvr_EstSp,Drvr_Injur,Drvr_Race,Drvr_Sex,Drvr_VehTy,ExcsSpdInd,Hit_Run,Latitude,Light_Cond,Locality,Longitude,Num_Lanes,Num_Units,Rd_Charact,Rd_Class,Rd_Conditi,Rd_Config,Rd_Defects,Rd_Feature,Rd_Surface,Region,Rural_Urba,Speed_Limi,Traff_Cntr,Weather,Workzone_I,Location +0,1,Yes,,11,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,Black,Male,Durham,Durham,No,Tuesday,01/02/2007 12:00:00 AM,Parallel Paths - Other Circumstances,16,Non-Intersection,January,16:41,Bicyclist Ride Out - Parallel Path,331225,2007,B: Evident Injury,Residential,30-39,35,No,31-35 mph,O: No Injury,White,Male,Passenger Car,No,No,36.03949,Daylight,Urban (>70% Developed),-78.883896,1 lane,2,Straight - Level,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,No Control Present,Clear,No,"(36.03949, -78.883896)" +1,2,Yes,20-24,20,No,Facing Traffic,C: Possible Injury,Sidewalk / Crosswalk / Driveway Crossing,Hispanic,Male,Cary,Wake,No,Friday,01/12/2007 12:00:00 AM,Motorist Failed to Yield - Signalized Intersection,9,Intersection,January,9:34,Motorist Drive Out - Right Turn on Red,132151,2007,C: Possible Injury,Residential,60-69,64,No,0-5 mph,O: No Injury,White,Male,Passenger Car,No,No,35.751118,Daylight,Urban (>70% Developed),-78.7828,3 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,Stop And Go Signal,Clear,No,"(35.751118, -78.7828)" +2,3,Yes,30-39,37,No,,B: Evident Injury,Non-Roadway,Black,Male,Stallings,Union,No,Monday,01/15/2007 12:00:00 AM,Non-Roadway,17,Non-Roadway,January,17:55,Non-Roadway,460910,2007,B: Evident Injury,Commercial,30-39,39,No,6-10 mph,O: No Injury,White,Female,Passenger Car,No,No,35.084732,Dusk,Urban (>70% Developed),-80.69782,2 lanes,2,Straight - Level,Public Vehicular Area,Dry,"Two-Way, Not Divided",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,20 - 25 MPH,No Control Present,Cloudy,No,"(35.084732, -80.69782)" +3,4,Yes,30-39,30,No,With Traffic,C: Possible Injury,Travel Lane,White,Male,Salisbury,Rowan,No,Friday,01/19/2007 12:00:00 AM,Motorist Left Turn / Merge,17,Intersection,January,17:25,Motorist Left Turn - Opposite Direction,111212,2007,C: Possible Injury,Commercial,,,No,31-35 mph, Injury,/Missing,,Sport Utility,No,Yes,35.6844,Daylight,Urban (>70% Developed),-80.47932,2 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Not Divided",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,No Control Present,Cloudy,No,"(35.6844, -80.47932)" +4,5,Yes,40-49,45,No,With Traffic,B: Evident Injury,Travel Lane,Black,Male,Fayetteville,Cumberland,No,Friday,01/12/2007 12:00:00 AM,Motorist Overtaking Bicyclist,12,Non-Intersection,January,12:41,Motorist Overtaking - Bicyclist Swerved,311235,2007,B: Evident Injury,Commercial,50-59,51,No,31-35 mph,O: No Injury,Black,Female,Van,No,No,34.999428,Daylight,Urban (>70% Developed),-78.90445,2 lanes,2,Straight - Level,Local Street,Dry,"Two-Way, Not Divided",None,No Special Feature,Coarse Asphalt,Coastal,Urban,30 - 35 MPH,"Double Yellow Line, No Passing Zone",Clear,No,"(34.999428, -78.90445)" +5,6,Yes,50-59,58,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,White,Male,Salisbury,Rowan,No,Wednesday,01/31/2007 12:00:00 AM,Bicyclist Failed to Yield - Signalized Intersection,9,Intersection,January,9:25,Bicyclist Ride Out - Signalized Intersection,131153,2007,B: Evident Injury,Residential,,,No,21-25 mph, Injury,/Missing,,Passenger Car,No,Yes,35.666668,Daylight,Urban (>70% Developed),-80.47759,2 lanes,2,Straight - Grade,Local Street,Dry,"Two-Way, Not Divided",None,Four-Way Intersection,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,Stop And Go Signal,Clear,No,"(35.666668, -80.47759)" diff --git a/spec/lib/testrow.csv b/spec/lib/testrow.csv new file mode 100644 index 0000000..3c9ab10 --- /dev/null +++ b/spec/lib/testrow.csv @@ -0,0 +1,2 @@ +FID,OBJECTID,AmbulanceR,BikeAge_Gr,Bike_Age,Bike_Alc_D,Bike_Dir,Bike_Injur,Bike_Pos,Bike_Race,Bike_Sex,City,County,CrashAlcoh,CrashDay,Crash_Date,Crash_Grp,Crash_Hour,Crash_Loc,Crash_Mont,Crash_Time,Crash_Type,Crash_Ty_1,Crash_Year,Crsh_Sevri,Developmen,DrvrAge_Gr,Drvr_Age,Drvr_Alc_D,Drvr_EstSp,Drvr_Injur,Drvr_Race,Drvr_Sex,Drvr_VehTy,ExcsSpdInd,Hit_Run,Latitude,Light_Cond,Locality,Longitude,Num_Lanes,Num_Units,Rd_Charact,Rd_Class,Rd_Conditi,Rd_Config,Rd_Defects,Rd_Feature,Rd_Surface,Region,Rural_Urba,Speed_Limi,Traff_Cntr,Weather,Workzone_I,Location +0,1,Yes,,11,No,With Traffic,B: Evident Injury,Sidewalk / Crosswalk / Driveway Crossing,Black,Male,Durham,Durham,No,Tuesday,01/02/2007 12:00:00 AM,Parallel Paths - Other Circumstances,16,Non-Intersection,January,16:41,Bicyclist Ride Out - Parallel Path,331225,2007,B: Evident Injury,Residential,30-39,35,No,31-35 mph,O: No Injury,White,Male,Passenger Car,No,No,36.03949,Daylight,Urban (>70% Developed),-78.883896,1 lane,2,Straight - Level,Local Street,Dry,"Two-Way, Divided, Unprotected Median",None,No Special Feature,Smooth Asphalt,Piedmont,Urban,30 - 35 MPH,No Control Present,Clear,No,"(36.03949, -78.883896)"