Merge pull request #17 from wbruno/master

Recursive seed (1 minute interval) and module debug

index.js

@@ -1,6 +1,7 @@
 (function () {
   'use strict';
   var express = require('express'),
+      debug   = require('debug')('sabesp:app'),
       app     = express();
 
   // Heroku port settings
@@ -18,7 +19,7 @@
   app.use('/', require('./routes'));
 
   app.listen(app.get('port'), function () {
-    console.log('Magic happens on port: ' + app.get('port'));
+    debug('Magic happens on port: ' + app.get('port'));
   });
 
   module.exports = app;

lib/APIVersions.js

@@ -1,3 +1,4 @@
+var debug  = require('debug')('sabesp:api');
 var api = {
   v0: function(resolve, res) {
     res.json(resolve.dams || []);
@@ -17,9 +18,10 @@ var api = {
     });
     res.json(ret);
   }
-}
-api.reject = function(reject) {
-  console.log(reject);
-}
+};
+
+api.reject = function(reject, res) {
+  res.json(reject);
+};
 
 module.exports = api;

lib/Sabesp.js

@@ -3,6 +3,7 @@
   var request = require('request'),
       cheerio = require('cheerio'),
       Promise = require('promise'),
+      debug   = require('debug')('sabesp:crawler'),
       Helper  = require('./Helper');
 
   Helper.dams = {
@@ -34,7 +35,11 @@
           'jar': true,
           'form': data,
         }, function(error, response, html){
-          resolve(Helper.parserHTML(html, date));
+          if (error) {
+            reject(error)
+          } else {
+            resolve(Helper.parserHTML(html, date));
+          }
         }
       );
 

package.json

@@ -11,17 +11,20 @@
     "água"
   ],
   "scripts": {
+    "prestart": "export DEBUG=sabesp:*",
     "start": "node index.js",
     "pre-test": "./node_modules/jshint/bin/jshint *.js lib/*.js",
     "test": "node_modules/istanbul/lib/cli.js cover node_modules/.bin/_mocha tests/unit/*",
-    "test-api": "node_modules/istanbul/lib/cli.js cover node_modules/.bin/_mocha tests/api/*"
+    "test-api": "node_modules/istanbul/lib/cli.js cover node_modules/.bin/_mocha tests/api/*",
+    "seed": "export DEBUG=sabesp:* && node seed.js"
   },
   "engines": {
     "node": "0.10.x"
   },
   "dependencies": {
     "cheerio": "^0.18.0",
     "config": "^1.12.0",
+    "debug": "^2.1.3",
     "express": "^4.11.2",
     "moment": "^2.10.2",
     "mongodb": "^1.4.34",

routes.js

@@ -1,6 +1,7 @@
 var express = require('express'),
     router  = express.Router(),
     Promise = require('promise'),
+    debug   = require('debug')('sabesp:routes'),
     Mongo   = require('./lib/Mongo'),
     Helper  = require('./lib/Helper'),
     Sabesp  = require('./lib/Sabesp'),
@@ -18,7 +19,9 @@ router.get('/v1/:date?', function (req, res) {
     .then(function(resolve) {
       api.v1(resolve, res);
     })
-    .catch(api.reject);
+    .catch(function(err) {
+      api.reject(err, res);
+    });
 });
 
 router.get('/v2/:date?', function (req, res) {
@@ -27,16 +30,22 @@ router.get('/v2/:date?', function (req, res) {
     .then(function(resolve) {
       api.v2(resolve, res);
     })
-    .catch(api.reject);
+    .catch(function(err) {
+      api.reject(err, res);
+    });
 });
 
 router.get('/:date?', function (req, res) {
   var date = req.params.date || Helper.today();
+  debug('date', date);
+
   _isCached(date)
     .then(function(resolve) {
       api.v0(resolve, res);
     })
-    .catch(api.reject);
+    .catch(function(err) {
+      api.reject(err, res);
+    });
 
 });
 
@@ -47,12 +56,15 @@ function _isCached (date) {
         resolve(result);
       } else {
         Sabesp.fetch(date, token).then(function(data) {
-          if ((date !== '') && (date !== Helper.today())) {
+          if ((date !== '') && (data.date !== '') && (date !== Helper.today())) {
             Mongo.insert('dams', data, function(err, result) {
-              console.log(err, result);
+              if (err) { debug('err', err); }
             });
           }
           resolve(data);
+        })
+        .catch(function(err) {
+          reject(err);
         });
       }
     });

seed.js

@@ -1,37 +1,35 @@
 'use strict';
 var moment = require('moment'),
-    Mongo   = require('./lib/Mongo'),
-    Sabesp  = require('./lib/Sabesp'),
-    date = moment('2014-01-01'),
-    today = moment().format('YYYY-MM-DD');
+    http   = require('http'),
+    debug  = require('debug')('sabesp:seed'),
+    today  = moment(),
+    start  = moment('2003-01-01'),
+    next   = moment('2003-01-01').add(4, 'day');
 
-Sabesp.getToken().then(seedDatabase);
+debug('first crawler with:', start.toString(), next.toString());
+seedDatabase(start, next);
 
-function seedDatabase (token) {
-  var i = 0;
-  while (date.format('YYYY-MM-DD') !== today) {
-    var currentDate = date.format('YYYY-MM-DD');
-    setTimeout(function() {
-      isCached(currentDate, token);
-    }, 2000 * i);
+setInterval(function() {
+  next = next.add(4, 'day');
+  debug('next crawler with:', start.toString(), next.toString());
+  seedDatabase(start, next);
+}, 60 * 1000);
 
+function seedDatabase (firstDay, lastDay) {
+  var date = firstDay,
+      i = 0;
+
+  while (date <= lastDay) {
+    var url = 'http://localhost:8080/' + date.format('YYYY-MM-DD');
+    doRequest(url);
     date = date.add(1, 'day');
-    i++;
   }
 }
 
-function isCached (date, token) {
-  Mongo.findOne('dams', { date: date }, function(err, result) {
-      if (!result) {
-        insertData(date, token);
-      }
+function doRequest(url) {
+  http.get(url, function(res) {
+    debug("Got response: " + res.statusCode);
+  }).on('error', function(e) {
+    debug("Got error: " + e.message);
   });
 }
-
-function insertData (date, token) {
-  Sabesp.fetch(date, token).then(function(data) {
-    Mongo.insert('dams', data, function(err, result) {
-      console.log(err, result);
-    });
-  });
-}