refactorizing the code

it's nice and readable now
the class TR can be reused for accessing trustroots.org API
library co used for nice async syntax with Promises

.gitignore

@@ -4,7 +4,6 @@ config/*
 services/mailer/config.json
 npm-debug.log
 _*
-secret
 output
 
 #Allow

Graph.js

@@ -0,0 +1,136 @@
+'use strict';
+
+let co = require('co');
+let fs = require('fs');
+let denodeify = require('denodeify');
+
+let unlink = denodeify(fs.unlink);
+let writeFile = denodeify(fs.writeFile);
+
+let User = require('./User');
+
+class Graph {
+  constructor() {
+    this._users = [];
+    this._links = [];
+  }
+
+  get users() {
+    return this._users;
+  }
+
+  get links() {
+    //get connections of every user (and not duplicate, or yes?)
+    let links = [];
+    let users = this.users;
+    //create connections from the user and her contacts
+    for(let user of users) {
+      let contacts = user.contacts;
+      for(let contact of contacts) {
+        links.push([user.username, contact.username]);      
+      }
+    }
+    return links;
+  }
+
+  //returns boolean indicating whether user is already in the graph._users
+  hasUser (username) {
+    for(let usr of this._users) {
+      if(usr.username === username) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  //async
+
+  //filling the graph with the data
+  scrape(usernames) {
+    return co.call(this, function * () {
+      //TODO there may be also ids stored for efficiency
+
+      let currentLevel = usernames;
+      let nextLevel;
+      while(currentLevel.length > 0) {
+        nextLevel = [];
+        for(let username of currentLevel) {
+          let user = new User(username);
+
+          //get connections of each user from nextLevel
+          try {
+            yield user.scrape();
+
+            //save the user to this._users;
+
+            this._users.push(user);
+            console.log(`${this.users.length}  ${user.username}  ${user.contacts.length}`);
+
+            //fill first-time users to _nextLevel;
+            let contacts = user.contacts;
+            for(let contact of contacts) {
+              let isNotAnywhereYet = nextLevel.indexOf(contact.username) === -1 && currentLevel.indexOf(contact.username) === -1 && this.hasUser(contact.username) === false;
+              if(isNotAnywhereYet) {
+                nextLevel.push(contact.username);
+              }
+            }
+          }
+          catch(e) {
+            //there are some inconsistencies in the database - some contacts don't exist 
+            if(e.status === 404) {}
+            else{
+              throw e;
+            }
+          }
+        }
+        currentLevel = nextLevel;
+      }
+    });
+  }
+
+  outputGraph (filename) {
+    return co.call(this, function * () {
+      var path = `./output/${filename || 'graph'}.gdf`;
+
+      //outputString will be eventually written to the gdf file
+      //first the node header
+      var outputString = 'nodedef>name VARCHAR,label VARCHAR\n';
+      //write users to the outputString;
+      let users = this.users;
+      for(let user of users) {
+        outputString += `${user.username},${user.username}\n`;
+      }
+
+      //write edge header to the outputString
+      outputString += 'edgedef>node1 VARCHAR,node2 VARCHAR\n'
+      //write links to the outputString
+      let links = this.links;
+      for(let link of links) {
+        outputString += `${link[0]},${link[1]}\n`;
+      }
+
+      //write the data to the file
+      yield unlink(path);
+      yield writeFile(path, outputString);
+    });
+  }
+
+  outputUsers (filename) {
+    return co.call(this, function * () {
+      var path = `./output/${filename || 'users'}.txt`;
+
+      //get usernames from the list of user objects
+      var usernames = [];
+      var users = this.users;
+      for(let user of users) {
+        usernames.push(user.username);
+      }
+
+      //write to output file
+      yield unlink(path);
+      yield writeFile(path, usernames.join('\n')+'\n');
+    });
+  }
+}
+
+module.exports = Graph;

README.md

@@ -7,40 +7,30 @@ Users without connection to the main network will not be found.
 
 ##Prerequisities
 
-nodejs installed
+Node.js supporting ES2016 installed
 
-node package manager (npm) installed
+Node Package Manager (npm) installed
 
 ##Installation
 
 1. clone this repository
 2. run the following commands in terminal in the folder of the repository
 
 		npm install
-		mkdir secret
-		mkdir output
-		touch output/users.txt
-		touch output/graph.gdf
-
-3. create a file secret/login.json with following content
-
-		{
-			"username": "[valid trustroots username]",
-			"password": "[valid password]"
-		}
-
+                npm run prepare
+
 ##Usage
 
 - run `npm start`
-- watch the data scraper work. When it finishes, you'll find list of found users in `output/users.txt` and a graph in `.gdf` format in `output/graph.gdf`
+- provide your login data
+- provide a username where the scraper should start (defaults to your username)
+- watch the data scraper work. When it finishes, you'll find list of connected users in `output/users.txt` and a graph in `.gdf` format in `output/graph.gdf`
 - play with the data (i.e. with [gephi](https://gephi.org/))
 
 ##To do
 
 - time development of the network
 - shortest path between users
-- easier installation
-- comments and nicer code
 
 ##License
 MIT

TR.js

@@ -0,0 +1,96 @@
+'use strict';
+
+let denodeify = require('denodeify');
+let request = require('request').defaults({jar: true});
+let co = require('co');
+
+let get = denodeify(request.get, function (err, resp, body) {
+  return [err, body];
+});
+
+let post = denodeify(request.post, function (err, resp, body) {
+  return [err, body];
+});
+
+class TR {
+  //async
+  static login(username, password) {
+    return co(function * () {
+      let body = yield post({url: 'https://www.trustroots.org/api/auth/signin', form:{username: username, password: password}});
+      let jsonBody = JSON.parse(body);
+      let isNotLoggedIn = jsonBody && jsonBody.hasOwnProperty('message') && jsonBody.message === 'Unknown user or invalid password';
+      let isLoggedIn = jsonBody && jsonBody.hasOwnProperty('_id');
+
+      if(isNotLoggedIn) {
+        let e = new Error('login not successful');
+        e.status = 403;
+        throw e;
+      }
+      else if (isLoggedIn) {
+        return;
+      }
+      else {
+        let e = new Error(jsonBody.message || 'other error');
+        e.status = 500;
+        throw e;
+      }
+    });
+  }
+
+  //async
+  static logout() {
+
+  }
+
+  //show all existing hosts
+  static hosts() {
+  }
+
+  static user(username) {
+    return co(function * () {
+      let body = yield get({url: `https://www.trustroots.org/api/users/${username}`});
+      let jsonBody = JSON.parse(body);
+      if(jsonBody && jsonBody.hasOwnProperty('message') && jsonBody.message === 'Not found.') {
+        let e = new Error('Not found.');
+        e.status = 404;
+        throw e;
+      }
+      return ({
+        id: jsonBody._id,
+        username: jsonBody.username,
+        name: jsonBody.displayName,
+        created: new Date(jsonBody.created),
+        gender: jsonBody.gender
+      });
+    });
+  }
+
+  static contacts(id) {
+    return co(function * () {
+      let body = yield get({url: `https://www.trustroots.org/api/contacts/${id}`});
+      let jsonBody = JSON.parse(body);
+
+      let contacts = [];
+      for(let rawContact of jsonBody){
+        //include only confirmed and not deleted
+        if(rawContact && rawContact.confirmed === true) {
+          let users = rawContact.users;
+          if(users.length === 2) { //protection from deleted contact
+            let contact = users[0]._id === id ? users[1] : users[0];
+            delete rawContact.users;
+            let finalContact = {
+              id: contact._id,
+              username: contact.username,
+              name: contact.displayName,
+              created: new Date(rawContact.created)
+            }
+            contacts.push(finalContact);
+          }
+        }
+      }
+      return contacts;
+    });
+  }
+}
+
+module.exports = TR;

User.js

@@ -0,0 +1,41 @@
+'use strict';
+
+let co = require('co');
+
+let TR = require('./TR');
+//object User represents each user
+//async User.scrape(): read User data
+class User {
+  constructor (username) {
+    this._username = username;
+    this._contacts = [];
+  }
+
+  get username () {
+    return this._username;
+  }
+
+  get contacts () {
+    let contacts = [];
+    for(let contact of this._contacts) {
+      contacts.push({
+        username: contact.username,
+        id: contact.id
+      });
+    }
+    return contacts;
+  }
+
+  //async
+  scrape () {
+    return co.call(this, function * () {
+      let user = yield TR.user(this._username);
+      let contacts = yield TR.contacts(user.id);
+
+      this._id = user.id;
+      this._contacts = contacts;
+    });
+  }
+}
+
+module.exports = User;