From 465e0c572f4e2494a3f60d54229ab2c46d55cbea Mon Sep 17 00:00:00 2001
From: Matt <mmccollow@gmail.com>
Date: Thu, 19 Jun 2014 14:19:06 -0400
Subject: [PATCH] New backend to scrape redesigned NASDAQ website

---
 app.py               |  6 +++---
 nd_csv.py            | 40 ++++++++++++++++++++++++++++++++++++++++
 templates/index.html |  5 +++--
 3 files changed, 46 insertions(+), 5 deletions(-)
 create mode 100644 nd_csv.py

diff --git a/app.py b/app.py
index 2121254..51baf99 100644
--- a/app.py
+++ b/app.py
@@ -2,13 +2,13 @@
 
 from flask import Flask
 from flask import render_template, url_for
-from scrape import load_data, format_json
+from nd_csv import get_stocks_data
 
 app = Flask(__name__)
 
 @app.route("/data.json")
 def data_json():
-	return format_json(load_data())
+	return get_stocks_data()
 
 @app.route("/")
 def index():
@@ -19,4 +19,4 @@ def index():
 	return render_template("index.html", d3=d3, bootstrapjs=bootstrapjs, bootstrapcss=bootstrapcss, qr=qr)
 
 if __name__ == "__main__":
-	app.run(debug=True)
\ No newline at end of file
+	app.run(debug=True)
diff --git a/nd_csv.py b/nd_csv.py
new file mode 100644
index 0000000..3ae786d
--- /dev/null
+++ b/nd_csv.py
@@ -0,0 +1,40 @@
+import csv
+import urllib2
+import StringIO
+import json
+
+URL = "http://www.nasdaq.com/quotes/nasdaq-100-stocks.aspx?render=download"
+
+# try to simulate Chrome
+HEADERS= {
+	"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+	"User-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
+	"Accept-Encoding": "gzip,deflate,sdch",
+	"Accept-Language": "en-US,en;q=0.8",
+	"Cookie": "clientPrefs=||||lightg; \
+	__atuvc=1%7C24%2C1%7C25; \
+	s_sess=%20s_cc%3Dtrue%3B%20s_sq%3Dnasdaqprod%253D%252526pid%25253DFlashQuotes%25252520-%25252520Nasdaq%25252520100%252526pidt%25253D1%252526oid%25253Dhttp%2525253A//www.nasdaq.com/quotes/nasdaq-100-stocks.aspx%2525253Frender%2525253Ddownload%252526ot%25253DA%3B; \
+	s_pers=%20bc%3D1%7C1403018677068%3B%20s_nr%3D1403105222488-Repeat%7C1410881222488%3B; \
+	NSC_W.TJUFEFGFOEFS.OBTEBR.80=ffffffffc3a08e3745525d5f4f58455e445a4a423660"
+}
+
+def get_stocks_data():
+	req = urllib2.Request(URL, headers=HEADERS)
+	page = urllib2.urlopen(req)
+
+	csvfile = StringIO.StringIO(page.read())
+	csvdict = csv.DictReader(csvfile)
+
+	output = {'children': []}
+	for item in csvdict:
+		output['children'].append({
+			'symbol': item['Symbol'].strip(),
+			'name': item[' Name'].strip(),
+			'price': item[' lastsale'].strip(),
+			'change_d': item[' netchange'].strip(),
+			'change_p': item['pctchange'].strip(),
+			'volume': item[' share_volume'].strip(),
+			'value': item[' Nasdaq100_points'].strip()
+		})
+	return json.dumps(output)
+
diff --git a/templates/index.html b/templates/index.html
index 947c60c..7cea441 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -31,12 +31,13 @@ <h2>NASDAQ-100 Stock Performance</h2>
     var diameter = 960,
       format = d3.format(",d"),
       color = d3.scale.category20c();
+			rSize = d3.scale.pow().domain([-100,100]).range([-50,50]);
 
     var bubble = d3.layout.pack()
       .sort(null)
       .size([diameter, diameter])
       .padding(1.5)
-      .radius(function(d) { return 20 + (1 + (d * 7)); });
+      .radius(function(d) { return 20 + (rSize(d) * 30); });
 
     var svg = d3.select("#graph").append("svg")
       .attr("width", diameter)
@@ -64,4 +65,4 @@ <h2>NASDAQ-100 Stock Performance</h2>
   setInterval(drawGraph, 900000);
   </script>
 </body>
-</html>
\ No newline at end of file
+</html>