From 6d09638522904745313b6d721aa0766d32a60855 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Thu, 18 Sep 2014 20:04:37 -0400 Subject: [PATCH 01/12] add fab file that creates a virtual environment and installs dependencies --- fabfile.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 fabfile.py diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 00000000..bf3581fb --- /dev/null +++ b/fabfile.py @@ -0,0 +1,14 @@ +from fabric.api import local, task + +@task +def create_env(): + local("virtualenv venv") + virtualenv("pip install -r requirements.txt --allow-external mysql-connector-python") + + +def virtualenv(command): + """ + Runs a command in the virtual environment + """ + local('source venv/bin/activate && {command}'.format(command=command), + shell='/bin/bash') From 836939c990f62fe12e2f2d0647025c54c1a4672a Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 09:50:52 -0400 Subject: [PATCH 02/12] generate a default configuration file from the fabfile --- .gitignore | 1 + fabfile.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 143f9b3e..fbb5104b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ venv *.db .env src +*.config diff --git a/fabfile.py b/fabfile.py index bf3581fb..1319c724 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1,14 +1,161 @@ +from __future__ import print_function + +import os +import copy + +try: + import configparser +except ImportError: + import ConfigParser as configparser + from fabric.api import local, task + +CONFIG_FILE_PATH = "{}{}".format(os.path.abspath(os.path.dirname(__file__)), + "/call_congress.config") + +DEVELOPMENT_ENV = "development" +PRODUCTION_ENV = "production" + +DEVELOPMENT_ENV_VARIABLES = [ + "SUNLIGHTLABS_KEY", + "TWILIO_ACCOUNT_SID", + "TWILIO_AUTH_TOKEN", + "TW_NUMBER", + "REDISTOGO_URL" +] + +PRODUCTION_ENV_VARIABLES = [ + "SUNLIGHTLABS_KEY", + "TWILIO_ACCOUNT_SID", + "TWILIO_AUTH_TOKEN", + "APPLICATION_ROOT", + "REDISTOGO_URL" +] + @task def create_env(): + """ + Sets up the virtualenv and downloads the dependencies with pip and creates + a default config file + """ local("virtualenv venv") - virtualenv("pip install -r requirements.txt --allow-external mysql-connector-python") + # When using pip install it complained about the mysql-connector-python lib + # not being in a standard location + virtualenv("pip install -r requirements.txt --allow-external " + \ + "mysql-connector-python") + + if not os.path.isfile(CONFIG_FILE_PATH): + generate_example_config() + +@task +def run(env=None): + """ + Runs the application in a specified environment + """ + if env is None: + print("[Warning] No environment specified.") + env = DEVELOPMENT_ENV + + create_env() + print("[Info] Running {} environment.".format(env)) + if env is DEVELOPMENT_ENV: + virtualenv("python app.py", env) + else: + virtualenv("foreman start", env) + +@task +def generate_example_config(): + """ + Generates a default config file at CONFIG_FILE_PATH + """ + if os.path.isfile(CONFIG_FILE_PATH): + print("[Warning] Config file already exists, not generating a new one.") + return + + print("Generating config file at {}".format(CONFIG_FILE_PATH)) + config = configparser.RawConfigParser() + + def __set_section(config, section, variables): + """ + Helper function to set up a config section from an array + """ + config.add_section(section) + for item in variables: + config.set(section, item, "") + __set_section(config, PRODUCTION_ENV, PRODUCTION_ENV_VARIABLES) + __set_section(config, DEVELOPMENT_ENV, DEVELOPMENT_ENV_VARIABLES) -def virtualenv(command): + with open(CONFIG_FILE_PATH, 'wb') as config_file: + config.write(config_file) + + +@task +def clean(): + """ + Removes all compiled python files from the project + """ + local("rm -rf *.pyc") + + +def virtualenv(command, env=None): """ Runs a command in the virtual environment """ - local('source venv/bin/activate && {command}'.format(command=command), + if env is None: + local('source venv/bin/activate && {}'.format(command), + shell='/bin/bash') + return + + env_prefix = generate_env_command_prefix(env) + local('source venv/bin/activate && {}{}'.format(env_prefix, command), shell='/bin/bash') + +def generate_env_command_prefix(env=None): + """ + Reads the Config file and sets up an env command based on their values + """ + if env is None: + env = DEVELOPMENT_ENV + + export_str = "env " + env_variables = parse_config(env) + for variable in env_variables: + if not env_variables[variable]: + print("[WARNING] {} not set in your config file.".format(variable)) + continue + export_str += "{}={} ".format(variable.upper(), env_variables[variable]) + + return export_str + +def parse_config(section): + """ + Looks for the default config file (ENV_FILE_PATH) and defaults to the + example config file defined in this repository. + + Returns a dictionary with the config values + """ + if not os.path.isfile(CONFIG_FILE_PATH): + print("[Warning] No config file found.") + generate_example_config() + + def __parse_config_for(section, variables): + """ + Helper function to parse a set of variables out of a given config file + and section. + """ + config = configparser.RawConfigParser() + config.read(CONFIG_FILE_PATH) + + values = {} + for key in variables: + values[key] = config.get(section, key) + + return values + + if section == PRODUCTION_ENV: + return __parse_config_for(section, PRODUCTION_ENV_VARIABLES) + else: + return __parse_config_for(section, DEVELOPMENT_ENV_VARIABLES) + From 241758a4b063e14aacff193d49cb1405351619a6 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 10:32:50 -0400 Subject: [PATCH 03/12] add notes on running with fabric --- docs/running_with_fabric.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/running_with_fabric.md diff --git a/docs/running_with_fabric.md b/docs/running_with_fabric.md new file mode 100644 index 00000000..83fd52ff --- /dev/null +++ b/docs/running_with_fabric.md @@ -0,0 +1,28 @@ +# Setting up with Fabric + +First install fabric to your local system, not your virtualenv (`sudo pip install fabric`) + +To generate your virtual environment, you can run either create a default environment manually with + + $ fab create_env + +## Config File + +When creating your environment, a configuration file (call_congress.config) will be generated in the git root. Edit this file to add the appropriate values before trying to run the application. When you run the application (with `fab run`) all of these values will be exported as environment variables. + +## Running with Fabric + +Once you've set up the config file, you should be able to run the application by running: + + $ fab run + +If you want to specify that the app should run in development or production, then run the following: + + $ fab run:production + $ fab run:development + +By default, the app runs in development. + +## Other Fabric operations + +- `fab clean` removes all .pyc files from the project From bd8a6dca2a282ed35f8d0c8c938edcf823b34cf1 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 10:49:36 -0400 Subject: [PATCH 04/12] fix typo in docs --- docs/running_with_fabric.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_with_fabric.md b/docs/running_with_fabric.md index 83fd52ff..add5854b 100644 --- a/docs/running_with_fabric.md +++ b/docs/running_with_fabric.md @@ -2,7 +2,7 @@ First install fabric to your local system, not your virtualenv (`sudo pip install fabric`) -To generate your virtual environment, you can run either create a default environment manually with +To generate your virtual environment, you should run the following: $ fab create_env From d4ccbe59251401996dbe8e115f99d4ca8ee672ae Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 10:51:09 -0400 Subject: [PATCH 05/12] add a second note about where the environment variables come from --- docs/running_with_fabric.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_with_fabric.md b/docs/running_with_fabric.md index add5854b..bf3097ba 100644 --- a/docs/running_with_fabric.md +++ b/docs/running_with_fabric.md @@ -16,7 +16,7 @@ Once you've set up the config file, you should be able to run the application by $ fab run -If you want to specify that the app should run in development or production, then run the following: +This command will automatically export the environment variables from your config file. If you want to specify that the app should run in development or production, then run the following: $ fab run:production $ fab run:development From 72584bdbaffb543d15f3b7a143562dd718b79704 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 11:06:23 -0400 Subject: [PATCH 06/12] add 'update_data' fab command --- docs/running_with_fabric.md | 1 + fabfile.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/running_with_fabric.md b/docs/running_with_fabric.md index bf3097ba..c260c635 100644 --- a/docs/running_with_fabric.md +++ b/docs/running_with_fabric.md @@ -26,3 +26,4 @@ By default, the app runs in development. ## Other Fabric operations - `fab clean` removes all .pyc files from the project +- `fab update_data` will download the updated data from the Sunlight Congress API diff --git a/fabfile.py b/fabfile.py index 1319c724..857c87c6 100644 --- a/fabfile.py +++ b/fabfile.py @@ -8,11 +8,11 @@ except ImportError: import ConfigParser as configparser -from fabric.api import local, task +from fabric.api import local, task, lcd -CONFIG_FILE_PATH = "{}{}".format(os.path.abspath(os.path.dirname(__file__)), - "/call_congress.config") +FILE_ROOT = os.path.abspath(os.path.dirname(__file__)) +CONFIG_FILE_PATH = "{}{}".format(FILE_ROOT, "/call_congress.config") DEVELOPMENT_ENV = "development" PRODUCTION_ENV = "production" @@ -91,6 +91,15 @@ def __set_section(config, section, variables): config.write(config_file) +@task +def update_data(): + """ + Updates the districts and legislators CSV files in the `data/` folder + """ + with lcd(FILE_ROOT + "/data"): + local('curl -kO "http://unitedstates.sunlightfoundation.com/legislators/legislators.csv"') + local('curl -kO "http://assets.sunlightfoundation.com/data/districts.csv"') + @task def clean(): """ From 6eccff00f51355f4a50064a351bde868117a7f6c Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 11:12:31 -0400 Subject: [PATCH 07/12] add taskforce key to production config section --- fabfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 857c87c6..7aaf0505 100644 --- a/fabfile.py +++ b/fabfile.py @@ -30,7 +30,8 @@ "TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "APPLICATION_ROOT", - "REDISTOGO_URL" + "REDISTOGO_URL", + "TASKFORCE_KEY", ] @task From 98b7f6d664ed435c98224e280ed28147ed9c5a81 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 13:23:35 -0400 Subject: [PATCH 08/12] remove twilio_dev env variables since the app still depends on loading vanilla twilio credentials (look at the configuration in top of app.py) --- config.py | 8 ++------ fabfile.py | 10 +++++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 742d97cb..1c12f968 100755 --- a/config.py +++ b/config.py @@ -4,7 +4,6 @@ import twilio.rest - class Config(object): DEBUG = True @@ -13,8 +12,8 @@ class Config(object): APPLICATION_ROOT = 'http://1cf55a5a.ngrok.com' TW_CLIENT = twilio.rest.TwilioRestClient( - os.environ.get('TWILIO_DEV_ACCOUNT_SID'), - os.environ.get('TWILIO_DEV_AUTH_TOKEN')) + os.environ.get('TWILIO_ACCOUNT_SID'), + os.environ.get('TWILIO_AUTH_TOKEN')) TASKFORCE_KEY = os.environ.get('TASKFORCE_KEY') SUNLIGHTLABS_KEY = os.environ.get('SUNLIGHTLABS_KEY') @@ -43,9 +42,6 @@ class ConfigProduction(Config): APPLICATION_ROOT = os.environ.get('APPLICATION_ROOT') - TW_CLIENT = twilio.rest.TwilioRestClient( - os.environ.get('TWILIO_ACCOUNT_SID'), - os.environ.get('TWILIO_AUTH_TOKEN')) SECRET_KEY = os.environ.get('SECRET_KEY') diff --git a/fabfile.py b/fabfile.py index 7aaf0505..fb4717d8 100644 --- a/fabfile.py +++ b/fabfile.py @@ -58,9 +58,16 @@ def run(env=None): print("[Warning] No environment specified.") env = DEVELOPMENT_ENV + env = str(env) + + if env != DEVELOPMENT_ENV and env != PRODUCTION_ENV: + print("[Warning] Invalid environment specified, use {} or {}.".format( + DEVELOPMENT_ENV, PRODUCTION_ENV)) + env = DEVELOPMENT_ENV + create_env() print("[Info] Running {} environment.".format(env)) - if env is DEVELOPMENT_ENV: + if env == DEVELOPMENT_ENV: virtualenv("python app.py", env) else: virtualenv("foreman start", env) @@ -137,6 +144,7 @@ def generate_env_command_prefix(env=None): continue export_str += "{}={} ".format(variable.upper(), env_variables[variable]) + export_str += "ENV={} ".format(env) return export_str def parse_config(section): From a5db87edc3e4bd38e3d9ca45329d8022597e5d71 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 13:54:53 -0400 Subject: [PATCH 09/12] colorize warnings --- fabfile.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fabfile.py b/fabfile.py index fb4717d8..5dd813c7 100644 --- a/fabfile.py +++ b/fabfile.py @@ -9,6 +9,7 @@ import ConfigParser as configparser from fabric.api import local, task, lcd +from fabric.colors import red FILE_ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -55,19 +56,21 @@ def run(env=None): Runs the application in a specified environment """ if env is None: - print("[Warning] No environment specified.") + print(red("[Warning] ") + "No environment specified.") env = DEVELOPMENT_ENV env = str(env) - if env != DEVELOPMENT_ENV and env != PRODUCTION_ENV: - print("[Warning] Invalid environment specified, use {} or {}.".format( - DEVELOPMENT_ENV, PRODUCTION_ENV)) + tag = red("[Warning] ") + environments = "{} or {}".format(DEVELOPMENT_ENV, PRODUCTION_ENV) + print(tag + "Invalid environment specified, use " + environments) env = DEVELOPMENT_ENV create_env() print("[Info] Running {} environment.".format(env)) if env == DEVELOPMENT_ENV: + print(red("[Warning] ") + "ngrok not started automatically, you " + \ + "need to start it manually") virtualenv("python app.py", env) else: virtualenv("foreman start", env) @@ -78,7 +81,8 @@ def generate_example_config(): Generates a default config file at CONFIG_FILE_PATH """ if os.path.isfile(CONFIG_FILE_PATH): - print("[Warning] Config file already exists, not generating a new one.") + tag = red("[Warning] ") + print(tag + "Config file already exists, not generating a new one.") return print("Generating config file at {}".format(CONFIG_FILE_PATH)) @@ -115,7 +119,6 @@ def clean(): """ local("rm -rf *.pyc") - def virtualenv(command, env=None): """ Runs a command in the virtual environment @@ -140,7 +143,8 @@ def generate_env_command_prefix(env=None): env_variables = parse_config(env) for variable in env_variables: if not env_variables[variable]: - print("[WARNING] {} not set in your config file.".format(variable)) + tag = red("[Warning] ") + print("{}{} not set in your config file.".format(tag, variable)) continue export_str += "{}={} ".format(variable.upper(), env_variables[variable]) From 8084064c569b2bec307f2075a30f59aa9f0a6c83 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 14:03:47 -0400 Subject: [PATCH 10/12] make README non-executable --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 README.md diff --git a/README.md b/README.md old mode 100755 new mode 100644 From cea32e7505188151cb1d17ca2b95aa60a3c00a42 Mon Sep 17 00:00:00 2001 From: Jack Wink Date: Sat, 20 Sep 2014 14:07:59 -0400 Subject: [PATCH 11/12] remove old directory, it already exists in version control --- old/test_server.py | 239 --------------------------------------------- 1 file changed, 239 deletions(-) delete mode 100644 old/test_server.py diff --git a/old/test_server.py b/old/test_server.py deleted file mode 100644 index edd7c27a..00000000 --- a/old/test_server.py +++ /dev/null @@ -1,239 +0,0 @@ -import app as server -from utils import load_data, locate_member_ids, SQLAlchemy -import models -import unittest -from urlparse import parse_qs -from requests.compat import urlencode, urlparse -import lxml - -def url_for(action, **params): - url = ('/' if action[0] != '/' else '') + action - if params: - url += '?' + urlencode(params, doseq=True) - return url - -# hack for testing -server.url_for = url_for - -id_pelosi = 'P000197' -id_boxer = 'B000711' -id_cardenas = 'C001097' - - -class FlaskrTestCase(unittest.TestCase): - - def setUp(self): - server.app.config.from_object('config.ConfigTesting') - self.db = SQLAlchemy() - self.db.app = server.app - models.setUp(server.app) - self.app = server.app.test_client() - self.campaigns, self.legislators, self.districts = load_data() - - # a default example where just Rep. Nacy Pelosi is called - self.example_params = dict( - campaignId='default', - repIds=[id_pelosi], - userPhone='415-000-1111', - zipcode='95110') - - def post_tree(self, path, **params): - req = self.app.post(url_for(path, **params)) - tree = lxml.etree.fromstring(req.data) - return tree - - - def parse_url(self, urlstring): - url = urlparse(urlstring) - return url, dict((k, v if len(v)>1 else v[0]) - for k, v in parse_qs(url.query).iteritems()) - - def assert_is_xml(self, req): - assert(req.data.startswith(' Date: Sat, 20 Sep 2014 14:15:07 -0400 Subject: [PATCH 12/12] add 'fab test' to fabfile --- docs/running_with_fabric.md | 1 + fabfile.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/running_with_fabric.md b/docs/running_with_fabric.md index c260c635..b1f5f1ae 100644 --- a/docs/running_with_fabric.md +++ b/docs/running_with_fabric.md @@ -27,3 +27,4 @@ By default, the app runs in development. - `fab clean` removes all .pyc files from the project - `fab update_data` will download the updated data from the Sunlight Congress API +- `fab test` will execute `nosetests` in the virtual environment diff --git a/fabfile.py b/fabfile.py index 5dd813c7..b2ccbc75 100644 --- a/fabfile.py +++ b/fabfile.py @@ -102,6 +102,13 @@ def __set_section(config, section, variables): with open(CONFIG_FILE_PATH, 'wb') as config_file: config.write(config_file) +@task +def test(): + """ + Runs the test suite locally + """ + virtualenv("nosetests", env="development") + @task def update_data():