refactored to put state in treasury_config.json, updated README and s…

…hell script accordingly

README.md

@@ -188,4 +188,28 @@ Example: Retrieve all accounts with balances greater than 50M from the network d
 
 ```sh
 python -m network.richlist_symbol --resources templates/symbol.mainnet.yaml --min-balance 50000000 --output 50M.csv
-````
+```
+
+## treasury
+
+### serve
+
+_runs a self-contained webapp for monitoring account balances, scraping price data, and visualizing future account values in aggregate_
+
+Default location for storage is `treasury/data` but any alternative location can be provided. Configuration is located at `treasury/treasury_config.json`. **All config fields other than cm_key are required.** 
+
+Price data download is relatively lightweight for default configuration. If no data is present, the app will attempt to collect prices for all assets defined in configuration on first load. Price data is cached to disk as collected, so data for any particular asset/date combination should be download no more than once. 
+
+Server can be invoked with defaults via:
+
+```sh
+treasury/serve.sh
+```
+
+or manually with:
+
+```sh
+python -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0'
+```
+
+With default settings the app will listen for requests at [http:\\\\localhost:8080](http:\\\\localhost:8080)

treasury/data/accounts.csv

@@ -0,0 +1,6 @@
+Asset,Name,Address
+XYM,XYM Treasury,NCHEST3QRQS4JZGOO64TH7NFJ2A63YA7TPM5PXI
+XYM,XYM Sink (Mosaic),NAVORTEX3IPBAUWQBBI3I3BDIOS4AVHPZLCFC7Y
+XYM,XYM Sink (Fees),NCVORTEX4XD5IQASZQEHDWUXT33XBOTBMKFDCLI
+XEM,XEM Treasury,NCHESTYVD2P6P646AMY7WSNG73PCPZDUQNSD6JAK
+XEM,XEM Rewards,NCPAYOUTH2BGEGT3Q7K75PV27QKMVNN2IZRVZWMD

treasury/serve.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python3 -m treasury.app --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0'
+python3 -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0'

treasury/treasury/app.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 
 import dash
 import dash_bootstrap_components as dbc
@@ -8,34 +9,49 @@
 
 from treasury.callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances,
                                 get_update_prices, update_forecast_chart, update_price_chart, update_summary)
-from treasury.data import get_gecko_spot, lookup_balance
+from treasury.data import get_gecko_spot, get_gecko_prices, lookup_balance
 
-# defaults for startup
-FORECAST_PERIODS = 90
-NUM_SIMS = 1000
-REF_TICKER = 'XEM'
 THEME = dbc.themes.VAPOR
 TITLE = 'Symbol Treasury Analysis Tool v1.0'
 
 
-def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_date, auto_update_delay_seconds=600):
+def get_app(price_data_loc, account_data_loc, config, serve, base_path, start_date, end_date, auto_update_delay_seconds=600):
 
     app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME])
     app.title = TITLE
 
     # preprocess data for fast load
-    prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True)
+    try:
+        prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True)
+    except FileNotFoundError:
+        print('No price data found, pulling fresh data for assets in config (this may take a while)')
+        if len(config['assets']) > 0:
+            prices = []
+            for asset in config['assets']:
+                prices.append(get_gecko_prices(
+                    asset,
+                    start_date,
+                    end_date,
+                    config['max_api_tries'],
+                    config['retry_delay_seconds']))
+            prices = pd.concat(prices, axis=1).sort_index(axis=0).sort_index(axis=1)
+            print(f'Prices acquired successfully; writing to {price_data_loc}')
+            prices.to_csv(price_data_loc)
+        else:
+            print('No assets found in config; aborting!')
+            raise
+
     lookback_prices = prices.loc[start_date:end_date]
 
     accounts = pd.read_csv(account_data_loc, header=0, index_col=None)
-    accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()]
+    accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, config['api_hosts'])) for row in accounts.itertuples()]
     asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict()
 
     summary_df = pd.DataFrame.from_records({
         'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'],
         'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'],
-        'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'],
-        'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']})
+        'Reference Trend (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().mean():.3%}'],
+        'Reference Vol (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().std():.3%}']})
 
     app.layout = dbc.Container([
         dbc.Row([html.H1(TITLE)], justify='center'),
@@ -51,15 +67,25 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_
                 dbc.InputGroup(
                     [
                         dbc.InputGroupText('Reference Asset:'),
-                        dbc.Select(id='ref-ticker', options=[{'label': ticker, 'value': ticker} for ticker in prices], value='XYM')
+                        dbc.Select(
+                            id='ref-ticker',
+                            options=[{'label': ticker, 'value': ticker} for ticker in prices],
+                            value=config['default_ref_ticker'])
                     ],
                     className='mb-3',
                 ),
                 dbc.FormText('Choose how many days into the future you wish to forecast.'),
                 dbc.InputGroup(
                     [
                         dbc.InputGroupText('Forecast Days:'),
-                        dbc.Input(id='forecast-days', value=FORECAST_PERIODS, type='number', min=1, max=1000, step=1, debounce=True)
+                        dbc.Input(
+                            id='forecast-days',
+                            value=config['default_forecast_periods'],
+                            type='number',
+                            min=1,
+                            max=1000,
+                            step=1,
+                            debounce=True)
                     ],
                     className='mb-3',
                 ),
@@ -69,7 +95,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_
                 dbc.InputGroup(
                     [
                         dbc.InputGroupText('Number of Simulations:'),
-                        dbc.Input(id='num-sims', value=NUM_SIMS, type='number', min=1, step=1, debounce=True)
+                        dbc.Input(id='num-sims', value=config['default_num_sims'], type='number', min=1, step=1, debounce=True)
                     ],
                     className='mb-3',
                 ),
@@ -201,15 +227,16 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_
     app.callback(
         Output('address-table', 'children'),
         Output('asset-values', 'data'),
-        Input('auto-update-trigger', 'n_intervals'))(get_update_balances(account_data_loc))
+        Input('auto-update-trigger', 'n_intervals'))(
+            get_update_balances(account_data_loc, config['api_hosts'],  config['explorer_url_map']))
 
     app.callback(
         Output('ref-prices', 'data'),
         Output('lookback-prices', 'data'),
         Input('start-date', 'value'),
         Input('end-date', 'value'),
         State('ref-prices', 'data'),
-        State('lookback-prices', 'data'))(get_update_prices(price_data_loc))
+        State('lookback-prices', 'data'))(get_update_prices(price_data_loc, config['max_api_tries'], config['retry_delay_seconds']))
 
     app.callback(
         Output('forecast-graph', 'figure'),
@@ -236,7 +263,8 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_
 
 
 def main():
-    parser = argparse.ArgumentParser(description='webapp that processes data files and renders fork information')
+    parser = argparse.ArgumentParser(description='webapp that monitors treasury balances and crypto asset prices')
+    parser.add_argument('--config', '-c', help='configuration file location', default='../treasury_config.json')
     parser.add_argument('--host', help='host ip, defaults to localhost', default='127.0.0.1')
     parser.add_argument('--port', type=int, help='port for webserver', default=8080)
     parser.add_argument('--proxy', help='proxy spec of the form ip:port::gateway to render urls', default=None)
@@ -251,7 +279,15 @@ def main():
     if args.end_date is None:
         args.end_date = (pd.to_datetime('today')-pd.Timedelta(1, unit='D')).strftime('%Y-%m-%d')
 
-    app = get_app(args.price_data_loc, args.account_data_loc, args.serve, args.base_path, args.start_date, args.end_date)
+    try:
+        with open(args.config) as config_file:
+            args.config = json.load(config_file)
+    except FileNotFoundError:
+        print(f'No configuration file found at {args.config}')
+        print('Configuration is required to run the app!')
+        raise
+
+    app = get_app(args.price_data_loc, args.account_data_loc, args.config, args.serve, args.base_path, args.start_date, args.end_date)
     app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True)
 
 

treasury/treasury/callbacks.py

@@ -6,11 +6,6 @@
 from treasury.data import get_gecko_prices, get_gecko_spot, lookup_balance
 from treasury.models import get_mean_variance_forecasts
 
-EXPLORER_URL_MAP = {
-    'XYM': 'https://symbol.fyi/accounts/',
-    'XEM': 'https://explorer.nemtool.com/#/s_account?account='
-}
-
 
 def download_full_prices(_, full_prices):
     """Callback to feed the full price simulation download feature"""
@@ -48,23 +43,23 @@ def update_summary(lookback_prices, ref_ticker):
     return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')]
 
 
-def get_update_balances(account_data_loc):
+def get_update_balances(account_data_loc, api_hosts, explorer_url_map):
     """Wrapper to inject location dependency into account balance callback"""
 
     def update_balances(_):
         accounts = pd.read_csv(account_data_loc, header=0, index_col=None)
-        accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()]
+        accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, api_hosts)) for row in accounts.itertuples()]
         asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict()
         updated_addresses = []
         for _, row in accounts.iterrows():
-            updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{EXPLORER_URL_MAP[row.Asset]}{row.Address}'))
+            updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{explorer_url_map[row.Asset]}{row.Address}'))
         accounts['Address'] = updated_addresses
         return [dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark')], asset_values
 
     return update_balances
 
 
-def get_update_prices(price_data_loc):
+def get_update_prices(price_data_loc, max_api_tries, retry_delay_seconds):
     """Wrapper to inject location dependency into price data callback"""
 
     def update_prices(
@@ -83,13 +78,23 @@ def update_prices(
         if start_date < ref_prices.index[0]:
             new_prices = []
             for asset in ref_prices.columns:
-                new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1)))
+                new_prices.append(get_gecko_prices(
+                    asset,
+                    start_date,
+                    ref_prices.index[0]-pd.Timedelta(days=1),
+                    max_api_tries,
+                    retry_delay_seconds))
             new_prices = pd.concat(new_prices, axis=1)
             ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates()
         if end_date > ref_prices.index[-1]:
             new_prices = []
             for asset in ref_prices.columns:
-                new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date))
+                new_prices.append(get_gecko_prices(
+                    asset,
+                    ref_prices.index[-1]+pd.Timedelta(days=1),
+                    end_date,
+                    max_api_tries,
+                    retry_delay_seconds))
             new_prices = pd.concat(new_prices, axis=1)
             ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates()
 

treasury/treasury/data.py

@@ -1,15 +1,10 @@
 import datetime
-import os
 import time
 
 import pandas as pd
 import requests
 from tqdm import tqdm
 
-XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp')
-XEM_API_HOST = os.getenv('XEM_API_HOST', 'bigalice3.nem.ninja')
-CM_KEY = os.getenv('CM_KEY', '')
-
 GECKO_TICKER_MAP = {
    'ADA': 'cardano',
    'AVAX': 'avalanche-2',
@@ -26,24 +21,20 @@
 }
 
 XYM_MOSAIC_ID = '6BED913FA20223F8'
-MAX_TRIES = 6
-RETRY_S = 15
 
 
-def lookup_balance(address, asset):
+def lookup_balance(address, asset, api_hosts):
     asset = asset.lower()
     if asset in ['symbol', 'xym']:
-        return lookup_xym_balance(address)
+        return lookup_xym_balance(address, api_hosts['XYM'])
     if asset in ['nem', 'xem']:
-        return lookup_xem_balance(address)
-    else:
-        raise ValueError(f'Asset not supported: {asset}')
-
+        return lookup_xem_balance(address, api_hosts['XEM'])
+    raise ValueError(f'Asset not supported for balance lookup: {asset}')
 
-def lookup_xym_balance(address):
 
+def lookup_xym_balance(address, xym_api_host):
     balance = 0
-    json_account = requests.get('https://' + XYM_API_HOST + ':3001/accounts/' + address).json()
+    json_account = requests.get('https://' + xym_api_host + ':3001/accounts/' + address).json()
     json_mosaics = json_account['account']['mosaics']
     for json_mosaic in json_mosaics:
         if XYM_MOSAIC_ID == json_mosaic['id']:
@@ -53,25 +44,25 @@ def lookup_xym_balance(address):
     return balance
 
 
-def lookup_xem_balance(address):
-    response = requests.get('http://' + XEM_API_HOST + ':7890/account/get?address=' + address).json()
+def lookup_xem_balance(address, xem_api_host):
+    response = requests.get('http://' + xem_api_host + ':7890/account/get?address=' + address).json()
     balance = float(response['account']['balance']) / 1000000
     return balance
 
 
-def get_cm_prices(ticker, date_time):
+def get_cm_prices(ticker, date_time, cm_key):
 
     response = requests.get(
         'https://api.coinmetrics.io/v4/timeseries/asset-metrics?assets=' +
         ticker +
         '&start_time=' +
         date_time +
-        '&limit_per_asset=1&metrics=PriceUSD&api_key=' + CM_KEY).json()
+        '&limit_per_asset=1&metrics=PriceUSD&api_key=' + cm_key).json()
     ref_rate = response['data'][0]['PriceUSD']
     return ref_rate
 
 
-def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, frequency='1d'):
+def get_cm_metrics(assets, metrics, cm_key, start_time='2016-01-01', end_time=None, frequency='1d'):
     if end_time is None:
         end_time = datetime.date.today()
     data = []
@@ -82,7 +73,7 @@ def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, freq
         f'end_time={end_time}&' +
         f'metrics={",".join(metrics)}&' +
         f'frequency={frequency}&' +
-        f'pretty=true&api_key={CM_KEY}')
+        f'pretty=true&api_key={cm_key}')
     while True:
         response = requests.get(url).json()
         data.extend(response['data'])
@@ -97,23 +88,23 @@ def fix_ticker(ticker):
     return GECKO_TICKER_MAP.get(ticker, ticker)
 
 
-def get_gecko_spot(ticker, currency='usd'):
+def get_gecko_spot(ticker, max_api_tries=6, retry_delay_seconds=15, currency='usd'):
     ticker = fix_ticker(ticker)
     tries = 1
-    while tries <= MAX_TRIES:
+    while tries <= max_api_tries:
         try:
             response = requests.get('https://api.coingecko.com/api/v3/simple/price?ids=' + ticker + '&vs_currencies=' + currency).json()
             return response[ticker][currency]
         except (KeyError, requests.exceptions.RequestException) as _:
-            time.sleep(RETRY_S)
+            time.sleep(retry_delay_seconds)
             tries += 1
     return None
 
 
-def get_gecko_price(ticker, date, currency='usd'):
+def get_gecko_price(ticker, date, max_api_tries=6, retry_delay_seconds=15, currency='usd'):
     ticker = fix_ticker(ticker)
     tries = 1
-    while tries <= MAX_TRIES:
+    while tries <= max_api_tries:
         try:
             response = requests.get(
                 'https://api.coingecko.com/api/v3/coins/' +
@@ -125,15 +116,15 @@ def get_gecko_price(ticker, date, currency='usd'):
                 return None
             return response['market_data']['current_price'][currency]
         except (KeyError, requests.exceptions.RequestException) as _:
-            print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {RETRY_S}s')
-            time.sleep(RETRY_S)
+            print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {retry_delay_seconds}s')
+            time.sleep(retry_delay_seconds)
             tries += 1
     return None
 
 
-def get_gecko_prices(ticker, start_date, end_date, currency='usd'):
+def get_gecko_prices(ticker, start_date, end_date, max_api_tries=6, retry_delay_seconds=15, currency='usd'):
     dates = pd.date_range(start_date, end_date, freq='D')
     prices = {'date': dates, ticker: []}
     for date_time in tqdm(dates):
-        prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), currency))
+        prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), max_api_tries, retry_delay_seconds, currency))
     return pd.DataFrame.from_records(prices).set_index('date')