diff --git a/README.md b/README.md index c17e327..c3b2583 100644 --- a/README.md +++ b/README.md @@ -188,4 +188,28 @@ Example: Retrieve all accounts with balances greater than 50M from the network d ```sh python -m network.richlist_symbol --resources templates/symbol.mainnet.yaml --min-balance 50000000 --output 50M.csv -```` +``` + +## treasury + +### serve + +_runs a self-contained webapp for monitoring account balances, scraping price data, and visualizing future account values in aggregate_ + +Default location for storage is `treasury/data` but any alternative location can be provided. Configuration is located at `treasury/treasury_config.json`. **All config fields other than cm_key are required.** + +Price data download is relatively lightweight for default configuration. If no data is present, the app will attempt to collect prices for all assets defined in configuration on first load. Price data is cached to disk as collected, so data for any particular asset/date combination should be download no more than once. + +Server can be invoked with defaults via: + +```sh +treasury/serve.sh +``` + +or manually with: + +```sh +python -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' +``` + +With default settings the app will listen for requests at [http:\\\\localhost:8080](http:\\\\localhost:8080) diff --git a/treasury/data/accounts.csv b/treasury/data/accounts.csv new file mode 100644 index 0000000..57ed85d --- /dev/null +++ b/treasury/data/accounts.csv @@ -0,0 +1,6 @@ +Asset,Name,Address +XYM,XYM Treasury,NCHEST3QRQS4JZGOO64TH7NFJ2A63YA7TPM5PXI +XYM,XYM Sink (Mosaic),NAVORTEX3IPBAUWQBBI3I3BDIOS4AVHPZLCFC7Y +XYM,XYM Sink (Fees),NCVORTEX4XD5IQASZQEHDWUXT33XBOTBMKFDCLI +XEM,XEM Treasury,NCHESTYVD2P6P646AMY7WSNG73PCPZDUQNSD6JAK +XEM,XEM Rewards,NCPAYOUTH2BGEGT3Q7K75PV27QKMVNN2IZRVZWMD \ No newline at end of file diff --git a/treasury/serve.sh b/treasury/serve.sh index c2af419..b695de6 100755 --- a/treasury/serve.sh +++ b/treasury/serve.sh @@ -1,3 +1,3 @@ #!/bin/bash -python3 -m treasury.app --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' +python3 -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index 907a4a5..de9f02b 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -1,4 +1,5 @@ import argparse +import json import dash import dash_bootstrap_components as dbc @@ -8,34 +9,49 @@ from treasury.callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances, get_update_prices, update_forecast_chart, update_price_chart, update_summary) -from treasury.data import get_gecko_spot, lookup_balance +from treasury.data import get_gecko_spot, get_gecko_prices, lookup_balance -# defaults for startup -FORECAST_PERIODS = 90 -NUM_SIMS = 1000 -REF_TICKER = 'XEM' THEME = dbc.themes.VAPOR TITLE = 'Symbol Treasury Analysis Tool v1.0' -def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_date, auto_update_delay_seconds=600): +def get_app(price_data_loc, account_data_loc, config, serve, base_path, start_date, end_date, auto_update_delay_seconds=600): app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME]) app.title = TITLE # preprocess data for fast load - prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) + try: + prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) + except FileNotFoundError: + print('No price data found, pulling fresh data for assets in config (this may take a while)') + if len(config['assets']) > 0: + prices = [] + for asset in config['assets']: + prices.append(get_gecko_prices( + asset, + start_date, + end_date, + config['max_api_tries'], + config['retry_delay_seconds'])) + prices = pd.concat(prices, axis=1).sort_index(axis=0).sort_index(axis=1) + print(f'Prices acquired successfully; writing to {price_data_loc}') + prices.to_csv(price_data_loc) + else: + print('No assets found in config; aborting!') + raise + lookback_prices = prices.loc[start_date:end_date] accounts = pd.read_csv(account_data_loc, header=0, index_col=None) - accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, config['api_hosts'])) for row in accounts.itertuples()] asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() summary_df = pd.DataFrame.from_records({ 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], - 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], - 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) + 'Reference Trend (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().std():.3%}']}) app.layout = dbc.Container([ dbc.Row([html.H1(TITLE)], justify='center'), @@ -51,7 +67,10 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Reference Asset:'), - dbc.Select(id='ref-ticker', options=[{'label': ticker, 'value': ticker} for ticker in prices], value='XYM') + dbc.Select( + id='ref-ticker', + options=[{'label': ticker, 'value': ticker} for ticker in prices], + value=config['default_ref_ticker']) ], className='mb-3', ), @@ -59,7 +78,14 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Forecast Days:'), - dbc.Input(id='forecast-days', value=FORECAST_PERIODS, type='number', min=1, max=1000, step=1, debounce=True) + dbc.Input( + id='forecast-days', + value=config['default_forecast_periods'], + type='number', + min=1, + max=1000, + step=1, + debounce=True) ], className='mb-3', ), @@ -69,7 +95,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Number of Simulations:'), - dbc.Input(id='num-sims', value=NUM_SIMS, type='number', min=1, step=1, debounce=True) + dbc.Input(id='num-sims', value=config['default_num_sims'], type='number', min=1, step=1, debounce=True) ], className='mb-3', ), @@ -201,7 +227,8 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ app.callback( Output('address-table', 'children'), Output('asset-values', 'data'), - Input('auto-update-trigger', 'n_intervals'))(get_update_balances(account_data_loc)) + Input('auto-update-trigger', 'n_intervals'))( + get_update_balances(account_data_loc, config['api_hosts'], config['explorer_url_map'])) app.callback( Output('ref-prices', 'data'), @@ -209,7 +236,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ Input('start-date', 'value'), Input('end-date', 'value'), State('ref-prices', 'data'), - State('lookback-prices', 'data'))(get_update_prices(price_data_loc)) + State('lookback-prices', 'data'))(get_update_prices(price_data_loc, config['max_api_tries'], config['retry_delay_seconds'])) app.callback( Output('forecast-graph', 'figure'), @@ -236,7 +263,8 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ def main(): - parser = argparse.ArgumentParser(description='webapp that processes data files and renders fork information') + parser = argparse.ArgumentParser(description='webapp that monitors treasury balances and crypto asset prices') + parser.add_argument('--config', '-c', help='configuration file location', default='../treasury_config.json') parser.add_argument('--host', help='host ip, defaults to localhost', default='127.0.0.1') parser.add_argument('--port', type=int, help='port for webserver', default=8080) parser.add_argument('--proxy', help='proxy spec of the form ip:port::gateway to render urls', default=None) @@ -251,7 +279,15 @@ def main(): if args.end_date is None: args.end_date = (pd.to_datetime('today')-pd.Timedelta(1, unit='D')).strftime('%Y-%m-%d') - app = get_app(args.price_data_loc, args.account_data_loc, args.serve, args.base_path, args.start_date, args.end_date) + try: + with open(args.config) as config_file: + args.config = json.load(config_file) + except FileNotFoundError: + print(f'No configuration file found at {args.config}') + print('Configuration is required to run the app!') + raise + + app = get_app(args.price_data_loc, args.account_data_loc, args.config, args.serve, args.base_path, args.start_date, args.end_date) app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True) diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index 4de9d02..3c1489b 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -6,11 +6,6 @@ from treasury.data import get_gecko_prices, get_gecko_spot, lookup_balance from treasury.models import get_mean_variance_forecasts -EXPLORER_URL_MAP = { - 'XYM': 'https://symbol.fyi/accounts/', - 'XEM': 'https://explorer.nemtool.com/#/s_account?account=' -} - def download_full_prices(_, full_prices): """Callback to feed the full price simulation download feature""" @@ -48,23 +43,23 @@ def update_summary(lookback_prices, ref_ticker): return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] -def get_update_balances(account_data_loc): +def get_update_balances(account_data_loc, api_hosts, explorer_url_map): """Wrapper to inject location dependency into account balance callback""" def update_balances(_): accounts = pd.read_csv(account_data_loc, header=0, index_col=None) - accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, api_hosts)) for row in accounts.itertuples()] asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() updated_addresses = [] for _, row in accounts.iterrows(): - updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{EXPLORER_URL_MAP[row.Asset]}{row.Address}')) + updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{explorer_url_map[row.Asset]}{row.Address}')) accounts['Address'] = updated_addresses return [dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark')], asset_values return update_balances -def get_update_prices(price_data_loc): +def get_update_prices(price_data_loc, max_api_tries, retry_delay_seconds): """Wrapper to inject location dependency into price data callback""" def update_prices( @@ -83,13 +78,23 @@ def update_prices( if start_date < ref_prices.index[0]: new_prices = [] for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) + new_prices.append(get_gecko_prices( + asset, + start_date, + ref_prices.index[0]-pd.Timedelta(days=1), + max_api_tries, + retry_delay_seconds)) new_prices = pd.concat(new_prices, axis=1) ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() if end_date > ref_prices.index[-1]: new_prices = [] for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) + new_prices.append(get_gecko_prices( + asset, + ref_prices.index[-1]+pd.Timedelta(days=1), + end_date, + max_api_tries, + retry_delay_seconds)) new_prices = pd.concat(new_prices, axis=1) ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py index ab1155e..18a2dbd 100644 --- a/treasury/treasury/data.py +++ b/treasury/treasury/data.py @@ -1,15 +1,10 @@ import datetime -import os import time import pandas as pd import requests from tqdm import tqdm -XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp') -XEM_API_HOST = os.getenv('XEM_API_HOST', 'bigalice3.nem.ninja') -CM_KEY = os.getenv('CM_KEY', '') - GECKO_TICKER_MAP = { 'ADA': 'cardano', 'AVAX': 'avalanche-2', @@ -26,24 +21,20 @@ } XYM_MOSAIC_ID = '6BED913FA20223F8' -MAX_TRIES = 6 -RETRY_S = 15 -def lookup_balance(address, asset): +def lookup_balance(address, asset, api_hosts): asset = asset.lower() if asset in ['symbol', 'xym']: - return lookup_xym_balance(address) + return lookup_xym_balance(address, api_hosts['XYM']) if asset in ['nem', 'xem']: - return lookup_xem_balance(address) - else: - raise ValueError(f'Asset not supported: {asset}') - + return lookup_xem_balance(address, api_hosts['XEM']) + raise ValueError(f'Asset not supported for balance lookup: {asset}') -def lookup_xym_balance(address): +def lookup_xym_balance(address, xym_api_host): balance = 0 - json_account = requests.get('https://' + XYM_API_HOST + ':3001/accounts/' + address).json() + json_account = requests.get('https://' + xym_api_host + ':3001/accounts/' + address).json() json_mosaics = json_account['account']['mosaics'] for json_mosaic in json_mosaics: if XYM_MOSAIC_ID == json_mosaic['id']: @@ -53,25 +44,25 @@ def lookup_xym_balance(address): return balance -def lookup_xem_balance(address): - response = requests.get('http://' + XEM_API_HOST + ':7890/account/get?address=' + address).json() +def lookup_xem_balance(address, xem_api_host): + response = requests.get('http://' + xem_api_host + ':7890/account/get?address=' + address).json() balance = float(response['account']['balance']) / 1000000 return balance -def get_cm_prices(ticker, date_time): +def get_cm_prices(ticker, date_time, cm_key): response = requests.get( 'https://api.coinmetrics.io/v4/timeseries/asset-metrics?assets=' + ticker + '&start_time=' + date_time + - '&limit_per_asset=1&metrics=PriceUSD&api_key=' + CM_KEY).json() + '&limit_per_asset=1&metrics=PriceUSD&api_key=' + cm_key).json() ref_rate = response['data'][0]['PriceUSD'] return ref_rate -def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, frequency='1d'): +def get_cm_metrics(assets, metrics, cm_key, start_time='2016-01-01', end_time=None, frequency='1d'): if end_time is None: end_time = datetime.date.today() data = [] @@ -82,7 +73,7 @@ def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, freq f'end_time={end_time}&' + f'metrics={",".join(metrics)}&' + f'frequency={frequency}&' + - f'pretty=true&api_key={CM_KEY}') + f'pretty=true&api_key={cm_key}') while True: response = requests.get(url).json() data.extend(response['data']) @@ -97,23 +88,23 @@ def fix_ticker(ticker): return GECKO_TICKER_MAP.get(ticker, ticker) -def get_gecko_spot(ticker, currency='usd'): +def get_gecko_spot(ticker, max_api_tries=6, retry_delay_seconds=15, currency='usd'): ticker = fix_ticker(ticker) tries = 1 - while tries <= MAX_TRIES: + while tries <= max_api_tries: try: response = requests.get('https://api.coingecko.com/api/v3/simple/price?ids=' + ticker + '&vs_currencies=' + currency).json() return response[ticker][currency] except (KeyError, requests.exceptions.RequestException) as _: - time.sleep(RETRY_S) + time.sleep(retry_delay_seconds) tries += 1 return None -def get_gecko_price(ticker, date, currency='usd'): +def get_gecko_price(ticker, date, max_api_tries=6, retry_delay_seconds=15, currency='usd'): ticker = fix_ticker(ticker) tries = 1 - while tries <= MAX_TRIES: + while tries <= max_api_tries: try: response = requests.get( 'https://api.coingecko.com/api/v3/coins/' + @@ -125,15 +116,15 @@ def get_gecko_price(ticker, date, currency='usd'): return None return response['market_data']['current_price'][currency] except (KeyError, requests.exceptions.RequestException) as _: - print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {RETRY_S}s') - time.sleep(RETRY_S) + print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {retry_delay_seconds}s') + time.sleep(retry_delay_seconds) tries += 1 return None -def get_gecko_prices(ticker, start_date, end_date, currency='usd'): +def get_gecko_prices(ticker, start_date, end_date, max_api_tries=6, retry_delay_seconds=15, currency='usd'): dates = pd.date_range(start_date, end_date, freq='D') prices = {'date': dates, ticker: []} for date_time in tqdm(dates): - prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), currency)) + prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), max_api_tries, retry_delay_seconds, currency)) return pd.DataFrame.from_records(prices).set_index('date') diff --git a/treasury/treasury_config.json b/treasury/treasury_config.json new file mode 100644 index 0000000..e049d88 --- /dev/null +++ b/treasury/treasury_config.json @@ -0,0 +1,30 @@ +{ + "assets": [ + "ADA", + "AVAX", + "BTC", + "DOT", + "ETH", + "LTC", + "MATIC", + "SOL", + "TRX", + "XEM", + "XMR", + "XYM" + ], + "api_hosts" : { + "XYM" : "wolf.importance.jp", + "XEM" : "bigalice3.nem.ninja" + }, + "cm_key" : "", + "default_forecast_periods" : 90, + "default_num_sims" : 1000, + "default_ref_ticker" : "XEM", + "explorer_url_map" : { + "XYM": "https://symbol.fyi/accounts/", + "XEM": "https://explorer.nemtool.com/#/s_account?account=" + }, + "max_api_tries" : 6, + "retry_delay_seconds" : 15 +} \ No newline at end of file