From 7fe2e5c929ac48580d3c7c72e640777477100c02 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sun, 20 Aug 2017 17:34:04 -0400 Subject: [PATCH] Using files for exchange connection and symbol map --- catalyst/examples/buy_the_dip_live.py | 33 +++++++--------- catalyst/exchange/bitfinex.py | 4 +- catalyst/exchange/exchange.py | 14 +++---- catalyst/exchange/exchange_errors.py | 14 +++++++ catalyst/exchange/exchange_utils.py | 57 ++++++++++++++++++++++++++- catalyst/utils/run_algo.py | 21 +++++----- 6 files changed, 102 insertions(+), 41 deletions(-) diff --git a/catalyst/examples/buy_the_dip_live.py b/catalyst/examples/buy_the_dip_live.py index 5ecf87e0..30d81ea6 100644 --- a/catalyst/examples/buy_the_dip_live.py +++ b/catalyst/examples/buy_the_dip_live.py @@ -8,6 +8,8 @@ from catalyst.api import ( get_open_orders, ) from catalyst.errors import ZiplineError +from catalyst.exchange.exchange_utils import get_exchange_folder, \ + download_exchange_symbols, get_exchange_auth import talib algo_namespace = 'buy_the_dip_live' @@ -23,11 +25,12 @@ def initialize(context): context.PROFIT_TARGET = 0.1 context.SLIPPAGE_ALLOWED = 0.02 - context.retry_check_open_orders = 2 - context.retry_update_portfolio = 2 - context.retry_order = 2 + context.retry_check_open_orders = 10 + context.retry_update_portfolio = 10 + context.retry_order = 5 context.errors = [] + pass def _handle_data(context, data): @@ -54,6 +57,11 @@ def _handle_data(context, data): price = data.current(context.asset, 'price') log.info('got price {price}'.format(price=price)) + record( + price=price, + rsi=rsi, + ) + orders = get_open_orders(context.asset) if orders: log.info('skipping bar until all open orders execute') @@ -102,13 +110,6 @@ def _handle_data(context, data): limit_price=price * (1 + context.SLIPPAGE_ALLOWED) ) - record( - price=price, - cash=cash, - starting_cash=context.portfolio.starting_cash, - leverage=context.account.leverage, - ) - def handle_data(context, data): log.info('handling bar {}'.format(data.current_dt)) @@ -132,18 +133,12 @@ def analyze(context, stats): pass -exchange_conn = dict( - name='bitfinex', - key='', - secret=b'', - base_currency='usd' -) run_algorithm( initialize=initialize, handle_data=handle_data, analyze=analyze, - capital_base=100000, - exchange_conn=exchange_conn, + exchange_name='bitfinex', live=True, - algo_namespace=algo_namespace + algo_namespace=algo_namespace, + base_currency='usd' ) diff --git a/catalyst/exchange/bitfinex.py b/catalyst/exchange/bitfinex.py index 9a8aad48..e7812af4 100644 --- a/catalyst/exchange/bitfinex.py +++ b/catalyst/exchange/bitfinex.py @@ -21,6 +21,7 @@ from catalyst.finance.execution import (MarketOrder, StopOrder, StopLimitOrder) from catalyst.exchange.exchange_portfolio import ExchangePortfolio +from catalyst.exchange.exchange_utils import get_exchange_symbols from catalyst.errors import ( IncompatibleHistoryFrequency, ) @@ -34,7 +35,6 @@ from catalyst.exchange.exchange_errors import ( requests.adapters.DEFAULT_RETRIES = 20 BITFINEX_URL = 'https://api.bitfinex.com' -ASSETS = '{ "USDT_BTC": {"symbol":"btc_usd", "start_date": "2010-01-01"}, "ltcusd": {"symbol":"ltc_usd", "start_date": "2010-01-01"}, "ltcbtc": {"symbol":"ltc_btc", "start_date": "2010-01-01"}, "ethusd": {"symbol":"eth_usd", "start_date": "2010-01-01"}, "ethbtc": {"symbol":"eth_btc", "start_date": "2010-01-01"}, "etcbtc": {"symbol":"etc_btc", "start_date": "2010-01-01"}, "etcusd": {"symbol":"etc_usd", "start_date": "2010-01-01"}, "rrtusd": {"symbol":"rrt_usd", "start_date": "2010-01-01"}, "rrtbtc": {"symbol":"rrt_btc", "start_date": "2010-01-01"}, "zecusd": {"symbol":"zec_usd", "start_date": "2010-01-01"}, "zecbtc": {"symbol":"zec_btc", "start_date": "2010-01-01"}, "xmrusd": {"symbol":"xmr_usd", "start_date": "2010-01-01"}, "xmrbtc": {"symbol":"xmr_btc", "start_date": "2010-01-01"}, "dshusd": {"symbol":"dsh_usd", "start_date": "2010-01-01"}, "dshbtc": {"symbol":"dsh_btc", "start_date": "2010-01-01"}, "bccbtc": {"symbol":"bcc_btc", "start_date": "2010-01-01"}, "bcubtc": {"symbol":"bcu_btc", "start_date": "2010-01-01"}, "bccusd": {"symbol":"bcc_usd", "start_date": "2010-01-01"}, "bcuusd": {"symbol":"bcu_usd", "start_date": "2010-01-01"}, "xrpusd": {"symbol":"xrp_usd", "start_date": "2010-01-01"}, "xrpbtc": {"symbol":"xrp_btc", "start_date": "2010-01-01"}, "iotusd": {"symbol":"iot_usd", "start_date": "2010-01-01"}, "iotbtc": {"symbol":"iot_btc", "start_date": "2010-01-01"}, "ioteth": {"symbol":"iot_eth", "start_date": "2010-01-01"}, "eosusd": {"symbol":"eos_usd", "start_date": "2010-01-01"}, "eosbtc": {"symbol":"eos_btc", "start_date": "2010-01-01"}, "eoseth": {"symbol":"eos_eth", "start_date": "2010-01-01"} }' log = Logger('Bitfinex') warning_logger = Logger('AlgoWarning') @@ -48,7 +48,7 @@ class Bitfinex(Exchange): self.id = 'b' self.name = 'bitfinex' self.assets = {} - self.load_assets(ASSETS) + self.load_assets(get_exchange_symbols(self.name)) self.base_currency = base_currency self.store = store diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 1c41c312..c7d888d3 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -74,18 +74,14 @@ class Exchange: asset[key] = pd.to_datetime(asset[key], utc=True) return asset - def load_assets(self, assets_json): - assets = json.loads( - assets_json, - object_hook=Exchange.asset_parser - ) - - for exchange_symbol in assets: + def load_assets(self, symbol_map): + for exchange_symbol in symbol_map: asset_obj = Asset( - sid=abs(hash(assets[exchange_symbol]['symbol'])) % (10 ** 4), + sid=abs(hash(symbol_map[exchange_symbol]['symbol'])) + % (10 ** 4), exchange=self.name, end_date=pd.Timestamp.utcnow() + timedelta(minutes=300000), - **assets[exchange_symbol] + **symbol_map[exchange_symbol] ) self.assets[exchange_symbol] = asset_obj diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index e9cf44f7..63eb13b2 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -34,6 +34,20 @@ class ExchangeTransactionError(ZiplineError): ).strip() +class ExchangeAuthNotFound(ZiplineError): + msg = ( + 'Please create an auth.json file containing the api token and key for ' + 'exchange {exchange}. Place the file here: {filename}' + ).strip() + + +class ExchangeSymbolsNotFound(ZiplineError): + msg = ( + 'Unable to download or find a local copy of symbols.json for exchange ' + '{exchange}. The file should be here: {filename}' + ).strip() + + class InvalidHistoryFrequencyError(ZiplineError): msg = ( 'History frequency {frequency} not supported by the exchange.' diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index c9ffee57..9ebe637b 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -1,9 +1,62 @@ import os -from catalyst.utils.paths import data_root +import urllib +import json +from catalyst.utils.paths import data_root, ensure_directory +from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ + ExchangeSymbolsNotFound + +SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'live-trading/catalyst/exchange/symbols/{exchange}.json' -def get_exchange_folder(environ=None): +def get_exchange_folder(exchange_name, environ=None): if not environ: environ = os.environ root = data_root(environ) + exchange_folder = os.path.join(root, 'exchanges', exchange_name) + ensure_directory(exchange_folder) + + return exchange_folder + + +def download_exchange_symbols(exchange_name, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, 'symbols.json') + + url = SYMBOLS_URL.format(exchange=exchange_name) + response = urllib.urlretrieve(url=url, filename=filename) + return response + + +def get_exchange_symbols(exchange_name, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, 'symbols.json') + + if not os.path.isfile(filename): + download_exchange_symbols(exchange_name, environ) + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + return data + else: + raise ExchangeSymbolsNotFound( + exchange=exchange_name, + filename=filename + ) + + +def get_exchange_auth(exchange_name, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, 'auth.json') + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + return data + else: + raise ExchangeAuthNotFound( + exchange=exchange_name, + filename=filename + ) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 252af115..082f3c54 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -44,6 +44,8 @@ from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangeRequestErrorTooManyAttempts ) +from catalyst.exchange.exchange_utils import get_exchange_auth, \ + get_exchange_folder from logbook import Logger log = Logger('run_algo') @@ -369,7 +371,7 @@ def load_extensions(default, extensions, strict, environ, reload=False): def run_algorithm(initialize, - capital_base, + capital_base=None, start=None, end=None, handle_data=None, @@ -384,7 +386,8 @@ def run_algorithm(initialize, strict_extensions=True, environ=os.environ, live=False, - exchange_conn=None, + exchange_name=None, + base_currency=None, algo_namespace=None): """Run a trading algorithm. @@ -480,19 +483,19 @@ def run_algorithm(initialize, 'cannot specify `bundle_timestamp` without passing `bundle`', ) else: - if exchange_conn is not None: + if exchange_name is not None: store = PortfolioMemoryStore(algo_namespace) - - if exchange_conn['name'] == 'bitfinex': + exchange_auth = get_exchange_auth(exchange_name) + if exchange_name == 'bitfinex': exchange = Bitfinex( - key=exchange_conn['key'], - secret=exchange_conn['secret'], - base_currency=exchange_conn['base_currency'], + key=exchange_auth['key'], + secret=exchange_auth['secret'].encode('UTF-8'), + base_currency=base_currency, store=store ) else: raise NotImplementedError( - 'exchange not supported: %s' % exchange_conn['name']) + 'exchange not supported: %s' % exchange_name) return _run( handle_data=handle_data,