diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 1edb53f4..d58b5c30 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -509,6 +509,40 @@ def ingest_exchange(exchange_name, data_frequency, start, end, ) +@main.command(name='clean-exchange') +@click.option( + '-x', + '--exchange-name', + type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}), + help='The name of the exchange bundle to ingest (supported: bitfinex,' + ' bittrex, poloniex).', +) +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute'}), + default=None, + help='The bundle data frequency to remove. If not specified, it will ' + 'remove both daily and minute bundles.', +) +@click.pass_context +def clean_exchange(ctx, exchange_name, data_frequency): + """Clean up bundles from 'ingest-exchange'. + """ + + if exchange_name is None: + ctx.fail("must specify an exchange name '-x'") + + exchange = get_exchange(exchange_name) + exchange_bundle = ExchangeBundle(exchange) + + click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) + exchange_bundle.clean( + data_frequency=data_frequency, + ) + click.echo('Done') + + @main.command() @click.option( '-b', @@ -598,7 +632,7 @@ def ingest(ctx, bundle, exchange_name, compile_locally, assets_version, ' This may not be passed with -e / --before or -a / --after', ) def clean(bundle, before, after, keep_last): - """Clean up data downloaded with the ingest command. + """Clean up bundles from 'ingest'. """ bundles_module.clean( bundle, diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index 986d2174..bd67526e 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -17,6 +17,8 @@ """ Cythonized Asset object. """ +import hashlib + cimport cython from cpython.number cimport PyNumber_Index from cpython.object cimport ( @@ -501,7 +503,11 @@ cdef class TradingPair(Asset): if sid == 0 or sid is None: try: - sid = abs(hash(symbol)) % (10 ** 4) + # sid = abs(hash(symbol)) % (10 ** 4) + # TODO: try to encode the symbol in the main scope + sid = int( + hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16 + ) % 10 ** 6 except Exception as e: raise SidHashError(symbol=symbol) diff --git a/catalyst/curate/poloniex.py b/catalyst/curate/poloniex.py index e2a88476..c81ddf04 100644 --- a/catalyst/curate/poloniex.py +++ b/catalyst/curate/poloniex.py @@ -212,32 +212,32 @@ class PoloniexCurator(object): def write_ohlcv_file(self, currencyPair): csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' - if( os.path.isfile(csv_1min) ): - log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.') - else: - df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], - dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } ) - df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True) - df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) - ohlcv = self.generate_ohlcv(df) - try: - with open(csv_1min, 'ab') as csvfile: - csvwriter = csv.writer(csvfile) - for item in ohlcv.itertuples(): - if item.Index == 0: - continue - csvwriter.writerow([ - item.Index.value // 10 ** 9, - item.open, - item.high, - item.low, - item.close, - item.volume, - ]) - except Exception as e: - log.error('Error opening %s' % csv_fn) - log.exception(e) - log.debug(currencyPair+': Generated 1min OHLCV data.') + #if( os.path.isfile(csv_1min) ): + # log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.') + #else: + df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], + dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } ) + df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True) + df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) + ohlcv = self.generate_ohlcv(df) + try: + with open(csv_1min, 'w') as csvfile: + csvwriter = csv.writer(csvfile) + for item in ohlcv.itertuples(): + if item.Index == 0: + continue + csvwriter.writerow([ + item.Index.value // 10 ** 9, + item.open, + item.high, + item.low, + item.close, + item.volume, + ]) + except Exception as e: + log.error('Error opening %s' % csv_fn) + log.exception(e) + log.debug(currencyPair+': Generated 1min OHLCV data.') ''' diff --git a/catalyst/examples/buy_and_hodl.py b/catalyst/examples/buy_and_hodl.py index b3f411f4..b2b6a7ec 100644 --- a/catalyst/examples/buy_and_hodl.py +++ b/catalyst/examples/buy_and_hodl.py @@ -24,7 +24,7 @@ from catalyst.api import ( ) def initialize(context): - context.ASSET_NAME = 'USDT_BTC' + context.ASSET_NAME = 'BTC_USDT' context.TARGET_HODL_RATIO = 0.8 context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO @@ -49,14 +49,14 @@ def handle_data(context, data): orders = get_open_orders(context.asset) or [] for order in orders: cancel_order(order) - + # Stop buying after passing the reserve threshold cash = context.portfolio.cash if cash <= reserve_value: context.is_buying = False # Retrieve current asset price from pricing data - price = data[context.asset].price + price = data.current(context.asset, 'price') # Check if still buying and could (approximately) afford another purchase if context.is_buying and cash > price: @@ -70,7 +70,7 @@ def handle_data(context, data): record( price=price, - volume=data[context.asset].volume, + volume=data.current(context.asset, 'volume'), cash=cash, starting_cash=context.portfolio.starting_cash, leverage=context.account.leverage, diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index feadf49b..acf481e0 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -27,7 +27,7 @@ log = Logger(algo_namespace) def initialize(context): log.info('initializing algo') - context.ASSET_NAME = 'XRP_USD' + context.ASSET_NAME = 'XRP_USDT' context.asset = symbol(context.ASSET_NAME) context.TARGET_POSITIONS = 5000 diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index dc130b3a..d9bf988e 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -1,13 +1,13 @@ -import pandas as pd import talib +import pandas as pd from catalyst import run_algorithm from catalyst.api import symbol def initialize(context): print('initializing') - context.asset = symbol('xrp_btc') + context.asset = symbol('btc_usd') def handle_data(context, data): @@ -20,32 +20,32 @@ def handle_data(context, data): context.asset, fields='price', bar_count=15, - frequency='1d' + frequency='1m' ) rsi = talib.RSI(prices.values, timeperiod=14)[-1] print('got rsi: {}'.format(rsi)) pass -# run_algorithm( -# capital_base=250, -# start=pd.to_datetime('2015-08-01', utc=True), -# end=pd.to_datetime('2017-9-30', utc=True), -# data_frequency='daily', -# initialize=initialize, -# handle_data=handle_data, -# analyze=None, -# exchange_name='poloniex', -# algo_namespace='simple_loop', -# base_currency='eth' -# ) run_algorithm( + capital_base=250, + start=pd.to_datetime('2017-08-01', utc=True), + end=pd.to_datetime('2017-9-30', utc=True), + data_frequency='minute', initialize=initialize, handle_data=handle_data, analyze=None, exchange_name='bitfinex', - live=True, algo_namespace='simple_loop', - base_currency='eth', - live_graph=False + base_currency='btc' ) +# run_algorithm( +# initialize=initialize, +# handle_data=handle_data, +# analyze=None, +# exchange_name='bitfinex', +# live=True, +# algo_namespace='simple_loop', +# base_currency='eth', +# live_graph=False +# ) diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index 6df3638e..1bca9422 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -24,7 +24,7 @@ URL2 = 'https://bittrex.com/Api/v2.0' class Bittrex(Exchange): def __init__(self, key, secret, base_currency, portfolio=None): - self.api = Bittrex_api(key=key, secret=secret.encode('UTF-8')) + self.api = Bittrex_api(key=key, secret=secret) self.name = 'bittrex' self.color = 'blue' self.base_currency = base_currency @@ -65,10 +65,10 @@ class Bittrex(Exchange): return exchange_symbol.lower() def get_balances(self): + balances = self.api.getbalances() try: log.debug('retrieving wallet balances') self.ask_request() - balances = self.api.getbalances() except Exception as e: raise ExchangeRequestError(error=e) @@ -208,7 +208,7 @@ class Bittrex(Exchange): ) def get_candles(self, data_frequency, assets, bar_count=None, - start_date=None): + start_dt=None, end_dt=None): """ Supported Intervals ------------------- diff --git a/catalyst/exchange/bittrex/bittrex_api.py b/catalyst/exchange/bittrex/bittrex_api.py index cda7581e..bc31607d 100644 --- a/catalyst/exchange/bittrex/bittrex_api.py +++ b/catalyst/exchange/bittrex/bittrex_api.py @@ -4,10 +4,10 @@ import time import hmac import hashlib -from six.moves import urllib # Workaround for backwards compatibility # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7 +from six.moves import urllib urlopen = urllib.request.urlopen @@ -39,7 +39,10 @@ class Bittrex_api(object): if method not in self.public: url += '&apikey=' + self.key url += '&nonce=' + str(int(time.time())) - signature = hmac.new(self.secret, url, hashlib.sha512).hexdigest() + + signature = hmac.new(self.secret.encode('utf-8'), + url.encode('utf-8'), + hashlib.sha512).hexdigest() headers = {'apisign': signature} else: headers = {} diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index fe622b52..b1aa1a07 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -103,42 +103,6 @@ def get_start_dt(end_dt, bar_count, data_frequency): return start_dt -def get_adj_dates(start, end, assets, data_frequency): - """ - Contains a date range to the trading availability of the specified pairs. - - :param start: - :param end: - :param assets: - :param data_frequency: - :return: - """ - earliest_trade = None - last_entry = None - for asset in assets: - if earliest_trade is None or earliest_trade > asset.start_date: - earliest_trade = asset.start_date - - end_asset = asset.end_minute if data_frequency == 'minute' else \ - asset.end_daily - if end_asset is not None and \ - (last_entry is None or end_asset > last_entry): - last_entry = end_asset - - if start is None or earliest_trade > start: - start = earliest_trade - - if end is None or (last_entry is not None and end > last_entry): - end = last_entry - - if end is None or start >= end: - raise NoDataAvailableOnExchange( - exchange=asset.exchange.title(), - symbol=[asset.symbol.encode('utf-8')], - data_frequency=data_frequency, - ) - - return start, end def get_month_start_end(dt): @@ -243,12 +207,12 @@ def find_most_recent_time(bundle_name): for folder in bundle_folders: date = from_bundle_ingest_dirname(folder) if not most_recent_bundle or date > \ - most_recent_bundle[most_recent_bundle.keys()[0]]: + most_recent_bundle[list(most_recent_bundle.keys())[0]]: most_recent_bundle = dict() most_recent_bundle[folder] = date if most_recent_bundle: - return most_recent_bundle.keys()[0] + return list(most_recent_bundle.keys())[0] else: return None diff --git a/catalyst/exchange/data_portal_exchange.py b/catalyst/exchange/data_portal_exchange.py index ce454303..3eebf3aa 100644 --- a/catalyst/exchange/data_portal_exchange.py +++ b/catalyst/exchange/data_portal_exchange.py @@ -80,7 +80,7 @@ class DataPortalExchangeBase(DataPortal): return pd.concat(df_list) else: - exchange = self.exchanges[exchange_assets.keys()[0]] + exchange = self.exchanges[list(exchange_assets.keys())[0]] return self.get_exchange_history_window( exchange, assets, @@ -165,8 +165,8 @@ class DataPortalExchangeBase(DataPortal): exchange_assets[asset.exchange].append(asset) - if len(exchange_assets.keys()) == 1: - exchange = self.exchanges[exchange_assets.keys()[0]] + if len(list(exchange_assets.keys())) == 1: + exchange = self.exchanges[list(exchange_assets.keys())[0]] return self.get_exchange_spot_value( exchange, assets, field, dt, data_frequency) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 25a11594..bb4018d5 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -87,7 +87,7 @@ class Exchange: self.request_cpt[now] = 0 return True - cpt_date = self.request_cpt.keys()[0] + cpt_date = list(self.request_cpt.keys())[0] cpt = self.request_cpt[cpt_date] if now > cpt_date + timedelta(minutes=1): @@ -167,8 +167,10 @@ class Exchange: asset = self.assets[key] if not asset: - supported_symbols = [pair.symbol.encode('utf-8') for pair in - self.assets.values()] + supported_symbols = [ + pair.symbol for pair in list(self.assets.values()) + ] + raise SymbolNotFoundOnExchange( symbol=symbol, exchange=self.name.title(), @@ -552,7 +554,7 @@ class Exchange: portfolio.starting_cash = portfolio.cash if portfolio.positions: - assets = portfolio.positions.keys() + assets = list(portfolio.positions.keys()) tickers = self.tickers(assets) portfolio.positions_value = 0.0 diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 22c26ef2..6650fda2 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -113,7 +113,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): else self.sim_params.end_session if exchange_name is None: - exchange = self.exchanges.values()[0] + exchange = list(self.exchanges.values())[0] else: exchange = self.exchanges[exchange_name] @@ -524,7 +524,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.add_pnl_stats(minute_stats) if self.recorded_vars: self.add_custom_signals_stats(minute_stats) - recorded_cols = self.recorded_vars.keys() + recorded_cols = list(self.recorded_vars.keys()) else: recorded_cols = None @@ -556,6 +556,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): except Exception as e: log.warn('unable to calculate performance: {}'.format(e)) + # TODO: pickle does not seem to work in python 3 try: save_algo_object( algo_name=self.algo_namespace, diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 7aab5dc6..b86639aa 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -10,7 +10,7 @@ from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ BcolzMinuteBarMetadata from catalyst.exchange.bundle_utils import range_in_bundle, \ - get_bcolz_chunk, get_delta, get_adj_dates, get_month_start_end, \ + get_bcolz_chunk, get_delta, get_month_start_end, \ get_year_start_end, get_periods_range, get_df_from_arrays, get_start_dt from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ BcolzExchangeBarWriter @@ -24,11 +24,13 @@ from catalyst.utils.paths import ensure_directory log = Logger('exchange_bundle', level=LOG_LEVEL) -BUNDLE_NAME_TEMPLATE = os.path.join('{root}','{frequency}_bundle') +BUNDLE_NAME_TEMPLATE = os.path.join('{root}', '{frequency}_bundle') + def _cachpath(symbol, type_): return '-'.join([symbol, type_]) + class ExchangeBundle: def __init__(self, exchange): self.exchange = exchange @@ -177,10 +179,7 @@ class ExchangeBundle: # This is workaround, there is an issue with empty # session_label when using a newly created writer - key = writer._rootdir if data_frequency == 'minute' \ - else writer._filename - - del self._writers[key] + del self._writers[writer._rootdir] writer = self.get_writer(writer._start_session, writer._end_session, data_frequency) @@ -224,12 +223,18 @@ class ExchangeBundle: if reader is None: raise TempBundleNotFoundError(path=path) - arrays = reader.load_raw_arrays( - sids=[asset.sid], - fields=['open', 'high', 'low', 'close', 'volume'], - start_dt=start_dt, - end_dt=end_dt - ) + arrays = None + try: + arrays = reader.load_raw_arrays( + sids=[asset.sid], + fields=['open', 'high', 'low', 'close', 'volume'], + start_dt=start_dt, + end_dt=end_dt + ) + except Exception as e: + log.warn('skipping ctable for {} from {} to {}: {}'.format( + asset.symbol, start_dt, end_dt, e + )) if not arrays: return path @@ -287,6 +292,7 @@ class ExchangeBundle: if not df.empty: df.sort_index(inplace=True) data.append((asset.sid, df)) + self._write(data, writer, data_frequency) if cleanup: @@ -296,6 +302,45 @@ class ExchangeBundle: return path + def get_adj_dates(self, start, end, assets, data_frequency): + """ + Contains a date range to the trading availability of the specified pairs. + + :param start: + :param end: + :param assets: + :param data_frequency: + :return: + """ + earliest_trade = None + last_entry = None + for asset in assets: + if (earliest_trade is None or earliest_trade > asset.start_date) \ + and asset.start_date >= self.calendar.first_session: + earliest_trade = asset.start_date + + end_asset = asset.end_minute if data_frequency == 'minute' else \ + asset.end_daily + if end_asset is not None and \ + (last_entry is None or end_asset > last_entry): + last_entry = end_asset + + if start is None or \ + (earliest_trade is not None and earliest_trade > start): + start = earliest_trade + + if end is None or (last_entry is not None and end > last_entry): + end = last_entry + + if end is None or start is None or start >= end: + raise NoDataAvailableOnExchange( + exchange=asset.exchange.title(), + symbol=[asset.symbol], + data_frequency=data_frequency, + ) + + return start, end + def prepare_chunks(self, assets, data_frequency, start_dt, end_dt): """ Split a price data request into chunks corresponding to individual @@ -312,26 +357,26 @@ class ExchangeBundle: chunks = [] for asset in assets: try: - asset_start, asset_end = \ - get_adj_dates(start_dt, end_dt, [asset], data_frequency) + # Checking if the the asset has price data in the specified + # date range + adj_start, adj_end = self.get_adj_dates( + start_dt, end_dt, [asset], data_frequency + ) except NoDataAvailableOnExchange: + # If not, we continue to the next asset continue - start_dt = max(start_dt, self.calendar.first_trading_session) - start_dt = max(start_dt, asset_start) + # This is either the first trading day of the asset or the + # first session available in the calendar + first_trading_dt = asset.start_date \ + if asset.start_date > self.calendar.first_session \ + else self.calendar.first_session # Aligning start / end dates with the daily calendar - sessions = get_periods_range(start_dt, end_dt, data_frequency) \ - if data_frequency == 'minute' \ - else self.calendar.sessions_in_range(start_dt, end_dt) - - if asset_start < sessions[0]: - asset_start = sessions[0] - - if asset_end > sessions[-1]: - asset_end = sessions[-1] + sessions = self.calendar.sessions_in_range(adj_start, adj_end) + # We loop through each session to create chunks for each period chunk_labels = [] dt = sessions[0] while dt <= sessions[-1]: @@ -345,29 +390,49 @@ class ExchangeBundle: # of the trading pair if data_frequency == 'minute': period_start, period_end = get_month_start_end(dt) - asset_start_month, _ = get_month_start_end(asset_start) + # TODO: redundant gate, we are already filtering dates + if first_trading_dt > period_start: + dt += timedelta(days=1) + continue + + asset_start_month, _ = get_month_start_end( + first_trading_dt + ) if asset_start_month == period_start \ - and period_start < asset_start: - period_start = asset_start + and period_start < first_trading_dt: + period_start = first_trading_dt - _, asset_end_month = get_month_start_end(asset_end) + # TODO: need to filter closed pairs? + _, asset_end_month = get_month_start_end( + asset.end_minute + ) if asset_end_month == period_end \ - and period_end > asset_end: - period_end = asset_end + and period_end > asset.end_minute: + period_end = asset.end_minute elif data_frequency == 'daily': period_start, period_end = get_year_start_end(dt) - asset_start_year, _ = get_year_start_end(asset_start) + # TODO: redundant gate, we are already filtering dates + if first_trading_dt > period_start: + dt += timedelta(days=1) + continue + + asset_start_year, _ = get_year_start_end( + first_trading_dt + ) if asset_start_year == period_start \ - and period_start < asset_start: - period_start = asset_start + and period_start < first_trading_dt: + period_start = first_trading_dt - _, asset_end_year = get_year_start_end(asset_end) + _, asset_end_year = get_year_start_end( + asset.end_minute + ) if asset_end_year == period_end \ - and period_end > asset_end: - period_end = asset_end + and period_end > asset.end_minute: + period_end = asset.end_minute + else: raise InvalidHistoryFrequencyError( frequency=data_frequency @@ -377,10 +442,13 @@ class ExchangeBundle: # Checking the last minute of the day instead. range_start = period_start.replace(hour=23, minute=59) \ if data_frequency == 'minute' else period_start + + # Checking if the data already exists in the bundle + # for the date range of the chunk. If not, we create + # a chunk for ingestion. has_data = range_in_bundle( asset, range_start, period_end, reader ) - if not has_data: log.debug('adding period: {}'.format(label)) chunks.append( @@ -394,6 +462,7 @@ class ExchangeBundle: dt += timedelta(days=1) + # We sort the chunks by end date to ingest most recent data first chunks.sort(key=lambda chunk: chunk['period_end']) return chunks @@ -408,13 +477,24 @@ class ExchangeBundle: :param end_dt: :return: """ - writer = self.get_writer(start_dt, end_dt, data_frequency) chunks = self.prepare_chunks( assets=assets, data_frequency=data_frequency, start_dt=start_dt, end_dt=end_dt ) + + # Since chunks are either monthly or yearly, it is possible that + # our ingestion data range is greater than specified. We adjust + # the boundaries to ensure that the writer can write all data. + for chunk in chunks: + if chunk['period_start'] < start_dt: + start_dt = chunk['period_start'] + + if chunk['period_end'] > end_dt: + end_dt = chunk['period_end'] + + writer = self.get_writer(start_dt, end_dt, data_frequency) with maybe_show_progress( chunks, show_progress, @@ -430,7 +510,8 @@ class ExchangeBundle: start_dt=chunk['period_start'], end_dt=chunk['period_end'], writer=writer, - empty_rows_behavior='strip' + empty_rows_behavior='strip', + cleanup=True ) def ingest(self, data_frequency, include_symbols=None, @@ -448,7 +529,9 @@ class ExchangeBundle: :return: """ assets = self.get_assets(include_symbols, exclude_symbols) - start_dt, end_dt = get_adj_dates(start, end, assets, data_frequency) + start_dt, end_dt = self.get_adj_dates( + start, end, assets, data_frequency + ) for frequency in data_frequency.split(','): self.ingest_assets(assets, start_dt, end_dt, frequency, @@ -517,7 +600,7 @@ class ExchangeBundle: return values except Exception: - symbols = [asset.symbol.encode('utf-8') for asset in assets] + symbols = [asset.symbol for asset in assets] raise PricingDataNotLoadedError( field=field, first_trading_day=min([asset.start_date for asset in assets]), @@ -535,8 +618,9 @@ class ExchangeBundle: data_frequency, reset_reader=False): start_dt = get_start_dt(end_dt, bar_count, data_frequency) - start_dt, end_dt = \ - get_adj_dates(start_dt, end_dt, assets, data_frequency) + start_dt, end_dt = self.get_adj_dates( + start_dt, end_dt, assets, data_frequency + ) reader = self.get_reader(data_frequency) if reset_reader: @@ -544,7 +628,7 @@ class ExchangeBundle: reader = self.get_reader(data_frequency) if reader is None: - symbols = [asset.symbol.encode('utf-8') for asset in assets] + symbols = [asset.symbol for asset in assets] raise PricingDataNotLoadedError( field=field, first_trading_day=min([asset.start_date for asset in assets]), @@ -555,8 +639,9 @@ class ExchangeBundle: ) for asset in assets: - asset_start_dt, asset_end_dt = \ - get_adj_dates(start_dt, end_dt, assets, data_frequency) + asset_start_dt, asset_end_dt = self.get_adj_dates( + start_dt, end_dt, assets, data_frequency + ) in_bundle = range_in_bundle( asset, asset_start_dt, asset_end_dt, reader @@ -602,3 +687,34 @@ class ExchangeBundle: series[asset] = value_series return series + + def clean(self, data_frequency): + log.debug('cleaning exchange {}, frequency {}'.format( + self.exchange.name, data_frequency + )) + root = get_exchange_folder(self.exchange.name) + + symbols = os.path.join(root, 'symbols.json') + if os.path.isfile(symbols): + os.remove(symbols) + + temp_bundles = os.path.join(root, 'temp_bundles') + + if os.path.isdir(temp_bundles): + log.debug('removing folder and content: {}'.format(temp_bundles)) + shutil.rmtree(temp_bundles) + log.debug('{} removed'.format(temp_bundles)) + + frequencies = ['daily', 'minute'] if data_frequency is None \ + else [data_frequency] + + for frequency in frequencies: + label = '{}_bundle'.format(frequency) + frequency_bundle = os.path.join(root, label) + + if os.path.isdir(frequency_bundle): + log.debug( + 'removing folder and content: {}'.format(frequency_bundle) + ) + shutil.rmtree(frequency_bundle) + log.debug('{} removed'.format(frequency_bundle)) diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index c5baadec..6cd55014 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -6,12 +6,12 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError, - SymbolNotFoundOnExchange, NoDataAvailableOnExchange, - ExchangeAuthEmpty ]: + SymbolNotFoundOnExchange, NoDataAvailableOnExchange, + ExchangeAuthEmpty]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] - print "Error traceback: {1} (line {2})\n" \ - "{0.__name__}: {3}".format(exctype, fn, ln, excvalue) + print("Error traceback: {1} (line {2})\n" + "{0.__name__}: {3}".format(exctype, fn, ln, excvalue)) else: sys.__excepthook__(exctype, excvalue, exctraceback) @@ -214,7 +214,9 @@ class PricingDataNotLoadedError(ZiplineError): class ApiCandlesError(ZiplineError): msg = ('Unable to fetch candles from the remote API: {error}.').strip() + class NoDataAvailableOnExchange(ZiplineError): - msg = ('Requested data for trading pair {symbol} is not available on exchange {exchange} ' - 'in `{data_frequency}` frequency at this time. ' - 'Check `http://enigma.co/catalyst/status` for market coverage.').strip() + msg = ( + 'Requested data for trading pair {symbol} is not available on exchange {exchange} ' + 'in `{data_frequency}` frequency at this time. ' + 'Check `http://enigma.co/catalyst/status` for market coverage.').strip() diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index b9e6cf21..2e3982f0 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -1,13 +1,12 @@ import json import os import pickle -import urllib +from six.moves.urllib import request from datetime import date, datetime import pandas as pd -from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ - ExchangeSymbolsNotFound +from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time @@ -34,7 +33,7 @@ def get_exchange_symbols_filename(exchange_name, environ=None): def download_exchange_symbols(exchange_name, environ=None): filename = get_exchange_symbols_filename(exchange_name) url = SYMBOLS_URL.format(exchange=exchange_name) - response = urllib.urlretrieve(url=url, filename=filename) + response = request.urlretrieve(url=url, filename=filename) return response @@ -42,7 +41,9 @@ def get_exchange_symbols(exchange_name, environ=None): filename = get_exchange_symbols_filename(exchange_name) if not os.path.isfile(filename) or \ - pd.Timedelta(pd.Timestamp('now', tz='UTC') - last_modified_time(filename)).days > 1: + pd.Timedelta(pd.Timestamp('now', + tz='UTC') - last_modified_time( + filename)).days > 1: download_exchange_symbols(exchange_name, environ) if os.path.isfile(filename): @@ -67,9 +68,11 @@ def get_exchange_auth(exchange_name, environ=None): else: data = dict(name=exchange_name, key='', secret='') with open(filename, 'w') as f: - json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) return data + def get_algo_folder(algo_name, environ=None): if not environ: environ = os.environ @@ -151,8 +154,8 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None): filename = os.path.join(folder, key + '.csv') - with open(filename, 'wb') as handle: - df.to_csv(handle) + with open(filename, 'wt') as handle: + df.to_csv(handle, encoding='UTF_8') def get_exchange_minute_writer_root(exchange_name, environ=None): @@ -163,6 +166,7 @@ def get_exchange_minute_writer_root(exchange_name, environ=None): return minute_data_folder + def get_exchange_bundles_folder(exchange_name, environ=None): exchange_folder = get_exchange_folder(exchange_name, environ) diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py index 43954036..76d51332 100644 --- a/catalyst/exchange/poloniex/poloniex.py +++ b/catalyst/exchange/poloniex/poloniex.py @@ -33,7 +33,7 @@ log = Logger('Poloniex', level=LOG_LEVEL) class Poloniex(Exchange): def __init__(self, key, secret, base_currency, portfolio=None): - self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8')) + self.api = Poloniex_api(key=key, secret=secret) self.name = 'poloniex' self.assets = {} self.load_assets() @@ -119,9 +119,9 @@ class Poloniex(Exchange): return order, executed_price def get_balances(self): - log.debug('retrieving wallets balances') + balances = self.api.returnbalances() try: - balances = self.api.returnbalances() + log.debug('retrieving wallets balances') except Exception as e: log.debug(e) raise ExchangeRequestError(error=e) diff --git a/catalyst/exchange/poloniex/poloniex_api.py b/catalyst/exchange/poloniex/poloniex_api.py index 599a0b65..8bf6bb83 100644 --- a/catalyst/exchange/poloniex/poloniex_api.py +++ b/catalyst/exchange/poloniex/poloniex_api.py @@ -19,19 +19,25 @@ class Poloniex_api(object): self.max_requests_per_second = 6 self.request_cpt = dict() - self.public = ['returnTicker', 'return24Volume', 'returnOrderBook', - 'returnTradeHistory', 'returnChartData', - 'returnCurrencies', 'returnLoanOrders'] - self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses', - 'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders', - 'returnTradeHistory','returnOrderTrades', + self.public = ['returnTicker', 'return24Volume', 'returnOrderBook', + 'returnTradeHistory', 'returnChartData', + 'returnCurrencies', 'returnLoanOrders'] + self.trading = ['returnBalances', 'returnCompleteBalances', + 'returnDepositAddresses', + 'generateNewAddress', 'returnDepositsWithdrawals', + 'returnOpenOrders', + 'returnTradeHistory', 'returnOrderTrades', 'buy', 'sell', 'cancelOrder', 'moveOrder', - 'withdraw', 'returnFeeInfo','returnAvailableAccountBalances', + 'withdraw', 'returnFeeInfo', + 'returnAvailableAccountBalances', 'returnTradableBalances', 'transferBalance', - 'returnMarginAccountSummary','marginBuy','marginSell', - 'getMarginPosition', 'closeMarginPosition','createLoanOffer', - 'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans', - 'returnLendingHistory','toggleAutoRenew'] + 'returnMarginAccountSummary', 'marginBuy', + 'marginSell', + 'getMarginPosition', 'closeMarginPosition', + 'createLoanOffer', + 'cancelLoanOffer', 'returnOpenLoanOffers', + 'returnActiveLoans', + 'returnLendingHistory', 'toggleAutoRenew'] def ask_request(self): """ @@ -50,7 +56,7 @@ class Poloniex_api(object): self.request_cpt[now] = 0 return True - cpt_date = self.request_cpt.keys()[0] + cpt_date = list(self.request_cpt.keys())[0] cpt = self.request_cpt[cpt_date] if now > cpt_date + 1: @@ -59,9 +65,8 @@ class Poloniex_api(object): return True if cpt >= self.max_requests_per_second: - - log.debug('max requests 6 reached, sleeping for 1 seconds') - sleep(1) + + time.sleep(1) now = time.time() self.request_cpt = dict() @@ -73,21 +78,34 @@ class Poloniex_api(object): def query(self, method, req={}): if method in self.public: - url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req) + url = 'https://poloniex.com/public?command=' + method + '&' + \ + urllib.parse.urlencode(req) headers = {} post_data = None elif method in self.trading: url = 'https://poloniex.com/tradingApi' req['command'] = method - req['nonce'] = int(time.time()*1000) - post_data = urllib.parse.urlencode(req) - signature = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest() - headers = { 'Sign': signature, 'Key': self.key} + req['nonce'] = int(time.time() * 1000) + post_data = urllib.parse.urlencode(req) + + signature = hmac.new(self.secret.encode('utf-8'), + post_data.encode('utf-8'), + hashlib.sha512).hexdigest() + headers = {'Sign': signature, 'Key': self.key} + + post_data = post_data.encode('utf-8') else: - raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints') + raise ValueError( + 'Method "' + method + '" not found in neither the Public API ' + 'or Trading API endpoints' + ) self.ask_request() - req = urllib.request.Request(url, data=post_data, headers=headers) + req = urllib.request.Request( + url, + data=post_data, + headers=headers + ) return json.loads(urlopen(req).read()) def returnticker(self): @@ -100,15 +118,17 @@ class Poloniex_api(object): return self.query('returnOrderBook', {'currencyPair': market}) def returntradehistory(self, market, start=None, end=None): - if(start is not None and end is not None): - return self.query('returntradehistory', - {'currencyPair': market, 'start': start, 'end': end }) + if (start is not None and end is not None): + return self.query('returntradehistory', + {'currencyPair': market, 'start': start, + 'end': end}) else: - return self.query('returntradehistory', {'currencyPair': market }) + return self.query('returntradehistory', {'currencyPair': market}) def returnchartdata(self, market, period, start, end=9999999999): - return self.query('returnChartData', {'currencyPair': market, 'period': period, - 'start': start, 'end': end}) + return self.query('returnChartData', + {'currencyPair': market, 'period': period, + 'start': start, 'end': end}) def returncurrencies(self): return self.query('returnCurrencies', {}) @@ -120,7 +140,7 @@ class Poloniex_api(object): return self.query('returnBalances') def returncompletebalances(self, account): - if(account): + if (account): return self.query('returnCompleteBalances', {'account': account}) else: return self.query('returnCompleteBalances') @@ -132,43 +152,54 @@ class Poloniex_api(object): return self.query('generateNewAddress', {'currency': currency}) def returnDepositsWithdrawals(self, start, end): - return self.query('returnDepositsWithdrawals', {'start': start, 'end': end}) + return self.query('returnDepositsWithdrawals', + {'start': start, 'end': end}) def returnopenorders(self, market): return self.query('returnOpenOrders', {'currencyPair': market}) def returntradehistory(self, market): - #TODO: optional start and/or end and limit + # TODO: optional start and/or end and limit return self.query('returnTradeHistory', {'currencyPair': market}) def returnordertrades(self, ordernumber): return self.query('returnOrderTrades', {'orderNumber': ordernumber}) - def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0): - if(fillorkill): - return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, + def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, + postonly=0): + if (fillorkill): + return self.query('buy', {'currencyPair': market, 'rate': rate, + 'amount': amount, 'fillOrKill': fillorkill, }) - elif(immediateorcancel): - return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, + elif (immediateorcancel): + return self.query('buy', {'currencyPair': market, 'rate': rate, + 'amount': amount, 'immediateOrCancel': immediateorcancel, }) - elif(postonly): - return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, + elif (postonly): + return self.query('buy', {'currencyPair': market, 'rate': rate, + 'amount': amount, 'postOnly': postonly, }) else: - return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, }) + return self.query('buy', {'currencyPair': market, 'rate': rate, + 'amount': amount, }) - def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0): - if(fillorkill): - return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, - 'fillOrKill': fillorkill, }) - elif(immediateorcancel): - return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, - 'immediateOrCancel': immediateorcancel, }) - elif(postonly): - return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, - 'postOnly': postonly, }) + def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, + postonly=0): + if (fillorkill): + return self.query('sell', {'currencyPair': market, 'rate': rate, + 'amount': amount, + 'fillOrKill': fillorkill, }) + elif (immediateorcancel): + return self.query('sell', {'currencyPair': market, 'rate': rate, + 'amount': amount, + 'immediateOrCancel': immediateorcancel, }) + elif (postonly): + return self.query('sell', {'currencyPair': market, 'rate': rate, + 'amount': amount, + 'postOnly': postonly, }) else: - return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, }) + return self.query('sell', {'currencyPair': market, 'rate': rate, + 'amount': amount, }) def cancelorder(self, ordernumber): return self.query('cancelOrder', {'orderNumber': ordernumber}) @@ -180,4 +211,3 @@ class Poloniex_api(object): def returnfeeinfo(self): return self.query('returnFeeInfo') - diff --git a/catalyst/utils/paths.py b/catalyst/utils/paths.py index 6ba10a29..8ec87c7e 100644 --- a/catalyst/utils/paths.py +++ b/catalyst/utils/paths.py @@ -126,7 +126,7 @@ def catalyst_root(environ=None): root = environ.get('ZIPLINE_ROOT', None) if root is None: - root = expanduser('~/.catalyst') + root = os.path.join(expanduser('~'),'.catalyst') return root diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py new file mode 100644 index 00000000..8c76799a --- /dev/null +++ b/tests/exchange/test_bcolz.py @@ -0,0 +1,150 @@ +import shutil +import random +import tempfile +import pandas as pd + +from catalyst.exchange.exchange_bundle import ExchangeBundle +from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \ + BcolzExchangeBarReader + +from catalyst.exchange.bundle_utils import get_df_from_arrays + +from nose.tools import assert_equals + + +class TestBcolzWriter(object): + @classmethod + def setup_class(cls): + cls.columns = ['open', 'high', 'low', 'close', 'volume'] + + def setUp(self): + self.root_dir = tempfile.mkdtemp() # Create a temporary directory + + def tearDown(self): + shutil.rmtree(self.root_dir) # Remove the directory after the test + + def generate_df(self, exchange_name, freq, start, end): + bundle = ExchangeBundle(exchange_name) + index = bundle.get_calendar_periods_range(start, end, freq) + df = pd.DataFrame(index=index, columns=self.columns) + df.fillna(random.random(), inplace=True) + return df + + def test_bcolz_write_daily_past(self): + start = pd.to_datetime('2016-01-01') + end = pd.to_datetime('2016-12-31') + freq = 'daily' + + df = self.generate_df('bitfinex', freq, start, end) + + writer = BcolzExchangeBarWriter( + rootdir=self.root_dir, + start_session=start, + end_session=end, + data_frequency=freq, + write_metadata=True) + + data = [] + data.append((1, df)) + writer.write(data) + pass + + def test_bcolz_write_daily_present(self): + start = pd.to_datetime('2017-01-01') + end = pd.to_datetime('today') + freq = 'daily' + + df = self.generate_df('bitfinex', freq, start, end) + + writer = BcolzExchangeBarWriter( + rootdir=self.root_dir, + start_session=start, + end_session=end, + data_frequency=freq, + write_metadata=True) + + data = [] + data.append((1, df)) + writer.write(data) + pass + + def test_bcolz_write_minute_past(self): + start = pd.to_datetime('2015-04-01 00:00') + end = pd.to_datetime('2015-04-30 23:59') + freq = 'minute' + + df = self.generate_df('bitfinex', freq, start, end) + + writer = BcolzExchangeBarWriter( + rootdir=self.root_dir, + start_session=start, + end_session=end, + data_frequency=freq, + write_metadata=True) + + data = [] + data.append((1, df)) + writer.write(data) + + pass + + def test_bcolz_write_minute_present(self): + start = pd.to_datetime('2017-10-01 00:00') + end = pd.to_datetime('today') + freq = 'minute' + + df = self.generate_df('bitfinex', freq, start, end) + + writer = BcolzExchangeBarWriter( + rootdir=self.root_dir, + start_session=start, + end_session=end, + data_frequency=freq, + write_metadata=True) + + data = [] + data.append((1, df)) + writer.write(data) + pass + + def bcolz_exchange_daily_write_read(self, exchange_name): + start = pd.to_datetime('2017-10-01 00:00') + end = pd.to_datetime('today') + freq = 'daily' + + bundle = ExchangeBundle(exchange_name) + + df = self.generate_df(exchange_name, freq, start, end) + + print df.index[0],df.index[-1] + + writer = BcolzExchangeBarWriter( + rootdir=self.root_dir, + start_session=df.index[0], + end_session=df.index[-1], + data_frequency=freq, + write_metadata=True) + + data = [] + data.append((1, df)) + writer.write(data) + + reader = BcolzExchangeBarReader(rootdir=self.root_dir, + data_frequency=freq) + + arrays = reader.load_raw_arrays(self.columns, start, end, [1, ]) + + periods = bundle.get_calendar_periods_range( + start, end, freq + ) + + dx = get_df_from_arrays(arrays, periods) + + assert_equals(df.equals(df), True) + pass + + def test_bcolz_bitfinex_daily_write_read(self): + self.bcolz_exchange_daily_write_read('bitfinex') + + def test_bcolz_poloniex_daily_write_read(self): + self.bcolz_exchange_daily_write_read('poloniex') diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py index ded6c8ca..94968e9e 100644 --- a/tests/exchange/test_bitfinex.py +++ b/tests/exchange/test_bitfinex.py @@ -8,7 +8,7 @@ from catalyst.finance.execution import (LimitOrder) log = Logger('test_bitfinex') -class BitfinexTestCase(BaseExchangeTestCase): +class TestBitfinexTestCase(BaseExchangeTestCase): @classmethod def setup(self): log.info('creating bitfinex object') diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py index f1becbcc..bbf52fa1 100644 --- a/tests/exchange/test_bittrex.py +++ b/tests/exchange/test_bittrex.py @@ -7,7 +7,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_bittrex') -class BittrexTestCase(BaseExchangeTestCase): +class TestBittrexTestCase(BaseExchangeTestCase): @classmethod def setup(self): print ('creating bittrex object') diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index c4575fe8..bd79c16b 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -1,9 +1,10 @@ -from logging import Logger +import hashlib +from logging import getLogger import pandas as pd from catalyst import get_calendar -from catalyst.exchange.bundle_utils import get_bcolz_chunk, get_periods, \ +from catalyst.exchange.bundle_utils import get_bcolz_chunk, \ get_periods_range from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ BcolzExchangeBarWriter @@ -13,10 +14,10 @@ from catalyst.exchange.exchange_utils import get_exchange_folder from catalyst.exchange.init_utils import get_exchange from catalyst.utils.paths import ensure_directory -log = Logger('test_exchange_bundle') +log = getLogger('test_exchange_bundle') -class ExchangeBundleTestCase: +class TestExchangeBundle: def test_spot_value(self): data_frequency = 'daily' exchange_name = 'poloniex' @@ -93,16 +94,39 @@ class ExchangeBundleTestCase: ) pass + def test_ingest_exchange(self): + # exchange_name = 'bitfinex' + # data_frequency = 'daily' + # include_symbols = 'neo_btc,bch_btc,eth_btc' + + exchange_name = 'bitfinex' + data_frequency = 'minute' + + exchange = get_exchange(exchange_name) + exchange_bundle = ExchangeBundle(exchange) + + log.info('ingesting exchange bundle {}'.format(exchange_name)) + exchange_bundle.ingest( + data_frequency=data_frequency, + include_symbols=None, + exclude_symbols=None, + start=None, + end=None, + show_progress=True + ) + + pass + def test_ingest_daily(self): # exchange_name = 'bitfinex' # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' - exchange_name = 'poloniex' + exchange_name = 'bittrex' data_frequency = 'daily' - include_symbols = 'btc_usdt' + include_symbols = 'wings_eth' - start = pd.to_datetime('2016-1-1', utc=True) + start = pd.to_datetime('2017-1-1', utc=True) end = pd.to_datetime('2017-10-16', utc=True) periods = get_periods_range(start, end, data_frequency) @@ -274,7 +298,7 @@ class ExchangeBundleTestCase: data_frequency = 'minute' exchange = get_exchange(exchange_name) - asset = exchange.get_asset('neo_btc') + asset = exchange.get_asset('neos_btc') path = get_bcolz_chunk( exchange_name=exchange_name, @@ -284,3 +308,10 @@ class ExchangeBundleTestCase: ) pass + + def test_hash_symbol(self): + symbol = 'etc_btc' + sid = int( + hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16 + ) % 10 ** 6 + pass diff --git a/tests/exchange/test_clock.py b/tests/exchange/test_clock.py deleted file mode 100644 index ff74986b..00000000 --- a/tests/exchange/test_clock.py +++ /dev/null @@ -1,50 +0,0 @@ -from unittest import TestCase -from logbook import Logger -from mock import patch, sentinel -from catalyst.exchange.simple_clock import SimpleClock -from catalyst.utils.calendars.trading_calendar import days_at_time -from datetime import time -from collections import defaultdict -from catalyst.utils.calendars import get_calendar -import pandas as pd - -log = Logger('ExchangeClockTestCase') - - -class ExchangeClockTestCase(TestCase): - @classmethod - def setUpClass(cls): - cls.open_calendar = get_calendar("OPEN") - - cls.sessions = pd.Timestamp.utcnow() - - def setUp(self): - self.internal_clock = None - self.events = defaultdict(list) - - def advance_clock(self, x): - """Mock function for sleep. Advances the internal clock by 1 min""" - # The internal clock advance time must be 1 minute to match - # MinutesSimulationClock's update frequency - self.internal_clock += pd.Timedelta('1 min') - - def get_clock(self, arg, *args, **kwargs): - """Mock function for pandas.to_datetime which is used to query the - current time in RealtimeClock""" - assert arg == "now" - return self.internal_clock - - def test_clock(self): - with patch('catalyst.exchange.simple_clock.pd.to_datetime') as to_dt, \ - patch('catalyst.exchange.simple_clock.sleep') as sleep: - clock = SimpleClock(sessions=self.sessions) - to_dt.side_effect = self.get_clock - sleep.side_effect = self.advance_clock - start_time = pd.Timestamp.utcnow() - self.internal_clock = start_time - - events = list(clock) - - # Event 0 is SESSION_START which always happens at 00:00. - ts, event_type = events[1] - pass diff --git a/tests/exchange/test_data_portal.py b/tests/exchange/test_data_portal.py index 80ed5531..8a605a0b 100644 --- a/tests/exchange/test_data_portal.py +++ b/tests/exchange/test_data_portal.py @@ -12,7 +12,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_bitfinex') -class ExchangeDataPortalTestCase: +class TestExchangeDataPortalTestCase: @classmethod def setup(self): log.info('creating bitfinex exchange') diff --git a/tests/exchange/test_poloniex.py b/tests/exchange/test_poloniex.py index 4f2f12a7..4a883701 100644 --- a/tests/exchange/test_poloniex.py +++ b/tests/exchange/test_poloniex.py @@ -8,7 +8,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_poloniex') -class PoloniexTestCase(BaseExchangeTestCase): +class TestPoloniexTestCase(BaseExchangeTestCase): @classmethod def setup(self): print ('creating poloniex object') @@ -21,7 +21,7 @@ class PoloniexTestCase(BaseExchangeTestCase): def test_order(self): log.info('creating order') - asset = self.exchange.get_asset('neo_btc') + asset = self.exchange.get_asset('neos_btc') order_id = self.exchange.order( asset=asset, limit_price=0.0005, @@ -33,7 +33,7 @@ class PoloniexTestCase(BaseExchangeTestCase): def test_open_orders(self): log.info('retrieving open orders') - asset = self.exchange.get_asset('neo_btc') + asset = self.exchange.get_asset('neos_btc') orders = self.exchange.get_open_orders(asset) pass @@ -53,13 +53,13 @@ class PoloniexTestCase(BaseExchangeTestCase): log.info('retrieving candles') ohlcv_neo = self.exchange.get_candles( data_frequency='5m', - assets=self.exchange.get_asset('neo_btc') + assets=self.exchange.get_asset('neos_btc') ) ohlcv_neo_ubq = self.exchange.get_candles( data_frequency='5m', assets=[ - self.exchange.get_asset('neo_btc'), - self.exchange.get_asset('ubq_btc') + self.exchange.get_asset('neos_btc'), + self.exchange.get_asset('via_btc') ], bar_count=14 )