diff --git a/catalyst/__init__.py b/catalyst/__init__.py index 508137df..7a8a26a4 100644 --- a/catalyst/__init__.py +++ b/catalyst/__init__.py @@ -29,11 +29,14 @@ from ._version import get_versions from . algorithm import TradingAlgorithm from . import api +from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher + +__version__ = get_versions()['version'] +del get_versions # PERF: Fire a warning if calendars were instantiated during catalyst import. # Having calendars doesn't break anything per-se, but it makes catalyst imports # noticeably slower, which becomes particularly noticeable in the Zipline CLI. -from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher if global_calendar_dispatcher._calendars: import warnings warnings.warn( @@ -44,10 +47,6 @@ if global_calendar_dispatcher._calendars: del global_calendar_dispatcher -__version__ = get_versions()['version'] -del get_versions - - def load_ipython_extension(ipython): from .__main__ import catalyst_magic ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst') @@ -69,7 +68,6 @@ if os.name == 'nt': _() del _ - __all__ = [ 'TradingAlgorithm', 'api', @@ -80,7 +78,3 @@ __all__ = [ 'run_algorithm', 'utils', ] - -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 3a38ce23..47c7dc55 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -10,7 +10,6 @@ from six import text_type from catalyst.data import bundles as bundles_module from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_utils import delete_algo_folder -from catalyst.exchange.factory import get_exchange from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions @@ -520,7 +519,7 @@ def live(ctx, default=False, help='Report potential anomalies found in data bundles.' ) -def ingest_exchange(exchange_name, data_frequency, start, end, +def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ diff --git a/catalyst/algorithm.py b/catalyst/algorithm.py index 8af375b4..1cf9476a 100644 --- a/catalyst/algorithm.py +++ b/catalyst/algorithm.py @@ -124,7 +124,6 @@ from catalyst.utils.events import ( from catalyst.utils.factory import create_simulation_parameters from catalyst.utils.math_utils import ( tolerant_equals, - round_if_near_integer, round_nearest ) from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches @@ -1485,7 +1484,6 @@ class TradingAlgorithm(object): """ Converts the number of shares to the smallest tradable lot size for the asset being ordered. - """ return round_nearest(amount, asset.min_trade_size) @@ -1523,6 +1521,7 @@ class TradingAlgorithm(object): self.updated_portfolio(), self.get_datetime(), self.trading_client.current_data) + @staticmethod def __convert_order_params_for_blotter(limit_price, stop_price, style): """ diff --git a/catalyst/constants.py b/catalyst/constants.py index 6372b11f..c4111fdd 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -7,8 +7,7 @@ import logbook For example, if you want to see the DEBUG messages, run: $ export CATALYST_LOG_LEVEL=10 ''' -# LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) -LOG_LEVEL = logbook.DEBUG +LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' @@ -16,4 +15,4 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' -AUTO_INGEST = False \ No newline at end of file +AUTO_INGEST = False diff --git a/catalyst/curate/poloniex.py b/catalyst/curate/poloniex.py index 2e51e4fb..d09afcc6 100644 --- a/catalyst/curate/poloniex.py +++ b/catalyst/curate/poloniex.py @@ -1,25 +1,33 @@ -import json, time, csv +import os +import time +import shutil +import json +import csv from datetime import datetime + import pandas as pd -import os, time, shutil, requests, logbook +import requests +import logbook + from catalyst.exchange.exchange_utils import get_exchange_symbols_filename -DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())) -DT_END = pd.to_datetime('today').value // 10 ** 9 -CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/') -CONN_RETRIES = 2 +DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())) +DT_END = pd.to_datetime('today').value // 10 ** 9 +CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/') +CONN_RETRIES = 2 logbook.StderrHandler().push_application() log = logbook.Logger(__name__) + class PoloniexCurator(object): ''' OHLCV data feed generator for crypto data. Based on Poloniex market data ''' - _api_path = 'https://poloniex.com/public?' - currency_pairs = [] + _api_path = 'https://poloniex.com/public?' + currency_pairs = [] def __init__(self): if not os.path.exists(CSV_OUT_FOLDER): @@ -30,10 +38,9 @@ class PoloniexCurator(object): CSV_OUT_FOLDER)) log.exception(e) - def get_currency_pairs(self): ''' - Retrieves and returns all currency pairs from the exchange + Retrieves and returns all currency pairs from the exchange ''' url = self._api_path + 'command=returnTicker' @@ -45,7 +52,7 @@ class PoloniexCurator(object): return None data = response.json() - self.currency_pairs = [] + self.currency_pairs = [] for ticker in data: self.currency_pairs.append(ticker) self.currency_pairs.sort() @@ -54,54 +61,60 @@ class PoloniexCurator(object): len(self.currency_pairs) )) - - def _retrieve_tradeID_date(self, row): ''' Helper function that reads tradeID and date fields from CSV readline ''' tId = int(row.split(',')[0]) - d = pd.to_datetime(row.split(',')[1], - infer_datetime_format=True).value // 10 ** 9 + d = pd.to_datetime(row.split(',')[1], + infer_datetime_format=True).value // 10 ** 9 return tId, d - - def retrieve_trade_history(self, currencyPair, start=DT_START, + def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None): ''' - Retrieves TradeHistory from exchange for a given currencyPair - between start and end dates. If no start date is provided, uses + Retrieves TradeHistory from exchange for a given currencyPair + between start and end dates. If no start date is provided, uses a system-wide one (beginning of time for cryptotrading). If no end date is provided, 'now' is used. Stores results in CSV file on disk. - - This function is called recursively to work around the + + This function is called recursively to work around the limitations imposed by the provider API. ''' csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' ''' - Check what data we already have on disk, reading first and last + Check what data we already have on disk, reading first and last lines from file. Data is stored on file from NEWEST to OLDEST. ''' try: - with open(csv_fn, 'ab+') as f: + with open(csv_fn, 'ab+') as f: f.seek(0, os.SEEK_END) if(f.tell() > 2): # Check file size is not 0 - f.seek(0) # Go to start to read - last_tradeID, end_file = self._retrieve_tradeID_date(f.readline()) + f.seek(0) # Go to start to read + last_tradeID, end_file = self._retrieve_tradeID_date( + f.readline()) f.seek(-2, os.SEEK_END) # Jump to the 2nd last byte while f.read(1) != b"\n": # Until EOL is found... - f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. - first_tradeID, start_file = self._retrieve_tradeID_date(f.readline()) + # ...jump back the read byte plus one more. + f.seek(-2, os.SEEK_CUR) + first_tradeID, start_file = self._retrieve_tradeID_date( + f.readline()) - if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1 - or (currencyPair == 'BTC_HUC' and first_tradeID == 2) - or (currencyPair == 'BTC_RIC' and first_tradeID == 2) - or (currencyPair == 'BTC_XCP' and first_tradeID == 2) - or (currencyPair == 'BTC_NAV' and first_tradeID == 4569) - or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ): + if(end_file + 3600 * 6 > DT_END + and (first_tradeID == 1 + or (currencyPair == 'BTC_HUC' + and first_tradeID == 2) + or (currencyPair == 'BTC_RIC' + and first_tradeID == 2) + or (currencyPair == 'BTC_XCP' + and first_tradeID == 2) + or (currencyPair == 'BTC_NAV' + and first_tradeID == 4569) + or (currencyPair == 'BTC_POT' + and first_tradeID == 23511))): return except Exception as e: @@ -109,11 +122,11 @@ class PoloniexCurator(object): log.exception(e) ''' - Poloniex API limits querying TradeHistory to intervals smaller + Poloniex API limits querying TradeHistory to intervals smaller than 1 month, so we make sure that start date is never more than 1 month apart from end date ''' - if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days + if(end - start > 2419200): # 60s/min * 60min/hr * 24hr/day * 28days newstart = end - 2419200 else: newstart = start @@ -124,12 +137,11 @@ class PoloniexCurator(object): url = '{path}command=returnTradeHistory¤cyPair={pair}' \ '&start={start}&end={end}'.format( - path = self._api_path, - pair = currencyPair, - start = str(newstart), - end = str(end) + path=self._api_path, + pair=currencyPair, + start=str(newstart), + end=str(end) ) - print(url) attempts = 0 success = 0 @@ -137,14 +149,14 @@ class PoloniexCurator(object): try: response = requests.get(url) except Exception as e: - log.error('Failed to retrieve trade history data for {}'.format( - currencyPair - )) + log.error('Failed to retrieve trade history data' + 'for {}'.format(currencyPair)) log.exception(e) attempts += 1 else: try: - if isinstance(response.json(), dict) and response.json()['error']: + if(isinstance(response.json(), dict) + and response.json()['error']): log.error('Failed to to retrieve trade history data ' 'for {}: {}'.format( currencyPair, @@ -161,33 +173,32 @@ class PoloniexCurator(object): if not success: return None - ''' - If we get to transactionId == 1, and we already have that on + If we get to transactionId == 1, and we already have that on disk, we got to the end of TradeHistory for this coin. ''' - if('first_tradeID' in locals() - and response.json()[-1]['tradeID'] == first_tradeID): + if('first_tradeID' in locals() + and response.json()[-1]['tradeID'] == first_tradeID): return ''' There are primarily two scenarios: - a) There is newer data available that we need to add at - the beginning of the file. We'll retrieve all what we - need until we get to what we already have, writing it - to a temporary file; and we will write that at the + a) There is newer data available that we need to add at + the beginning of the file. We'll retrieve all what we + need until we get to what we already have, writing it + to a temporary file; and we will write that at the beginning of our existing file. - b) We are going back in time, appending at the end of - our existing TradeHistory until the first transaction + b) We are going back in time, appending at the end of + our existing TradeHistory until the first transaction for this currencyPair ''' - try: - if( 'end_file' in locals() and end_file + 3600 < end): + try: + if('end_file' in locals() and end_file + 3600 < end): if (temp is None): temp = os.tmpfile() tempcsv = csv.writer(temp) for item in response.json(): - if( item['tradeID'] <= last_tradeID ): + if(item['tradeID'] <= last_tradeID): continue tempcsv.writerow([ item['tradeID'], @@ -196,27 +207,28 @@ class PoloniexCurator(object): item['rate'], item['amount'], item['total'], - item['globalTradeID'] + item['globalTradeID'], ]) - if( response.json()[-1]['tradeID'] > last_tradeID ): - end = pd.to_datetime( response.json()[-1]['date'], - infer_datetime_format=True).value // 10 ** 9 - self.retrieve_trade_history(currencyPair, start, - end, temp=temp) + if(response.json()[-1]['tradeID'] > last_tradeID): + end = pd.to_datetime(response.json()[-1]['date'], + infer_datetime_format=True + ).value // 10**9 + self.retrieve_trade_history(currencyPair, start, + end, temp=temp) else: - with open(csv_fn,'rb+') as f: - shutil.copyfileobj(f,temp) + with open(csv_fn, 'rb+') as f: + shutil.copyfileobj(f, temp) f.seek(0) temp.seek(0) - shutil.copyfileobj(temp,f) + shutil.copyfileobj(temp, f) temp.close() end = start_file else: with open(csv_fn, 'ab') as csvfile: csvwriter = csv.writer(csvfile) for item in response.json(): - if( 'first_tradeID' in locals() - and item['tradeID'] >= first_tradeID ): + if('first_tradeID' in locals() + and item['tradeID'] >= first_tradeID): continue csvwriter.writerow([ item['tradeID'], @@ -227,70 +239,66 @@ class PoloniexCurator(object): item['total'], item['globalTradeID'] ]) - end = pd.to_datetime(response.json()[-1]['date'], - infer_datetime_format=True).value // 10 ** 9 + end = pd.to_datetime(response.json()[-1]['date'], + infer_datetime_format=True).value//10**9 except Exception as e: log.error('Error opening {}'.format(csv_fn)) log.exception(e) ''' - If we got here, we aren't done yet. Call recursively with + If we got here, we aren't done yet. Call recursively with 'end' times that go sequentially back in time. ''' self.retrieve_trade_history(currencyPair, start, end) - - def generate_ohlcv(self, df): ''' Generates OHLCV dataframe from a dataframe containing all TradeHistory by resampling with 1-minute period ''' - df.set_index('date', inplace=True) # Index by date - vol = df['total'].to_frame('volume') # set Vol aside - df.drop('total', axis=1, inplace=True) # Drop volume data - ohlc = df.resample('T').ohlc() # Resample OHLC 1min - ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate' - closes = ohlc['close'].fillna(method='pad') # Pad fwd missing 'close' - ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close - vol = vol.resample('T').sum().fillna(0) # Add volumes by bin - ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Vol + df.set_index('date', inplace=True) # Index by date + vol = df['total'].to_frame('volume') # set Vol aside + df.drop('total', axis=1, inplace=True) # Drop volume data + ohlc = df.resample('T').ohlc() # Resample OHLC 1min + ohlc.cols = ohlc.cols.map(lambda t: t[1]) # Raname cols + closes = ohlc['close'].fillna(method='pad') # Pad fwd missing close + ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill NA w/ last close + vol = vol.resample('T').sum().fillna(0) # Add volumes by bin + ohlcv = pd.concat([ohlc, vol], axis=1) # Concat OHLC + Vol return ohlcv - - - def write_ohlcv_file(self, currencyPair): + def write_ohlcv_file(self, currencyPair): ''' Generates OHLCV data file with 1minute bars from TradeHistory on disk - ''' + ''' csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' - csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' - if( os.path.getmtime(csv_1min) > time.time() - 7200 ): + csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' + if(os.path.getmtime(csv_1min) > time.time() - 7200): log.debug(currencyPair+': 1min data file already up to date. ' 'Delete the file if you want to rebuild it.') else: - df = pd.read_csv(csv_trades, - names=['tradeID', - 'date', - 'type', - 'rate', - 'amount', - 'total', - 'globalTradeID'], - dtype = {'tradeID': int, - 'date': str, - 'type': str, - 'rate': float, - 'amount': float, - 'total': float, - 'globalTradeID': int } - ) - df.drop(['tradeID','type','amount','globalTradeID'], + df = pd.read_csv(csv_trades, + names=['tradeID', + 'date', + 'type', + 'rate', + 'amount', + 'total', + 'globalTradeID'], + dtype={'tradeID': int, + 'date': str, + 'type': str, + 'rate': float, + 'amount': float, + 'total': float, + 'globalTradeID': int} + ) + df.drop(['tradeID', 'type', 'amount', 'globalTradeID'], axis=1, inplace=True) - df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) + df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) ohlcv = self.generate_ohlcv(df) - try: + try: with open(csv_1min, 'w') as csvfile: csvwriter = csv.writer(csvfile) for item in ohlcv.itertuples(): @@ -305,32 +313,28 @@ class PoloniexCurator(object): item.volume, ]) except Exception as e: - log.error('Error opening {}'.format(csv_fn)) + log.error('Error opening {}'.format(csv_1min)) log.exception(e) log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair)) - - def onemin_to_dataframe(self, currencyPair, start, end): ''' Returns a data frame for a given currencyPair from data on disk ''' - csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' - df = pd.read_csv(csv_fn, names=['date', - 'open', - 'high', - 'low', - 'close', - 'volume'] - ) - df['date'] = pd.to_datetime(df['date'],unit='s') + csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' + df = pd.read_csv(csv_fn, names=['date', + 'open', + 'high', + 'low', + 'close', + 'volume']) + df['date'] = pd.to_datetime(df['date'], unit='s') df.set_index('date', inplace=True) - return df[start : end] - + return df[start:end] def generate_symbols_json(self, filename=None): ''' - Generates a symbols.json file with corresponding start_date + Generates a symbols.json file with corresponding start_date for each currencyPair ''' symbol_map = {} @@ -341,36 +345,37 @@ class PoloniexCurator(object): with open(filename, 'w') as symbols: for currencyPair in self.currency_pairs: start = None - csv_fn = '{}crypto_trades-{}.csv'.format( - CSV_OUT_FOLDER, currencyPair) - with open(csv_fn, 'r') as f: + csv_fn = '{}crypto_trades-{}.csv'.format( + CSV_OUT_FOLDER, + currencyPair) + with open(csv_fn, 'r') as f: f.seek(0, os.SEEK_END) if(f.tell() > 2): # Check file size is not 0 f.seek(-2, os.SEEK_END) # Jump to 2nd last byte while f.read(1) != b"\n": # Until EOL is found... - f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. - start = pd.to_datetime( f.readline().split(',')[1], - infer_datetime_format=True) + # ...jump back the read byte plus one more. + f.seek(-2, os.SEEK_CUR) + start = pd.to_datetime(f.readline().split(',')[1], + infer_datetime_format=True) if(start is None): start = time.gmtime() base, market = currencyPair.lower().split('_') - symbol = '{market}_{base}'.format( market=market, base=base ) + symbol = '{market}_{base}'.format(market=market, base=base) symbol_map[currencyPair] = dict( - symbol = symbol, - start_date = start.strftime("%Y-%m-%d") + symbol=symbol, + start_date=start.strftime("%Y-%m-%d") ) - json.dump(symbol_map, symbols, sort_keys=True, indent=2, - separators=(',',':')) + json.dump(symbol_map, symbols, sort_keys=True, indent=2, + separators=(',', ':')) if __name__ == '__main__': pc = PoloniexCurator() pc.get_currency_pairs() - #pc.generate_symbols_json() - + # pc.generate_symbols_json() + for currencyPair in pc.currency_pairs: pc.retrieve_trade_history(currencyPair) log.debug('{} up to date.'.format(currencyPair)) pc.write_ohlcv_file(currencyPair) - diff --git a/catalyst/data/bundles/__init__.py b/catalyst/data/bundles/__init__.py index 17343b9c..b8e463dd 100644 --- a/catalyst/data/bundles/__init__.py +++ b/catalyst/data/bundles/__init__.py @@ -1,6 +1,5 @@ # These imports are necessary to force module-scope register calls to happen. from . import quandl # noqa -from . import poloniex from .core import ( UnknownBundle, bundles, diff --git a/catalyst/data/bundles/base.py b/catalyst/data/bundles/base.py index 73b5058b..c3700003 100644 --- a/catalyst/data/bundles/base.py +++ b/catalyst/data/bundles/base.py @@ -13,10 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. - from itertools import count import tarfile -from time import time, sleep +from time import sleep from abc import abstractmethod, abstractproperty import logbook @@ -37,6 +36,7 @@ log = logbook.Logger(__name__, level=LOG_LEVEL) DEFAULT_RETRIES = 5 + class BaseBundle(object): def __init__(self, asset_filter=[]): self._asset_filter = asset_filter @@ -104,11 +104,11 @@ class BaseBundle(object): def post_process_symbol_metadata(self, metadata, data): return metadata - + @abstractmethod def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date): raise NotImplementedError() - + def ingest(self, environ, asset_db_writer, @@ -128,7 +128,7 @@ class BaseBundle(object): retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5) if is_compile: - # User has instructed local compilation and ingestion of bundle. + # User has instructed local compilation & ingestion of bundle. # Fetch raw metadata for all symbols. raw_metadata = self._fetch_metadata_frame( api_key, @@ -157,9 +157,9 @@ class BaseBundle(object): show_progress=show_progress, ) - # Post-process metadata using cached symbol frames, and write to - # disk. This metadata must be written before any attempt to write - # minute data. + # Post-process metadata using cached symbol frames, and write + # to disk. This metadata must be written before any attempt + # to write minute data. metadata = self._post_process_metadata( raw_metadata, cache, @@ -184,10 +184,11 @@ class BaseBundle(object): show_progress=show_progress, ) - # For legacy purposes, this call is required to ensure the database - # contains an appropriately initialized file structure. We don't - # forsee a usecase for adjustments at this time, but may later - # choose to expose this functionality in the future. + # For legacy purposes, this call is required to ensure the + # database contains an appropriately initialized file + # structure. We don't forsee a usecase for adjustments at + # this time, but may later choose to expose this functionality + # in the future. adjustment_writer.write( splits=( pd.concat(self.splits, ignore_index=True) @@ -232,12 +233,12 @@ class BaseBundle(object): tar.extractall(output_dir) def _fetch_metadata_frame(self, - api_key, - cache, - retries=DEFAULT_RETRIES, - environ=None, - show_progress=False): - + api_key, + cache, + retries=DEFAULT_RETRIES, + environ=None, + show_progress=False): + # Setup raw metadata iterator to fetch pages if necessary. raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ) @@ -251,7 +252,7 @@ class BaseBundle(object): show_percent=False, ) as blocks: metadata = pd.concat(blocks, ignore_index=True) - + return metadata def _fetch_metadata_iter(self, api_key, cache, retries, environ): @@ -269,21 +270,20 @@ class BaseBundle(object): page_number, ) break - except ValueError as e: + except ValueError: raw = pd.DataFrame([]) break - except Exception as e: + except Exception: log.exception( 'Failed to load metadata from {}. ' 'Retrying.'.format(self.name) - ) + ) else: raise ValueError( 'Failed to download metadata page {} after {} ' 'attempts.'.format(page_number, retries) ) - if raw.empty: # Empty DataFrame signals completion. break @@ -305,7 +305,7 @@ class BaseBundle(object): columns=self.md_column_names, index=metadata.index, ) - + # Iterate over the available symbols, loading the asset's raw symbol # data from the cache. The final metadata is computed and recorded in # the appropriate row depending on the asset's id. @@ -318,22 +318,22 @@ class BaseBundle(object): show_percent=False, ) as symbols_map: for asset_id, symbol in symbols_map: - # Attempt to load data from disk, the cache should have an entry - # for each symbol at this point of the execution. If one does - # not exist, we should fail. + # Attempt to load data from disk, the cache should have an + # entry for each symbol at this point of the execution. If one + # does not exist, we should fail. key = '{sym}.daily.frame'.format(sym=symbol) try: raw_data = cache[key] except KeyError: raise ValueError( - 'Unable to find cached data for symbol: {0}'.format(symbol) - ) + 'Unable to find cached data for symbol:' + ' {0}'.format(symbol)) # Perform and require post-processing of metadata. final_symbol_metadata = self.post_process_symbol_metadata( asset_id, metadata.iloc[asset_id], - raw_data, + raw_data, ) # Record symbol's final metadata. @@ -363,8 +363,8 @@ class BaseBundle(object): # returns the cached data unaltered. The `should_sleep` flag # indicates that an API call was attempted, and that we should be # ensure aren't exceeding our rate limit before proceeding to the - # next symbol. If the raw_data is updated, it is cached before being - # returned. + # next symbol. If the raw_data is updated, it is cached before + # being returned. raw_data, should_sleep = self._maybe_update_symbol_frame( start_time, api_key, @@ -414,7 +414,7 @@ class BaseBundle(object): last = start_session if raw_data is not None and len(raw_data) > 0: last = raw_data.index[-1].tz_localize('UTC') - + should_sleep = False # Determine time at which cached data will be considered stale. @@ -455,7 +455,7 @@ class BaseBundle(object): retries=DEFAULT_RETRIES): # Data for symbol is old enough to attempt an update or is not - # present in the cache. Fetch raw data for a single symbol + # present in the cache. Fetch raw data for a single symbol # with requested intervals and frequency. Retry as necessary. for _ in range(retries): try: @@ -468,7 +468,6 @@ class BaseBundle(object): data_frequency, ) raw_data.index = pd.to_datetime(raw_data.index, utc=True) - #raw_data.index = raw_data.index.tz_localize('UTC') # Filter incoming data to fit start and end sessions. raw_data = raw_data[ @@ -482,7 +481,7 @@ class BaseBundle(object): return raw_data - except Exception as e: + except Exception: log.exception( 'Exception raised fetching {name} data. Retrying.' .format(name=self.name) diff --git a/catalyst/data/bundles/base_pricing.py b/catalyst/data/bundles/base_pricing.py index 7b94e4bc..ecb9b778 100644 --- a/catalyst/data/bundles/base_pricing.py +++ b/catalyst/data/bundles/base_pricing.py @@ -16,6 +16,7 @@ from catalyst.data.bundles.base import BaseBundle from catalyst.utils.memoize import lazyval + class BasePricingBundle(BaseBundle): @lazyval def md_dtypes(self): @@ -38,6 +39,7 @@ class BasePricingBundle(BaseBundle): ('volume', 'float64'), ] + class BaseCryptoPricingBundle(BasePricingBundle): @lazyval def calendar_name(self): @@ -55,6 +57,7 @@ class BaseCryptoPricingBundle(BasePricingBundle): def dividends(self): return [] + class BaseEquityPricingBundle(BasePricingBundle): @lazyval def calendar_name(self): diff --git a/catalyst/data/bundles/core.py b/catalyst/data/bundles/core.py index a25591de..58160ab1 100644 --- a/catalyst/data/bundles/core.py +++ b/catalyst/data/bundles/core.py @@ -37,6 +37,7 @@ from catalyst.utils.cli import maybe_show_progress ONE_MEGABYTE = 1024 * 1024 + def asset_db_path(bundle_name, timestr, environ=None, db_version=None): return pth.data_path( asset_db_relative(bundle_name, timestr, environ, db_version), @@ -135,6 +136,7 @@ def ingestions_for_bundle(bundle, environ=None): reverse=True, ) + def download_with_progress(url, chunk_size, **progress_kwargs): """ Download streaming data from a URL, printing progress information to the @@ -705,4 +707,5 @@ def _make_bundle_core(): ) -bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core() +bundles, register_bundle, register, unregister, ingest, load, clean = \ + _make_bundle_core() diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index 64cc2d27..6428dbb0 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -14,19 +14,17 @@ # limitations under the License. import sys - -from datetime import datetime +from six.moves.urllib.parse import urlencode import pandas as pd -from six.moves.urllib.parse import urlencode - from catalyst.data.bundles.core import register_bundle from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle from catalyst.utils.memoize import lazyval from catalyst.curate.poloniex import PoloniexCurator + class PoloniexBundle(BaseCryptoPricingBundle): @lazyval def name(self): @@ -46,7 +44,8 @@ class PoloniexBundle(BaseCryptoPricingBundle): @lazyval def tar_url(self): return ( - 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz' + 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' + 'poloniex/poloniex-bundle.tar.gz' ) @lazyval @@ -67,12 +66,11 @@ class PoloniexBundle(BaseCryptoPricingBundle): raw = raw.sort_index().reset_index() raw.rename( - columns={'index':'symbol'}, + columns={'index': 'symbol'}, inplace=True, ) raw = raw[raw['isFrozen'] == 0] - return raw def post_process_symbol_metadata(self, asset_id, sym_md, sym_data): @@ -98,7 +96,8 @@ class PoloniexBundle(BaseCryptoPricingBundle): frequency): # TODO: replace this with direct exchange call - # The end date and frequency should be used to calculate the number of bars + # The end date and frequency should be used to + # calculate the number of bars if(frequency == 'minute'): pc = PoloniexCurator() raw = pc.onemin_to_dataframe(symbol, start_date, end_date) @@ -116,8 +115,9 @@ class PoloniexBundle(BaseCryptoPricingBundle): ) raw.set_index('date', inplace=True) - # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored - # on disk, which we compensate here to get the right pricing amounts + # BcolzDailyBarReader introduces a 1/1000 factor in the way + # pricing is stored on disk, which we compensate here to get + # the right pricing amounts # ref: data/us_equity_pricing.py scale = 1 raw.loc[:, 'open'] /= scale @@ -139,7 +139,6 @@ class PoloniexBundle(BaseCryptoPricingBundle): return self._format_polo_query(query_params) - def _format_data_url(self, api_key, symbol, @@ -162,27 +161,26 @@ class PoloniexBundle(BaseCryptoPricingBundle): ('end', end_date.value / 10**9), ('period', period), ] - + return self._format_polo_query(query_params) - + def _format_polo_query(self, query_params): # TODO: got against the exchange object return 'https://poloniex.com/public?{query}'.format( query=urlencode(query_params), ) -''' -As a second parameter, you can pass an array of currency pairs -that will be processed as an asset_filter to only process that + +''' +As a second parameter, you can pass an array of currency pairs +that will be processed as an asset_filter to only process that subset of assets in the bundle, such as: register_bundle(PoloniexBundle, ['USDT_BTC',]) For a production environment make sure to use (to bundle all pairs): register_bundle(PoloniexBundle) ''' - if 'ingest' in sys.argv and '-c' in sys.argv: register_bundle(PoloniexBundle) else: register_bundle(PoloniexBundle, create_writers=False) - diff --git a/catalyst/data/bundles/quandl.py b/catalyst/data/bundles/quandl.py index 1255449c..79e3eba8 100644 --- a/catalyst/data/bundles/quandl.py +++ b/catalyst/data/bundles/quandl.py @@ -16,7 +16,6 @@ from datetime import datetime import pandas as pd - from six.moves.urllib.parse import urlencode from catalyst.data.bundles.core import register_bundle @@ -26,25 +25,16 @@ from catalyst.utils.memoize import lazyval """ Module for building a complete daily dataset from Quandl's WIKI dataset. """ -from itertools import count -import tarfile -from time import time, sleep -from datetime import datetime - from logbook import Logger -import pandas as pd -from six.moves.urllib.parse import urlencode - -from catalyst.utils.calendars import register_calendar_alias -from catalyst.utils.cli import maybe_show_progress - -from . import core as bundles from catalyst.constants import LOG_LEVEL +from catalyst.utils.calendars import register_calendar_alias + log = Logger(__name__, level=LOG_LEVEL) seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds() + class QuandlBundle(BaseEquityPricingBundle): @lazyval def name(self): @@ -109,8 +99,8 @@ class QuandlBundle(BaseEquityPricingBundle): # Filter out invalid symbols raw = raw[~raw.symbol.isin(self._excluded_symbols)] - # cut out all the other stuff in the name column - # we need to escape the paren because it is actually splitting on a regex + # cut out all the other stuff in the name column. We need to + # escape the paren because it is actually splitting on a regex raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0) return raw @@ -175,7 +165,6 @@ class QuandlBundle(BaseEquityPricingBundle): df['sid'] = asset_id self.splits.append(df) - def _update_dividends(self, asset_id, raw_data): divs = raw_data.ex_dividend df = pd.DataFrame({'amount': divs[divs != 0]}) @@ -186,7 +175,6 @@ class QuandlBundle(BaseEquityPricingBundle): df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT self.dividends.append(df) - def _format_metadata_url(self, api_key, page_number): """Build the query RL for the quandl WIKI metadata. """ @@ -200,10 +188,10 @@ class QuandlBundle(BaseEquityPricingBundle): query_params = [('api_key', api_key)] + query_params return ( - 'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params) + 'https://www.quandl.com/api/v3/datasets.csv?' + + urlencode(query_params) ) - def _format_wiki_url(self, api_key, symbol, @@ -229,5 +217,6 @@ class QuandlBundle(BaseEquityPricingBundle): ) ) + register_calendar_alias('QUANDL', 'NYSE') register_bundle(QuandlBundle) diff --git a/catalyst/data/data_portal.py b/catalyst/data/data_portal.py index 1f7ec6b3..34c720d0 100644 --- a/catalyst/data/data_portal.py +++ b/catalyst/data/data_portal.py @@ -656,11 +656,11 @@ class DataPortal(object): return spot_value def _get_minutely_spot_value(self, - asset, - column, - dt, - data_frequency, - ffill=False): + asset, + column, + dt, + data_frequency, + ffill=False): reader = self._get_pricing_reader(data_frequency) @@ -706,7 +706,7 @@ class DataPortal(object): asset, column, dt, - ffill, + ffill, 'minute', ) diff --git a/catalyst/data/dispatch_bar_reader.py b/catalyst/data/dispatch_bar_reader.py index 7dfd7e95..a8e7429b 100644 --- a/catalyst/data/dispatch_bar_reader.py +++ b/catalyst/data/dispatch_bar_reader.py @@ -133,11 +133,13 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)): return results + class AssetDispatchMinuteBarReader(AssetDispatchBarReader): def _dt_window_size(self, start_dt, end_dt): return len(self.trading_calendar.minutes_in_range(start_dt, end_dt)) + class AssetDispatchSessionBarReader(AssetDispatchBarReader): def _dt_window_size(self, start_dt, end_dt): diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index e58127f7..b9227b3e 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import datetime import os from collections import OrderedDict @@ -129,11 +128,13 @@ def load_crypto_market_data(trading_day=None, trading_days=None, # before this date. ''' if(bundle_data): - # If we are using the bundle to retrieve the cryptobenchmark, find the last - # date for which there is trading data in the bundle - asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None) + # If we are using the bundle to retrieve the cryptobenchmark, find + # the last date for which there is trading data in the bundle + asset = bundle_data.asset_finder.lookup_symbol( + symbol=bm_symbol,as_of_date=None) ix = bundle_data.daily_bar_reader._last_rows[asset.sid] - last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s') + last_date = pd.to_datetime( + bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s') else: last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2] ''' @@ -164,8 +165,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None, br.loc[start_dt] = 0 br = br.sort_index() - # Override first_date for treasury data since we have it for many more years - # and is independent of crypto data + # Override first_date for treasury data since we have it for many more + # years and is independent of crypto data first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC') tc = ensure_treasury_data( bm_symbol, @@ -301,14 +302,14 @@ def ensure_crypto_benchmark_data(symbol, if (bundle == 'poloniex'): ''' - If we're using the Poloniex bundle, we'll get the benchmark from the bundle - instead of downloading it from Poloniex every time we need it. - Poloniex has a captcha for API queries originating from outside the US that - prevents users abroad from getting Catalyst to work + If we're using the Poloniex bundle, we'll get the benchmark from the + bundle instead of downloading it from Poloniex every time we need it. + Poloniex has a captcha for API queries originating from outside the US + that prevents users abroad from getting Catalyst to work ''' logger.info( - ( - 'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'), + ('Retrieving benchmark data from bundle for {symbol!r}' + ' from {first_date} to {last_date}'), symbol=symbol, first_date=first_date, last_date=last_date) asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol, @@ -330,11 +331,12 @@ def ensure_crypto_benchmark_data(symbol, last_date)] else: - # This is how it used to be: downloading the benchmark everytime. - # Leaving this code here to be repurposed in the future for other bundles. + # This is how it used to be: downloading the benchmark everytime. + # Leaving this code here to be repurposed in the future for + # other bundles. logger.info( - ( - 'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'), + ('Downloading benchmark data for {symbol!r}' + ' from {first_date} to {last_date}'), symbol=symbol, first_date=first_date, last_date=last_date) raise DeprecationWarning('poloniex bundle deprecated') @@ -431,67 +433,6 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day, return data -def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day, - environ=None): - """ - Ensure we have benchmark data for `symbol` from `first_date` to `last_date` - - Parameters - ---------- - symbol : str - The symbol for the benchmark to load. - first_date : pd.Timestamp - First required date for the cache. - last_date : pd.Timestamp - Last required date for the cache. - now : pd.Timestamp - The current time. This is used to prevent repeated attempts to - re-download data that isn't available due to scheduling quirks or other - failures. - trading_day : pd.CustomBusinessDay - A trading day delta. Used to find the day before first_date so we can - get the close of the day prior to first_date. - - We attempt to download data unless we already have data stored at the data - cache for `symbol` whose first entry is before or on `first_date` and whose - last entry is on or after `last_date`. - - If we perform a download and the cache criteria are not satisfied, we wait - at least one hour before attempting a redownload. This is determined by - comparing the current time to the result of os.path.getmtime on the cache - path. - """ - filename = get_benchmark_filename(symbol) - data = _load_cached_data(filename, first_date, last_date, now, 'benchmark', - environ) - if data is not None: - return data - - # If no cached data was found or it was missing any dates then download the - # necessary data. - logger.info( - ('Downloading benchmark data for {symbol!r} ' - 'from {first_date} to {last_date}'), - symbol=symbol, - first_date=first_date - trading_day, - last_date=last_date - ) - - try: - data = get_benchmark_returns( - symbol, - first_date - trading_day, - last_date, - ) - data.to_csv(get_data_filepath(filename, environ)) - except (OSError, IOError, HTTPError): - logger.exception('Failed to cache the new benchmark returns') - raise - if not has_data_for_dates(data, first_date, last_date): - logger.warn("Still don't have expected data after redownload!") - return data - - def ensure_treasury_data(symbol, first_date, last_date, now, environ=None): """ Ensure we have treasury data from treasury module associated with diff --git a/catalyst/data/minute_bars.py b/catalyst/data/minute_bars.py index b2491a5a..3ef13cc2 100644 --- a/catalyst/data/minute_bars.py +++ b/catalyst/data/minute_bars.py @@ -341,12 +341,10 @@ class BcolzMinuteBarMetadata(object): 'end_session': str(self.end_session.date()), # Write these values for backwards compatibility 'first_trading_day': str(self.start_session.date()), - 'market_opens': ( - market_opens.values.astype('datetime64[m]'). - astype(np.int64).tolist()), - 'market_closes': ( - market_closes.values.astype('datetime64[m]'). - astype(np.int64).tolist()), + 'market_opens': (market_opens.values.astype('datetime64[m]'). + astype(np.int64).tolist()), + 'market_closes': (market_closes.values.astype('datetime64[m]'). + astype(np.int64).tolist()), } with open(self.metadata_path(rootdir), 'w+') as fp: json.dump(metadata, fp) @@ -1256,8 +1254,8 @@ class BcolzMinuteBarReader(MinuteBarReader): values = carray[start_idx:end_idx + 1] if indices_to_exclude is not None: for excl_start, excl_stop in indices_to_exclude[::-1]: - excl_slice = np.s_[ - excl_start - start_idx:excl_stop - start_idx + 1] + excl_slice = np.s_[excl_start - start_idx:excl_stop + - start_idx + 1] values = np.delete(values, excl_slice) where = values != 0 @@ -1320,9 +1318,8 @@ class H5MinuteBarUpdateWriter(object): def __init__(self, path, complevel=None, complib=None): self._complevel = complevel if complevel \ - is not None else self._COMPLEVEL - self._complib = complib if complib \ - is not None else self._COMPLIB + is not None else self._COMPLEVEL + self._complib = complib if complib is not None else self._COMPLIB self._path = path def write(self, frames): diff --git a/catalyst/data/us_equity_pricing.py b/catalyst/data/us_equity_pricing.py index 8e74ab43..8bb8427b 100644 --- a/catalyst/data/us_equity_pricing.py +++ b/catalyst/data/us_equity_pricing.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import division # Python2 req to have division of ints yield float +from __future__ import division # Python2 req for division of ints yield float from errno import ENOENT from functools import partial @@ -120,7 +120,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = { UINT32_MAX = iinfo(uint32).max UINT64_MAX = iinfo(uint64).max -PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220 +# Provides 9 decimals resolution. Also affects _equities.pyx L220 +PRICE_ADJUSTMENT_FACTOR = 1000000000 def check_uint32_safe(value, colname): @@ -130,6 +131,7 @@ def check_uint32_safe(value, colname): "for uint32" % (value, colname) ) + def check_uint64_safe(value, colname): if value >= UINT64_MAX: raise ValueError( @@ -322,8 +324,8 @@ class BcolzDailyBarWriter(object): # Maps column name -> output carray. columns = { k: carray(array([], dtype=uint64)) - if k in OHLCV - else carray(array([], dtype=uint32)) + if k in OHLCV + else carray(array([], dtype=uint32)) for k in US_EQUITY_PRICING_BCOLZ_COLUMNS } @@ -439,11 +441,13 @@ class BcolzDailyBarWriter(object): return raw_data winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC) - processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64') + processed = (raw_data[list(OHLC)] + * PRICE_ADJUSTMENT_FACTOR).astype('uint64') dates = raw_data.index.values.astype('datetime64[s]') check_uint32_safe(dates.max().view(np.int64), 'day') processed['day'] = dates.astype('uint32') - processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64') + processed['volume'] = (raw_data.volume + * PRICE_ADJUSTMENT_FACTOR).astype('uint64') return ctable.fromdataframe(processed) @@ -496,7 +500,7 @@ class BcolzDailyBarReader(SessionBarReader): The data in these columns is interpreted as follows: - - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted + - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted as 10^9 * as-traded dollar value. - Day is interpreted as seconds since midnight UTC, Jan 1, 1970. - Id is the asset id of the row. diff --git a/catalyst/examples/buy_and_hodl.py b/catalyst/examples/buy_and_hodl.py index 74b04238..6722665f 100644 --- a/catalyst/examples/buy_and_hodl.py +++ b/catalyst/examples/buy_and_hodl.py @@ -19,7 +19,7 @@ import matplotlib.pyplot as plt from catalyst import run_algorithm from catalyst.api import (order_target_value, symbol, record, - cancel_order, get_open_orders, ) + cancel_order, get_open_orders, ) def initialize(context): diff --git a/catalyst/examples/buy_btc_simple.py b/catalyst/examples/buy_btc_simple.py index 5e4ebd57..52463d58 100644 --- a/catalyst/examples/buy_btc_simple.py +++ b/catalyst/examples/buy_btc_simple.py @@ -1,30 +1,34 @@ ''' - This is a very simple example referenced in the beginner's tutorial: - https://enigmampc.github.io/catalyst/beginner-tutorial.html + This is a very simple example referenced in the beginner's tutorial: + https://enigmampc.github.io/catalyst/beginner-tutorial.html - Run this example, by executing the following from your terminal: - catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle + Run this example, by executing the following from your terminal: + catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt + catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \ + --end 2017-9-30 -o buy_btc_simple_out.pickle - If you want to run this code using another exchange, make sure that - the asset is available on that exchange. For example, if you were to run - it for exchange Poloniex, you would need to edit the following line: + If you want to run this code using another exchange, make sure that + the asset is available on that exchange. For example, if you were to run + it for exchange Poloniex, you would need to edit the following line: - context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' + context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' - and specify exchange poloniex as follows: - catalyst ingest-exchange -x poloniex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle + and specify exchange poloniex as follows: + catalyst ingest-exchange -x poloniex -f daily -i btc_usdt + catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \ + --end 2017-9-30 -o buy_btc_simple_out.pickle - To see which assets are available on each exchange, visit: - https://www.enigma.co/catalyst/status + To see which assets are available on each exchange, visit: + https://www.enigma.co/catalyst/status ''' from catalyst.api import order, record, symbol + def initialize(context): context.asset = symbol('btc_usd') + def handle_data(context, data): order(context.asset, 1) - record(btc = data.current(context.asset, 'price')) \ No newline at end of file + record(btc=data.current(context.asset, 'price')) diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index acf481e0..b3afb723 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -1,12 +1,13 @@ ''' -This algorithm requires an additional library (ta-lib) beyond those required by catalyst. -Install it first by running: +This algorithm requires an additional library (ta-lib) beyond those +required by catalyst. Install it first by running: $ pip install TA-Lib -If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory" -it typically means that it can't find the underlying TA-Lib library and needs to be installed. -See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install -the required dependencies. +If you get build errors like: + "fatal error: ta-lib/ta_libc.h: No such file or directory" +it typically means that it can't find the underlying TA-Lib library and it +needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for +instructions on how to install the required dependencies. ''' import talib @@ -100,8 +101,8 @@ def _handle_data(context, data): if price < cost_basis: is_buy = True - elif position.amount > 0 and \ - price > cost_basis * (1 + context.PROFIT_TARGET): + elif(position.amount > 0 + and price > cost_basis * (1 + context.PROFIT_TARGET)): profit = (price * position.amount) - (cost_basis * position.amount) log.info('closing position, taking profit: {}'.format(profit)) order_target_percent( diff --git a/catalyst/examples/buy_low_sell_high_live.py b/catalyst/examples/buy_low_sell_high_live.py index a1fd326c..422033a6 100644 --- a/catalyst/examples/buy_low_sell_high_live.py +++ b/catalyst/examples/buy_low_sell_high_live.py @@ -88,8 +88,8 @@ def _handle_data(context, data): if price < cost_basis: is_buy = True - elif position.amount > 0 and \ - price > cost_basis * (1 + context.PROFIT_TARGET): + elif(position.amount > 0 + and price > cost_basis * (1 + context.PROFIT_TARGET)): profit = (price * position.amount) - (cost_basis * position.amount) log.info('closing position, taking profit: {}'.format(profit)) order_target_percent( diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index a73461f1..f54d91b6 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -4,13 +4,14 @@ from logbook import Logger import matplotlib.pyplot as plt from catalyst import run_algorithm -from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) +from catalyst.api import (record, symbol, order_target_percent, + get_open_orders) from catalyst.exchange.stats_utils import extract_transactions NAMESPACE = 'dual_moving_average' log = Logger(NAMESPACE) + def initialize(context): context.i = 0 context.asset = symbol('ltc_usd') @@ -25,16 +26,22 @@ def handle_data(context, data): # Skip as many bars as long_window to properly compute the average context.i += 1 if context.i < long_window: - return + return # Compute moving averages calling data.history() for each # moving average with the appropriate parameters. We choose to use # minute bars for this simulation -> freq="1m" # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() + short_mavg = data.history(context.asset, + 'price', + bar_count=short_window, + frequency="1m", + ).mean() + long_mavg = data.history(context.asset, + 'price', + bar_count=long_window, + frequency="1m", + ).mean() # Let's keep the price of our asset in a more handy variable price = data.current(context.asset, 'price') @@ -67,11 +74,11 @@ def handle_data(context, data): # Trading logic if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) + # we buy 100% of our portfolio for this asset + order_target_percent(context.asset, 1) elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) + # we sell all our positions for this asset + order_target_percent(context.asset, 0) def analyze(context, perf): @@ -89,11 +96,13 @@ def analyze(context, perf): # Second chart: Plot asset price, moving averages and buys/sells ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') + perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot( + ax=ax2, + label='Price') ax2.legend_.remove() ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency + asset=context.asset.symbol, + base=base_currency )) start, end = ax2.get_ylim() ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) @@ -150,4 +159,4 @@ if __name__ == '__main__': base_currency='usd', start=pd.to_datetime('2017-9-22', utc=True), end=pd.to_datetime('2017-9-23', utc=True), - ) \ No newline at end of file + ) diff --git a/catalyst/examples/dual_vwap.py b/catalyst/examples/dual_vwap.py index 52c1789d..7059b865 100644 --- a/catalyst/examples/dual_vwap.py +++ b/catalyst/examples/dual_vwap.py @@ -52,13 +52,14 @@ def initialize(context): schedule_function( rebalance, - time_rules=times_rules.every_minute(), + time_rules=date_rules.every_minute(), ) def before_trading_start(context, data): context.pipeline_data = pipeline_output('vwap_pipeline') + def make_pipeline(context): return Pipeline( columns={ @@ -69,6 +70,7 @@ def make_pipeline(context): } ) + def rebalance(context, data): context.i += 1 @@ -111,7 +113,6 @@ def rebalance(context, data): long_mavg=long_mavg, volume=volume, ) - def analyze(context=None, results=None): @@ -124,10 +125,11 @@ def analyze(context=None, results=None): ax2 = plt.subplot(612, sharex=ax1) ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME)) - (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2) + (context.TICK_SIZE*results[['price', + 'short_mavg', + 'long_mavg']]).plot(ax=ax2) trans = results.ix[[t != [] for t in results.transactions]] - amounts = [t[0]['amount'] for t in trans.transactions] buys = trans.ix[ [t[0]['amount'] > 0 for t in trans.transactions] diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 450a73f7..4508c8e5 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -1,4 +1,4 @@ -# For this example, we're going to write a simple momentum script. When the +# For this example, we're going to write a simple momentum script. When the # stock goes up quickly, we're going to buy; when it goes down quickly, we're # going to sell. Hopefully we'll ride the waves. import os @@ -12,8 +12,8 @@ from logbook import Logger from catalyst import run_algorithm from catalyst.api import symbol, record, order_target_percent, get_open_orders -from catalyst.exchange.stats_utils import extract_transactions, \ - get_pretty_stats +from catalyst.exchange.stats_utils import extract_transactions + # We give a name to the algorithm which Catalyst will use to persist its state. # In this example, Catalyst will create the `.catalyst/data/live_algos` # directory. If we stop and start the algorithm, Catalyst will resume its @@ -122,7 +122,7 @@ def handle_data(context, data): # Another powerful built-in feature of the Catalyst backtester is the # portfolio object. The portfolio object tracks your positions, cash, # cost basis of specific holdings, and more. In this line, we calculate - # how long or short our position is at this minute. + # how long or short our position is at this minute. pos_amount = context.portfolio.positions[context.market].amount if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: @@ -250,7 +250,9 @@ if __name__ == '__main__': timestr = time.strftime('%Y%m%d-%H%M%S') out = os.path.join(folder, '{}.p'.format(timestr)) - # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000 + # catalyst run -f catalyst/examples/mean_reversion_simple.py \ + # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ + # --data-frequency minute --capital-base 10000 run_algorithm( capital_base=0.1, data_frequency='minute', diff --git a/catalyst/examples/portfolio_optimization.py b/catalyst/examples/portfolio_optimization.py index bb1660a2..e93b2daf 100644 --- a/catalyst/examples/portfolio_optimization.py +++ b/catalyst/examples/portfolio_optimization.py @@ -1,7 +1,7 @@ -'''Use this code to execute a portfolio optimization model. This code - will select the portfolio with the maximum Sharpe Ratio. The parameters +'''Use this code to execute a portfolio optimization model. This code + will select the portfolio with the maximum Sharpe Ratio. The parameters are set to use 180 days of historical data and rebalance every 30 days. - + This is the code used in the following article: https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556 @@ -15,115 +15,130 @@ import os import pytz import numpy as np import pandas as pd -from scipy.optimize import minimize import matplotlib.pyplot as plt from datetime import datetime -from catalyst.api import record, symbol, symbols, order_target_percent +from catalyst.api import record, symbols, order_target_percent from catalyst.utils.run_algo import run_algorithm np.set_printoptions(threshold='nan', suppress=True) def initialize(context): - # Portfolio assets list - context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', - 'xmr_usdt') - context.nassets = len(context.assets) - # Set the time window that will be used to compute expected return - # and asset correlations - context.window = 180 - # Set the number of days between each portfolio rebalancing - context.rebalance_period = 30 - context.i = 0 + # Portfolio assets list + context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', + 'xmr_usdt') + context.nassets = len(context.assets) + # Set the time window that will be used to compute expected return + # and asset correlations + context.window = 180 + # Set the number of days between each portfolio rebalancing + context.rebalance_period = 30 + context.i = 0 + - def handle_data(context, data): - # Only rebalance at the beggining of the algorithm execution and - # every multiple of the rebalance period - if context.i == 0 or context.i%context.rebalance_period == 0: - n = context.window - prices = data.history(context.assets, fields='price', - bar_count=n+1, frequency='1d') - pr = np.asmatrix(prices) - t_prices = prices.iloc[1:n+1] - t_val = t_prices.values - tminus_prices = prices.iloc[0:n] - tminus_val = tminus_prices.values - # Compute daily returns (r) - r = np.asmatrix(t_val/tminus_val-1) - # Compute the expected returns of each asset with the average - # daily return for the selected time window - m = np.asmatrix(np.mean(r, axis=0)) - # ### - stds = np.std(r, axis=0) - # Compute excess returns matrix (xr) - xr = r - m - # Matrix algebra to get variance-covariance matrix - cov_m = np.dot(np.transpose(xr),xr)/n - # Compute asset correlation matrix (informative only) - corr_m = cov_m/np.dot(np.transpose(stds),stds) - - # Define portfolio optimization parameters - n_portfolios = 50000 - results_array = np.zeros((3+context.nassets,n_portfolios)) - for p in xrange(n_portfolios): - weights = np.random.random(context.nassets) - weights /= np.sum(weights) - w = np.asmatrix(weights) - p_r = np.sum(np.dot(w,np.transpose(m)))*365 - p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365) - - #store results in results array - results_array[0,p] = p_r - results_array[1,p] = p_std - #store Sharpe Ratio (return / volatility) - risk free rate element - #excluded for simplicity - results_array[2,p] = results_array[0,p] / results_array[1,p] - i = 0 - for iw in weights: - results_array[3+i,p] = weights[i] - i += 1 - - #convert results array to Pandas DataFrame - results_frame = pd.DataFrame(np.transpose(results_array), - columns=['r','stdev','sharpe']+context.assets) - #locate position of portfolio with highest Sharpe Ratio - max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] - #locate positon of portfolio with minimum standard deviation - min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] - - #order optimal weights for each asset - for asset in context.assets: - if data.can_trade(asset): - order_target_percent(asset, max_sharpe_port[asset]) - - #create scatter plot coloured by Sharpe Ratio - plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn') - plt.xlabel('Volatility') - plt.ylabel('Returns') - plt.colorbar() - #plot red star to highlight position of portfolio with highest Sharpe Ratio - plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200) - #plot green star to highlight position of minimum variance portfolio - plt.show() - print(max_sharpe_port) - record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m) - context.i += 1 - - + # Only rebalance at the beggining of the algorithm execution and + # every multiple of the rebalance period + if context.i == 0 or context.i % context.rebalance_period == 0: + n = context.window + prices = data.history(context.assets, fields='price', + bar_count=n+1, frequency='1d') + pr = np.asmatrix(prices) + t_prices = prices.iloc[1:n+1] + t_val = t_prices.values + tminus_prices = prices.iloc[0:n] + tminus_val = tminus_prices.values + # Compute daily returns (r) + r = np.asmatrix(t_val/tminus_val-1) + # Compute the expected returns of each asset with the average + # daily return for the selected time window + m = np.asmatrix(np.mean(r, axis=0)) + # ### + stds = np.std(r, axis=0) + # Compute excess returns matrix (xr) + xr = r - m + # Matrix algebra to get variance-covariance matrix + cov_m = np.dot(np.transpose(xr), xr)/n + # Compute asset correlation matrix (informative only) + corr_m = cov_m/np.dot(np.transpose(stds), stds) + + # Define portfolio optimization parameters + n_portfolios = 50000 + results_array = np.zeros((3+context.nassets, n_portfolios)) + for p in xrange(n_portfolios): + weights = np.random.random(context.nassets) + weights /= np.sum(weights) + w = np.asmatrix(weights) + p_r = np.sum(np.dot(w, np.transpose(m)))*365 + p_std = np.sqrt(np.dot(np.dot(w, cov_m), + np.transpose(w)))*np.sqrt(365) + + # store results in results array + results_array[0, p] = p_r + results_array[1, p] = p_std + # store Sharpe Ratio (return / volatility) - risk free rate element + # excluded for simplicity + results_array[2, p] = results_array[0, p] / results_array[1, p] + i = 0 + for iw in weights: + results_array[3+i, p] = weights[i] + i += 1 + + # convert results array to Pandas DataFrame + results_frame = pd.DataFrame(np.transpose(results_array), + columns=['r', 'stdev', 'sharpe'] + + context.assets) + # locate position of portfolio with highest Sharpe Ratio + max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] + # locate positon of portfolio with minimum standard deviation + # min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] + + # order optimal weights for each asset + for asset in context.assets: + if data.can_trade(asset): + order_target_percent(asset, max_sharpe_port[asset]) + + # create scatter plot coloured by Sharpe Ratio + plt.scatter(results_frame.stdev, + results_frame.r, + c=results_frame.sharpe, + cmap='RdYlGn') + plt.xlabel('Volatility') + plt.ylabel('Returns') + plt.colorbar() + # plot red star to highlight position of portfolio + # with highest Sharpe Ratio + plt.scatter(max_sharpe_port[1], + max_sharpe_port[0], + marker='o', + color='b', + s=200) + # plot green star to highlight position of minimum variance portfolio + plt.show() + print(max_sharpe_port) + record(pr=pr, + r=r, + m=m, + stds=stds, + max_sharpe_port=max_sharpe_port, + corr_m=corr_m) + context.i += 1 + + def analyze(context=None, results=None): - # Form DataFrame with selected data - data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']] - - # Save results in CSV file - filename = os.path.splitext(os.path.basename(__file__))[0] - data.to_csv(filename + '.csv') + # Form DataFrame with selected data + data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m', + 'portfolio_value']] + + # Save results in CSV file + filename = os.path.splitext(os.path.basename(__file__))[0] + data.to_csv(filename + '.csv') -# Bitcoin data is available from 2015-3-2. Dates vary for other tokens. +# Bitcoin data is available from 2015-3-2. Dates vary for other tokens. start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) -end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) +end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) results = run_algorithm(initialize=initialize, handle_data=handle_data, analyze=analyze, diff --git a/catalyst/examples/rsi_profit_target.py b/catalyst/examples/rsi_profit_target.py index 5e24f122..7b8ac868 100644 --- a/catalyst/examples/rsi_profit_target.py +++ b/catalyst/examples/rsi_profit_target.py @@ -11,7 +11,6 @@ from catalyst.api import ( record, get_open_orders, ) -from catalyst.exchange.stats_utils import crossover, crossunder from catalyst.utils.run_algo import run_algorithm algo_namespace = 'rsi' @@ -55,7 +54,7 @@ def _handle_buy_sell_decision(context, data, signal, price): stop=None ) - action = None + # action = None if context.position is not None: cost_basis = context.position['cost_basis'] amount = context.position['amount'] @@ -80,7 +79,7 @@ def _handle_buy_sell_decision(context, data, signal, price): amount=-amount, limit_price=price * (1 - context.SLIPPAGE_ALLOWED), ) - action = 0 + # action = 0 context.position = None else: @@ -97,7 +96,7 @@ def _handle_buy_sell_decision(context, data, signal, price): amount=buy_amount, stop=None ) - action = 0 + # action = 0 def _handle_data_rsi_only(context, data): diff --git a/catalyst/examples/simple_universe.py b/catalyst/examples/simple_universe.py index a0abc535..7539a76f 100644 --- a/catalyst/examples/simple_universe.py +++ b/catalyst/examples/simple_universe.py @@ -2,26 +2,26 @@ Requires Catalyst version 0.3.0 or above Tested on Catalyst version 0.3.3 -This example aims to provide an easy way for users to learn how to +This example aims to provide an easy way for users to learn how to collect data from any given exchange and select a subset of the available -currency pairs for trading. You simply need to specify the exchange and -the market (base_currency) that you want to focus on. You will then see -how to create a universe of assets, and filter it based the market you +currency pairs for trading. You simply need to specify the exchange and +the market (base_currency) that you want to focus on. You will then see +how to create a universe of assets, and filter it based the market you desire. -The example prints out the closing price of all the pairs for a given -market in a given exchange every 30 minutes. The example also contains -the OHLCV data with minute-resolution for the past seven days which -could be used to create indicators. Use this code as the backbone to +The example prints out the closing price of all the pairs for a given +market in a given exchange every 30 minutes. The example also contains +the OHLCV data with minute-resolution for the past seven days which +could be used to create indicators. Use this code as the backbone to create your own trading strategy. -The lookback_date variable is used to ensure data for a coin existed on +The lookback_date variable is used to ensure data for a coin existed on the lookback period specified. -To run, execute the following two commands in a terminal (inside catalyst +To run, execute the following two commands in a terminal (inside catalyst environment). The first one retrieves all the pricing data needed for this script to run (only needs to be run once), and the second one executes this -script with the parameters specified in the run_algorithm() call at the end +script with the parameters specified in the run_algorithm() call at the end of the file: catalyst ingest-exchange -x bitfinex -f minute @@ -41,8 +41,8 @@ from catalyst.api import (symbols, ) def initialize(context): context.i = -1 # minute counter - context.exchange = context.exchanges.values()[0].name.lower() - context.base_currency = context.exchanges.values()[0].base_currency.lower() + context.exchange = context.exchanges.values()[0].name.lower() + context.base_currency = context.exchanges.values()[0].base_currency.lower() def handle_data(context, data): @@ -52,9 +52,9 @@ def handle_data(context, data): # current date & time in each iteration formatted into a string now = data.current_dt date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ') - lookback_date = now - timedelta(days=lookback_days) + lookback_date = now - timedelta(days=lookback_days) # keep only the date as a string, discard the time - lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] + lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute' # update universe everyday at midnight @@ -64,39 +64,50 @@ def handle_data(context, data): # get data every 30 minutes minutes = 30 # get lookback_days of history data: that is 'lookback' number of bins - lookback = one_day_in_minutes / minutes * lookback_days + lookback = one_day_in_minutes / minutes * lookback_days if not context.i % minutes and context.universe: # we iterate for every pair in the current universe for coin in context.coins: pair = str(coin.symbol) - # Get 30 minute interval OHLCV data. This is the standard data + # Get 30 minute interval OHLCV data. This is the standard data # required for candlestick or indicators/signals. Return Pandas - # DataFrames. 30T means 30-minute re-sampling of one minute data. + # DataFrames. 30T means 30-minute re-sampling of one minute data. # Adjust it to your desired time interval as needed. - opened = fill(data.history(coin, 'open', - bar_count=lookback, frequency='30T')).values - high = fill(data.history(coin, 'high', - bar_count=lookback, frequency='30T')).values - low = fill(data.history(coin, 'low', - bar_count=lookback, frequency='30T')).values - close = fill(data.history(coin, 'price', - bar_count=lookback, frequency='30T')).values - volume = fill(data.history(coin, 'volume', - bar_count=lookback, frequency='30T')).values + opened = fill(data.history(coin, + 'open', + bar_count=lookback, + frequency='30T')).values + high = fill(data.history(coin, + 'high', + bar_count=lookback, + frequency='30T')).values + low = fill(data.history(coin, + 'low', + bar_count=lookback, + frequency='30T')).values + close = fill(data.history(coin, + 'price', + bar_count=lookback, + frequency='30T')).values + volume = fill(data.history(coin, + 'volume', + bar_count=lookback, + frequency='30T')).values - # close[-1] is the last value in the set, which is the equivalent + # close[-1] is the last value in the set, which is the equivalent # to current price (as in the most recent value) # displays the minute price for each pair every 30 minutes - print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format( - now=now, - pair=pair, - o=opened[-1], - h=high[-1], + print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},' + '\tV:{v}'.format( + now=now, + pair=pair, + o=opened[-1], + h=high[-1], l=low[-1], c=close[-1], v=volume[-1], - )) + )) # ------------------------------------------------------------- # --------------- Insert Your Strategy Here ------------------- @@ -111,16 +122,18 @@ def analyze(context=None, results=None): # Example: Poloniex BTC Market def universe(context, lookback_date, current_date): # get all the pairs for the given exchange - json_symbols = get_exchange_symbols(context.exchange) + json_symbols = get_exchange_symbols(context.exchange) # convert into a DataFrame for easier processing - df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) - df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1) - df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1) + df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) + df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1], + axis=1) + df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0], + axis=1) # Filter all the pairs to get only the ones for a given base_currency df = df[df['base_currency'] == context.base_currency] - # Filter all the pairs to ensure that pair existed in the current date range + # Filter all pairs to ensure that pair existed in the current date range df = df[df.start_date < lookback_date] df = df[df.end_daily >= current_date] context.coins = symbols(*df.symbol) # convert all the pairs to symbols @@ -155,4 +168,3 @@ if __name__ == '__main__': live=False, live_graph=False, algo_namespace='simple_universe') - diff --git a/catalyst/examples/talib_simple.py b/catalyst/examples/talib_simple.py index d4a7d807..5129dda9 100644 --- a/catalyst/examples/talib_simple.py +++ b/catalyst/examples/talib_simple.py @@ -1,9 +1,11 @@ # Run Command -# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex -# +# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \ +# -f talib_simple.py -x poloniex +# # Description -# Simple TALib Example showing how to use various indicators in you strategy -# Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py +# Simple TALib Example showing how to use various indicators +# in you strategy. Based loosly on +# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py import os @@ -88,7 +90,7 @@ def _handle_data(context, data): prices.close.as_matrix(), fastperiod=context.MACD_FAST, slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL) - # Stochastics %K %D + # Stochastics %K %D # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100 # %D = 3-day SMA of %K analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH( diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 12c051cd..fa6077ac 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -14,6 +14,7 @@ import six from catalyst.assets._assets import TradingPair from logbook import Logger +from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( @@ -33,7 +34,6 @@ requests.adapters.DEFAULT_RETRIES = 20 BITFINEX_URL = 'https://api.bitfinex.com' -from catalyst.constants import LOG_LEVEL log = Logger('Bitfinex', level=LOG_LEVEL) warning_logger = Logger('AlgoWarning') @@ -172,7 +172,8 @@ class Bitfinex(Exchange): executed_price = float(order_status['avg_execution_price']) - # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it. + # TODO: bitfinex does not specify comission. + # I could calculate it but not sure if it's worth it. commission = None date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) @@ -599,17 +600,17 @@ class Bitfinex(Exchange): else: try: start_date = cached_symbols[symbol]['start_date'] - except KeyError as e: + except KeyError: start_date = time.strftime('%Y-%m-%d') try: end_daily = cached_symbols[symbol]['end_daily'] - except KeyError as e: + except KeyError: end_daily = 'N/A' try: end_minute = cached_symbols[symbol]['end_minute'] - except KeyError as e: + except KeyError: end_minute = 'N/A' symbol_map[symbol] = dict( @@ -660,15 +661,16 @@ class Bitfinex(Exchange): """ Query again with daily resolution setting the start and end around - the startmonth we got above. Avoid end dates greater than now: time.time() + the startmonth we got above. Avoid end dates greater than + now: time.time() """ - url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format( - url=self.url, - symbol=symbol_v2, - start=startmonth - 3600 * 24 * 31 * 1000, - end=min(startmonth + 3600 * 24 * 31 * 1000, - int(time.time() * 1000)) - ) + url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}' + '&end={end}').format( + url=self.url, + symbol=symbol_v2, + start=startmonth - 3600 * 24 * 31 * 1000, + end=min(startmonth + 3600 * 24 * 31 * 1000, + int(time.time() * 1000))) try: self.ask_request() diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index af85aef1..5835b8e1 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -262,11 +262,10 @@ class Bittrex(Exchange): end = int(time.mktime(end_dt.timetuple())) url = '{url}/pub/market/GetTicks?marketName={symbol}' \ '&tickInterval={frequency}&_={end}'.format( - url=URL2, - symbol=self.get_symbol(asset), - frequency=frequency, - end=end - ) + url=URL2, + symbol=self.get_symbol(asset), + frequency=frequency, + end=end, ) try: data = json.loads(urllib.request.urlopen(url).read().decode()) @@ -359,12 +358,12 @@ class Bittrex(Exchange): try: end_daily = cached_symbols[exchange_symbol]['end_daily'] - except KeyError as e: + except KeyError: end_daily = 'N/A' try: end_minute = cached_symbols[exchange_symbol]['end_minute'] - except KeyError as e: + except KeyError: end_minute = 'N/A' symbol_map[exchange_symbol] = dict( diff --git a/catalyst/exchange/bittrex/extensions-example.py b/catalyst/exchange/bittrex/extensions-example.py index 33ffb4f5..4b087899 100644 --- a/catalyst/exchange/bittrex/extensions-example.py +++ b/catalyst/exchange/bittrex/extensions-example.py @@ -4,4 +4,4 @@ from catalyst.exchange.exchange_bundle import exchange_bundle symbols = ( 'neo_btc', ) -register('exchange_bitfinex', exchange_bundle('bitfinex', symbols)) \ No newline at end of file +register('exchange_bitfinex', exchange_bundle('bitfinex', symbols)) diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index 29509bfa..0d758c7d 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -6,11 +6,9 @@ from datetime import timedelta, datetime, date import numpy as np import pandas as pd import pytz -from catalyst.assets._assets import TradingPair from catalyst.data.bundles.core import download_without_progress -from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \ - get_exchange_symbols +from catalyst.exchange.exchange_utils import get_exchange_bundles_folder EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] API_URL = 'http://data.enigma.co/api/v1' @@ -80,9 +78,8 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period): if not os.path.isdir(path): url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \ 'exchange-{exchange}/{name}.tar.gz'.format( - exchange=exchange_name, - name=name - ) + exchange=exchange_name, + name=name) bytes = download_without_progress(url) with tarfile.open('r', fileobj=bytes) as tar: @@ -193,8 +190,10 @@ def get_period_label(dt, data_frequency): str """ - return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \ - else '{}'.format(dt.year) + if data_frequency == 'minute': + return '{}-{:02d}'.format(dt.year, dt.month) + else: + return '{}'.format(dt.year) def get_month_start_end(dt, first_day=None, last_day=None): @@ -315,7 +314,7 @@ def range_in_bundle(asset, start_dt, end_dt, reader): if np.isnan(close): has_data = False - except Exception as e: + except Exception: has_data = False return has_data diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 642d3612..3f2ac85a 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -5,7 +5,6 @@ from time import sleep import numpy as np import pandas as pd -from catalyst.assets._assets import TradingPair from logbook import Logger from catalyst.constants import LOG_LEVEL @@ -242,9 +241,7 @@ class Exchange: asset = a if asset is None: - supported_symbols = sorted([ - asset.symbol for asset in self.assets - ]) + supported_symbols = sorted([a.symbol for a in self.assets]) raise SymbolNotFoundOnExchange( symbol=symbol, @@ -551,7 +548,7 @@ class Exchange: start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ - if asset in series else start_dt + if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize @@ -693,7 +690,8 @@ class Exchange: display_price = style.get_limit_price(is_buy) log.debug( - 'issuing {side} order of {amount} {symbol} for {type}: {price}'.format( + 'issuing {side} order of {amount} {symbol} for {type}:' + ' {price}'.format( side='buy' if is_buy else 'sell', amount=amount, symbol=asset.symbol, diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 147d87e4..adf76d5d 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -29,19 +29,22 @@ from catalyst.exchange.exchange_blotter import ExchangeBlotter from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangePortfolioDataError, - OrderTypeNotSupported) + OrderTypeNotSupported, ) from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \ - get_algo_folder, get_algo_df, \ - save_algo_df, group_assets_by_exchange +from catalyst.exchange.exchange_utils import ( + save_algo_object, + get_algo_object, + get_algo_folder, + get_algo_df, + save_algo_df, + group_assets_by_exchange, ) from catalyst.exchange.live_graph_clock import LiveGraphClock from catalyst.exchange.simple_clock import SimpleClock from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3 from catalyst.finance.execution import MarketOrder from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator -from catalyst.utils.api_support import ( - api_method) +from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest from catalyst.utils.preprocess import preprocess @@ -394,7 +397,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # This method is taken from TradingAlgorithm. # The clock has been replaced to use RealtimeClock - # TODO: should we apply a time skew? not sure to understand the utility. + # TODO: should we apply time skew? not sure to understand the utility. log.debug('creating clock') if self.live_graph: @@ -616,7 +619,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # print_df = pd.DataFrame(list(self.frame_stats)) log.info( - 'statistics for the last {stats_minutes} minutes:\n{stats}'.format( + 'statistics for the last {stats_minutes} minutes:\n' + '{stats}'.format( stats_minutes=self.stats_minutes, stats=get_pretty_stats( stats=self.frame_stats, diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 59e5ce90..46c0c9e7 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -1,6 +1,6 @@ import os import shutil -from datetime import datetime, timedelta +from datetime import timedelta from functools import partial from itertools import chain from operator import is_not @@ -233,11 +233,13 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has empty ' \ 'periods: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - dates=[date.strftime(DATE_TIME_FORMAT) for date in dates] - ) + name=asset.symbol, + start_dt=asset.start_date.strftime( + DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + dates=[date.strftime( + DATE_TIME_FORMAT) for date in dates]) + if empty_rows_behavior == 'warn': log.warn(problem) @@ -245,8 +247,7 @@ class ExchangeBundle: raise EmptyValuesInBundleError( name=asset.symbol, end_minute=end_dt, - dates=dates - ) + dates=dates, ) else: ohlcv_df.dropna(inplace=True) @@ -286,13 +287,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \ 'identical close values on: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - threshold=threshold, - dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) - for date in dates] - ) + name=asset.symbol, + start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + threshold=threshold, + dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) + for date in dates]) problems.append(problem) @@ -630,8 +630,8 @@ class ExchangeBundle: show_progress, label='Ingesting {frequency} price data on ' '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, + exchange=self.exchange_name, + frequency=data_frequency, )) as it: for chunk in it: problems += self.ingest_ctable( diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 7bf66f6c..4f1acd9c 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -143,7 +143,8 @@ class OrphanOrderError(ZiplineError): class OrphanOrderReverseError(ZiplineError): msg = ( - 'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.' + 'Order {order_id} tracked by algorithm, but not found in exchange ' + '{exchange}.' ).strip() @@ -206,8 +207,9 @@ class EmptyValuesInBundleError(ZiplineError): class PricingDataBeforeTradingError(ZiplineError): msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} ' - 'starts on {first_trading_day}, but you are either trying to trade or ' - 'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip() + 'starts on {first_trading_day}, but you are either trying to trade ' + 'or retrieve pricing data on {dt}. Adjust your dates accordingly.' + ).strip() class PricingDataNotLoadedError(ZiplineError): @@ -238,9 +240,11 @@ class ApiCandlesError(ZiplineError): class NoDataAvailableOnExchange(ZiplineError): msg = ( - 'Requested data for trading pair {symbol} is not available on exchange {exchange} ' + 'Requested data for trading pair {symbol} is not available on ' + 'exchange {exchange} ' 'in `{data_frequency}` frequency at this time. ' - 'Check `http://enigma.co/catalyst/status` for market coverage.').strip() + 'Check `http://enigma.co/catalyst/status` for market coverage.' + ).strip() class NoValueForField(ZiplineError): diff --git a/catalyst/exchange/exchange_execution.py b/catalyst/exchange/exchange_execution.py index fe029e3c..536b526a 100644 --- a/catalyst/exchange/exchange_execution.py +++ b/catalyst/exchange/exchange_execution.py @@ -1,4 +1,4 @@ -from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder, MarketOrder +from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder class ExchangeLimitOrder(LimitOrder): diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py index 1df8f9b2..71ed9a35 100644 --- a/catalyst/exchange/exchange_portfolio.py +++ b/catalyst/exchange/exchange_portfolio.py @@ -90,7 +90,8 @@ class ExchangePortfolio(Portfolio): if order_position is None: raise ValueError( - 'Trying to execute order for a position not held: %s' % order.id + 'Trying to execute order for a position not held:' + ' {}'.format(order.id) ) self.capital_used += order.amount * transaction.price diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index af7f2fee..19938d3e 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -134,7 +134,7 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): if not is_local and (not os.path.isfile(filename) or pd.Timedelta( pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): + filename)).days > 1): download_exchange_symbols(exchange_name, environ) if os.path.isfile(filename): @@ -143,7 +143,7 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): data = json.load(data_file, object_hook=symbols_parser) return data - except ValueError as e: + except ValueError: return dict() else: raise ExchangeSymbolsNotFound( @@ -296,7 +296,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None): try: with open(filename, 'rb') as handle: return pickle.load(handle) - except Exception as e: + except Exception: return None else: return None diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py index 1c2028ee..b29b0e92 100644 --- a/catalyst/exchange/poloniex/poloniex.py +++ b/catalyst/exchange/poloniex/poloniex.py @@ -1,5 +1,4 @@ import json -import json import time from collections import defaultdict @@ -18,7 +17,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( ExchangeRequestError, InvalidHistoryFrequencyError, - InvalidOrderStyle, OrphanOrderReverseError) + InvalidOrderStyle, + OrphanOrderError, + OrphanOrderReverseError) from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ ExchangeStopLimitOrder from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ @@ -87,7 +88,6 @@ class Poloniex(Exchange): # filled = -filled price = float(order_status['rate']) - order_type = order_status['type'] stop_price = None limit_price = None @@ -101,11 +101,11 @@ class Poloniex(Exchange): # executed_price = float(order_status['avg_execution_price']) executed_price = price - # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it. + # TODO: Set Poloniex comission commission = None - # date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) - # date = pytz.utc.localize(date) + # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) + # date=pytz.utc.localize(date) date = None order = Order( @@ -292,8 +292,8 @@ class Poloniex(Exchange): """ exchange_symbol = self.get_symbol(asset) - if isinstance(style, ExchangeLimitOrder) or isinstance(style, - ExchangeStopLimitOrder): + if(isinstance(style, ExchangeLimitOrder) + or isinstance(style, ExchangeStopLimitOrder)): if isinstance(style, ExchangeStopLimitOrder): log.warn('{} will ignore the stop price'.format(self.name)) @@ -350,8 +350,8 @@ class Poloniex(Exchange): return self.portfolio.open_orders """ - TODO: Why going to the exchange if we already have this info locally? - And why creating all these Orders if we later discard them? + TODO: Why going to the exchange if we already have this info locally? + And why creating all these Orders if we later discard them? """ try: @@ -365,7 +365,7 @@ class Poloniex(Exchange): if 'error' in response: raise ExchangeRequestError( error='Unable to retrieve open orders: {}'.format( - order_statuses['message']) + response['message']) ) print(self.portfolio.open_orders) @@ -373,8 +373,8 @@ class Poloniex(Exchange): # TODO: Need to handle openOrders for 'all' orders = list() for order_status in response: - order, executed_price = self._create_order( - order_status) # will Throw error b/c Polo doesn't track order['symbol'] + # will Throw error b/c Polo doesn't track order['symbol'] + order, executed_price = self._create_order(order_status) if asset is None or asset == order.sid: orders.append(order) @@ -437,7 +437,8 @@ class Poloniex(Exchange): if 'error' in response: log.info( - 'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format( + 'Unable to cancel order {order_id} on exchange {exchange} ' + '{error}.'.format( order_id=order.id, exchange=self.name, error=response['error'] @@ -512,17 +513,17 @@ class Poloniex(Exchange): else: try: start_date = cached_symbols[exchange_symbol]['start_date'] - except KeyError as e: + except KeyError: start_date = time.strftime('%Y-%m-%d') try: end_daily = cached_symbols[exchange_symbol]['end_daily'] - except KeyError as e: + except KeyError: end_daily = 'N/A' try: end_minute = cached_symbols[exchange_symbol]['end_minute'] - except KeyError as e: + except KeyError: end_minute = 'N/A' symbol_map[exchange_symbol] = dict( @@ -593,19 +594,21 @@ class Poloniex(Exchange): else: for tx in response: """ - We maintain a list of dictionaries of transactions that correspond to - partially filled orders, indexed by order_id. Every time we query - executed transactions from the exchange, we check if we had that - transaction for that order already. If not, we process it. + We maintain a list of dictionaries of transactions that + correspond to partially filled orders, indexed by + order_id. Every time we query executed transactions + from the exchange, we check if we had that transaction + for that order already. If not, we process it. - When an order if fully filled, we flush the dict of transactions - associated with that order. + When an order if fully filled, we flush the dict of + transactions associated with that order. """ if (not filter( lambda item: item['order_id'] == tx['tradeID'], self.transactions[order_id])): log.debug( - 'Got new transaction for order {}: amount {}, price {}'.format( + 'Got new transaction for order {}: amount {}, ' + 'price {}'.format( order_id, tx['amount'], tx['rate'])) tx['amount'] = float(tx['amount']) if (tx['type'] == 'sell'): @@ -616,7 +619,7 @@ class Poloniex(Exchange): dt=pd.to_datetime(tx['date'], utc=True), price=float(tx['rate']), order_id=tx['tradeID'], - # it's a misnomer, but keeping it for compatibility + # it's a misnomer, but keep for compatibility commission=float(tx['fee']) ) self.transactions[order_id].append(transaction) @@ -626,7 +629,8 @@ class Poloniex(Exchange): if (not order_open): """ Since transactions have been executed individually - the only thing left to do is remove them from list of open_orders + the only thing left to do is remove them from list + of open_orders """ del self.portfolio.open_orders[order_id] del self.transactions[order_id] diff --git a/catalyst/exchange/poloniex/poloniex_api.py b/catalyst/exchange/poloniex/poloniex_api.py index ce0831bc..6f339192 100644 --- a/catalyst/exchange/poloniex/poloniex_api.py +++ b/catalyst/exchange/poloniex/poloniex_api.py @@ -108,7 +108,7 @@ class Poloniex_api(object): headers=headers, ) resource = urlopen(req, context=ssl._create_unverified_context()) - content = resource.read().decode('utf-8') + content = resource.read().decode('utf-8') return json.loads(content) def returnticker(self): @@ -161,10 +161,6 @@ class Poloniex_api(object): def returnopenorders(self, market): return self.query('returnOpenOrders', {'currencyPair': market}) - def returntradehistory(self, market): - # TODO: optional start and/or end and limit - return self.query('returnTradeHistory', {'currencyPair': market}) - def returnordertrades(self, ordernumber): return self.query('returnOrderTrades', {'orderNumber': ordernumber}) @@ -177,7 +173,7 @@ class Poloniex_api(object): elif (immediateorcancel): return self.query('buy', {'currencyPair': market, 'rate': rate, 'amount': amount, - 'immediateOrCancel': immediateorcancel, }) + 'immediateOrCancel': immediateorcancel}) elif (postonly): return self.query('buy', {'currencyPair': market, 'rate': rate, 'amount': amount, @@ -195,7 +191,7 @@ class Poloniex_api(object): elif (immediateorcancel): return self.query('sell', {'currencyPair': market, 'rate': rate, 'amount': amount, - 'immediateOrCancel': immediateorcancel, }) + 'immediateOrCancel': immediateorcancel}) elif (postonly): return self.query('sell', {'currencyPair': market, 'rate': rate, 'amount': amount, diff --git a/catalyst/exchange/simple_clock.py b/catalyst/exchange/simple_clock.py index f79a5ab7..cede9429 100644 --- a/catalyst/exchange/simple_clock.py +++ b/catalyst/exchange/simple_clock.py @@ -31,7 +31,8 @@ class SimpleClock(object): This class is a drop-in replacement for :class:`zipline.gens.sim_engine.MinuteSimulationClock`. - This is a stripped down version because crypto exchanges run around the clock. + This is a stripped down version because crypto exchanges run + around the clock. The :param:`time_skew` parameter represents the time difference between the Broker and the live trading machine's clock. diff --git a/catalyst/finance/execution.py b/catalyst/finance/execution.py index 8fec0af6..f4fe95fe 100644 --- a/catalyst/finance/execution.py +++ b/catalyst/finance/execution.py @@ -15,13 +15,8 @@ import abc -from sys import float_info - -from six import with_metaclass - -import catalyst.utils.math_utils as zp_math - from numpy import isfinite +from six import with_metaclass from catalyst.errors import BadOrderParameters diff --git a/catalyst/finance/risk/period.py b/catalyst/finance/risk/period.py index 283bd3c5..987b8d72 100644 --- a/catalyst/finance/risk/period.py +++ b/catalyst/finance/risk/period.py @@ -154,8 +154,8 @@ class RiskMetricsPeriod(object): self.algorithm_returns.values, self.benchmark_returns.values, ) - self.excess_return = self.algorithm_period_returns - \ - self.treasury_period_return + self.excess_return = self.algorithm_period_returns \ + - self.treasury_period_return self.max_drawdown = max_drawdown(self.algorithm_returns.values) self.max_leverage = self.calculate_max_leverage() diff --git a/catalyst/finance/risk/risk.py b/catalyst/finance/risk/risk.py index e447617e..07c29030 100644 --- a/catalyst/finance/risk/risk.py +++ b/catalyst/finance/risk/risk.py @@ -160,7 +160,8 @@ def choose_treasury(select_treasury, treasury_curves, start_session, ) break - if search_day and trading_calendar.name != 'OPEN': # Supress warning for 'OPEN' calendar + # Supress warning for 'OPEN' calendar + if search_day and trading_calendar.name != 'OPEN': if (search_dist is None or search_dist > 1) and \ search_days[0] <= end_session <= search_days[-1]: message = "No rate within 1 trading day of end date = \ diff --git a/catalyst/finance/slippage.py b/catalyst/finance/slippage.py index 36de4ec7..98784f50 100644 --- a/catalyst/finance/slippage.py +++ b/catalyst/finance/slippage.py @@ -41,7 +41,6 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05 - class LiquidityExceeded(Exception): pass diff --git a/catalyst/pipeline/factors/equity/__init__.py b/catalyst/pipeline/factors/equity/__init__.py index 663d025f..e8d29cf0 100644 --- a/catalyst/pipeline/factors/equity/__init__.py +++ b/catalyst/pipeline/factors/equity/__init__.py @@ -1,9 +1,6 @@ from .statistical import ( - RollingPearson, - RollingLinearRegression, RollingLinearRegressionOfReturns, RollingPearsonOfReturns, - RollingSpearman, RollingSpearmanOfReturns, ) from .technical import ( diff --git a/catalyst/pipeline/loaders/equity_pricing_loader.py b/catalyst/pipeline/loaders/equity_pricing_loader.py index c7dbfd41..648bd5f5 100644 --- a/catalyst/pipeline/loaders/equity_pricing_loader.py +++ b/catalyst/pipeline/loaders/equity_pricing_loader.py @@ -38,9 +38,11 @@ class USEquityPricingLoader(PipelineLoader): def __init__(self, bundle, data_frequency, dataset): - if data_frequency == 'daily': - reader = bundle.daily_bar_reader - elif daily_bar_reader == 'minute': + # TODO: This is currently broken, No Pipeline support for Catalyst + # if data_frequency == 'daily': + # reader = bundle.daily_bar_reader + # elif daily_bar_reader == 'minute': + if data_frequency == 'minute': reader = bundle.minute_bar_reader else: raise ValueError( @@ -51,7 +53,9 @@ class USEquityPricingLoader(PipelineLoader): if data_frequency == 'daily': all_sessions = cal.all_sessions - elif daily_bar_reader == 'minute': + # TODO: this cannot be right, but no pipeline support at the moment + # elif daily_bar_reader == 'minute': + elif data_frequency == 'minute': reader = bundle.minute_bar_reader all_sessions = cal.all_minutes diff --git a/catalyst/pipeline/loaders/events.py b/catalyst/pipeline/loaders/events.py index ece7b25e..512c772d 100644 --- a/catalyst/pipeline/loaders/events.py +++ b/catalyst/pipeline/loaders/events.py @@ -231,7 +231,7 @@ class EventsLoader(PipelineLoader): self.load_next_events(n, dates, sids, mask), self.load_previous_events(p, dates, sids, mask), ) - + @property def columns(self): return self._columns diff --git a/catalyst/pipeline/loaders/frame.py b/catalyst/pipeline/loaders/frame.py index 827c12be..c663176e 100644 --- a/catalyst/pipeline/loaders/frame.py +++ b/catalyst/pipeline/loaders/frame.py @@ -180,4 +180,3 @@ class DataFrameLoader(PipelineLoader): @property def columns(self): return self._columns - diff --git a/catalyst/pipeline/loaders/synthetic.py b/catalyst/pipeline/loaders/synthetic.py index 1f88396a..fef8be24 100644 --- a/catalyst/pipeline/loaders/synthetic.py +++ b/catalyst/pipeline/loaders/synthetic.py @@ -163,7 +163,7 @@ class SeededRandomLoader(PrecomputedLoader): bool_dtype: self._bool_values, object_dtype: self._object_values, }[dtype](shape) - + @property def columns(self): return self._columns diff --git a/catalyst/support/issue_44.py b/catalyst/support/issue_44.py deleted file mode 100644 index b6d1277a..00000000 --- a/catalyst/support/issue_44.py +++ /dev/null @@ -1,109 +0,0 @@ -import pandas as pd -from catalyst import run_algorithm -from catalyst.exchange.exchange_utils import get_exchange_symbols - -from catalyst.api import ( - symbols, -) - - -def initialize(context): - context.i = -1 - context.base_currency = 'btc' - - -def handle_data(context, data): - lookback = 60 * 24 * 7 # (minutes, hours, days) - context.i += 1 - if context.i < lookback: - return - - today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S') - - try: - # update universe everyday - new_day = 60 * 24 - if not context.i % new_day: - context.universe = universe(context, today) - - # get data every 30 minutes - minutes = 30 - if not context.i % minutes and context.universe: - for coin in context.coins: - pair = str(coin.symbol) - - # ohlcv data - open = data.history(coin, 'open', lookback, - '1m').ffill().bfill().resample( - '30T').first() - high = data.history(coin, 'high', lookback, - '1m').ffill().bfill().resample('30T').max() - low = data.history(coin, 'low', lookback, - '1m').ffill().bfill().resample('30T').min() - close = data.history(coin, 'price', lookback, - '1m').ffill().bfill().resample( - '30T').last() - volume = data.history(coin, 'volume', lookback, - '1m').ffill().bfill().resample( - '30T').sum() - - print(today, pair, close[-1]) - - except Exception as e: - print(e) - - -def analyze(context=None, results=None): - pass - - -def universe(context, today): - json_symbols = get_exchange_symbols('poloniex') - poloniex_universe_df = pd.DataFrame.from_dict( - json_symbols).transpose().astype(str) - poloniex_universe_df['base_currency'] = poloniex_universe_df.apply( - lambda row: row.symbol.split('_')[1], - axis=1) - poloniex_universe_df['market_currency'] = poloniex_universe_df.apply( - lambda row: row.symbol.split('_')[0], - axis=1) - poloniex_universe_df = poloniex_universe_df[ - poloniex_universe_df['base_currency'] == context.base_currency] - poloniex_universe_df = poloniex_universe_df[ - poloniex_universe_df.symbol != 'gas_btc'] - - # Markets currently not working on Catalyst 0.3.1 - # 2017-01-01 - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc'] - # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc'] - print(poloniex_universe_df.head()) - - date = str(today).split(' ')[0] - - poloniex_universe_df = poloniex_universe_df[ - poloniex_universe_df.start_date < date] - context.coins = symbols(*poloniex_universe_df.symbol) - print(len(poloniex_universe_df)) - return poloniex_universe_df.symbol.tolist() - - -if __name__ == '__main__': - start_date = pd.to_datetime('2017-01-01', utc=True) - end_date = pd.to_datetime('2017-10-15', utc=True) - - performance = run_algorithm(start=start_date, end=end_date, - capital_base=10000.0, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='poloniex', - data_frequency='minute', - base_currency='btc', - live=False, - live_graph=False, - algo_namespace='test') diff --git a/catalyst/support/issue_47.py b/catalyst/support/issue_47.py deleted file mode 100644 index 3ebc0bc7..00000000 --- a/catalyst/support/issue_47.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -Requires Catalyst version 0.3.0 or above -Tested on Catalyst version 0.3.3 - -These example aims to provide and easy way for users to learn how to collect data from the different exchanges. -You simply need to specify the exchange and the market that you want to focus on. -You will all see how to create a universe and filter it base on the exchange and the market you desire. - -The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes. -The example also contains the ohlcv minute data for the past seven days which could be used to create indicators -Use this as the backbone to create your own trading strategies. - -Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified. -""" - -import numpy as np -import pandas as pd -from datetime import timedelta -from catalyst import run_algorithm -from catalyst.exchange.exchange_utils import get_exchange_symbols - -from catalyst.api import ( - symbols, -) - - -def initialize(context): - context.i = -1 # counts the minutes - context.exchange = 'poloniex' # must match the exchange specified in run_algorithm - context.base_currency = 'btc' # must match the base currency specified in run_algorithm - - -def handle_data(context, data): - lookback = 60 * 24 * 7 # (minutes, hours, days) of how far to lookback in the data history - context.i += 1 - - # current date formatted into a string - today = context.blotter.current_dt - date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ') - lookback_date = today - timedelta(days=( - lookback / (60 * 24))) # subtract the amount of days specified in lookback - lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[ - 0] # get only the date as a string - - # update universe everyday - new_day = 60 * 24 - if not context.i % new_day: - context.universe = universe(context, lookback_date, date) - - # get data every 30 minutes - minutes = 30 - if not context.i % minutes and context.universe: - # we iterate for every pair in the current universe - for coin in context.coins: - pair = str(coin.symbol) - - # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals) - # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval. - opened = fill(data.history(coin, 'open', bar_count=lookback, - frequency='30T')).values - high = fill(data.history(coin, 'high', bar_count=lookback, - frequency='30T')).values - low = fill(data.history(coin, 'low', bar_count=lookback, - frequency='30T')).values - close = fill(data.history(coin, 'price', bar_count=lookback, - frequency='30T')).values - volume = fill(data.history(coin, 'volume', bar_count=lookback, - frequency='30T')).values - - # close[-1] is the equivalent to current price - # displays the minute price for each pair every 30 minutes - print( - today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1]) - - # ---------------------------------------------------------------------------------------------------------- - # -------------------------------------- Insert Your Strategy Here ----------------------------------------- - # ---------------------------------------------------------------------------------------------------------- - - -def analyze(context=None, results=None): - pass - - -# Get the universe for a given exchange and a given base_currency market -# Example: Poloniex btc Market -def universe(context, lookback_date, current_date): - json_symbols = get_exchange_symbols( - context.exchange) # get all the pairs for the exchange - universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype( - str) # convert into a dataframe - universe_df['base_currency'] = universe_df.apply( - lambda row: row.symbol.split('_')[1], - axis=1) - universe_df['market_currency'] = universe_df.apply( - lambda row: row.symbol.split('_')[0], - axis=1) - # Filter all the exchange pairs to only the ones for a give base currency - universe_df = universe_df[ - universe_df['base_currency'] == context.base_currency] - - # Filter all the pairs to ensure that pair existed in the current date range - universe_df = universe_df[universe_df.start_date < lookback_date] - universe_df = universe_df[universe_df.end_daily >= current_date] - context.coins = symbols( - *universe_df.symbol) # convert all the pairs to symbols - return universe_df.symbol.tolist() - - -# Replace all NA, NAN or infinite values with its nearest value -def fill(series): - if isinstance(series, pd.Series): - return series.replace([np.inf, -np.inf], np.nan).ffill().bfill() - elif isinstance(series, np.ndarray): - return pd.Series(series).replace([np.inf, -np.inf], - np.nan).ffill().bfill().values - else: - return series - - -if __name__ == '__main__': - start_date = pd.to_datetime('2017-01-08', utc=True) - end_date = pd.to_datetime('2017-11-13', utc=True) - - performance = run_algorithm(start=start_date, end=end_date, - capital_base=10000.0, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='poloniex', - data_frequency='minute', - base_currency='btc', - live=False, - live_graph=False, - algo_namespace='simple_universe') - -""" -Run in Terminal (inside catalyst environment): -python simple_universe.py -""" diff --git a/catalyst/support/issue_55.py b/catalyst/support/issue_55.py index 195f5019..733a1585 100644 --- a/catalyst/support/issue_55.py +++ b/catalyst/support/issue_55.py @@ -1,4 +1,3 @@ -import talib import pandas as pd from catalyst import run_algorithm diff --git a/catalyst/support/issue_57.py b/catalyst/support/issue_57.py deleted file mode 100644 index f7bfcd18..00000000 --- a/catalyst/support/issue_57.py +++ /dev/null @@ -1,46 +0,0 @@ -import talib -import pandas as pd - -from catalyst import run_algorithm -from catalyst.api import symbol - - -def initialize(context): - print('initializing') - context.asset = symbol('btc_usdt') - - -def handle_data(context, data): - print('handling bar: {}'.format(data.current_dt)) - - price = data.current(context.asset, 'close') - print('got price {price}'.format(price=price)) - - try: - prices = data.history( - context.asset, - fields='close', - bar_count=60, - frequency='1D' - ) - print('got {} price entries\n'.format(len(prices), prices)) - except Exception as e: - print(e) - - -run_algorithm( - capital_base=1, - start=pd.to_datetime('2016-2-11', utc=True), - end=pd.to_datetime('2017-8-31', utc=True), - data_frequency='daily', - initialize=initialize, - handle_data=handle_data, - analyze=None, - exchange_name='bittrex', - algo_namespace='issue_57', - base_currency='btc' -<<<<<<< HEAD -) -======= -) ->>>>>>> develop diff --git a/catalyst/support/issue_74.py b/catalyst/support/issue_74.py deleted file mode 100644 index ad6d6fee..00000000 --- a/catalyst/support/issue_74.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import division -import os -import pytz -import numpy as np -import pandas as pd -from scipy.optimize import minimize -import matplotlib.pyplot as plt -from datetime import datetime - -from catalyst.api import record, symbol, symbols, order_target_percent -from catalyst.utils.run_algo import run_algorithm - -np.set_printoptions(threshold='nan', suppress=True) - - -def initialize(context): - # Portfolio assets list - context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', - 'xmr_usdt') - context.nassets = len(context.assets) - # Set the time window that will be used to compute expected return - # and asset correlations - context.window = 180 - # Set the number of days between each portfolio rebalancing - context.rebalance_period = 30 - context.i = 0 - - -def handle_data(context, data): - # Only rebalance at the beggining of the algorithm execution and - # every multiple of the rebalance period - if context.i == 0 or context.i % context.rebalance_period == 0: - n = context.window - prices = data.history(context.assets, fields='price', - bar_count=n + 1, frequency='daily') - pr = np.asmatrix(prices) - t_prices = prices.iloc[1:n + 1] - t_val = t_prices.values - tminus_prices = prices.iloc[0:n] - tminus_val = tminus_prices.values - # Compute daily returns (r) - r = np.asmatrix(t_val / tminus_val - 1) - # Compute the expected returns of each asset with the average - # daily return for the selected time window - m = np.asmatrix(np.mean(r, axis=0)) - # ### - stds = np.std(r, axis=0) - # Compute excess returns matrix (xr) - xr = r - m - # Matrix algebra to get variance-covariance matrix - cov_m = np.dot(np.transpose(xr), xr) / n - # Compute asset correlation matrix (informative only) - corr_m = cov_m / np.dot(np.transpose(stds), stds) - - # Define portfolio optimization parameters - n_portfolios = 50000 - results_array = np.zeros((3 + context.nassets, n_portfolios)) - for p in xrange(n_portfolios): - weights = np.random.random(context.nassets) - weights /= np.sum(weights) - w = np.asmatrix(weights) - p_r = np.sum(np.dot(w, np.transpose(m))) * 365 - p_std = np.sqrt( - np.dot(np.dot(w, cov_m), np.transpose(w))) * np.sqrt(365) - - # store results in results array - results_array[0, p] = p_r - results_array[1, p] = p_std - # store Sharpe Ratio (return / volatility) - risk free rate element - # excluded for simplicity - results_array[2, p] = results_array[0, p] / results_array[1, p] - i = 0 - for iw in weights: - results_array[3 + i, p] = weights[i] - i += 1 - - # convert results array to Pandas DataFrame - results_frame = pd.DataFrame(np.transpose(results_array), - columns=['r', 'stdev', - 'sharpe'] + context.assets) - # locate position of portfolio with highest Sharpe Ratio - max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] - # locate positon of portfolio with minimum standard deviation - min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] - - # order optimal weights for each asset - for asset in context.assets: - if data.can_trade(asset): - order_target_percent(asset, max_sharpe_port[asset]) - - # create scatter plot coloured by Sharpe Ratio - plt.scatter(results_frame.stdev, results_frame.r, - c=results_frame.sharpe, cmap='RdYlGn') - plt.xlabel('Volatility') - plt.ylabel('Returns') - plt.colorbar() - # plot red star to highlight position of portfolio with highest Sharpe Ratio - plt.scatter(max_sharpe_port[1], max_sharpe_port[0], marker='o', - color='b', s=200) - # plot green star to highlight position of minimum variance portfolio - plt.show() - print(max_sharpe_port) - record(pr=pr, r=r, m=m, stds=stds, max_sharpe_port=max_sharpe_port, - corr_m=corr_m) - context.i += 1 - - -def analyze(context=None, results=None): - # Form DataFrame with selected data - data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m', - 'portfolio_value']] - - # Save results in CSV file - filename = os.path.splitext(os.path.basename(__file__))[0] - data.to_csv(filename + '.csv') - - -# Bitcoin data is available from 2015-3-2. Dates vary for other tokens. -start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) -end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) -results = run_algorithm(initialize=initialize, - handle_data=handle_data, - analyze=analyze, - start=start, - end=end, - exchange_name='poloniex', - capital_base=100000, ) diff --git a/catalyst/support/rodrigo_1.py b/catalyst/support/rodrigo_1.py deleted file mode 100644 index ec32c617..00000000 --- a/catalyst/support/rodrigo_1.py +++ /dev/null @@ -1,153 +0,0 @@ -import pandas as pd -from logbook import Logger, DEBUG - -from catalyst import run_algorithm -from catalyst.api import (schedule_function, order_target_percent, symbol, - date_rules, get_open_orders, cancel_order, record, - set_commission, set_slippage) - -log = Logger('rodrigo_1', level=DEBUG) -""" -The initialize function sets any data or variables that -you'll use in your algorithm. -It's only called once at the beginning of your algorithm. -""" - - -def initialize(context): - # Select asset of interest - context.asset = symbol('BTC_USD') - - # set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5)) - # set_slippage(TradingPairFixedSlippage(spread=0.5)) - # Set up a rebalance method to run every day - schedule_function(rebalance, date_rule=date_rules.every_day()) - - -""" -Rebalance function scheduled to run once per day. -""" - - -def rebalance(context, data): - # To make market decisions, we're calculating the token's - # moving average for the last 5 days. - - # We get the price history for the last 5 days. - price_history = data.history(context.asset, fields='price', bar_count=5, - frequency='1d') - - # Then we take an average of those 5 days. - average_price = price_history.mean() - - # We also get the coin's current price. - price = data.current(context.asset, 'price') - - # Cancel any outstanding orders - orders = get_open_orders(context.asset) or [] - for order in orders: - cancel_order(order) - - # If our coin is currently listed on a major exchange - if data.can_trade(context.asset): - # If the current price is 1% above the 5-day average price, - # we open a long position. If the current price is below the - # average price, then we want to close our position to 0 shares. - if price > (1.01 * average_price): - # Place the buy order (positive means buy, negative means sell) - order_target_percent(context.asset, .99) - log.info("Buying %s" % (context.asset.symbol)) - elif price < average_price: - # Sell all of our shares by setting the target position to zero - order_target_percent(context.asset, 0) - log.info("Selling %s" % (context.asset.symbol)) - - # Use the record() method to track up to five custom signals. - # Record Apple's current price and the average price over the last - # five days. - cash = context.portfolio.cash - leverage = context.account.leverage - - record(price=price, average_price=average_price, cash=cash, - leverage=leverage) - - -def analyze(context=None, results=None): - import matplotlib.pyplot as plt - - # Plot the portfolio and asset data. - ax1 = plt.subplot(511) - results[['portfolio_value']].plot(ax=ax1) - ax1.set_ylabel('Portfolio Value (USD)') - - ax2 = plt.subplot(512, sharex=ax1) - ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset)) - (results[[ - 'price', - ]]).plot(ax=ax2) - - trans = results.ix[[t != [] for t in results.transactions]] - buys = trans.ix[ - [t[0]['amount'] > 0 for t in trans.transactions] - ] - sells = trans.ix[ - [t[0]['amount'] < 0 for t in trans.transactions] - ] - - ax2.plot( - buys.index, - results.price[buys.index], - '^', - markersize=10, - color='g', - ) - ax2.plot( - sells.index, - results.price[sells.index], - 'v', - markersize=10, - color='r', - ) - - ax3 = plt.subplot(513, sharex=ax1) - results[['leverage']].plot(ax=ax3) - ax3.set_ylabel('Leverage ') - - ax4 = plt.subplot(514, sharex=ax1) - results[['cash']].plot(ax=ax4) - ax4.set_ylabel('Cash (USD)') - - results[[ - 'algorithm', - 'benchmark', - ]] = results[[ - 'algorithm_period_return', - 'benchmark_period_return', - ]] - - ax5 = plt.subplot(515, sharex=ax1) - results[[ - 'algorithm', - 'benchmark', - ]].plot(ax=ax5) - ax5.set_ylabel('Percent Change') - - plt.legend(loc=3) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - - -run_algorithm( - capital_base=100000, - start=pd.to_datetime('2017-1-1', utc=True), - end=pd.to_datetime('2017-10-22', utc=True), - data_frequency='minute', - initialize=initialize, - handle_data=None, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace='rodrigo_1', - base_currency='usd' -) diff --git a/catalyst/utils/calendars/exchange_calendar_open.py b/catalyst/utils/calendars/exchange_calendar_open.py index 54be460f..f7be8e22 100644 --- a/catalyst/utils/calendars/exchange_calendar_open.py +++ b/catalyst/utils/calendars/exchange_calendar_open.py @@ -31,4 +31,5 @@ class OpenExchangeCalendar(TradingCalendar): return DateOffset(days=1) def __init__(self, *args, **kwargs): - super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs) + super(OpenExchangeCalendar, self).__init__( + start=Timestamp('2015-3-1', tz='UTC'), **kwargs) diff --git a/catalyst/utils/cli.py b/catalyst/utils/cli.py index 51e519e0..59a7300a 100644 --- a/catalyst/utils/cli.py +++ b/catalyst/utils/cli.py @@ -9,6 +9,7 @@ DEFAULT_BAR_TEMPLATE = ' [%(bar)s] %(label)s: %(info)s' DEFAULT_EMPTY_CHAR = ' ' DEFAULT_FILL_CHAR = '=' + def item_show_count(total=None): def maybe_show_total(index): if total is not None: @@ -17,12 +18,13 @@ def item_show_count(total=None): def item_show_func(item, _it=iter(count())): if item is not None: - starting = False + # starting = False return maybe_show_total(next(_it)) return 'DONE' return item_show_func + def maybe_show_progress(it, show_progress, empty_char=DEFAULT_EMPTY_CHAR, diff --git a/catalyst/utils/math_utils.py b/catalyst/utils/math_utils.py index 7981a52b..8e3e086c 100644 --- a/catalyst/utils/math_utils.py +++ b/catalyst/utils/math_utils.py @@ -17,9 +17,11 @@ import math from numpy import isnan + def round_nearest(x, a): return round(round(x / a) * a, -int(math.floor(math.log10(a)))) + def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False): """Check if a and b are equal with some tolerance. diff --git a/catalyst/utils/paths.py b/catalyst/utils/paths.py index 8ec87c7e..2764b623 100644 --- a/catalyst/utils/paths.py +++ b/catalyst/utils/paths.py @@ -126,7 +126,7 @@ def catalyst_root(environ=None): root = environ.get('ZIPLINE_ROOT', None) if root is None: - root = os.path.join(expanduser('~'),'.catalyst') + root = os.path.join(expanduser('~'), '.catalyst') return root diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 92a5e335..3c62ea2d 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -8,6 +8,7 @@ from time import sleep import click import pandas as pd +from logbook import Logger from catalyst.data.bundles import load from catalyst.data.data_portal import DataPortal @@ -30,17 +31,16 @@ from catalyst.utils.factory import create_simulation_parameters from catalyst.data.loader import load_crypto_market_data import catalyst.utils.paths as pth -from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \ - ExchangeTradingAlgorithmBacktest +from catalyst.exchange.exchange_algorithm import ( + ExchangeTradingAlgorithmLive, + ExchangeTradingAlgorithmBacktest, +) from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \ DataPortalExchangeBacktest from catalyst.exchange.asset_finder_exchange import AssetFinderExchange -from catalyst.exchange.exchange_portfolio import ExchangePortfolio from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangeRequestErrorTooManyAttempts, BaseCurrencyNotFoundError) -from catalyst.exchange.exchange_utils import get_algo_object -from logbook import Logger from catalyst.constants import LOG_LEVEL @@ -172,7 +172,7 @@ def _run(handle_data, asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = AssetFinderExchange() - choose_loader = None # TODO: use the DataPortal for in the algorithm class for this + choose_loader = None # TODO: use the DataPortal in the algo class for this if live: start = pd.Timestamp.utcnow() diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py index 8c76799a..a842bee7 100644 --- a/tests/exchange/test_bcolz.py +++ b/tests/exchange/test_bcolz.py @@ -116,7 +116,7 @@ class TestBcolzWriter(object): df = self.generate_df(exchange_name, freq, start, end) - print df.index[0],df.index[-1] + print(df.index[0], df.index[-1]) writer = BcolzExchangeBarWriter( rootdir=self.root_dir, @@ -140,7 +140,7 @@ class TestBcolzWriter(object): dx = get_df_from_arrays(arrays, periods) - assert_equals(df.equals(df), True) + assert_equals(df.equals(dx), True) pass def test_bcolz_bitfinex_daily_write_read(self): diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py index 194422f5..1859f38f 100644 --- a/tests/exchange/test_bitfinex.py +++ b/tests/exchange/test_bitfinex.py @@ -34,7 +34,7 @@ class TestBitfinex(BaseExchangeTestCase): def test_open_orders(self): log.info('retrieving open orders') - orders = self.exchange.get_open_orders() + # orders = self.exchange.get_open_orders() pass def test_get_order(self): @@ -47,18 +47,17 @@ class TestBitfinex(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') - ohlcv_neo = self.exchange.get_candles( - freq='1T', - assets=self.exchange.get_asset('neo_btc') - ) + # ohlcv_neo = self.exchange.get_candles( + # freq='1T', + # assets=self.exchange.get_asset('neo_btc')) pass def test_tickers(self): log.info('retrieving tickers') - tickers = self.exchange.tickers([ - self.exchange.get_asset('eth_btc'), - self.exchange.get_asset('etc_btc') - ]) + # tickers = self.exchange.tickers([ + # self.exchange.get_asset('eth_btc'), + # self.exchange.get_asset('etc_btc') + # ]) pass def test_get_account(self): @@ -67,11 +66,11 @@ class TestBitfinex(BaseExchangeTestCase): def test_get_balances(self): log.info('testing exchange balances') - balances = self.exchange.get_balances() + # balances = self.exchange.get_balances() pass def test_orderbook(self): log.info('testing order book for bitfinex') - asset = self.exchange.get_asset('eth_btc') - orderbook = self.exchange.get_orderbook(asset) + # asset = self.exchange.get_asset('eth_btc') + # orderbook = self.exchange.get_orderbook(asset) pass diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py index bf17970d..47d54068 100644 --- a/tests/exchange/test_bittrex.py +++ b/tests/exchange/test_bittrex.py @@ -1,4 +1,4 @@ -import pandas as pd +# import pandas as pd from catalyst.exchange.bittrex.bittrex import Bittrex from catalyst.finance.order import Order from base import BaseExchangeTestCase @@ -33,8 +33,8 @@ class TestBittrex(BaseExchangeTestCase): def test_open_orders(self): log.info('retrieving open orders') - asset = self.exchange.get_asset('neo_btc') - orders = self.exchange.get_open_orders(asset) + # asset = self.exchange.get_asset('neo_btc') + # orders = self.exchange.get_open_orders(asset) pass def test_get_order(self): @@ -51,21 +51,21 @@ class TestBittrex(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') - ohlcv_neo = self.exchange.get_candles( - freq='5T', - assets=self.exchange.get_asset('neo_btc'), - bar_count=20, - end_dt=pd.to_datetime('2017-10-20', utc=True) - ) - ohlcv_neo_ubq = self.exchange.get_candles( - freq='1D', - assets=[ - self.exchange.get_asset('neo_btc'), - self.exchange.get_asset('ubq_btc') - ], - bar_count=14, - end_dt=pd.to_datetime('2017-10-20', utc=True) - ) + # ohlcv_neo = self.exchange.get_candles( + # freq='5T', + # assets=self.exchange.get_asset('neo_btc'), + # bar_count=20, + # end_dt=pd.to_datetime('2017-10-20', utc=True) + # ) + # ohlcv_neo_ubq = self.exchange.get_candles( + # freq='1D', + # assets=[ + # self.exchange.get_asset('neo_btc'), + # self.exchange.get_asset('ubq_btc') + # ], + # bar_count=14, + # end_dt=pd.to_datetime('2017-10-20', utc=True) + # ) pass def test_tickers(self): @@ -79,7 +79,7 @@ class TestBittrex(BaseExchangeTestCase): def test_get_balances(self): log.info('testing wallet balances') - balances = self.exchange.get_balances() + # balances = self.exchange.get_balances() pass def test_get_account(self): @@ -88,6 +88,6 @@ class TestBittrex(BaseExchangeTestCase): def test_orderbook(self): log.info('testing order book for bittrex') - asset = self.exchange.get_asset('eth_btc') - orderbook = self.exchange.get_orderbook(asset) + # asset = self.exchange.get_asset('eth_btc') + # orderbook = self.exchange.get_orderbook(asset) pass diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index 89ef3062..a0d23319 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -1,11 +1,10 @@ -import hashlib +# import hashlib import os import tempfile from logging import getLogger import pandas as pd -from catalyst import get_calendar from catalyst.exchange.bundle_utils import get_bcolz_chunk, \ get_start_dt, get_df_from_arrays from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ @@ -22,22 +21,22 @@ log = getLogger('test_exchange_bundle') class TestExchangeBundle: def test_spot_value(self): - data_frequency = 'daily' - exchange_name = 'poloniex' + # data_frequency = 'daily' + # exchange_name = 'poloniex' - exchange = get_exchange(exchange_name) - exchange_bundle = ExchangeBundle(exchange) - assets = [ - exchange.get_asset('btc_usdt') - ] - dt = pd.to_datetime('2017-10-14', utc=True) + # exchange = get_exchange(exchange_name) + # exchange_bundle = ExchangeBundle(exchange) + # assets = [ + # exchange.get_asset('btc_usdt') + # ] + # dt = pd.to_datetime('2017-10-14', utc=True) - values = exchange_bundle.get_spot_values( - assets=assets, - field='close', - dt=dt, - data_frequency=data_frequency - ) + # values = exchange_bundle.get_spot_values( + # assets=assets, + # field='close', + # dt=dt, + # data_frequency=data_frequency + # ) pass def test_ingest_minute(self): @@ -215,7 +214,7 @@ class TestExchangeBundle: # encounter these problems as I have been focusing on minute data. reader = exchange_bundle.get_reader(data_frequency) for asset in assets: - # Since this pair was loaded last. It should be there in daily mode. + # Since this pair was loaded last. It should be here in daily mode. arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], @@ -252,7 +251,6 @@ class TestExchangeBundle: ensure_directory(path) exchange_bundle = ExchangeBundle(exchange) - calendar = get_calendar('OPEN') # We are using a BcolzMinuteBarWriter even though the data is daily # Each day has a maximum of one bar @@ -304,26 +302,25 @@ class TestExchangeBundle: pass def test_minute_bundle(self): - exchange_name = 'poloniex' - data_frequency = 'minute' + # exchange_name = 'poloniex' + # data_frequency = 'minute' - exchange = get_exchange(exchange_name) - asset = exchange.get_asset('neos_btc') - - path = get_bcolz_chunk( - exchange_name=exchange_name, - symbol=asset.symbol, - data_frequency=data_frequency, - period='2017-5', - ) + # exchange = get_exchange(exchange_name) + # asset = exchange.get_asset('neos_btc') + # path = get_bcolz_chunk( + # exchange_name=exchange_name, + # symbol=asset.symbol, + # data_frequency=data_frequency, + # period='2017-5', + # ) pass def test_hash_symbol(self): - symbol = 'etc_btc' - sid = int( - hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16 - ) % 10 ** 6 + # symbol = 'etc_btc' + # sid = int( + # hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16 + # ) % 10 ** 6 pass def test_validate_data(self): diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index b1df6a19..3ff44a8b 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -1,13 +1,10 @@ -import os -import tempfile - import pandas as pd +from logbook import Logger +from base import BaseExchangeTestCase + from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.finance.order import Order -from base import BaseExchangeTestCase -from logbook import Logger from catalyst.exchange.exchange_utils import get_exchange_auth -from catalyst.utils.paths import ensure_directory log = Logger('test_ccxt') @@ -38,9 +35,9 @@ class TestCCXT(BaseExchangeTestCase): pass def test_open_orders(self): - log.info('retrieving open orders') - asset = self.exchange.get_asset('neo_eth') - orders = self.exchange.get_open_orders(asset) + # log.info('retrieving open orders') + # asset = self.exchange.get_asset('neo_eth') + # orders = self.exchange.get_open_orders(asset) pass def test_get_order(self): @@ -79,7 +76,7 @@ class TestCCXT(BaseExchangeTestCase): def test_get_balances(self): log.info('testing wallet balances') - balances = self.exchange.get_balances() + # balances = self.exchange.get_balances() pass def test_get_account(self): @@ -88,8 +85,8 @@ class TestCCXT(BaseExchangeTestCase): def test_orderbook(self): log.info('testing order book for bittrex') - asset = self.exchange.get_asset('eth_btc') - orderbook = self.exchange.get_orderbook(asset, 'all', limit=10) + # asset = self.exchange.get_asset('eth_btc') + # orderbook = self.exchange.get_orderbook(asset, 'all', limit=10) pass def test_get_fees(self): diff --git a/tests/exchange/test_data_portal.py b/tests/exchange/test_data_portal.py index 7b2b4720..29ef4d46 100644 --- a/tests/exchange/test_data_portal.py +++ b/tests/exchange/test_data_portal.py @@ -3,11 +3,13 @@ from logbook import Logger from catalyst import get_calendar from catalyst.exchange.asset_finder_exchange import AssetFinderExchange -from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \ +from catalyst.exchange.exchange_data_portal import ( + DataPortalExchangeBacktest, DataPortalExchangeLive +) from catalyst.exchange.exchange_utils import get_common_assets -from catalyst.exchange.factory import get_exchange, get_exchanges -from test_utils import rnd_history_date_days, rnd_bar_count, output_df +from catalyst.exchange.factory import get_exchanges +from test_utils import rnd_history_date_days, rnd_bar_count log = Logger('test_bitfinex') @@ -35,31 +37,31 @@ class TestExchangeDataPortal: ) def test_get_history_window_live(self): - asset_finder = self.data_portal_live.asset_finder + # asset_finder = self.data_portal_live.asset_finder - assets = [ - asset_finder.lookup_symbol('eth_btc', self.bitfinex), - asset_finder.lookup_symbol('eth_btc', self.bittrex) - ] - now = pd.Timestamp.utcnow() - data = self.data_portal_live.get_history_window( - assets, - now, - 10, - '1m', - 'price') + # assets = [ + # asset_finder.lookup_symbol('eth_btc', self.bitfinex), + # asset_finder.lookup_symbol('eth_btc', self.bittrex) + # ] + # now = pd.Timestamp.utcnow() + # data = self.data_portal_live.get_history_window( + # assets, + # now, + # 10, + # '1m', + # 'price') pass def test_get_spot_value_live(self): - asset_finder = self.data_portal_live.asset_finder + # asset_finder = self.data_portal_live.asset_finder - assets = [ - asset_finder.lookup_symbol('eth_btc', self.bitfinex), - asset_finder.lookup_symbol('eth_btc', self.bittrex) - ] - now = pd.Timestamp.utcnow() - value = self.data_portal_live.get_spot_value( - assets, 'price', now, '1m') + # assets = [ + # asset_finder.lookup_symbol('eth_btc', self.bitfinex), + # asset_finder.lookup_symbol('eth_btc', self.bittrex) + # ] + # now = pd.Timestamp.utcnow() + # value = self.data_portal_live.get_spot_value( + # assets, 'price', now, '1m') pass def test_get_history_window_backtest(self): diff --git a/tests/exchange/test_poloniex.py b/tests/exchange/test_poloniex.py index d84d1010..62800468 100644 --- a/tests/exchange/test_poloniex.py +++ b/tests/exchange/test_poloniex.py @@ -34,8 +34,8 @@ class TestPoloniex(BaseExchangeTestCase): def test_open_orders(self): log.info('retrieving open orders') - asset = self.exchange.get_asset('neos_btc') - orders = self.exchange.get_open_orders(asset) + # asset = self.exchange.get_asset('neos_btc') + # orders = self.exchange.get_open_orders(asset) pass def test_get_order(self): @@ -79,7 +79,7 @@ class TestPoloniex(BaseExchangeTestCase): def test_get_balances(self): log.info('testing wallet balances') - balances = self.exchange.get_balances() + # balances = self.exchange.get_balances() pass def test_get_account(self): @@ -88,7 +88,6 @@ class TestPoloniex(BaseExchangeTestCase): def test_orderbook(self): log.info('testing order book for poloniex') - asset = self.exchange.get_asset('eth_btc') - - orderbook = self.exchange.get_orderbook(asset) + # asset = self.exchange.get_asset('eth_btc') + # orderbook = self.exchange.get_orderbook(asset) pass diff --git a/tests/exchange/test_server_bundle.py b/tests/exchange/test_server_bundle.py index 42b2d3f3..ea90c0f2 100644 --- a/tests/exchange/test_server_bundle.py +++ b/tests/exchange/test_server_bundle.py @@ -1,21 +1,16 @@ import os -import tarfile import importlib + import pandas as pd - -from catalyst import get_calendar - -from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader -from catalyst.data.minute_bars import BcolzMinuteBarMetadata -from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk - import matplotlib import matplotlib.pyplot as plt from matplotlib.finance import candlestick2_ohlc -from matplotlib.finance import volume_overlay +# from matplotlib.finance import volume_overlay import matplotlib.ticker as ticker +from catalyst.exchange.exchange_bundle import ExchangeBundle +from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader +from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk from catalyst.exchange.factory import get_exchange EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] @@ -85,8 +80,8 @@ class ValidateChunks(object): matplotlib.transforms.Bbox([[0.125, 0.1], [0.9, 0.26]])) # Plot the volume overlay - bc = volume_overlay(ax2, df['open'], df['close'], df['volume'], - colorup='g', alpha=0.5, width=1) + # bc = volume_overlay(ax2, df['open'], df['close'], df['volume'], + # colorup='g', alpha=0.5, width=1) ax.xaxis.set_major_locator(ticker.MaxNLocator(6)) diff --git a/tests/exchange/test_utils.py b/tests/exchange/test_utils.py index ddf76215..17a774fd 100644 --- a/tests/exchange/test_utils.py +++ b/tests/exchange/test_utils.py @@ -26,8 +26,7 @@ def rnd_history_date_minutes(max_minutes=1440): def rnd_bar_count(max_bars=21): - now = pd.Timestamp.utcnow() - + # now = pd.Timestamp.utcnow() return randint(0, max_bars)