diff --git a/catalyst/algorithm.py b/catalyst/algorithm.py index 93ea60bb..51fdfdba 100644 --- a/catalyst/algorithm.py +++ b/catalyst/algorithm.py @@ -125,6 +125,7 @@ from catalyst.utils.factory import create_simulation_parameters from catalyst.utils.math_utils import ( tolerant_equals, round_if_near_integer, + round_nearest ) from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches from catalyst.utils.preprocess import preprocess @@ -1488,7 +1489,7 @@ class TradingAlgorithm(object): def _calculate_order(self, asset, amount, limit_price=None, stop_price=None, style=None): - amount = self.round_order(amount) + amount = self.round_order(amount, asset) # Raises a ZiplineError if invalid parameters are detected. self.validate_order_params(asset, @@ -1505,16 +1506,13 @@ class TradingAlgorithm(object): return amount, style @staticmethod - def round_order(amount): + def round_order(amount, asset): """ - Convert number of shares to an integer. - - By default, truncates to the integer share count that's either within - .0001 of amount or closer to zero. - - E.g. 3.9999 -> 4.0; 5.5 -> 5.0; -5.5 -> -5.0 + Converts the number of shares to the smallest tradable lot size for + the asset being ordered. + """ - return int(round_if_near_integer(amount)) + return round_nearest(amount, asset.min_trade_size) def validate_order_params(self, asset, @@ -1550,7 +1548,6 @@ class TradingAlgorithm(object): self.updated_portfolio(), self.get_datetime(), self.trading_client.current_data) - @staticmethod def __convert_order_params_for_blotter(limit_price, stop_price, style): """ diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index cdc34008..05144223 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -59,6 +59,7 @@ cdef class Asset: cdef readonly object exchange cdef readonly object exchange_full + cdef readonly object min_trade_size _kwargnames = frozenset({ 'sid', @@ -70,6 +71,7 @@ cdef class Asset: 'auto_close_date', 'exchange', 'exchange_full', + 'min_trade_size', }) def __init__(self, @@ -81,7 +83,8 @@ cdef class Asset: object end_date=None, object first_traded=None, object auto_close_date=None, - object exchange_full=None): + object exchange_full=None, + object min_trade_size=None): self.sid = sid self.sid_hash = hash(sid) @@ -94,6 +97,7 @@ cdef class Asset: self.end_date = end_date self.first_traded = first_traded self.auto_close_date = auto_close_date + self.min_trade_size = min_trade_size def __int__(self): return self.sid @@ -148,7 +152,8 @@ cdef class Asset: def __repr__(self): attrs = ('symbol', 'asset_name', 'exchange', - 'start_date', 'end_date', 'first_traded', 'auto_close_date') + 'start_date', 'end_date', 'first_traded', 'auto_close_date', + 'min_trade_size') tuples = ((attr, repr(getattr(self, attr, None))) for attr in attrs) strings = ('%s=%s' % (t[0], t[1]) for t in tuples) @@ -170,7 +175,8 @@ cdef class Asset: self.end_date, self.first_traded, self.auto_close_date, - self.exchange_full)) + self.exchange_full, + self.min_trade_size)) cpdef to_dict(self): """ @@ -186,6 +192,7 @@ cdef class Asset: 'auto_close_date': self.auto_close_date, 'exchange': self.exchange, 'exchange_full': self.exchange_full, + 'min_trade_size': self.min_trade_size } @classmethod @@ -234,7 +241,7 @@ cdef class Equity(Asset): def __repr__(self): attrs = ('symbol', 'asset_name', 'exchange', 'start_date', 'end_date', 'first_traded', 'auto_close_date', - 'exchange_full') + 'exchange_full', 'min_trade_size') tuples = ((attr, repr(getattr(self, attr, None))) for attr in attrs) strings = ('%s=%s' % (t[0], t[1]) for t in tuples) diff --git a/catalyst/assets/asset_db_schema.py b/catalyst/assets/asset_db_schema.py index 9c9c98c2..35e062f9 100644 --- a/catalyst/assets/asset_db_schema.py +++ b/catalyst/assets/asset_db_schema.py @@ -39,7 +39,8 @@ equities = sa.Table( sa.Column('first_traded', sa.Integer), sa.Column('auto_close_date', sa.Integer), sa.Column('exchange', sa.Text), - sa.Column('exchange_full', sa.Text) + sa.Column('exchange_full', sa.Text), + sa.Column('min_trade_size', sa.Float) ) equity_symbol_mappings = sa.Table( diff --git a/catalyst/assets/asset_writer.py b/catalyst/assets/asset_writer.py index 9910db72..ec30ffab 100644 --- a/catalyst/assets/asset_writer.py +++ b/catalyst/assets/asset_writer.py @@ -73,6 +73,7 @@ _equities_defaults = { 'exchange': None, # optional, something like "New York Stock Exchange" 'exchange_full': None, + 'min_trade_size': 1 } # Default values for the futures DataFrame @@ -390,6 +391,8 @@ class AssetDBWriter(object): The date on which to close any positions in this asset. exchange : str The exchange where this asset is traded. + min_trade_size: float, optional + The minimum denomination this asset can be traded. The index of this dataframe should contain the sids. futures : pd.DataFrame, optional diff --git a/catalyst/curate/poloniex.py b/catalyst/curate/poloniex.py index e92a9a11..e2a88476 100644 --- a/catalyst/curate/poloniex.py +++ b/catalyst/curate/poloniex.py @@ -1,22 +1,23 @@ import json, time, csv from datetime import datetime import pandas as pd -import os -import time -import requests -import logbook +import os, time, shutil, requests, logbook +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename -DT_START = time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()) + +DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())) +DT_END = int(time.time()) CSV_OUT_FOLDER = '/var/tmp/catalyst/data/poloniex/' +CSV_OUT_FOLDER = '/Volumes/enigma/data/poloniex/' CONN_RETRIES = 2 logbook.StderrHandler().push_application() log = logbook.Logger(__name__) class PoloniexCurator(object): - """ + ''' OHLCV data feed generator for crypto data. Based on Poloniex market data - """ + ''' _api_path = 'https://poloniex.com/public?' currency_pairs = [] @@ -29,6 +30,9 @@ class PoloniexCurator(object): log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER) log.exception(e) + ''' + Retrieves and returns all currency pairs from the exchange + ''' def get_currency_pairs(self): url = self._api_path + 'command=returnTicker' @@ -47,98 +51,244 @@ class PoloniexCurator(object): log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs))) - def _get_start_date(self, csv_fn): - ''' Function returns latest appended date, if the file has been previously written - the last line is an empty one, so we have to read the second to last line + + ''' + Helper function that reads tradeID and date fields from CSV readline + ''' + def _retrieve_tradeID_date(self, row): + tId = int(row.split(',')[0]) + d = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9 + return tId, d + + ''' + Retrieves TradeHistory from exchange for a given currencyPair between start and end dates. + If no start date is provided, uses a system-wide one (beginning of time for cryptotrading) + If no end date is provided, 'now' is used + Stores results in CSV file on disk. + This function is called recursively to work around the limitations imposed by the provider API. + ''' + def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None): + csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' + + ''' + Check what data we already have on disk, reading first and last lines from file. + Data is stored on file from NEWEST to OLDEST. ''' try: with open(csv_fn, 'ab+') as f: - f.seek(0, os.SEEK_END) # First check file is not zero size - if(f.tell() > 2): - f.seek(-2, os.SEEK_END) # Jump to the second last byte. - while f.read(1) != b"\n": # Until EOL is found... - f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. - lastrow = f.readline() - return int(lastrow.split(',')[0]) + 300 + f.seek(0, os.SEEK_END) + if(f.tell() > 2): # First check file is not zero size + f.seek(0) # Go to the beginning to read first line + last_tradeID, end_file = self._retrieve_tradeID_date(f.readline()) + f.seek(-2, os.SEEK_END) # Jump to the second last byte. + while f.read(1) != b"\n": # Until EOL is found... + f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. + first_tradeID, start_file = self._retrieve_tradeID_date(f.readline()) + + if( first_tradeID == 1 and end_file + 3600 > DT_END ): + return except Exception as e: log.error('Error opening file: %s' % csv_fn) log.exception(e) - return DT_START + ''' + Poloniex API limits querying TradeHistory to intervals smaller than 1 month, + so we make sure that start date is never more than 1 month apart from end date + ''' + if( end - start > 2419200 ): # 60 s/min * 60 min/hr * 24 hr/day * 28 days + newstart = end - 2419200 + else: + newstart = start - def get_data(self, currencyPair, start, end=9999999999, period=300): - url = self._api_path + 'command=returnChartData¤cyPair=' + currencyPair + '&start=' + str(start) + '&end=' + str(end) + '&period=' + str(period) + log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t ' + + time.ctime(newstart) + ' - '+ time.ctime(end)) + + url = self._api_path + 'command=returnTradeHistory¤cyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end) try: response = requests.get(url) except Exception as e: - log.error('Failed to retrieve candlestick chart data for %s' % currencyPair) + log.error('Failed to retrieve trade history data for %s' % currencyPair) log.exception(e) return None + else: + if isinstance(response.json(), dict) and response.json()['error']: + log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error'])) + exit(1) + + ''' + If we get to transactionId == 1, and we already have that on disk, + we got to the end of TradeHistory for this coin. + ''' + if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID): + return + + ''' + There are primarily two scenarios: + a) There is newer data available that we need to add at the beginning + of the file. We'll retrieve all what we need until we get to what + we already have, writing it to a temporary file; and we will write + that at the beginning of our existing file. + b) We are going back in time, appending at the end of our existing + TradeHistory until the first transaction for this currencyPair + ''' + try: + if( 'end_file' in locals() and end_file + 3600 < end): + if (temp is None): + temp = os.tmpfile() + tempcsv = csv.writer(temp) + for item in response.json(): + if( item['tradeID'] <= last_tradeID ): + continue + tempcsv.writerow([ + item['tradeID'], + item['date'], + item['type'], + item['rate'], + item['amount'], + item['total'], + item['globalTradeID'] + ]) + if( response.json()[-1]['tradeID'] > last_tradeID ): + end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9 + self.retrieve_trade_history(currencyPair, start, end, temp=temp) + else: + with open(csv_fn,'rb+') as f: + shutil.copyfileobj(f,temp) + f.seek(0) + temp.seek(0) + shutil.copyfileobj(temp,f) + temp.close() + end = start_file + else: + with open(csv_fn, 'ab') as csvfile: + csvwriter = csv.writer(csvfile) + for item in response.json(): + if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ): + continue + csvwriter.writerow([ + item['tradeID'], + item['date'], + item['type'], + item['rate'], + item['amount'], + item['total'], + item['globalTradeID'] + ]) + end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9 + + except Exception as e: + log.error('Error opening %s' % csv_fn) + log.exception(e) + + ''' + If we got here, we aren't done yet. Call recursively with 'end' times + that go sequentially back in time. + ''' + self.retrieve_trade_history(currencyPair, start, end) - return response.json() ''' - Pulls latest data for a single pair + Generates OHLCV dataframe from a dataframe containing all TradeHistory + by resampling with 1-minute period ''' - def append_data_single_pair(self, currencyPair, repeat=0): - log.debug('Getting data for %s' % currencyPair) - csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv' - start = self._get_start_date(csv_fn) - # Only fetch data if more than 5min have passed since last fetch - if (time.time() > start): - data = self.get_data(currencyPair, start) - if data is not None: - try: - with open(csv_fn, 'ab') as csvfile: - csvwriter = csv.writer(csvfile) - for item in data: - if item['date'] == 0: - continue - csvwriter.writerow([ - item['date'], - item['open'], - item['high'], - item['low'], - item['close'], - item['volume'], - ]) - except Exception as e: - log.error('Error opening %s' % csv_fn) - log.exception(e) - elif (repeat < CONN_RETRIES): - log.debug('Retrying: attemt %d' % (repeat+1) ) - self.append_data_single_pair(currencyPair, repeat + 1) + def generate_ohlcv(self, df): + df.set_index('date', inplace=True) # Index by date + vol = df['total'].to_frame('volume') # Will deal with vol separately, as ohlc() messes it up + df.drop('total', axis=1, inplace=True) # Drop volume data from dataframe + ohlc = df.resample('T').ohlc() # Resample OHLC in 1min bins + ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate' + closes = ohlc['close'].fillna(method='pad') # Pad forward missing 'close' + ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close + vol = vol.resample('T').sum().fillna(0) # Add volumes by bin + ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Volume + return ohlcv + ''' - Pulls latest data for all currency pairs + Generates OHLCV data file with 1minute bars from TradeHistory on disk ''' - def append_data(self): - for currencyPair in self.currency_pairs: - self.append_data_single_pair(currencyPair) - # Rate limit is 6 calls per second, sleep 1sec/6 to be safe - time.sleep(0.17) + def write_ohlcv_file(self, currencyPair): + csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' + csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' + if( os.path.isfile(csv_1min) ): + log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.') + else: + df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], + dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } ) + df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True) + df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) + ohlcv = self.generate_ohlcv(df) + try: + with open(csv_1min, 'ab') as csvfile: + csvwriter = csv.writer(csvfile) + for item in ohlcv.itertuples(): + if item.Index == 0: + continue + csvwriter.writerow([ + item.Index.value // 10 ** 9, + item.open, + item.high, + item.low, + item.close, + item.volume, + ]) + except Exception as e: + log.error('Error opening %s' % csv_fn) + log.exception(e) + log.debug(currencyPair+': Generated 1min OHLCV data.') + ''' - Returns a data frame for all pairs, or for the requests currency pair. - Makes sure data is up to date + Returns a data frame for a given currencyPair from data on disk ''' - def to_dataframe(self, start, end, currencyPair=None): - csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv' - last_date = self._get_start_date(csv_fn) - if last_date + 300 < end or not os.path.exists(csv_fn): - # get latest data - self.append_data_single_pair(currencyPair) - - # CSV holds the latest snapshot - df = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume']) - df['date']=pd.to_datetime(df['date'],unit='s') + def onemin_to_dataframe(self, currencyPair, start, end): + csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' + df = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume']) + df['date'] = pd.to_datetime(df['date'],unit='s') df.set_index('date', inplace=True) + return df[start : end] + + ''' + Generates a symbols.json file with corresponding start_date for each currencyPair + ''' + def generate_symbols_json(self, filename=None): + symbol_map = {} + + if(filename is None): + filename = get_exchange_symbols_filename('poloniex') + + with open(filename, 'w') as symbols: + for currencyPair in self.currency_pairs: + start = None + csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' + with open(csv_fn, 'r') as f: + f.seek(0, os.SEEK_END) + if(f.tell() > 2): # First check file is not zero size + f.seek(-2, os.SEEK_END) # Jump to the second last byte. + while f.read(1) != b"\n": # Until EOL is found... + f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. + start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True) + + if(start is None): + start = time.gmtime() + base, market = currencyPair.lower().split('_') + symbol = '{market}_{base}'.format( market=market, base=base ) + symbol_map[currencyPair] = dict( + symbol = symbol, + start_date = start.strftime("%Y-%m-%d") + ) + json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':')) - return df[datetime.fromtimestamp(start):datetime.fromtimestamp(end-1)] if __name__ == '__main__': pc = PoloniexCurator() pc.get_currency_pairs() - pc.append_data() + #pc.generate_symbols_json() + + for currencyPair in pc.currency_pairs: + pc.retrieve_trade_history(currencyPair) + pc.write_ohlcv_file(currencyPair) + + \ No newline at end of file diff --git a/catalyst/data/_equities.pyx b/catalyst/data/_equities.pyx index 81f9a66e..563fa56a 100644 --- a/catalyst/data/_equities.pyx +++ b/catalyst/data/_equities.pyx @@ -217,7 +217,7 @@ cpdef _read_bcolz_data(ctable_t table, if column_name in ['open', 'high', 'low', 'close']: where_nan = (outbuf == 0) - outbuf_as_float = outbuf.astype(float64) * .000001 + outbuf_as_float = outbuf.astype(float64) * .000000001 outbuf_as_float[where_nan] = NAN results.append(outbuf_as_float) elif column_name != 'volume': diff --git a/catalyst/data/bundles/base.py b/catalyst/data/bundles/base.py index 23640abd..135dd531 100644 --- a/catalyst/data/bundles/base.py +++ b/catalyst/data/bundles/base.py @@ -491,7 +491,7 @@ class BaseBundle(object): data_frequency, ) raw_data.index = pd.to_datetime(raw_data.index, utc=True) - raw_data.index = raw_data.index.tz_localize('UTC') + #raw_data.index = raw_data.index.tz_localize('UTC') # Filter incoming data to fit start and end sessions. raw_data = raw_data[ diff --git a/catalyst/data/bundles/base_pricing.py b/catalyst/data/bundles/base_pricing.py index a0abd51a..c5281fdd 100644 --- a/catalyst/data/bundles/base_pricing.py +++ b/catalyst/data/bundles/base_pricing.py @@ -24,6 +24,7 @@ class BasePricingBundle(BaseBundle): ('start_date', 'datetime64[ns]'), ('end_date', 'datetime64[ns]'), ('ac_date', 'datetime64[ns]'), + ('min_trade_size', 'float'), ] @lazyval diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index ad224492..e161df95 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + from datetime import datetime import pandas as pd @@ -23,6 +25,8 @@ from catalyst.data.bundles.core import register_bundle from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle from catalyst.utils.memoize import lazyval +from catalyst.curate.poloniex import PoloniexCurator + class PoloniexBundle(BaseCryptoPricingBundle): @lazyval def name(self): @@ -36,7 +40,7 @@ class PoloniexBundle(BaseCryptoPricingBundle): def frequencies(self): return set(( 'daily', - #'5-minute', + 'minute', )) @lazyval @@ -75,12 +79,14 @@ class PoloniexBundle(BaseCryptoPricingBundle): start_date = sym_data.index[0] end_date = sym_data.index[-1] ac_date = end_date + pd.Timedelta(days=1) + min_trade_size = 0.00000001 return ( sym_md.symbol, start_date, end_date, ac_date, + min_trade_size, ) def fetch_raw_symbol_frame(self, @@ -90,24 +96,30 @@ class PoloniexBundle(BaseCryptoPricingBundle): start_date, end_date, frequency): + # TODO: replace this with direct exchange call # The end date and frequency should be used to calculate the number of bars - raw = pd.read_json( - self._format_data_url( - api_key, - symbol, - start_date, - end_date, - frequency, - ), - orient='records', - ) - raw.set_index('date', inplace=True) + if(frequency == 'minute'): + pc = PoloniexCurator() + raw = pc.onemin_to_dataframe(symbol, start_date, end_date) + + else: + raw = pd.read_json( + self._format_data_url( + api_key, + symbol, + start_date, + end_date, + frequency, + ), + orient='records', + ) + raw.set_index('date', inplace=True) # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored # on disk, which we compensate here to get the right pricing amounts # ref: data/us_equity_pricing.py - scale = 1000 + scale = 1 raw.loc[:, 'open'] /= scale raw.loc[:, 'high'] /= scale raw.loc[:, 'low'] /= scale @@ -169,4 +181,9 @@ register_bundle(PoloniexBundle, ['USDT_BTC',]) For a production environment make sure to use (to bundle all pairs): register_bundle(PoloniexBundle) ''' -register_bundle(PoloniexBundle, create_writers=False) + +if 'ingest' in sys.argv and '-c' in sys.argv: + register_bundle(PoloniexBundle) +else: + register_bundle(PoloniexBundle, create_writers=False) + diff --git a/catalyst/data/history_loader.py b/catalyst/data/history_loader.py index 8bc00563..e33f5f6e 100644 --- a/catalyst/data/history_loader.py +++ b/catalyst/data/history_loader.py @@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype from catalyst.utils.pandas_utils import find_in_sorted_index # Default number of decimal places used for rounding asset prices. -DEFAULT_ASSET_PRICE_DECIMALS = 3 +DEFAULT_ASSET_PRICE_DECIMALS = 9 class HistoryCompatibleUSEquityAdjustmentReader(object): diff --git a/catalyst/data/resample.py b/catalyst/data/resample.py index 3154e590..dc325c39 100644 --- a/catalyst/data/resample.py +++ b/catalyst/data/resample.py @@ -156,7 +156,10 @@ class DailyHistoryAggregator(object): cache = self._caches[field] = (session, market_open, {}) _, market_open, entries = cache - market_open = market_open.tz_localize('UTC') + try: + market_open = market_open.tz_localize('UTC') + except TypeError: + market_open = market_open.tz_convert('UTC') if dt != market_open: prev_dt = dt_value - self._one_min else: diff --git a/catalyst/data/us_equity_pricing.py b/catalyst/data/us_equity_pricing.py index 03d01a4e..901a0e60 100644 --- a/catalyst/data/us_equity_pricing.py +++ b/catalyst/data/us_equity_pricing.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import division # Python2 req to have division of ints yield float + from errno import ENOENT from functools import partial from os import remove @@ -80,7 +83,6 @@ from catalyst.utils.cli import ( from ._equities import _compute_row_slices, _read_bcolz_data from ._adjustments import load_adjustments_from_sqlite - logger = logbook.Logger('UsEquityPricing') OHLC = frozenset(['open', 'high', 'low', 'close']) @@ -116,6 +118,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = { UINT32_MAX = iinfo(uint32).max UINT64_MAX = iinfo(uint64).max +PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220 + def check_uint32_safe(value, colname): if value >= UINT32_MAX: @@ -433,7 +437,7 @@ class BcolzDailyBarWriter(object): return raw_data winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC) - processed = (raw_data[list(OHLC)] * 1000000).astype('uint64') + processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64') dates = raw_data.index.values.astype('datetime64[s]') check_uint32_safe(dates.max().view(np.int64), 'day') processed['day'] = dates.astype('uint32') @@ -519,7 +523,6 @@ class BcolzDailyBarReader(SessionBarReader): # Need to test keeping the entire array in memory for the course of a # process first. self._spot_cols = {} - self.PRICE_ADJUSTMENT_FACTOR = 0.001 self._read_all_threshold = read_all_threshold @lazyval @@ -763,7 +766,7 @@ class BcolzDailyBarReader(SessionBarReader): if price == 0: return nan else: - return price * 0.001 + return price / PRICE_ADJUSTMENT_FACTOR else: return price diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 466e3e50..434fd933 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -24,6 +24,7 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ ExchangeStopLimitOrder, ExchangeStopOrder from catalyst.finance.order import Order, ORDER_STATUS from catalyst.protocol import Account +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename # Trying to account for REST api instability # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request @@ -559,3 +560,15 @@ class Bitfinex(Exchange): log.debug('got tickers {}'.format(ticks)) return ticks + + def generate_symbols_json(self, filename=None): + symbol_map = {} + response = self._request('symbols', None) + for symbol in response.json(): + symbol_map[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"} + + if(filename is None): + filename = get_exchange_symbols_filename(self.name) + + with open(filename,'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index 86f0db88..8ed92d85 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -12,6 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ CreateOrderError from catalyst.finance.execution import LimitOrder, StopLimitOrder from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename + log = Logger('Bittrex') @@ -56,32 +58,6 @@ class Bittrex(Exchange): """ return exchange_symbol.lower() - def fetch_symbol_map(self): - """ - Since Bittrex gives us a complete dictionary of symbols, - we can build the symbol map ad-hoc as opposed to maintaining - a static file. We must be careful with mapping any unconventional - symbol name as appropriate. - - :return symbol_map: - """ - symbol_map = dict() - - self.ask_request() - markets = self.api.getmarkets() - for market in markets: - exchange_symbol = market['MarketName'] - symbol = '{market}_{base}'.format( - market=self.sanitize_curency_symbol(market['MarketCurrency']), - base=self.sanitize_curency_symbol(market['BaseCurrency']) - ) - symbol_map[exchange_symbol] = dict( - symbol=symbol, - start_date=pd.to_datetime(market['Created'], utc=True) - ) - - return symbol_map - def get_balances(self): try: log.debug('retrieving wallet balances') @@ -330,3 +306,23 @@ class Bittrex(Exchange): def get_account(self): log.info('retrieving account data') pass + + def generate_symbols_json(self, filename=None): + symbol_map = {} + markets = self.api.getmarkets() + for market in markets: + exchange_symbol = market['MarketName'] + symbol = '{market}_{base}'.format( + market=self.sanitize_curency_symbol(market['MarketCurrency']), + base=self.sanitize_curency_symbol(market['BaseCurrency']) + ) + symbol_map[exchange_symbol] = dict( + symbol=symbol, + start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d") + ) + + if(filename is None): + filename = get_exchange_symbols_filename(self.name) + + with open(filename,'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 2638c1c9..ebb33023 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -9,9 +9,8 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ ExchangeSymbolsNotFound from catalyst.utils.paths import data_root, ensure_directory -# TODO: move to aws -SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'master/catalyst/exchange/{exchange}/symbols.json' +SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ + '{exchange}/symbols.json' def get_exchange_folder(exchange_name, environ=None): @@ -25,18 +24,20 @@ def get_exchange_folder(exchange_name, environ=None): return exchange_folder -def download_exchange_symbols(exchange_name, environ=None): +def get_exchange_symbols_filename(exchange_name, environ=None): exchange_folder = get_exchange_folder(exchange_name, environ) - filename = os.path.join(exchange_folder, 'symbols.json') + return os.path.join(exchange_folder, 'symbols.json') + +def download_exchange_symbols(exchange_name, environ=None): + filename = get_exchange_symbols_filename(exchange_name) url = SYMBOLS_URL.format(exchange=exchange_name) response = urllib.urlretrieve(url=url, filename=filename) return response def get_exchange_symbols(exchange_name, environ=None): - exchange_folder = get_exchange_folder(exchange_name, environ) - filename = os.path.join(exchange_folder, 'symbols.json') + filename = get_exchange_symbols_filename(exchange_name) if not os.path.isfile(filename): download_exchange_symbols(exchange_name, environ) diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index ce3aa7df..eab0d58c 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -12,22 +12,18 @@ # limitations under the License. from datetime import timedelta -import matplotlib.dates as mdates import pandas as pd from catalyst.gens.sim_engine import ( BAR, SESSION_START ) from logbook import Logger -from matplotlib import pyplot as plt -from matplotlib import style from catalyst.exchange.exchange_errors import \ MismatchingBaseCurrenciesExchanges -log = Logger('LiveGraphClock') -fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') +log = Logger('LiveGraphClock') class LiveGraphClock(object): @@ -58,13 +54,18 @@ class LiveGraphClock(object): def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')): - style.use('dark_background') + import matplotlib.dates as mdates + from matplotlib import pyplot as plt + from matplotlib import style self.sessions = sessions self.time_skew = time_skew self._last_emit = None self._before_trading_start_bar_yielded = True self.context = context + self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') + + style.use('dark_background') fig = plt.figure() fig.canvas.set_window_title('Enigma Catalyst: {}'.format( @@ -100,7 +101,7 @@ class LiveGraphClock(object): :return: """ ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) - ax.xaxis.set_major_formatter(fmt) + ax.xaxis.set_major_formatter(self.fmt) locator = mdates.HourLocator(interval=4) locator.MAXTICKS = 5000 diff --git a/catalyst/exchange/poloniex/__init__.py b/catalyst/exchange/poloniex/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py new file mode 100644 index 00000000..2b5b3586 --- /dev/null +++ b/catalyst/exchange/poloniex/poloniex.py @@ -0,0 +1,485 @@ +import base64 +import hashlib +import hmac +import json +import re +import time + +import numpy as np +import pandas as pd +import pytz +import requests +#import six +from six import iteritems +from catalyst.assets._assets import TradingPair +from logbook import Logger + +from catalyst.exchange.poloniex.poloniex_api import Poloniex_api + + +# from websocket import create_connection +from catalyst.exchange.exchange import Exchange +from catalyst.exchange.exchange_errors import ( + ExchangeRequestError, + InvalidHistoryFrequencyError, + InvalidOrderStyle, OrderCancelError) +from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ + ExchangeStopLimitOrder, ExchangeStopOrder +from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.protocol import Account +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename + + +log = Logger('Poloniex') + + +class Poloniex(Exchange): + def __init__(self, key, secret, base_currency, portfolio=None): + self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8')) + self.name = 'poloniex' + self.assets = {} + self.load_assets() + self.base_currency = base_currency + self._portfolio = portfolio + self.minute_writer = None + self.minute_reader = None + + + def sanitize_curency_symbol(self, exchange_symbol): + """ + Helper method used to build the universal pair. + Include any symbol mapping here if appropriate. + + :param exchange_symbol: + :return universal_symbol: + """ + return exchange_symbol.lower() + + + def _create_order(self, order_status): + """ + Create a Catalyst order object from the Exchange order dictionary + :param order_status: + :return: Order + """ + #if order_status['is_cancelled']: + # status = ORDER_STATUS.CANCELLED + #elif not order_status['is_live']: + # log.info('found executed order {}'.format(order_status)) + # status = ORDER_STATUS.FILLED + #else: + status = ORDER_STATUS.OPEN + + amount = float(order_status['amount']) + #filled = float(order_status['executed_amount']) + filled = None + + if order_status['type'] == 'sell': + amount = -amount + #filled = -filled + + price = float(order_status['rate']) + order_type = order_status['type'] + + stop_price = None + limit_price = None + + # TODO: is this comprehensive enough? + #if order_type.endswith('limit'): + # limit_price = price + #elif order_type.endswith('stop'): + # stop_price = price + + #executed_price = float(order_status['avg_execution_price']) + executed_price = price + + # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it. + commission = None + + #date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) + #date = pytz.utc.localize(date) + date = None + + order = Order( + dt=date, + asset=self.assets[order_status['symbol']], + amount=amount, + stop=stop_price, + limit=limit_price, + filled=filled, + id=str(order_status['orderNumber']), + commission=commission + ) + order.status = status + + return order, executed_price + + + def get_balances(self): + log.debug('retrieving wallets balances') + try: + balances = self.api.returnbalances() + except Exception as e: + log.debug(e) + raise ExchangeRequestError(error=e) + + if 'error' in balances: + raise ExchangeRequestError( + error='unable to fetch balance {}'.format(balances['error']) + ) + + std_balances = dict() + for (key, value) in iteritems(balances): + currency = key.lower() + std_balances[currency] = float(value) + + return std_balances + + + @property + def account(self): + account = Account() + + account.settled_cash = None + account.accrued_interest = None + account.buying_power = None + account.equity_with_loan = None + account.total_positions_value = None + account.total_positions_exposure = None + account.regt_equity = None + account.regt_margin = None + account.initial_margin_requirement = None + account.maintenance_margin_requirement = None + account.available_funds = None + account.excess_liquidity = None + account.cushion = None + account.day_trades_remaining = None + account.leverage = None + account.net_leverage = None + account.net_liquidation = None + + return account + + @property + def time_skew(self): + # TODO: research the time skew conditions + return pd.Timedelta('0s') + + def get_account(self): + # TODO: fetch account data and keep in cache + return None + + def get_candles(self, data_frequency, assets, bar_count=None): + """ + Retrieve OHLVC candles from Poloniex + + :param data_frequency: + :param assets: + :param bar_count: + :return: + + Available Frequencies + --------------------- + '5m', '15m', '30m', '2h', '4h', '1D' + """ + + # TODO: use BcolzMinuteBarReader to read from cache + if(data_frequency == '5m' or data_frequency == 'minute'): #TODO: Polo does not have '1m' + frequency = 300 + elif(data_frequency == '15m'): + frequency = 900 + elif(data_frequency == '30m'): + frequency = 1800 + elif(data_frequency == '2h'): + frequency = 7200 + elif(data_frequency == '4h'): + frequency = 14400 + elif(data_frequency == '1D' or data_frequency == 'daily'): + frequency = 86400 + else: + raise InvalidHistoryFrequencyError( + frequency=data_frequency + ) + + # Making sure that assets are iterable + asset_list = [assets] if isinstance(assets, TradingPair) else assets + ohlc_map = dict() + + for asset in asset_list: + + end = int(time.time()) + if(bar_count is None): + start = end - 2 * frequency + else: + start = end - bar_count * frequency + + try: + response = self.api.returnchartdata(self.get_symbol(asset),frequency, start, end) + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'error' in response: + raise ExchangeRequestError( + error='Unable to retrieve candles: {}'.format( + response.content) + ) + + def ohlc_from_candle(candle): + ohlc = dict( + open=np.float64(candle['open']), + high=np.float64(candle['high']), + low=np.float64(candle['low']), + close=np.float64(candle['close']), + volume=np.float64(candle['volume']), + price=np.float64(candle['close']), + last_traded=pd.Timestamp.utcfromtimestamp( candle['date'] ) + ) + + return ohlc + + if bar_count is None: + ohlc_map[asset] = ohlc_from_candle(response[0]) + else: + ohlc_bars = [] + for candle in response: + ohlc = ohlc_from_candle(candle) + ohlc_bars.append(ohlc) + ohlc_map[asset] = ohlc_bars + + return ohlc_map[assets] \ + if isinstance(assets, TradingPair) else ohlc_map + + + def create_order(self, asset, amount, is_buy, style): + pass + ''' + """ + Creating order on the exchange. + + :param asset: + :param amount: + :param is_buy: + :param style: + :return: + """ + exchange_symbol = self.get_symbol(asset) + if isinstance(style, ExchangeLimitOrder) \ + or isinstance(style, ExchangeStopLimitOrder): + price = style.get_limit_price(is_buy) + order_type = 'limit' + + elif isinstance(style, ExchangeStopOrder): + price = style.get_stop_price(is_buy) + order_type = 'stop' + + else: + raise InvalidOrderStyle(exchange=self.name, + style=style.__class__.__name__) + + req = dict( + symbol=exchange_symbol, + amount=str(float(abs(amount))), + price="{:.20f}".format(float(price)), + side='buy' if is_buy else 'sell', + type='exchange ' + order_type, # TODO: support margin trades + exchange=self.name, + is_hidden=False, + is_postonly=False, + use_all_available=0, + ocoorder=False, + buy_price_oco=0, + sell_price_oco=0 + ) + + date = pd.Timestamp.utcnow() + try: + response = self._request('order/new', req) + order_status = response.json() + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'message' in order_status: + raise ExchangeRequestError( + error='unable to create Bitfinex order {}'.format( + order_status['message']) + ) + + order_id = str(order_status['id']) + order = Order( + dt=date, + asset=asset, + amount=amount, + stop=style.get_stop_price(is_buy), + limit=style.get_limit_price(is_buy), + id=order_id + ) + + return order + ''' + + def get_open_orders(self, asset='all'): + """Retrieve all of the current open orders. + + Parameters + ---------- + asset : Asset + If passed and not None, return only the open orders for the given + asset instead of all open orders. + + Returns + ------- + open_orders : dict[list[Order]] or list[Order] + If no asset is passed this will return a dict mapping Assets + to a list containing all the open orders for the asset. + If an asset is passed then this will return a list of the open + orders for this asset. + """ + try: + if(asset=='all'): + response = self.api.returnopenorders('all') + else: + response = self.api.returnopenorders(self.get_symbol(asset)) + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'error' in response: + raise ExchangeRequestError( + error='Unable to retrieve open orders: {}'.format( + order_statuses['message']) + ) + + #TODO: Need to handle openOrders for 'all' + orders = list() + for order_status in response: + order, executed_price = self._create_order(order_status) + if asset is None or asset == order.sid: + orders.append(order) + + return orders + + + def get_order(self, order_id): + """Lookup an order based on the order id returned from one of the + order functions. + + Parameters + ---------- + order_id : str + The unique identifier for the order. + + Returns + ------- + order : Order + The order object. + """ + pass + ''' + try: + response = self._request( + 'order/status', {'order_id': int(order_id)}) + order_status = response.json() + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'message' in order_status: + raise ExchangeRequestError( + error='Unable to retrieve order status: {}'.format( + order_status['message']) + ) + return self._create_order(order_status) + ''' + + def cancel_order(self, order_param): + """Cancel an open order. + + Parameters + ---------- + order_param : str or Order + The order_id or order object to cancel. + """ + order_id = order_param.id \ + if isinstance(order_param, Order) else order_param + + try: + response = self.api.cancelorder(order_id) + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'error' in response: + raise OrderCancelError( + order_id=order_id, + exchange=self.name, + error=response['error'] + ) + + + def tickers(self, assets): + """ + Fetch ticket data for assets + https://docs.bitfinex.com/v2/reference#rest-public-tickers + + :param assets: + :return: + """ + pass + + ''' + symbols = self._get_v2_symbols(assets) + log.debug('fetching tickers {}'.format(symbols)) + + try: + response = requests.get( + '{url}/v2/tickers?symbols={symbols}'.format( + url=self.url, + symbols=','.join(symbols), + ) + ) + except Exception as e: + raise ExchangeRequestError(error=e) + + if 'error' in response.content: + raise ExchangeRequestError( + error='Unable to retrieve tickers: {}'.format( + response.content) + ) + + tickers = response.json() + + ticks = dict() + for index, ticker in enumerate(tickers): + if not len(ticker) == 11: + raise ExchangeRequestError( + error='Invalid ticker in response: {}'.format(ticker) + ) + + ticks[assets[index]] = dict( + timestamp=pd.Timestamp.utcnow(), + bid=ticker[1], + ask=ticker[3], + last_price=ticker[7], + low=ticker[10], + high=ticker[9], + volume=ticker[8], + ) + + log.debug('got tickers {}'.format(ticks)) + return ticks + ''' + + def generate_symbols_json(self, filename=None): + symbol_map = {} + response = self.api.returnticker() + for exchange_symbol in response: + base, market = self.sanitize_curency_symbol(exchange_symbol).split('_') + symbol = '{market}_{base}'.format( market=market, base=base ) + symbol_map[exchange_symbol] = dict( + symbol = symbol, + start_date = '2010-01-01' + ) + + if(filename is None): + filename = get_exchange_symbols_filename(self.name) + + with open(filename,'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) + diff --git a/catalyst/exchange/poloniex/poloniex_api.py b/catalyst/exchange/poloniex/poloniex_api.py new file mode 100644 index 00000000..2efdd858 --- /dev/null +++ b/catalyst/exchange/poloniex/poloniex_api.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +import json +import time +import hmac +import hashlib + +from six.moves import urllib + +# Workaround for backwards compatibility +# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7 +urlopen = urllib.request.urlopen + + +class Poloniex_api(object): + def __init__(self, key, secret): + self.key = key + self.secret = secret + self.public = ['returnTicker', 'return24Volume', 'returnOrderBook', + 'returnTradeHistory', 'returnChartData', + 'returnCurrencies', 'returnLoanOrders'] + self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses', + 'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders', + 'returnTradeHistory','returnOrderTrades', + 'buy', 'sell', 'cancelOrder', 'moveOrder', + 'withdraw', 'returnFeeInfo','returnAvailableAccountBalances', + 'returnTradableBalances', 'transferBalance', + 'returnMarginAccountSummary','marginBuy','marginSell', + 'getMarginPosition', 'closeMarginPosition','createLoanOffer', + 'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans', + 'returnLendingHistory','toggleAutoRenew'] + + def query(self, method, req={}): + + if method in self.public: + url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req) + headers = {} + post_data = None + elif method in self.trading: + url = 'https://poloniex.com/tradingApi' + req['command'] = method + req['nonce'] = int(time.time()*1000) + post_data = urllib.parse.urlencode(req) + print(post_data) + signature = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest() + headers = { 'Sign': signature, 'Key': self.key} + else: + raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints') + + req = urllib.request.Request(url, data=post_data, headers=headers) + return json.loads(urlopen(req).read()) + + def returnticker(self): + return self.query('returnTicker') + + def return24volume(self): + return self.query('return24Volume') + + def returnOrderBook(self, market='all'): + return self.query('returnOrderBook', {'currencyPair': market}) + + def returntradehistory(self, market, start=None, end=None): + if(start is not None and end is not None): + return self.query('returntradehistory', + {'currencyPair': market, 'start': start, 'end': end }) + else: + return self.query('returntradehistory', {'currencyPair': market }) + + def returnchartdata(self, market, period, start, end): + return self.query('returnChartData', {'currencyPair': market, 'period': period, + 'start': start, 'end': end}) + + def returncurrencies(self): + return self.query('returnCurrencies') + + def returnloadorders(self, market): + return self.query('returnLoanOrders', {'currency': market}) + + def returnbalances(self): + return self.query('returnBalances') + + def returnopenorders(self, market): + return self.query('returnOpenOrders', {'currencyPair': market}) + + def cancelorder(self, ordernumber): + return self.query('cancelOrder', {'orderNumber': ordernumber}) + + ''' + def buylimit(self, market, quantity, rate): + return self.query('buylimit', {'market': market, 'quantity': quantity, + 'rate': rate}) + + def buymarket(self, market, quantity): + return self.query('buymarket', + {'market': market, 'quantity': quantity}) + + def selllimit(self, market, quantity, rate): + return self.query('selllimit', {'market': market, 'quantity': quantity, + 'rate': rate}) + + def sellmarket(self, market, quantity): + return self.query('sellmarket', + {'market': market, 'quantity': quantity}) + + def getbalance(self, currency): + return self.query('getbalance', {'currency': currency}) + + def getdepositaddress(self, currency): + return self.query('getdepositaddress', {'currency': currency}) + + def withdraw(self, currency, quantity, address): + return self.query('withdraw', + {'currency': currency, 'quantity': quantity, + 'address': address}) + + def getorder(self, uuid): + return self.query('getorder', {'uuid': uuid}) + + def getorderhistory(self, market, count): + return self.query('getorderhistory', + {'market': market, 'count': count}) + + def getwithdrawalhistory(self, currency, count): + return self.query('getwithdrawalhistory', + {'currency': currency, 'count': count}) + + def getdeposithistory(self, currency, count): + return self.query('getdeposithistory', + {'currency': currency, 'count': count}) + ''' diff --git a/catalyst/finance/slippage.py b/catalyst/finance/slippage.py index 118a5b58..36de4ec7 100644 --- a/catalyst/finance/slippage.py +++ b/catalyst/finance/slippage.py @@ -41,6 +41,7 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05 + class LiquidityExceeded(Exception): pass @@ -205,20 +206,22 @@ class VolumeShareSlippage(SlippageModel): def process_order(self, data, order): volume = data.current(order.asset, "volume") + min_trade_size = order.asset.min_trade_size + max_volume = self.volume_limit * volume # price impact accounts for the total volume of transactions # created against the current minute bar remaining_volume = max_volume - self.volume_for_bar - if remaining_volume < 1: + if remaining_volume < min_trade_size: # we can't fill any more transactions raise LiquidityExceeded() # the current order amount will be the min of the # volume available in the bar or the open amount. - cur_volume = int(min(remaining_volume, abs(order.open_amount))) + cur_volume = min(remaining_volume, abs(order.open_amount)) - if cur_volume < 1: + if cur_volume < min_trade_size: return None, None # tally the current amount into our total amount ordered. diff --git a/catalyst/finance/transaction.py b/catalyst/finance/transaction.py index 3a388890..f9d72c38 100644 --- a/catalyst/finance/transaction.py +++ b/catalyst/finance/transaction.py @@ -65,14 +65,10 @@ def create_transaction(order, dt, price, amount): # floor the amount to protect against non-whole number orders # TODO: Investigate whether we can add a robust check in blotter # and/or tradesimulation, as well. - amount_magnitude = int(abs(amount)) - - if amount_magnitude < 1: - raise Exception("Transaction magnitude must be at least 1.") transaction = Transaction( asset=order.asset, - amount=int(amount), + amount=amount, dt=dt, price=price, order_id=order.id diff --git a/catalyst/utils/math_utils.py b/catalyst/utils/math_utils.py index 16fdb99d..7981a52b 100644 --- a/catalyst/utils/math_utils.py +++ b/catalyst/utils/math_utils.py @@ -17,6 +17,8 @@ import math from numpy import isnan +def round_nearest(x, a): + return round(round(x / a) * a, -int(math.floor(math.log10(a)))) def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False): """Check if a and b are equal with some tolerance. diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 6a4e6f7c..de8e64a5 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -9,6 +9,8 @@ import click import pandas as pd from catalyst.exchange.bittrex.bittrex import Bittrex +from catalyst.exchange.bitfinex.bitfinex import Bitfinex +from catalyst.exchange.poloniex.poloniex import Poloniex try: from pygments import highlight @@ -30,7 +32,6 @@ from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \ ExchangeTradingAlgorithmBacktest from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \ DataPortalExchangeBacktest -from catalyst.exchange.bitfinex.bitfinex import Bitfinex from catalyst.exchange.asset_finder_exchange import AssetFinderExchange from catalyst.exchange.exchange_portfolio import ExchangePortfolio from catalyst.exchange.exchange_errors import ( @@ -177,7 +178,13 @@ def _run(handle_data, base_currency=base_currency, portfolio=portfolio ) - + elif exchange_name == 'poloniex': + exchange = Poloniex( + key=exchange_auth['key'], + secret=exchange_auth['secret'], + base_currency=base_currency, + portfolio=portfolio + ) else: raise ExchangeNotFoundError(exchange_name=exchange_name) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml new file mode 100644 index 00000000..34fabcbb --- /dev/null +++ b/etc/python2.7-environment.yml @@ -0,0 +1,84 @@ +name: catalyst +channels: +- statiskit +- defaults +dependencies: +- certifi=2016.2.28=py27_0 +- coverage=4.4.1=py27_0 +- nose=1.3.7=py27_1 +- openssl=1.0.2l=0 +- path.py=10.3.1=py27_0 +- pip=9.0.1=py27_1 +- python=2.7.13=0 +- pyyaml=3.12=py27_0 +- readline=6.2=2 +- setuptools=36.4.0=py27_0 +- six=1.10.0=py27_0 +- sqlite=3.13.0=0 +- tk=8.5.18=0 +- wheel=0.29.0=py27_0 +- yaml=0.1.6=0 +- zlib=1.2.11=0 +- libdev=1.0.0=py27_0 +- python-dev=1.0.0=py27_0 +- python-scons=3.0.0=py27_0 +- pip: + - alembic==0.9.5 + - backports.shutil-get-terminal-size==1.0.0 + - bcolz==0.12.1 + - bottleneck==1.2.1 + - chardet==3.0.4 + - click==6.7 + - contextlib2==0.5.5 + - cycler==0.10.0 + - cyordereddict==1.0.0 + - cython==0.26.1 + - decorator==4.1.2 + - empyrical==0.2.1 + - enigma-catalyst>=0.2.dev2 + - enum34==1.1.6 + - functools32==3.2.3.post2 + - idna==2.6 + - intervaltree==2.1.0 + - ipdb==0.10.3 + - ipdbplugin==1.4.5 + - ipython==5.5.0 + - ipython-genutils==0.2.0 + - logbook==1.1.0 + - lru-dict==1.1.6 + - mako==1.0.7 + - markupsafe==1.0 + - matplotlib==2.0.2 + - multipledispatch==0.4.9 + - networkx==1.11 + - numexpr==2.6.4 + - numpy==1.13.1 + - pandas==0.19.2 + - pandas-datareader==0.5.0 + - pathlib2==2.3.0 + - patsy==0.4.1 + - pexpect==4.2.1 + - pickleshare==0.7.4 + - prompt-toolkit==1.0.15 + - ptyprocess==0.5.2 + - pygments==2.2.0 + - pyparsing==2.2.0 + - python-dateutil==2.6.1 + - python-editor==1.0.3 + - pytz==2017.2 + - requests==2.18.4 + - requests-file==1.4.2 + - requests-ftp==0.3.1 + - scandir==1.5 + - scipy==0.19.1 + - scons==3.0.0a20170821 + - simplegeneric==0.8.1 + - sortedcontainers==1.5.7 + - sqlalchemy==1.1.14 + - statsmodels==0.8.0 + - subprocess32==3.2.7 + - tables==3.4.2 + - toolz==0.8.2 + - traitlets==4.3.2 + - urllib3==1.22 + - wcwidth==0.1.7