wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-01 08:51:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'concurrent-exchanges' of github.com:enigmampc/catalyst into concurrent-exchanges

Browse Source
This commit is contained in:
fredfortier
2017-09-27 17:31:44 -04:00
parent 6bfe0eecd2 3362dbf95c
commit 9dfefec13c
28 changed files with 1186 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/algorithm.py
+7 -10
View File
@@ -125,6 +125,7 @@ from catalyst.utils.factory import create_simulation_parameters
from catalyst.utils.math_utils import (
tolerant_equals,
round_if_near_integer,
round_nearest
)
from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
from catalyst.utils.preprocess import preprocess
@@ -1488,7 +1489,7 @@ class TradingAlgorithm(object):
def _calculate_order(self, asset, amount,
limit_price=None, stop_price=None, style=None):
amount = self.round_order(amount)
amount = self.round_order(amount, asset)
# Raises a ZiplineError if invalid parameters are detected.
self.validate_order_params(asset,
@@ -1505,16 +1506,13 @@ class TradingAlgorithm(object):
return amount, style
@staticmethod
def round_order(amount):
def round_order(amount, asset):
"""
Convert number of shares to an integer.
By default, truncates to the integer share count that's either within
.0001 of amount or closer to zero.
E.g. 3.9999 -> 4.0; 5.5 -> 5.0; -5.5 -> -5.0
Converts the number of shares to the smallest tradable lot size for
the asset being ordered.
"""
return int(round_if_near_integer(amount))
return round_nearest(amount, asset.min_trade_size)
def validate_order_params(self,
asset,
@@ -1550,7 +1548,6 @@ class TradingAlgorithm(object):
self.updated_portfolio(),
self.get_datetime(),
self.trading_client.current_data)
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
catalyst/assets/_assets.pyx
+23 -9
View File
@@ -59,6 +59,7 @@ cdef class Asset:
cdef readonly object exchange
cdef readonly object exchange_full
cdef readonly object min_trade_size
_kwargnames = frozenset({
'sid',
@@ -70,6 +71,7 @@ cdef class Asset:
'auto_close_date',
'exchange',
'exchange_full',
'min_trade_size',
})
def __init__(self,
@@ -81,7 +83,8 @@ cdef class Asset:
object end_date=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None):
object exchange_full=None,
object min_trade_size=None):
self.sid = sid
self.sid_hash = hash(sid)
@@ -94,6 +97,7 @@ cdef class Asset:
self.end_date = end_date
self.first_traded = first_traded
self.auto_close_date = auto_close_date
self.min_trade_size = min_trade_size
def __int__(self):
return self.sid
@@ -148,7 +152,8 @@ cdef class Asset:
def __repr__(self):
attrs = ('symbol', 'asset_name', 'exchange',
'start_date', 'end_date', 'first_traded', 'auto_close_date')
'start_date', 'end_date', 'first_traded', 'auto_close_date',
'min_trade_size')
tuples = ((attr, repr(getattr(self, attr, None)))
for attr in attrs)
strings = ('%s=%s' % (t[0], t[1]) for t in tuples)
@@ -170,7 +175,8 @@ cdef class Asset:
self.end_date,
self.first_traded,
self.auto_close_date,
self.exchange_full))
self.exchange_full,
self.min_trade_size))
cpdef to_dict(self):
"""
@@ -186,6 +192,7 @@ cdef class Asset:
'auto_close_date': self.auto_close_date,
'exchange': self.exchange,
'exchange_full': self.exchange_full,
'min_trade_size': self.min_trade_size
}
@classmethod
@@ -234,7 +241,7 @@ cdef class Equity(Asset):
def __repr__(self):
attrs = ('symbol', 'asset_name', 'exchange',
'start_date', 'end_date', 'first_traded', 'auto_close_date',
'exchange_full')
'exchange_full', 'min_trade_size')
tuples = ((attr, repr(getattr(self, attr, None)))
for attr in attrs)
strings = ('%s=%s' % (t[0], t[1]) for t in tuples)
@@ -401,7 +408,8 @@ cdef class TradingPair(Asset):
'exchange_full',
'leverage',
'market_currency',
'base_currency'
'base_currency',
'min_trade_size',
})
def __init__(self,
object symbol,
@@ -413,7 +421,8 @@ cdef class TradingPair(Asset):
object end_date=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None):
object exchange_full=None,
object min_trade_size=None):
"""
Replicates the Asset constructor with some built-in conventions
and a new 'leverage' attribute.
@@ -469,6 +478,7 @@ cdef class TradingPair(Asset):
:param first_traded:
:param auto_close_date:
:param exchange_full:
:param min_trade_size:
"""
symbol = symbol.lower()
@@ -502,6 +512,7 @@ cdef class TradingPair(Asset):
first_traded=first_traded,
auto_close_date=auto_close_date,
exchange_full=exchange_full,
min_trade_size=min_trade_size
)
self.leverage = leverage
@@ -511,14 +522,16 @@ cdef class TradingPair(Asset):
'Introduced On: {start_date}, ' \
'Market Currency: {market_currency}, ' \
'Base Currency: {base_currency}, ' \
'Exchange Leverage: {leverage}'.format(
'Exchange Leverage: {leverage}, ' \
'Minimum Trade Size: {min_trade_size}'.format(
symbol=self.symbol,
sid=self.sid,
exchange=self.exchange,
start_date=self.start_date,
market_currency=self.market_currency,
base_currency=self.base_currency,
leverage=self.leverage
leverage=self.leverage,
min_trade_size=self.min_trade_size
)
cpdef __reduce__(self):
@@ -537,7 +550,8 @@ cdef class TradingPair(Asset):
self.end_date,
self.first_traded,
self.auto_close_date,
self.exchange_full))
self.exchange_full,
self.min_trade_size))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/assets/asset_db_schema.py
+2 -1
View File
@@ -39,7 +39,8 @@ equities = sa.Table(
sa.Column('first_traded', sa.Integer),
sa.Column('auto_close_date', sa.Integer),
sa.Column('exchange', sa.Text),
sa.Column('exchange_full', sa.Text)
sa.Column('exchange_full', sa.Text),
sa.Column('min_trade_size', sa.Float)
)
equity_symbol_mappings = sa.Table(
catalyst/assets/asset_writer.py
+3
View File
@@ -73,6 +73,7 @@ _equities_defaults = {
'exchange': None,
# optional, something like "New York Stock Exchange"
'exchange_full': None,
'min_trade_size': 1
}
# Default values for the futures DataFrame
@@ -390,6 +391,8 @@ class AssetDBWriter(object):
The date on which to close any positions in this asset.
exchange : str
The exchange where this asset is traded.
min_trade_size: float, optional
The minimum denomination this asset can be traded.
The index of this dataframe should contain the sids.
futures : pd.DataFrame, optional
catalyst/curate/poloniex.py
+221 -71
View File
@@ -1,22 +1,23 @@
import json, time, csv
from datetime import datetime
import pandas as pd
import os
import time
import requests
import logbook
import os, time, shutil, requests, logbook
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
DT_START = time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
DT_END = int(time.time())
CSV_OUT_FOLDER = '/var/tmp/catalyst/data/poloniex/'
CSV_OUT_FOLDER = '/Volumes/enigma/data/poloniex/'
CONN_RETRIES = 2
logbook.StderrHandler().push_application()
log = logbook.Logger(__name__)
class PoloniexCurator(object):
"""
'''
OHLCV data feed generator for crypto data. Based on Poloniex market data
"""
'''
_api_path = 'https://poloniex.com/public?'
currency_pairs = []
@@ -29,6 +30,9 @@ class PoloniexCurator(object):
log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
log.exception(e)
'''
Retrieves and returns all currency pairs from the exchange
'''
def get_currency_pairs(self):
url = self._api_path + 'command=returnTicker'
@@ -47,98 +51,244 @@ class PoloniexCurator(object):
log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
def _get_start_date(self, csv_fn):
''' Function returns latest appended date, if the file has been previously written
the last line is an empty one, so we have to read the second to last line
'''
Helper function that reads tradeID and date fields from CSV readline
'''
def _retrieve_tradeID_date(self, row):
tId = int(row.split(',')[0])
d = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9
return tId, d
'''
Retrieves TradeHistory from exchange for a given currencyPair between start and end dates.
If no start date is provided, uses a system-wide one (beginning of time for cryptotrading)
If no end date is provided, 'now' is used
Stores results in CSV file on disk.
This function is called recursively to work around the limitations imposed by the provider API.
'''
def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None):
csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
'''
Check what data we already have on disk, reading first and last lines from file.
Data is stored on file from NEWEST to OLDEST.
'''
try:
with open(csv_fn, 'ab+') as f:
f.seek(0, os.SEEK_END) # First check file is not zero size
if(f.tell() > 2):
f.seek(-2, os.SEEK_END) # Jump to the second last byte.
while f.read(1) != b"\n": # Until EOL is found...
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
lastrow = f.readline()
return int(lastrow.split(',')[0]) + 300
f.seek(0, os.SEEK_END)
if(f.tell() > 2): # First check file is not zero size
f.seek(0) # Go to the beginning to read first line
last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
f.seek(-2, os.SEEK_END) # Jump to the second last byte.
while f.read(1) != b"\n": # Until EOL is found...
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
if( first_tradeID == 1 and end_file + 3600 > DT_END ):
return
except Exception as e:
log.error('Error opening file: %s' % csv_fn)
log.exception(e)
return DT_START
'''
Poloniex API limits querying TradeHistory to intervals smaller than 1 month,
so we make sure that start date is never more than 1 month apart from end date
'''
if( end - start > 2419200 ): # 60 s/min * 60 min/hr * 24 hr/day * 28 days
newstart = end - 2419200
else:
newstart = start
def get_data(self, currencyPair, start, end=9999999999, period=300):
url = self._api_path + 'command=returnChartData&currencyPair=' + currencyPair + '&start=' + str(start) + '&end=' + str(end) + '&period=' + str(period)
log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t '
+ time.ctime(newstart) + ' - '+ time.ctime(end))
url = self._api_path + 'command=returnTradeHistory&currencyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end)
try:
response = requests.get(url)
except Exception as e:
log.error('Failed to retrieve candlestick chart data for %s' % currencyPair)
log.error('Failed to retrieve trade history data for %s' % currencyPair)
log.exception(e)
return None
else:
if isinstance(response.json(), dict) and response.json()['error']:
log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error']))
exit(1)
'''
If we get to transactionId == 1, and we already have that on disk,
we got to the end of TradeHistory for this coin.
'''
if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID):
return
'''
There are primarily two scenarios:
a) There is newer data available that we need to add at the beginning
of the file. We'll retrieve all what we need until we get to what
we already have, writing it to a temporary file; and we will write
that at the beginning of our existing file.
b) We are going back in time, appending at the end of our existing
TradeHistory until the first transaction for this currencyPair
'''
try:
if( 'end_file' in locals() and end_file + 3600 < end):
if (temp is None):
temp = os.tmpfile()
tempcsv = csv.writer(temp)
for item in response.json():
if( item['tradeID'] <= last_tradeID ):
continue
tempcsv.writerow([
item['tradeID'],
item['date'],
item['type'],
item['rate'],
item['amount'],
item['total'],
item['globalTradeID']
])
if( response.json()[-1]['tradeID'] > last_tradeID ):
end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
self.retrieve_trade_history(currencyPair, start, end, temp=temp)
else:
with open(csv_fn,'rb+') as f:
shutil.copyfileobj(f,temp)
f.seek(0)
temp.seek(0)
shutil.copyfileobj(temp,f)
temp.close()
end = start_file
else:
with open(csv_fn, 'ab') as csvfile:
csvwriter = csv.writer(csvfile)
for item in response.json():
if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ):
continue
csvwriter.writerow([
item['tradeID'],
item['date'],
item['type'],
item['rate'],
item['amount'],
item['total'],
item['globalTradeID']
])
end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
except Exception as e:
log.error('Error opening %s' % csv_fn)
log.exception(e)
'''
If we got here, we aren't done yet. Call recursively with 'end' times
that go sequentially back in time.
'''
self.retrieve_trade_history(currencyPair, start, end)
return response.json()
'''
Pulls latest data for a single pair
Generates OHLCV dataframe from a dataframe containing all TradeHistory
by resampling with 1-minute period
'''
def append_data_single_pair(self, currencyPair, repeat=0):
log.debug('Getting data for %s' % currencyPair)
csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv'
start = self._get_start_date(csv_fn)
# Only fetch data if more than 5min have passed since last fetch
if (time.time() > start):
data = self.get_data(currencyPair, start)
if data is not None:
try:
with open(csv_fn, 'ab') as csvfile:
csvwriter = csv.writer(csvfile)
for item in data:
if item['date'] == 0:
continue
csvwriter.writerow([
item['date'],
item['open'],
item['high'],
item['low'],
item['close'],
item['volume'],
])
except Exception as e:
log.error('Error opening %s' % csv_fn)
log.exception(e)
elif (repeat < CONN_RETRIES):
log.debug('Retrying: attemt %d' % (repeat+1) )
self.append_data_single_pair(currencyPair, repeat + 1)
def generate_ohlcv(self, df):
df.set_index('date', inplace=True) # Index by date
vol = df['total'].to_frame('volume') # Will deal with vol separately, as ohlc() messes it up
df.drop('total', axis=1, inplace=True) # Drop volume data from dataframe
ohlc = df.resample('T').ohlc() # Resample OHLC in 1min bins
ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate'
closes = ohlc['close'].fillna(method='pad') # Pad forward missing 'close'
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Volume
return ohlcv
'''
Pulls latest data for all currency pairs
Generates OHLCV data file with 1minute bars from TradeHistory on disk
'''
def append_data(self):
for currencyPair in self.currency_pairs:
self.append_data_single_pair(currencyPair)
# Rate limit is 6 calls per second, sleep 1sec/6 to be safe
time.sleep(0.17)
def write_ohlcv_file(self, currencyPair):
csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
if( os.path.isfile(csv_1min) ):
log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
else:
df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'],
dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
ohlcv = self.generate_ohlcv(df)
try:
with open(csv_1min, 'ab') as csvfile:
csvwriter = csv.writer(csvfile)
for item in ohlcv.itertuples():
if item.Index == 0:
continue
csvwriter.writerow([
item.Index.value // 10 ** 9,
item.open,
item.high,
item.low,
item.close,
item.volume,
])
except Exception as e:
log.error('Error opening %s' % csv_fn)
log.exception(e)
log.debug(currencyPair+': Generated 1min OHLCV data.')
'''
Returns a data frame for all pairs, or for the requests currency pair.
Makes sure data is up to date
Returns a data frame for a given currencyPair from data on disk
'''
def to_dataframe(self, start, end, currencyPair=None):
csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv'
last_date = self._get_start_date(csv_fn)
if last_date + 300 < end or not os.path.exists(csv_fn):
# get latest data
self.append_data_single_pair(currencyPair)
# CSV holds the latest snapshot
df = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
df['date']=pd.to_datetime(df['date'],unit='s')
def onemin_to_dataframe(self, currencyPair, start, end):
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
df = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
df['date'] = pd.to_datetime(df['date'],unit='s')
df.set_index('date', inplace=True)
return df[start : end]
'''
Generates a symbols.json file with corresponding start_date for each currencyPair
'''
def generate_symbols_json(self, filename=None):
symbol_map = {}
if(filename is None):
filename = get_exchange_symbols_filename('poloniex')
with open(filename, 'w') as symbols:
for currencyPair in self.currency_pairs:
start = None
csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
with open(csv_fn, 'r') as f:
f.seek(0, os.SEEK_END)
if(f.tell() > 2): # First check file is not zero size
f.seek(-2, os.SEEK_END) # Jump to the second last byte.
while f.read(1) != b"\n": # Until EOL is found...
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True)
if(start is None):
start = time.gmtime()
base, market = currencyPair.lower().split('_')
symbol = '{market}_{base}'.format( market=market, base=base )
symbol_map[currencyPair] = dict(
symbol = symbol,
start_date = start.strftime("%Y-%m-%d")
)
json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':'))
return df[datetime.fromtimestamp(start):datetime.fromtimestamp(end-1)]
if __name__ == '__main__':
pc = PoloniexCurator()
pc.get_currency_pairs()
pc.append_data()
#pc.generate_symbols_json()
for currencyPair in pc.currency_pairs:
pc.retrieve_trade_history(currencyPair)
pc.write_ohlcv_file(currencyPair)
catalyst/data/_equities.pyx
+1 -1
View File
@@ -217,7 +217,7 @@ cpdef _read_bcolz_data(ctable_t table,
if column_name in ['open', 'high', 'low', 'close']:
where_nan = (outbuf == 0)
outbuf_as_float = outbuf.astype(float64) * .000001
outbuf_as_float = outbuf.astype(float64) * .000000001
outbuf_as_float[where_nan] = NAN
results.append(outbuf_as_float)
elif column_name != 'volume':
catalyst/data/bundles/base.py
+1 -1
View File
@@ -491,7 +491,7 @@ class BaseBundle(object):
data_frequency,
)
raw_data.index = pd.to_datetime(raw_data.index, utc=True)
raw_data.index = raw_data.index.tz_localize('UTC')
#raw_data.index = raw_data.index.tz_localize('UTC')
# Filter incoming data to fit start and end sessions.
raw_data = raw_data[
catalyst/data/bundles/base_pricing.py
+1
View File
@@ -24,6 +24,7 @@ class BasePricingBundle(BaseBundle):
('start_date', 'datetime64[ns]'),
('end_date', 'datetime64[ns]'),
('ac_date', 'datetime64[ns]'),
('min_trade_size', 'float'),
]
@lazyval
catalyst/data/bundles/poloniex.py
+31 -14
View File
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from datetime import datetime
import pandas as pd
@@ -23,6 +25,8 @@ from catalyst.data.bundles.core import register_bundle
from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
from catalyst.utils.memoize import lazyval
from catalyst.curate.poloniex import PoloniexCurator
class PoloniexBundle(BaseCryptoPricingBundle):
@lazyval
def name(self):
@@ -36,7 +40,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
def frequencies(self):
return set((
'daily',
#'5-minute',
'minute',
))
@lazyval
@@ -75,12 +79,14 @@ class PoloniexBundle(BaseCryptoPricingBundle):
start_date = sym_data.index[0]
end_date = sym_data.index[-1]
ac_date = end_date + pd.Timedelta(days=1)
min_trade_size = 0.00000001
return (
sym_md.symbol,
start_date,
end_date,
ac_date,
min_trade_size,
)
def fetch_raw_symbol_frame(self,
@@ -90,24 +96,30 @@ class PoloniexBundle(BaseCryptoPricingBundle):
start_date,
end_date,
frequency):
# TODO: replace this with direct exchange call
# The end date and frequency should be used to calculate the number of bars
raw = pd.read_json(
self._format_data_url(
api_key,
symbol,
start_date,
end_date,
frequency,
),
orient='records',
)
raw.set_index('date', inplace=True)
if(frequency == 'minute'):
pc = PoloniexCurator()
raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
else:
raw = pd.read_json(
self._format_data_url(
api_key,
symbol,
start_date,
end_date,
frequency,
),
orient='records',
)
raw.set_index('date', inplace=True)
# BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
# on disk, which we compensate here to get the right pricing amounts
# ref: data/us_equity_pricing.py
scale = 1000
scale = 1
raw.loc[:, 'open'] /= scale
raw.loc[:, 'high'] /= scale
raw.loc[:, 'low'] /= scale
@@ -169,4 +181,9 @@ register_bundle(PoloniexBundle, ['USDT_BTC',])
For a production environment make sure to use (to bundle all pairs):
register_bundle(PoloniexBundle)
'''
register_bundle(PoloniexBundle, create_writers=False)
if 'ingest' in sys.argv and '-c' in sys.argv:
register_bundle(PoloniexBundle)
else:
register_bundle(PoloniexBundle, create_writers=False)
catalyst/data/history_loader.py
+1 -1
View File
@@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype
from catalyst.utils.pandas_utils import find_in_sorted_index
# Default number of decimal places used for rounding asset prices.
DEFAULT_ASSET_PRICE_DECIMALS = 3
DEFAULT_ASSET_PRICE_DECIMALS = 9
class HistoryCompatibleUSEquityAdjustmentReader(object):
catalyst/data/resample.py
+4 -1
View File
@@ -156,7 +156,10 @@ class DailyHistoryAggregator(object):
cache = self._caches[field] = (session, market_open, {})
_, market_open, entries = cache
market_open = market_open.tz_localize('UTC')
try:
market_open = market_open.tz_localize('UTC')
except TypeError:
market_open = market_open.tz_convert('UTC')
if dt != market_open:
prev_dt = dt_value - self._one_min
else:
catalyst/data/us_equity_pricing.py
+7 -4
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import division # Python2 req to have division of ints yield float
from errno import ENOENT
from functools import partial
from os import remove
@@ -80,7 +83,6 @@ from catalyst.utils.cli import (
from ._equities import _compute_row_slices, _read_bcolz_data
from ._adjustments import load_adjustments_from_sqlite
logger = logbook.Logger('UsEquityPricing')
OHLC = frozenset(['open', 'high', 'low', 'close'])
@@ -116,6 +118,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
UINT32_MAX = iinfo(uint32).max
UINT64_MAX = iinfo(uint64).max
PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220
def check_uint32_safe(value, colname):
if value >= UINT32_MAX:
@@ -433,7 +437,7 @@ class BcolzDailyBarWriter(object):
return raw_data
winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
processed = (raw_data[list(OHLC)] * 1000000).astype('uint64')
processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
dates = raw_data.index.values.astype('datetime64[s]')
check_uint32_safe(dates.max().view(np.int64), 'day')
processed['day'] = dates.astype('uint32')
@@ -519,7 +523,6 @@ class BcolzDailyBarReader(SessionBarReader):
# Need to test keeping the entire array in memory for the course of a
# process first.
self._spot_cols = {}
self.PRICE_ADJUSTMENT_FACTOR = 0.001
self._read_all_threshold = read_all_threshold
@lazyval
@@ -763,7 +766,7 @@ class BcolzDailyBarReader(SessionBarReader):
if price == 0:
return nan
else:
return price * 0.001
return price / PRICE_ADJUSTMENT_FACTOR
else:
return price
catalyst/exchange/bitfinex/bitfinex.py
+13
View File
@@ -24,6 +24,7 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder, ExchangeStopOrder
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.protocol import Account
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
# Trying to account for REST api instability
# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -559,3 +560,15 @@ class Bitfinex(Exchange):
log.debug('got tickers {}'.format(ticks))
return ticks
def generate_symbols_json(self, filename=None):
symbol_map = {}
response = self._request('symbols', None)
for symbol in response.json():
symbol_map[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"}
if(filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename,'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))
catalyst/exchange/bittrex/bittrex.py
+22 -26
View File
@@ -12,6 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
CreateOrderError
from catalyst.finance.execution import LimitOrder, StopLimitOrder
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
log = Logger('Bittrex')
@@ -56,32 +58,6 @@ class Bittrex(Exchange):
"""
return exchange_symbol.lower()
def fetch_symbol_map(self):
"""
Since Bittrex gives us a complete dictionary of symbols,
we can build the symbol map ad-hoc as opposed to maintaining
a static file. We must be careful with mapping any unconventional
symbol name as appropriate.
:return symbol_map:
"""
symbol_map = dict()
self.ask_request()
markets = self.api.getmarkets()
for market in markets:
exchange_symbol = market['MarketName']
symbol = '{market}_{base}'.format(
market=self.sanitize_curency_symbol(market['MarketCurrency']),
base=self.sanitize_curency_symbol(market['BaseCurrency'])
)
symbol_map[exchange_symbol] = dict(
symbol=symbol,
start_date=pd.to_datetime(market['Created'], utc=True)
)
return symbol_map
def get_balances(self):
try:
log.debug('retrieving wallet balances')
@@ -330,3 +306,23 @@ class Bittrex(Exchange):
def get_account(self):
log.info('retrieving account data')
pass
def generate_symbols_json(self, filename=None):
symbol_map = {}
markets = self.api.getmarkets()
for market in markets:
exchange_symbol = market['MarketName']
symbol = '{market}_{base}'.format(
market=self.sanitize_curency_symbol(market['MarketCurrency']),
base=self.sanitize_curency_symbol(market['BaseCurrency'])
)
symbol_map[exchange_symbol] = dict(
symbol=symbol,
start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d")
)
if(filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename,'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))
catalyst/exchange/exchange.py
+7 -1
View File
@@ -218,13 +218,19 @@ class Exchange:
else:
asset_name = None
if 'min_trade_size' in asset:
min_trade_size = asset['min_trade_size']
else:
min_trade_size = 0.0000001
trading_pair = TradingPair(
symbol=asset['symbol'],
exchange=self.name,
start_date=start_date,
end_date=end_date,
leverage=leverage,
asset_name=asset_name
asset_name=asset_name,
min_trade_size=min_trade_size
)
self.assets[exchange_symbol] = trading_pair
catalyst/exchange/exchange_algorithm.py
+3 -3
View File
@@ -52,6 +52,7 @@ from catalyst.utils.api_support import (
from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
expect_types
from catalyst.utils.preprocess import preprocess
from catalyst.utils.math_utils import round_nearest
log = logbook.Logger('exchange_algorithm')
@@ -67,15 +68,14 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
def round_order(self, amount):
def round_order(self, amount, asset):
"""
We need fractions with cryptocurrencies
:param amount:
:return:
"""
# TODO: is this good enough? Victor has a better solution.
return amount
return round_nearest(amount, asset.min_trade_size)
@api_method
@preprocess(symbol_str=ensure_upper_case)
catalyst/exchange/exchange_errors.py
+6
View File
@@ -101,6 +101,12 @@ class OrphanOrderError(ZiplineError):
).strip()
class OrphanOrderReverseError(ZiplineError):
msg = (
'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
).strip()
class OrderCancelError(ZiplineError):
msg = (
'Unable to cancel order {order_id} on exchange {exchange} {error}.'
catalyst/exchange/exchange_portfolio.py
+24
View File
@@ -70,6 +70,30 @@ class ExchangePortfolio(Portfolio):
log.debug('updated portfolio with executed order')
def execute_transaction(self, transaction):
log.debug('executing transaction {}'.format(transaction.order_id))
order_position = self.positions[transaction.asset] \
if transaction.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to execute transaction for a position not held: %s' % transaction.order_id
)
self.capital_used += transaction.amount * transaction.price
if transaction.amount > 0:
if order_position.cost_basis > 0:
order_position.cost_basis = np.average(
[order_position.cost_basis, transaction.price],
weights=[order_position.amount, transaction.amount]
)
else:
order_position.cost_basis = transaction.price
log.debug('updated portfolio with executed order')
def remove_order(self, order):
log.info('removing cancelled order {}'.format(order.id))
del self.open_orders[order.id]
catalyst/exchange/exchange_utils.py
+8 -7
View File
@@ -9,9 +9,8 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
ExchangeSymbolsNotFound
from catalyst.utils.paths import data_root, ensure_directory
# TODO: move to aws
SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
'master/catalyst/exchange/{exchange}/symbols.json'
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
def get_exchange_folder(exchange_name, environ=None):
@@ -25,18 +24,20 @@ def get_exchange_folder(exchange_name, environ=None):
return exchange_folder
def download_exchange_symbols(exchange_name, environ=None):
def get_exchange_symbols_filename(exchange_name, environ=None):
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, 'symbols.json')
return os.path.join(exchange_folder, 'symbols.json')
def download_exchange_symbols(exchange_name, environ=None):
filename = get_exchange_symbols_filename(exchange_name)
url = SYMBOLS_URL.format(exchange=exchange_name)
response = urllib.urlretrieve(url=url, filename=filename)
return response
def get_exchange_symbols(exchange_name, environ=None):
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, 'symbols.json')
filename = get_exchange_symbols_filename(exchange_name)
if not os.path.isfile(filename):
download_exchange_symbols(exchange_name, environ)
catalyst/exchange/live_graph_clock.py
+8 -7
View File
@@ -12,22 +12,18 @@
# limitations under the License.
from datetime import timedelta
import matplotlib.dates as mdates
import pandas as pd
from catalyst.gens.sim_engine import (
BAR,
SESSION_START
)
from logbook import Logger
from matplotlib import pyplot as plt
from matplotlib import style
from catalyst.exchange.exchange_errors import \
MismatchingBaseCurrenciesExchanges
log = Logger('LiveGraphClock')
fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
log = Logger('LiveGraphClock')
class LiveGraphClock(object):
@@ -58,13 +54,18 @@ class LiveGraphClock(object):
def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
style.use('dark_background')
import matplotlib.dates as mdates
from matplotlib import pyplot as plt
from matplotlib import style
self.sessions = sessions
self.time_skew = time_skew
self._last_emit = None
self._before_trading_start_bar_yielded = True
self.context = context
self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
style.use('dark_background')
fig = plt.figure()
fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
@@ -100,7 +101,7 @@ class LiveGraphClock(object):
:return:
"""
ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
ax.xaxis.set_major_formatter(fmt)
ax.xaxis.set_major_formatter(self.fmt)
locator = mdates.HourLocator(interval=4)
locator.MAXTICKS = 5000
catalyst/exchange/poloniex/__init__.py
View File
catalyst/exchange/poloniex/poloniex.py
+550
View File
@@ -0,0 +1,550 @@
import base64
import hashlib
import hmac
import json
import re
import time
from collections import defaultdict
import numpy as np
import pandas as pd
import pytz
import requests
#import six
from six import iteritems
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
# from websocket import create_connection
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
InvalidHistoryFrequencyError,
InvalidOrderStyle, OrderCancelError,
OrphanOrderReverseError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder, ExchangeStopOrder
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.protocol import Account
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
log = Logger('Poloniex')
class Poloniex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8'))
self.name = 'poloniex'
self.assets = {}
self.load_assets()
self.base_currency = base_currency
self._portfolio = portfolio
self.minute_writer = None
self.minute_reader = None
self.transactions = defaultdict(list)
def sanitize_curency_symbol(self, exchange_symbol):
"""
Helper method used to build the universal pair.
Include any symbol mapping here if appropriate.
:param exchange_symbol:
:return universal_symbol:
"""
return exchange_symbol.lower()
def _create_order(self, order_status):
"""
Create a Catalyst order object from the Exchange order dictionary
:param order_status:
:return: Order
"""
#if order_status['is_cancelled']:
# status = ORDER_STATUS.CANCELLED
#elif not order_status['is_live']:
# log.info('found executed order {}'.format(order_status))
# status = ORDER_STATUS.FILLED
#else:
status = ORDER_STATUS.OPEN
amount = float(order_status['amount'])
#filled = float(order_status['executed_amount'])
filled = None
if order_status['type'] == 'sell':
amount = -amount
#filled = -filled
price = float(order_status['rate'])
order_type = order_status['type']
stop_price = None
limit_price = None
# TODO: is this comprehensive enough?
#if order_type.endswith('limit'):
# limit_price = price
#elif order_type.endswith('stop'):
# stop_price = price
#executed_price = float(order_status['avg_execution_price'])
executed_price = price
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
commission = None
#date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
#date = pytz.utc.localize(date)
date = None
order = Order(
dt=date,
asset=self.assets[order_status['symbol']], # No such field in Poloniex
amount=amount,
stop=stop_price,
limit=limit_price,
filled=filled,
id=str(order_status['orderNumber']),
commission=commission
)
order.status = status
return order, executed_price
def get_balances(self):
log.debug('retrieving wallets balances')
try:
balances = self.api.returnbalances()
except Exception as e:
log.debug(e)
raise ExchangeRequestError(error=e)
if 'error' in balances:
raise ExchangeRequestError(
error='unable to fetch balance {}'.format(balances['error'])
)
std_balances = dict()
for (key, value) in iteritems(balances):
currency = key.lower()
std_balances[currency] = float(value)
return std_balances
@property
def account(self):
account = Account()
account.settled_cash = None
account.accrued_interest = None
account.buying_power = None
account.equity_with_loan = None
account.total_positions_value = None
account.total_positions_exposure = None
account.regt_equity = None
account.regt_margin = None
account.initial_margin_requirement = None
account.maintenance_margin_requirement = None
account.available_funds = None
account.excess_liquidity = None
account.cushion = None
account.day_trades_remaining = None
account.leverage = None
account.net_leverage = None
account.net_liquidation = None
return account
@property
def time_skew(self):
# TODO: research the time skew conditions
return pd.Timedelta('0s')
def get_account(self):
# TODO: fetch account data and keep in cache
return None
def get_candles(self, data_frequency, assets, bar_count=None):
"""
Retrieve OHLVC candles from Poloniex
:param data_frequency:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'5m', '15m', '30m', '2h', '4h', '1D'
"""
# TODO: use BcolzMinuteBarReader to read from cache
if(data_frequency == '5m' or data_frequency == 'minute'): #TODO: Polo does not have '1m'
frequency = 300
elif(data_frequency == '15m'):
frequency = 900
elif(data_frequency == '30m'):
frequency = 1800
elif(data_frequency == '2h'):
frequency = 7200
elif(data_frequency == '4h'):
frequency = 14400
elif(data_frequency == '1D' or data_frequency == 'daily'):
frequency = 86400
else:
raise InvalidHistoryFrequencyError(
frequency=data_frequency
)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
end = int(time.time())
if(bar_count is None):
start = end - 2 * frequency
else:
start = end - bar_count * frequency
try:
response = self.api.returnchartdata(self.get_symbol(asset),frequency, start, end)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content)
)
def ohlc_from_candle(candle):
ohlc = dict(
open=np.float64(candle['open']),
high=np.float64(candle['high']),
low=np.float64(candle['low']),
close=np.float64(candle['close']),
volume=np.float64(candle['volume']),
price=np.float64(candle['close']),
last_traded=pd.Timestamp.utcfromtimestamp( candle['date'] )
)
return ohlc
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(response[0])
else:
ohlc_bars = []
for candle in response:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def create_order(self, asset, amount, is_buy, style):
"""
Creating order on the exchange.
:param asset:
:param amount:
:param is_buy:
:param style:
:return:
"""
exchange_symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder) or isinstance(style, ExchangeStopLimitOrder):
if isinstance(style, ExchangeStopLimitOrder):
log.warn('{} will ignore the stop price'.format(self.name))
price = style.get_limit_price(is_buy)
try:
if(is_buy):
response = self.api.buy(exchange_symbol, amount, price)
else:
reponse = self.api.sell(exchange_symbol, amount, price)
except Exception as e:
raise ExchangeRequestError(error=e)
date = pd.Timestamp.utcnow()
print(response)
if('orderNumber' in response):
order_id = str(response['orderNumber'])
order = Order(
dt=date,
asset=asset,
amount=amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
else:
log.warn('{} order failed: {}'.format('buy' if is_buy else 'sell', response['error']))
return None
else:
raise InvalidOrderStyle(exchange=self.name,
style=style.__class__.__name__)
def get_open_orders(self, asset='all'):
"""Retrieve all of the current open orders.
Parameters
----------
asset : Asset
If passed and not 'all', return only the open orders for the given
asset instead of all open orders.
Returns
-------
open_orders : dict[list[Order]] or list[Order]
If 'all' is passed this will return a dict mapping Assets
to a list containing all the open orders for the asset.
If an asset is passed then this will return a list of the open
orders for this asset.
"""
return self.portfolio.open_orders
"""
TODO: Why going to the exchange if we already have this info locally?
And why creating all these Orders if we later discard them?
"""
try:
if(asset=='all'):
response = self.api.returnopenorders('all')
else:
response = self.api.returnopenorders(self.get_symbol(asset))
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve open orders: {}'.format(
order_statuses['message'])
)
print(self.portfolio.open_orders)
#TODO: Need to handle openOrders for 'all'
orders = list()
for order_status in response:
order, executed_price = self._create_order(order_status) # will Throw error b/c Polo doesn't track order['symbol']
if asset is None or asset == order.sid:
orders.append(order)
return orders
def get_order(self, order_id):
"""Lookup an order based on the order id returned from one of the
order functions.
Parameters
----------
order_id : str
The unique identifier for the order.
Returns
-------
order : Order
The order object.
"""
try:
order = self._portfolio.open_orders[order_id]
except Exception as e:
raise OrphanOrderError(order_id=order_id, exchange=self.name)
try:
response = self.api.returnopenorders(self.get_symbol(order.sid))
except Exception as e:
raise ExchangeRequestError(error=e)
for order in response:
if(int(order['orderNumber'])==int(order_id)):
return True
return None
def cancel_order(self, order_param):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
"""
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
try:
response = self.api.cancelorder(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise OrderCancelError(
order_id=order_id,
exchange=self.name,
error=response['error']
)
self.portfolio.remove_order(order_param) #TODO: Verify this works
def tickers(self, assets):
"""
Fetch ticket data for assets
https://docs.bitfinex.com/v2/reference#rest-public-tickers
:param assets:
:return:
"""
symbols = self.get_symbols(assets)
log.debug('fetching tickers {}'.format(symbols))
try:
response = self.api.returnticker()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve tickers: {}'.format(
response['error'])
)
ticks = dict()
for index, symbol in enumerate(symbols):
ticks[assets[index]] = dict(
timestamp=pd.Timestamp.utcnow(),
bid=float(response[symbol]['highestBid']),
ask=float(response[symbol]['lowestAsk']),
last_price=float(response[symbol]['last']),
low=float(response[symbol]['lowestAsk']), #TODO: Polo does not provide low
high=float(response[symbol]['highestBid']), #TODO: Polo does not provide high
volume=float(response[symbol]['baseVolume']),
)
log.debug('got tickers {}'.format(ticks))
return ticks
def generate_symbols_json(self, filename=None):
symbol_map = {}
response = self.api.returnticker()
for exchange_symbol in response:
base, market = self.sanitize_curency_symbol(exchange_symbol).split('_')
symbol = '{market}_{base}'.format( market=market, base=base )
symbol_map[exchange_symbol] = dict(
symbol = symbol,
start_date = '2010-01-01'
)
if(filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename,'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))
def check_open_orders(self):
"""
Need to override this function for Poloniex:
Loop through the list of open orders in the Portfolio object.
Check if any transactions have been executed:
If so, create a transaction and apply to the Portfolio.
Check if the order is still open:
If not, remove it from open orders
:return:
transactions: Transaction[]
"""
transactions = list()
if self.portfolio.open_orders:
for order_id in list(self.portfolio.open_orders):
order = self._portfolio.open_orders[order_id]
log.debug('found open order: {}'.format(order_id))
try:
order_open = self.get_order(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if(order_open):
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order_id,
delta=delta )
)
try:
response = self.api.returnordertrades(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if(response['error']):
if(not order_open):
raise OrphanOrderReverseError(order_id=order_id, exchange=self.name)
else:
for tx in response:
"""
We maintain a list of dictionaries of transactions that correspond to
partially filled orders, indexed by order_id. Every time we query
executed transactions from the exchange, we check if we had that
transaction for that order already. If not, we process it.
When an order if fully filled, we flush the dict of transactions
associated with that order.
"""
if(not filter(lambda item: item['order_id'] == tx['tradeID'], self.transactions[order_id])):
log.debug('Got new transaction for order {}: amount {}, price {}'.format(
order_id, tx.amount, tx.rate))
if(tx['type']=='sell'):
tx['amount'] = -tx['amount']
transaction = Transaction(
asset=order.asset,
amount=tx['amount'],
dt=pd.to_datetime(tx['date'], utc=True),
price=tx['rate'],
order_id=tx['tradeID'], # it's a misnomer, but keeping it for compatibility
commission=tx['fee']
)
self.transactions[order_id].append(transaction)
self.portfolio.execute_transaction(transaction)
transactions.append(transaction)
if(not order_open):
"""
Since transactions have been executed individually
the only thing left to do is remove them from list of open_orders
"""
del self.portfolio.open_orders[order_id]
del self.transactions[order_id]
return transactions
catalyst/exchange/poloniex/poloniex_api.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python
import json
import time
import hmac
import hashlib
from six.moves import urllib
# Workaround for backwards compatibility
# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
urlopen = urllib.request.urlopen
class Poloniex_api(object):
def __init__(self, key, secret):
self.key = key
self.secret = secret
self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
'returnTradeHistory', 'returnChartData',
'returnCurrencies', 'returnLoanOrders']
self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses',
'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders',
'returnTradeHistory','returnOrderTrades',
'buy', 'sell', 'cancelOrder', 'moveOrder',
'withdraw', 'returnFeeInfo','returnAvailableAccountBalances',
'returnTradableBalances', 'transferBalance',
'returnMarginAccountSummary','marginBuy','marginSell',
'getMarginPosition', 'closeMarginPosition','createLoanOffer',
'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans',
'returnLendingHistory','toggleAutoRenew']
def query(self, method, req={}):
if method in self.public:
url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req)
headers = {}
post_data = None
elif method in self.trading:
url = 'https://poloniex.com/tradingApi'
req['command'] = method
req['nonce'] = int(time.time()*1000)
post_data = urllib.parse.urlencode(req)
signature = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
headers = { 'Sign': signature, 'Key': self.key}
else:
raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints')
req = urllib.request.Request(url, data=post_data, headers=headers)
return json.loads(urlopen(req).read())
def returnticker(self):
return self.query('returnTicker', {})
def return24volume(self):
return self.query('return24Volume', {})
def returnOrderBook(self, market='all'):
return self.query('returnOrderBook', {'currencyPair': market})
def returntradehistory(self, market, start=None, end=None):
if(start is not None and end is not None):
return self.query('returntradehistory',
{'currencyPair': market, 'start': start, 'end': end })
else:
return self.query('returntradehistory', {'currencyPair': market })
def returnchartdata(self, market, period, start, end):
return self.query('returnChartData', {'currencyPair': market, 'period': period,
'start': start, 'end': end})
def returncurrencies(self):
return self.query('returnCurrencies', {})
def returnloadorders(self, market):
return self.query('returnLoanOrders', {'currency': market})
def returnbalances(self):
return self.query('returnBalances')
def returncompletebalances(self, account):
if(account):
return self.query('returnCompleteBalances', {'account': account})
else:
return self.query('returnCompleteBalances')
def returndepositaddresses(self):
return self.query('returnDepositAddresses')
def generatenewaddress(self, currency):
return self.query('generateNewAddress', {'currency': currency})
def returnDepositsWithdrawals(self, start, end):
return self.query('returnDepositsWithdrawals', {'start': start, 'end': end})
def returnopenorders(self, market):
return self.query('returnOpenOrders', {'currencyPair': market})
def returntradehistory(self, market):
#TODO: optional start and/or end and limit
return self.query('returnTradeHistory', {'currencyPair': market})
def returnordertrades(self, ordernumber):
return self.query('returnOrderTrades', {'orderNumber': ordernumber})
def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
if(fillorkill):
return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
'fillOrKill': fillorkill, })
elif(immediateorcancel):
return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
'immediateOrCancel': immediateorcancel, })
elif(postonly):
return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
'postOnly': postonly, })
else:
return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, })
def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
if(fillorkill):
return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
'fillOrKill': fillorkill, })
elif(immediateorcancel):
return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
'immediateOrCancel': immediateorcancel, })
elif(postonly):
return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
'postOnly': postonly, })
else:
return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, })
def cancelorder(self, ordernumber):
return self.query('cancelOrder', {'orderNumber': ordernumber})
def withdraw(self, currency, quantity, address):
return self.query('withdraw',
{'currency': currency, 'amount': quantity,
'address': address})
def returnfeeinfo(self):
return self.query('returnFeeInfo')
catalyst/finance/slippage.py
+6 -3
View File
@@ -41,6 +41,7 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
class LiquidityExceeded(Exception):
pass
@@ -205,20 +206,22 @@ class VolumeShareSlippage(SlippageModel):
def process_order(self, data, order):
volume = data.current(order.asset, "volume")
min_trade_size = order.asset.min_trade_size
max_volume = self.volume_limit * volume
# price impact accounts for the total volume of transactions
# created against the current minute bar
remaining_volume = max_volume - self.volume_for_bar
if remaining_volume < 1:
if remaining_volume < min_trade_size:
# we can't fill any more transactions
raise LiquidityExceeded()
# the current order amount will be the min of the
# volume available in the bar or the open amount.
cur_volume = int(min(remaining_volume, abs(order.open_amount)))
cur_volume = min(remaining_volume, abs(order.open_amount))
if cur_volume < 1:
if cur_volume < min_trade_size:
return None, None
# tally the current amount into our total amount ordered.
catalyst/finance/transaction.py
+1 -5
View File
@@ -65,14 +65,10 @@ def create_transaction(order, dt, price, amount):
# floor the amount to protect against non-whole number orders
# TODO: Investigate whether we can add a robust check in blotter
# and/or tradesimulation, as well.
amount_magnitude = int(abs(amount))
if amount_magnitude < 1:
raise Exception("Transaction magnitude must be at least 1.")
transaction = Transaction(
asset=order.asset,
amount=int(amount),
amount=amount,
dt=dt,
price=price,
order_id=order.id
catalyst/utils/math_utils.py
+2
View File
@@ -17,6 +17,8 @@ import math
from numpy import isnan
def round_nearest(x, a):
return round(round(x / a) * a, -int(math.floor(math.log10(a))))
def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False):
"""Check if a and b are equal with some tolerance.
catalyst/utils/run_algo.py
+9 -3
View File
@@ -9,6 +9,8 @@ import click
import pandas as pd
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.poloniex.poloniex import Poloniex
try:
from pygments import highlight
@@ -30,7 +32,6 @@ from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
ExchangeTradingAlgorithmBacktest
from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
DataPortalExchangeBacktest
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
from catalyst.exchange.exchange_errors import (
@@ -169,7 +170,6 @@ def _run(handle_data,
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'bittrex':
exchanges[exchange_name] = Bittrex(
key=exchange_auth['key'],
@@ -177,7 +177,13 @@ def _run(handle_data,
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'poloniex':
exchanges[exchange_name] = Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
etc/python2.7-environment.yml
+84
View File
@@ -0,0 +1,84 @@
name: catalyst
channels:
- statiskit
- defaults
dependencies:
- certifi=2016.2.28=py27_0
- coverage=4.4.1=py27_0
- nose=1.3.7=py27_1
- openssl=1.0.2l=0
- path.py=10.3.1=py27_0
- pip=9.0.1=py27_1
- python=2.7.13=0
- pyyaml=3.12=py27_0
- readline=6.2=2
- setuptools=36.4.0=py27_0
- six=1.10.0=py27_0
- sqlite=3.13.0=0
- tk=8.5.18=0
- wheel=0.29.0=py27_0
- yaml=0.1.6=0
- zlib=1.2.11=0
- libdev=1.0.0=py27_0
- python-dev=1.0.0=py27_0
- python-scons=3.0.0=py27_0
- pip:
- alembic==0.9.5
- backports.shutil-get-terminal-size==1.0.0
- bcolz==0.12.1
- bottleneck==1.2.1
- chardet==3.0.4
- click==6.7
- contextlib2==0.5.5
- cycler==0.10.0
- cyordereddict==1.0.0
- cython==0.26.1
- decorator==4.1.2
- empyrical==0.2.1
- enigma-catalyst>=0.2.dev2
- enum34==1.1.6
- functools32==3.2.3.post2
- idna==2.6
- intervaltree==2.1.0
- ipdb==0.10.3
- ipdbplugin==1.4.5
- ipython==5.5.0
- ipython-genutils==0.2.0
- logbook==1.1.0
- lru-dict==1.1.6
- mako==1.0.7
- markupsafe==1.0
- matplotlib==2.0.2
- multipledispatch==0.4.9
- networkx==1.11
- numexpr==2.6.4
- numpy==1.13.1
- pandas==0.19.2
- pandas-datareader==0.5.0
- pathlib2==2.3.0
- patsy==0.4.1
- pexpect==4.2.1
- pickleshare==0.7.4
- prompt-toolkit==1.0.15
- ptyprocess==0.5.2
- pygments==2.2.0
- pyparsing==2.2.0
- python-dateutil==2.6.1
- python-editor==1.0.3
- pytz==2017.2
- requests==2.18.4
- requests-file==1.4.2
- requests-ftp==0.3.1
- scandir==1.5
- scipy==0.19.1
- scons==3.0.0a20170821
- simplegeneric==0.8.1
- sortedcontainers==1.5.7
- sqlalchemy==1.1.14
- statsmodels==0.8.0
- subprocess32==3.2.7
- tables==3.4.2
- toolz==0.8.2
- traitlets==4.3.2
- urllib3==1.22
- wcwidth==0.1.7