From 41a4c7072f3e7f224d1d01eb156cb0b25db12c49 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 14 Mar 2018 09:44:24 +0200 Subject: [PATCH 1/8] DOC: fixed a mistake on the installation tutorial --- docs/source/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/install.rst b/docs/source/install.rst index e4ae99dd..87f3c64c 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -143,7 +143,7 @@ with the following steps: .. code-block:: bash - conda create --name catalyst python=2.7 scipy zlib + conda create --name catalyst python=3.6 scipy zlib 3. Activate the environment: From 7b796a4276588231412f3ebbe1b39443af7c64fb Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 15 Mar 2018 12:59:53 -0400 Subject: [PATCH 2/8] MAINT: [mktplace] sign_msg opens browser window --- catalyst/marketplace/utils/auth_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index ab3c668d..f4d893a8 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -1,5 +1,6 @@ import hashlib import hmac +import webbrowser import requests import time @@ -45,10 +46,17 @@ def get_key_secret(pubAddr, wallet='mew'): nonce = '0x{}'.format(d['nonce']) if wallet == 'mew': + url = 'https://www.myetherwallet.com/signmsg.html' + print('\nObtaining a key/secret pair to streamline all future ' 'requests with the authentication server.\n' - 'Visit https://www.myetherwallet.com/signmsg.html and sign the ' - 'following message:\n{}'.format(nonce)) + 'Visit {url} and sign the ' + 'following message:\n{nonce}'.format( + url=url, + nonce=nonce)) + + webbrowser.open_new(url) + signature = input('Copy and Paste the "sig" field from ' 'the signature here (without the double quotes, ' 'only the HEX value):\n') From 4cb8d54d97db10d69de4e0f35a4f6126a2d63af7 Mon Sep 17 00:00:00 2001 From: izokay Date: Thu, 15 Mar 2018 15:41:30 -0400 Subject: [PATCH 3/8] typo on creating env for python 3.6 --- docs/source/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/install.rst b/docs/source/install.rst index e4ae99dd..87f3c64c 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -143,7 +143,7 @@ with the following steps: .. code-block:: bash - conda create --name catalyst python=2.7 scipy zlib + conda create --name catalyst python=3.6 scipy zlib 3. Activate the environment: From 685ce25b854bef44d79bee7ca7f1f06502fccbe8 Mon Sep 17 00:00:00 2001 From: Albert De La Fuente Vigliotti Date: Fri, 16 Mar 2018 16:02:40 -0300 Subject: [PATCH 4/8] Fix H candle support --- catalyst/exchange/ccxt/ccxt_exchange.py | 3 +++ catalyst/exchange/exchange.py | 4 ---- catalyst/exchange/utils/datetime_utils.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 875661e9..0c8eb808 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -190,6 +190,9 @@ class CCXT(Exchange): if data_frequency == 'minute' and not freq.endswith('T'): continue + elif data_frequency == 'hourly' and not freq.endswith('D'): + continue + elif data_frequency == 'daily' and not freq.endswith('D'): continue diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 73593c6d..23681723 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -509,10 +509,6 @@ class Exchange: frequency, data_frequency, supported_freqs=['T', 'D', 'H'] ) - if unit == 'H': - raise InvalidHistoryFrequencyAlias( - freq=frequency) - # we want to avoid receiving empty candles # so we request more than needed # TODO: consider defining a const per asset diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index b5a03c49..03dee4f7 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -249,7 +249,7 @@ def get_year_start_end(dt, first_day=None, last_day=None): return year_start, year_end -def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']): +def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']): """ Get the frequency parameters. @@ -309,6 +309,7 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']): if 'H' in supported_freqs: unit = 'H' alias = '{}H'.format(candle_size) + data_frequency = 'hourly' else: candle_size = candle_size * 60 From 91d16aba3b6cca2cabe3a6f91585e285409cfc84 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 17 Mar 2018 18:32:28 -0400 Subject: [PATCH 5/8] DOC: documented the get_frequency function for additional clarity --- catalyst/exchange/utils/datetime_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index 03dee4f7..5b8d1b10 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -251,7 +251,10 @@ def get_year_start_end(dt, first_day=None, last_day=None): def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']): """ - Get the frequency parameters. + Takes an arbitrary candle size (e.g. 15T) and converts to the lowest + common denominator supported by the data bundles (e.g. 1T). The data + bundles only support 1T and 1D frequencies. If another frequency + is requested, Catalyst must request the underlying data and resample. Notes ----- From 98449b2088fc708e841f8ecbeea5ccd37100a8fe Mon Sep 17 00:00:00 2001 From: lenak25 Date: Mon, 19 Mar 2018 11:06:47 +0200 Subject: [PATCH 6/8] BLD: fix issue #274 - a bug in which a wrong bar number was returned when requesting day freq history candles in backtest --- catalyst/exchange/exchange_data_portal.py | 2 +- catalyst/support/issue_274.py | 72 +++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 catalyst/support/issue_274.py diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index c6523326..cd694f51 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -312,7 +312,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): algo_end_dt=self._last_available_session, ) - start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) + start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency) df = resample_history_df(pd.DataFrame(series), freq, field, start_dt) return df diff --git a/catalyst/support/issue_274.py b/catalyst/support/issue_274.py new file mode 100644 index 00000000..88ec83db --- /dev/null +++ b/catalyst/support/issue_274.py @@ -0,0 +1,72 @@ +import pytz +from datetime import datetime, timedelta +from catalyst.api import symbol +from catalyst.utils.run_algo import run_algorithm +#from utils.data import load_coin_data_from_csv + +exchange_name = 'bitfinex' +coin = 'btc' +base_currency = 'usd' +n_candles = 5 +resample_frequency_minutes = 5 + + +def initialize(context): + context.symbol = symbol('%s_%s' % (coin, base_currency)) + # very simple method that reads data from generated csv file + #context.all_data = load_coin_data_from_csv('btc', exchange_name, 'minute', + # from_date=datetime(2017, 6, 1, 0, 0, 0, 0, pytz.utc), + # resample='%dT' % resample_frequency_minutes) + + +def handle_data_polo_partial_candles(context, data): + # all I do is print the current last 2 candles (5T) + history = data.history(symbol('btc_usdt'), ['volume'], bar_count=10, frequency='1D') + print('\nnow: %s\n%s' % (data.current_dt, history)) + if not hasattr(context, 'i'): context.i = 0 + context.i += 1 + if context.i > 5: raise Exception('stop') + + +run_algorithm(initialize=lambda ctx: True, + handle_data=handle_data_polo_partial_candles, + exchange_name='poloniex', + base_currency='usdt', + algo_namespace='ns', + live=False, + data_frequency='minute', + capital_base=3000, + start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc), + end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc)) + + +""" +def handle_data(context, data): + #bundle_data = \ + #context.all_data[data.current_dt - timedelta(minutes=(n_candles - 1) * resample_frequency_minutes):][:n_candles][ + # 'close'] + backtest_data = data.history(context.symbol, 'close', bar_count=n_candles, + frequency='%dT' % resample_frequency_minutes) + print('current_time:', data.current_dt) + + #print('bundle_data:') + #print(bundle_data) + + print('backtest_data:') + print(backtest_data) + #raise Exception('fail') + + +run_algorithm( + initialize=initialize, + handle_data=handle_data, + exchange_name=exchange_name, + base_currency=base_currency, + algo_namespace='investigate_mismatching_final_candle', + live=False, + data_frequency='minute', + capital_base=3000, + simulate_orders=True, + start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc), + end=datetime(2018, 2, 3, 0, 0, 0, 0, pytz.utc)) +""" \ No newline at end of file From 9648767e9a63a714021b215891db9bf3bc4fc6a8 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Mon, 19 Mar 2018 15:53:06 +0200 Subject: [PATCH 7/8] STY: flake8 fixes --- catalyst/support/issue_274.py | 53 ++++++----------------------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/catalyst/support/issue_274.py b/catalyst/support/issue_274.py index 88ec83db..74a497d9 100644 --- a/catalyst/support/issue_274.py +++ b/catalyst/support/issue_274.py @@ -1,31 +1,26 @@ import pytz -from datetime import datetime, timedelta +from datetime import datetime from catalyst.api import symbol from catalyst.utils.run_algo import run_algorithm -#from utils.data import load_coin_data_from_csv -exchange_name = 'bitfinex' coin = 'btc' base_currency = 'usd' -n_candles = 5 -resample_frequency_minutes = 5 def initialize(context): context.symbol = symbol('%s_%s' % (coin, base_currency)) - # very simple method that reads data from generated csv file - #context.all_data = load_coin_data_from_csv('btc', exchange_name, 'minute', - # from_date=datetime(2017, 6, 1, 0, 0, 0, 0, pytz.utc), - # resample='%dT' % resample_frequency_minutes) def handle_data_polo_partial_candles(context, data): - # all I do is print the current last 2 candles (5T) - history = data.history(symbol('btc_usdt'), ['volume'], bar_count=10, frequency='1D') + history = data.history(symbol('btc_usdt'), ['volume'], + bar_count=10, + frequency='1D') print('\nnow: %s\n%s' % (data.current_dt, history)) - if not hasattr(context, 'i'): context.i = 0 + if not hasattr(context, 'i'): + context.i = 0 context.i += 1 - if context.i > 5: raise Exception('stop') + if context.i > 5: + raise Exception('stop') run_algorithm(initialize=lambda ctx: True, @@ -38,35 +33,3 @@ run_algorithm(initialize=lambda ctx: True, capital_base=3000, start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc), end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc)) - - -""" -def handle_data(context, data): - #bundle_data = \ - #context.all_data[data.current_dt - timedelta(minutes=(n_candles - 1) * resample_frequency_minutes):][:n_candles][ - # 'close'] - backtest_data = data.history(context.symbol, 'close', bar_count=n_candles, - frequency='%dT' % resample_frequency_minutes) - print('current_time:', data.current_dt) - - #print('bundle_data:') - #print(bundle_data) - - print('backtest_data:') - print(backtest_data) - #raise Exception('fail') - - -run_algorithm( - initialize=initialize, - handle_data=handle_data, - exchange_name=exchange_name, - base_currency=base_currency, - algo_namespace='investigate_mismatching_final_candle', - live=False, - data_frequency='minute', - capital_base=3000, - simulate_orders=True, - start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc), - end=datetime(2018, 2, 3, 0, 0, 0, 0, pytz.utc)) -""" \ No newline at end of file From 2a97ade68ef559ff606d7b8576ea89e2412c7242 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Mon, 19 Mar 2018 16:44:44 +0200 Subject: [PATCH 8/8] BLD: support hourly freq in live and backtest, as reported on issue #227 and issue #114 --- catalyst/exchange/exchange.py | 3 +- catalyst/exchange/exchange_data_portal.py | 2 +- catalyst/exchange/utils/datetime_utils.py | 5 +-- catalyst/support/issue_227.py | 49 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 catalyst/support/issue_227.py diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 23681723..e5bea0fe 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -13,7 +13,6 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ PricingDataNotLoadedError, \ NoDataAvailableOnExchange, NoValueForField, \ NoCandlesReceivedFromExchange, \ - InvalidHistoryFrequencyAlias, \ TickerNotFoundError, NotEnoughCashError from catalyst.exchange.utils.datetime_utils import get_delta, \ get_periods_range, \ @@ -612,7 +611,7 @@ class Exchange: # TODO: this function needs some work, # we're currently using it just for benchmark data freq, candle_size, unit, data_frequency = get_frequency( - frequency, data_frequency + frequency, data_frequency, supported_freqs=['T', 'D'] ) adj_bar_count = candle_size * bar_count try: diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index cd694f51..d2dab705 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -296,7 +296,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle freq, candle_size, unit, adj_data_frequency = get_frequency( - frequency, data_frequency + frequency, data_frequency, supported_freqs=['T', 'D'] ) adj_bar_count = candle_size * bar_count diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index 5b8d1b10..38269947 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -309,15 +309,14 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']): data_frequency = 'minute' elif unit.lower() == 'h': + data_frequency = 'minute' + if 'H' in supported_freqs: unit = 'H' alias = '{}H'.format(candle_size) - data_frequency = 'hourly' - else: candle_size = candle_size * 60 alias = '{}T'.format(candle_size) - data_frequency = 'minute' else: raise InvalidHistoryFrequencyAlias(freq=freq) diff --git a/catalyst/support/issue_227.py b/catalyst/support/issue_227.py new file mode 100644 index 00000000..09392f81 --- /dev/null +++ b/catalyst/support/issue_227.py @@ -0,0 +1,49 @@ +import pytz +from datetime import datetime +from catalyst.api import symbol +from catalyst.utils.run_algo import run_algorithm + +coin = 'btc' +base_currency = 'usd' +n_candles = 5 + + +def initialize(context): + context.symbol = symbol('%s_%s' % (coin, base_currency)) + + +def handle_data_polo_partial_candles(context, data): + history = data.history(symbol('btc_usdt'), ['volume'], + bar_count=10, + frequency='4H') + print('\nnow: %s\n%s' % (data.current_dt, history)) + if not hasattr(context, 'i'): + context.i = 0 + context.i += 1 + if context.i > 5: + raise Exception('stop') + + +live = False + +if live: + run_algorithm(initialize=lambda ctx: True, + handle_data=handle_data_polo_partial_candles, + exchange_name='poloniex', + base_currency='usdt', + algo_namespace='ns', + live=True, + data_frequency='minute', + capital_base=3000) +else: + run_algorithm(initialize=lambda ctx: True, + handle_data=handle_data_polo_partial_candles, + exchange_name='poloniex', + base_currency='usdt', + algo_namespace='ns', + live=False, + data_frequency='minute', + capital_base=3000, + start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc), + end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc) + )