diff --git a/catalyst/examples/buy_btc.py b/catalyst/examples/buy_btc.py new file mode 100644 index 00000000..e23b79b0 --- /dev/null +++ b/catalyst/examples/buy_btc.py @@ -0,0 +1,10 @@ +from catalyst.api import order, record, symbol + + +def initialize(context): + context.asset = symbol('btc_usd') + + +def handle_data(context, data): + order(context.asset, 1) + record(btc=data.current(context.asset, 'price')) diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index a5ebebbb..47e64cab 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -1,10 +1,10 @@ import base64 +import datetime import hashlib import hmac import json import re import time -import datetime import numpy as np import pandas as pd @@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import ( InvalidOrderStyle, OrderCancelError) from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ ExchangeStopLimitOrder, ExchangeStopOrder -from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.protocol import Account from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ download_exchange_symbols +from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.protocol import Account # Trying to account for REST api instability # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index 9b331b1a..6df3638e 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -5,18 +5,17 @@ from catalyst.assets._assets import TradingPair from logbook import Logger from six.moves import urllib +from catalyst.constants import LOG_LEVEL from catalyst.exchange.bittrex.bittrex_api import Bittrex_api from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \ CreateOrderError -from catalyst.finance.execution import LimitOrder, StopLimitOrder -from catalyst.finance.order import Order, ORDER_STATUS from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ download_exchange_symbols - -from catalyst.constants import LOG_LEVEL +from catalyst.finance.execution import LimitOrder, StopLimitOrder +from catalyst.finance.order import Order, ORDER_STATUS log = Logger('Bittrex', level=LOG_LEVEL) diff --git a/catalyst/exchange/data_portal_exchange.py b/catalyst/exchange/data_portal_exchange.py index 555ac93a..ce454303 100644 --- a/catalyst/exchange/data_portal_exchange.py +++ b/catalyst/exchange/data_portal_exchange.py @@ -19,17 +19,13 @@ import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger +from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import DataPortal -from catalyst.exchange.bundle_utils import get_start_dt from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangeBarDataError, - PricingDataBeforeTradingError, - PricingDataNotLoadedError, InvalidHistoryFrequencyError, - BundleNotFoundError) - -from catalyst.constants import LOG_LEVEL + PricingDataNotLoadedError) log = Logger('DataPortalExchange', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index e617fb17..25a11594 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -9,14 +9,14 @@ import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger +from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.bundle_utils import get_start_dt, \ - get_delta, get_periods, get_adj_dates + get_delta, get_periods from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ - InvalidHistoryFrequencyError, MismatchingFrequencyError, \ - BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError + InvalidHistoryFrequencyError, PricingDataNotLoadedError from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \ ExchangeLimitOrder, ExchangeStopOrder from catalyst.exchange.exchange_portfolio import ExchangePortfolio @@ -24,8 +24,6 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols from catalyst.finance.order import ORDER_STATUS from catalyst.finance.transaction import Transaction -from catalyst.constants import LOG_LEVEL - log = Logger('Exchange', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index be4e82c5..22c26ef2 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -26,6 +26,7 @@ from catalyst.assets._assets import TradingPair import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm +from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteBarWriter, \ BcolzMinuteBarReader from catalyst.errors import OrderInBeforeTradingStart @@ -51,10 +52,8 @@ from catalyst.utils.api_support import ( disallowed_in_before_trading_start) from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \ expect_types -from catalyst.utils.preprocess import preprocess from catalyst.utils.math_utils import round_nearest - -from catalyst.constants import LOG_LEVEL +from catalyst.utils.preprocess import preprocess log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_bcolz.py b/catalyst/exchange/exchange_bcolz.py index 0d19d04f..209f2d91 100644 --- a/catalyst/exchange/exchange_bcolz.py +++ b/catalyst/exchange/exchange_bcolz.py @@ -3,7 +3,6 @@ import numpy as np from catalyst import get_calendar from catalyst.data.minute_bars import BcolzMinuteBarReader, \ BcolzMinuteBarWriter -from catalyst.exchange.bundle_utils import get_periods, get_periods_range class BcolzExchangeBarWriter(BcolzMinuteBarWriter): diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index a45791c6..2ca2cb88 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -1,13 +1,12 @@ from catalyst.assets._assets import TradingPair from logbook import Logger +from catalyst.constants import LOG_LEVEL from catalyst.finance.blotter import Blotter from catalyst.finance.commission import CommissionModel from catalyst.finance.slippage import SlippageModel from catalyst.finance.transaction import Transaction -from catalyst.constants import LOG_LEVEL - log = Logger('exchange_blotter', level=LOG_LEVEL) # It seems like we need to accept greater slippage risk in cryptos diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 4cfa4016..7aab5dc6 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -3,9 +3,10 @@ import shutil from datetime import timedelta import pandas as pd -from logbook import Logger, INFO +from logbook import Logger from catalyst import get_calendar +from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ BcolzMinuteBarMetadata from catalyst.exchange.bundle_utils import range_in_bundle, \ @@ -14,18 +15,16 @@ from catalyst.exchange.bundle_utils import range_in_bundle, \ from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ BcolzExchangeBarWriter from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ - InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \ - TempBundleNotFoundError, NoDataAvailableOnExchange, \ + InvalidHistoryFrequencyError, TempBundleNotFoundError, \ + NoDataAvailableOnExchange, \ PricingDataNotLoadedError from catalyst.exchange.exchange_utils import get_exchange_folder from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory -from catalyst.constants import LOG_LEVEL - log = Logger('exchange_bundle', level=LOG_LEVEL) -BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle' +BUNDLE_NAME_TEMPLATE = os.path.join('{root}','{frequency}_bundle') def _cachpath(symbol, type_): return '-'.join([symbol, type_]) @@ -172,7 +171,7 @@ class ExchangeBundle: invalid_data_behavior='raise' ) except BcolzMinuteOverlappingData as e: - log.warn('chunk already exists: {}'.format(e)) + log.debug('chunk already exists: {}'.format(e)) except Exception as e: log.warn('error when writing data: {}, trying again'.format(e)) @@ -319,6 +318,9 @@ class ExchangeBundle: except NoDataAvailableOnExchange: continue + start_dt = max(start_dt, self.calendar.first_trading_session) + start_dt = max(start_dt, asset_start) + # Aligning start / end dates with the daily calendar sessions = get_periods_range(start_dt, end_dt, data_frequency) \ if data_frequency == 'minute' \ diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 602582d2..c5baadec 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -1,10 +1,13 @@ -import sys, traceback +import sys +import traceback + from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError, - SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]: + SymbolNotFoundOnExchange, NoDataAvailableOnExchange, + ExchangeAuthEmpty ]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print "Error traceback: {1} (line {2})\n" \ @@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError): ).strip() +class ExchangeAuthEmpty(ZiplineError): + msg = ( + 'Please enter your API token key and secret for exchange {exchange} ' + 'in the following file: {filename}' + ).strip() + + class ExchangeSymbolsNotFound(ZiplineError): msg = ( 'Unable to download or find a local copy of symbols.json for exchange ' diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py index da465165..2c0b1ac2 100644 --- a/catalyst/exchange/exchange_portfolio.py +++ b/catalyst/exchange/exchange_portfolio.py @@ -1,9 +1,8 @@ import numpy as np from logbook import Logger -from catalyst.protocol import Portfolio, Positions, Position - from catalyst.constants import LOG_LEVEL +from catalyst.protocol import Portfolio, Positions, Position log = Logger('ExchangePortfolio', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 1335561c..b9e6cf21 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -8,7 +8,8 @@ import pandas as pd from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ ExchangeSymbolsNotFound -from catalyst.utils.paths import data_root, ensure_directory, last_modified_time +from catalyst.utils.paths import data_root, ensure_directory, \ + last_modified_time SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' @@ -64,11 +65,10 @@ def get_exchange_auth(exchange_name, environ=None): data = json.load(data_file) return data else: - raise ExchangeAuthNotFound( - exchange=exchange_name, - filename=filename - ) - + data = dict(name=exchange_name, key='', secret='') + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) + return data def get_algo_folder(algo_name, environ=None): if not environ: diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index 1973a3ee..ecc83677 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -10,7 +10,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import timedelta import pandas as pd from catalyst.gens.sim_engine import ( @@ -19,11 +18,10 @@ from catalyst.gens.sim_engine import ( ) from logbook import Logger +from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange_errors import \ MismatchingBaseCurrenciesExchanges -from catalyst.constants import LOG_LEVEL - log = Logger('LiveGraphClock', level=LOG_LEVEL) diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py index 60e5b4b5..43954036 100644 --- a/catalyst/exchange/poloniex/poloniex.py +++ b/catalyst/exchange/poloniex/poloniex.py @@ -1,39 +1,32 @@ -import base64 -import hashlib -import hmac import json -import re +import json import time from collections import defaultdict import numpy as np import pandas as pd import pytz -import requests -# import six -from six import iteritems from catalyst.assets._assets import TradingPair from logbook import Logger +# import six +from six import iteritems -from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.poloniex.poloniex_api import Poloniex_api - +from catalyst.constants import LOG_LEVEL # from websocket import create_connection from catalyst.exchange.exchange import Exchange +from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( ExchangeRequestError, InvalidHistoryFrequencyError, - InvalidOrderStyle, OrderCancelError, - OrphanOrderReverseError) + InvalidOrderStyle, OrphanOrderReverseError) from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ - ExchangeStopLimitOrder, ExchangeStopOrder -from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.protocol import Account + ExchangeStopLimitOrder from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ download_exchange_symbols +from catalyst.exchange.poloniex.poloniex_api import Poloniex_api +from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction - -from catalyst.constants import LOG_LEVEL +from catalyst.protocol import Account log = Logger('Poloniex', level=LOG_LEVEL) diff --git a/catalyst/exchange/simple_clock.py b/catalyst/exchange/simple_clock.py index 1f4d3ffa..f79a5ab7 100644 --- a/catalyst/exchange/simple_clock.py +++ b/catalyst/exchange/simple_clock.py @@ -16,13 +16,13 @@ from time import sleep import pandas as pd from catalyst.gens.sim_engine import ( BAR, - SESSION_START, - MINUTE_END, - SESSION_END + SESSION_START ) from logbook import Logger -log = Logger('ExchangeClock') +from catalyst.constants import LOG_LEVEL + +log = Logger('ExchangeClock', level=LOG_LEVEL) class SimpleClock(object): diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index da2c3ef8..a5293ac5 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -36,11 +36,11 @@ from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \ from catalyst.exchange.asset_finder_exchange import AssetFinderExchange from catalyst.exchange.exchange_portfolio import ExchangePortfolio from catalyst.exchange.exchange_errors import ( - ExchangeRequestError, + ExchangeRequestError, ExchangeAuthEmpty, ExchangeRequestErrorTooManyAttempts, BaseCurrencyNotFoundError, ExchangeNotFoundError) from catalyst.exchange.exchange_utils import get_exchange_auth, \ - get_algo_object + get_algo_object, get_exchange_folder from logbook import Logger from catalyst.constants import LOG_LEVEL @@ -166,6 +166,12 @@ def _run(handle_data, # This corresponds to the json file containing api token info exchange_auth = get_exchange_auth(exchange_name) + + if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''): + raise ExchangeAuthEmpty( + exchange=exchange_name.title(), + filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') ) + if exchange_name == 'bitfinex': exchanges[exchange_name] = Bitfinex( key=exchange_auth['key'], @@ -237,8 +243,11 @@ def _run(handle_data, balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: - log.warn('exchange error when retrieving balances, {} ' - 'trying again in 5 seconds'.format(e)) + log.warn( + 'could not retrieve balances on {}: {}'.format( + exchange.name, e + ) + ) sleep(5) return fetch_capital_base(exchange, attempt_index + 1)