Refactoring exchanges

# Conflicts:
#	catalyst/__main__.py
#	catalyst/utils/run_algo.py
#	catalyst/utils/run_algo_original.py
#	zipline/utils/run_algo.py

.gitignore

@@ -40,7 +40,6 @@ develop-eggs
 coverage.xml
 htmlcov
 nosetests.xml
-.python-version
 
 # C Extensions
 *.o

Dockerfile

@@ -11,13 +11,13 @@
 #
 #    https://127.0.0.1
 #
-# Default password is 'jupyter'. To provide another, see:
+# default password is jupyter.  to provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# Once generated, you can pass the new value via `docker run --env` the first time
+# once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command. For example:
+# You can also run an algo using the docker exec command.  For example:
 #
 #    docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

Dockerfile-dev

@@ -5,7 +5,7 @@
 #
 # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile .
+#    docker build -t quantopian/catalyst -f Dockerfile
 #
 # To run the container:
 #
@@ -15,13 +15,13 @@
 #
 #    https://127.0.0.1
 #
-# Default password is 'jupyter'. To provide another, see:
+# default password is jupyter.  to provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# Once generated, you can pass the new value via `docker run --env` the first time
+# once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command. For example:
+# You can also run an algo using the docker exec command.  For example:
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

README.rst

@@ -1,70 +1 @@
-.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png
+All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_ can be found in the `catalyst-docs wiki <https://github.com/enigmampc/catalyst-docs/wiki>`_.
-    :target: https://enigmampc.github.io/catalyst
-    :align: center
-    :alt: Enigma | Catalyst
-
-|version tag|
-|version status|
-|discord|
-|twitter|
-
-|
-
-Catalyst is an algorithmic trading library for crypto-assets written in Python.
-It allows trading strategies to be easily expressed and backtested against 
-historical data (with daily and minute resolution), providing analytics and 
-insights regarding a particular strategy's performance. Catalyst also supports
-live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
-and Poloniex) with more being added over time. Catalyst empowers users to share 
-and curate data and build profitable, data-driven investment strategies. Please 
-visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
-
-Catalyst builds on top of the well-established 
-`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
-minimize structural changes to the general API to maximize compatibility with 
-existing trading algorithms, developer knowledge, and tutorials. Join us on 
-`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
-for questions around Catalyst, algorithmic trading and technical support.
-
-Overview
-========
-
--  Ease of use: Catalyst tries to get out of your way so that you can 
-   focus on algorithm development. See 
-   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
-   provided.
--  Support for several of the top crypto-exchanges by trading volume:
-   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
-   and `Poloniex <https://www.poloniex.com>`_. 
--  Secure: You and only you have access to each exchange API keys for your accounts.
--  Input of historical pricing data of all crypto-assets by exchange, 
-   with daily and minute resolution. See 
-   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
--  Backtesting and live-trading functionality, with a seamless transition
-   between the two modes.
--  Output of performance statistics are based on Pandas DataFrames to 
-   integrate nicely into the existing PyData eco-system.
--  Statistic and machine learning libraries like matplotlib, scipy, 
-   statsmodels, and sklearn support development, analysis, and 
-   visualization of state-of-the-art trading systems.
--  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
-   performance across trading algorithms.
-
-Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
-
-
-
-
-.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
-   :target: https://pypi.python.org/pypi/enigma-catalyst
-
-.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
-   :target: https://pypi.python.org/pypi/enigma-catalyst
-
-.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
-   :target: https://discordapp.com/invite/SJK32GY
-
-.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
-   :target: https://twitter.com/enigmampc
-
-

catalyst/__init__.py

@@ -29,14 +29,11 @@ from ._version import get_versions
 from . algorithm import TradingAlgorithm
 from . import api
 
-from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
-
-__version__ = get_versions()['version']
-del get_versions
 
 # PERF: Fire a warning if calendars were instantiated during catalyst import.
 # Having calendars doesn't break anything per-se, but it makes catalyst imports
 # noticeably slower, which becomes particularly noticeable in the Zipline CLI.
+from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
 if global_calendar_dispatcher._calendars:
     import warnings
     warnings.warn(
@@ -47,6 +44,10 @@ if global_calendar_dispatcher._calendars:
 del global_calendar_dispatcher
 
 
+__version__ = get_versions()['version']
+del get_versions
+
+
 def load_ipython_extension(ipython):
     from .__main__ import catalyst_magic
     ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -68,6 +69,7 @@ if os.name == 'nt':
     _()
     del _
 
+
 __all__ = [
     'TradingAlgorithm',
     'api',
@@ -77,4 +79,9 @@ __all__ = [
     'gens',
     'run_algorithm',
     'utils',
+    'exchange',
 ]
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions

catalyst/algorithm.py

@@ -124,7 +124,7 @@ from catalyst.utils.events import (
 from catalyst.utils.factory import create_simulation_parameters
 from catalyst.utils.math_utils import (
     tolerant_equals,
-    round_nearest
+    round_if_near_integer,
 )
 from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
 from catalyst.utils.preprocess import preprocess
@@ -133,13 +133,15 @@ from catalyst.utils.security_list import SecurityList
 import catalyst.protocol
 from catalyst.sources.requests_csv import PandasRequestsCSV
 
-from catalyst.gens.sim_engine import MinuteSimulationClock
+from catalyst.gens.sim_engine import (
+    MinuteSimulationClock,
+    FiveMinuteSimulationClock,
+)
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
-from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger("CatalystLog", level=LOG_LEVEL)
+log = logbook.Logger("ZiplineLog")
 
 
 class TradingAlgorithm(object):
@@ -171,7 +173,7 @@ class TradingAlgorithm(object):
     algo_filename : str, optional
         The filename for the algoscript. This will be used in exception
         tracebacks. default: '<string>'.
-    data_frequency : {'daily', 'minute'}, optional
+    data_frequency : {'daily', '5-minute', 'minute'}, optional
         The duration of the bars.
     instant_fill : bool, optional
         Whether to fill orders immediately or on next bar. default: False
@@ -224,7 +226,7 @@ class TradingAlgorithm(object):
             script : str
                 Algoscript that contains initialize and
                 handle_data function definition.
-            data_frequency : {'daily', 'minute'}
+            data_frequency : {'daily', '5-minute', 'minute'}
                The duration of the bars.
             capital_base : float <default: 1.0e5>
                How much capital to start with.
@@ -432,6 +434,8 @@ class TradingAlgorithm(object):
         if get_loader is not None:
             if data_frequency == 'daily':
                 all_dates = self.trading_calendar.all_sessions
+            elif data_frequency == '5-minute':
+                all_dates = self.trading_calendar.all_five_minutes
             elif data_frequency == 'minute':
                 all_dates = self.trading_calendar.all_minutes
             else:
@@ -463,7 +467,7 @@ class TradingAlgorithm(object):
         self._in_before_trading_start = True
 
         with handle_non_market_minutes(data) if \
-                self.data_frequency == 'minute' else ExitStack():
+                self.data_frequency in ('minute', '5-minute') else ExitStack():
             self._before_trading_start(self, data)
 
         self._in_before_trading_start = False
@@ -519,10 +523,11 @@ class TradingAlgorithm(object):
         market_closes = trading_o_and_c['market_close']
         minutely_emission = False
 
-        if self.sim_params.data_frequency == 'minute':
+        if self.sim_params.data_frequency in set(('minute', '5-minute')):
             market_opens = trading_o_and_c['market_open']
 
-            minutely_emission = self.sim_params.emission_rate == 'minute'
+            minutely_emission = self.sim_params.emission_rate in \
+                set(('minute', '5-minute'))
         else:
             # in daily mode, we want to have one bar per session, timestamped
             # as the last minute of the session.
@@ -546,6 +551,15 @@ class TradingAlgorithm(object):
             'UTC',
         )
 
+        if self.sim_params.data_frequency == '5-minute':
+            return FiveMinuteSimulationClock(
+                self.sim_params.sessions,
+                execution_opens,
+                execution_closes,
+                before_trading_start_minutes,
+                minute_emission=minutely_emission,
+            )
+
         return MinuteSimulationClock(
             self.sim_params.sessions,
             execution_opens,
@@ -677,6 +691,8 @@ class TradingAlgorithm(object):
                     time_count = times.nunique()
                     if time_count == 1:
                         self.sim_params.data_frequency = 'daily'
+                    elif time_count == 288:
+                        self.sim_params.data_frequency = '5-minute'
                     else:
                         self.sim_params.data_frequency = 'minute'
 
@@ -698,6 +714,8 @@ class TradingAlgorithm(object):
 
                 if self.sim_params.data_frequency == 'daily':
                     equity_reader_arg = 'equity_daily_reader'
+                elif self.sim_params.data_frequency == '5-minute':
+                    equity_daily_reader = 'equity_5_minute_reader'
                 elif self.sim_params.data_frequency == 'minute':
                     equity_reader_arg = 'equity_minute_reader'
                 equity_reader = PanelBarReader(
@@ -939,11 +957,11 @@ class TradingAlgorithm(object):
             The field to query. The options have the following meanings:
               arena : str
                   The arena from the simulation parameters. This will normally
-                  be ``backtest`` but some systems may use this distinguish
+                  be ``'backtest'`` but some systems may use this distinguish
                   live trading from backtesting.
-              data_frequency : {'daily', 'minute'}
+              data_frequency : {'daily', '5-minute', 'minute'}
                   data_frequency tells the algorithm if it is running with
-                  daily or minute mode.
+                  daily, minute, or five-minute mode.
               start : datetime
                   The start date for the simulation.
               end : datetime
@@ -954,7 +972,7 @@ class TradingAlgorithm(object):
                   The platform that the code is running on. By default this
                   will be the string 'catalyst'. This can allow algorithms to
                   know if they are running on the Quantopian platform instead.
-              \* : dict[str -> any]
+              * : dict[str -> any]
                   Returns all of the fields in a dictionary.
 
         Returns
@@ -1032,7 +1050,7 @@ class TradingAlgorithm(object):
             argument is the name of the column in the preprocessed dataframe
             containing the symbols. This will be used along with the date
             information to map the sids in the asset finder.
-        \*\*kwargs
+        **kwargs
             Forwarded to :func:`pandas.read_csv`.
 
         Returns
@@ -1118,11 +1136,20 @@ class TradingAlgorithm(object):
                           'time_rule= when calling schedule_function without '
                           'specifying a date_rule', stacklevel=3)
         
+        freq = self.sim_params.data_frequency
+
+        freq = self.sim_params.data_frequency
+
         date_rule = date_rule or date_rules.every_day()
-        time_rule = ((time_rule or time_rules.every_minute())
+        if freq is 'daily':
-                     if self.sim_params.data_frequency == 'minute' else
+            # ignore time rule in daily mode
-                     # If we are in daily mode the time_rule is ignored.
+            time_rule = time_rules.every_minute()
-                     time_rules.every_minute())
+        else:
+            # use provided time rule or default to every minute or 5 minutes
+            # based on desired data frequency.
+            time_rule = time_rule or (time_rules.every_5_minutes()
+                                      if freq is '5-minute' else
+                                      time_rules.every_minute())
 
         # Check the type of the algorithm's schedule before pulling calendar
         # Note that the ExchangeTradingSchedule is currently the only
@@ -1156,7 +1183,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        \*\*kwargs
+        **kwargs
             The names and values to record.
 
         Notes
@@ -1273,7 +1300,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        \*args : iterable[str]
+        *args : iterable[str]
             The ticker symbols to lookup.
 
         Returns
@@ -1463,7 +1490,7 @@ class TradingAlgorithm(object):
 
     def _calculate_order(self, asset, amount,
                          limit_price=None, stop_price=None, style=None):
-        amount = self.round_order(amount, asset)
+        amount = self.round_order(amount)
 
         # Raises a ZiplineError if invalid parameters are detected.
         self.validate_order_params(asset,
@@ -1480,12 +1507,16 @@ class TradingAlgorithm(object):
         return amount, style
 
     @staticmethod
-    def round_order(amount, asset):
+    def round_order(amount):
         """
-        Converts the number of shares to the smallest tradable lot size for
+        Convert number of shares to an integer.
-        the asset being ordered.
+
+        By default, truncates to the integer share count that's either within
+        .0001 of amount or closer to zero.
+
+        E.g. 3.9999 -> 4.0; 5.5 -> 5.0; -5.5 -> -5.0
         """
-        return round_nearest(amount, asset.min_trade_size)
+        return int(round_if_near_integer(amount))
 
     def validate_order_params(self,
                               asset,
@@ -1793,7 +1824,7 @@ class TradingAlgorithm(object):
 
     @data_frequency.setter
     def data_frequency(self, value):
-        assert value in ('daily', 'minute')
+        assert value in ('daily', '5-minute', 'minute')
         self.sim_params.data_frequency = value
 
     @api_method

catalyst/api.pyi

@@ -34,7 +34,6 @@ def attach_pipeline(pipeline, name, chunks=None):
     :func:`catalyst.api.pipeline_output`
     """
 
-
 def batch_market_order(share_counts):
     """Place a batch market order for multiple assets.
 
@@ -49,7 +48,6 @@ def batch_market_order(share_counts):
         Index of ids for newly-created orders.
     """
 
-
 def cancel_order(order_param):
     """Cancel an open order.
 
@@ -59,9 +57,7 @@ def cancel_order(order_param):
         The order_id or order object to cancel.
     """
 
-
+def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
-def continuous_future(root_symbol_str, offset=0, roll='volume',
-                      adjustment='mul'):
     """Create a specifier for a continuous contract.
 
     Parameters
@@ -85,10 +81,7 @@ def continuous_future(root_symbol_str, offset=0, roll='volume',
         The continuous future specifier.
     """
 
-
+def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
-def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
-              date_format=None, timezone='UTC', symbol=None, mask=True,
-              symbol_column=None, special_params_checker=None, **kwargs):
     """Fetch a csv from a remote url and register the data so that it is
     queryable from the ``data`` object.
 
@@ -132,7 +125,6 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
         A requests source that will pull data from the url specified.
     """
 
-
 def future_symbol(symbol):
     """Lookup a futures contract with a given symbol.
 
@@ -152,7 +144,6 @@ def future_symbol(symbol):
         Raised when no contract named 'symbol' is found.
     """
 
-
 def get_datetime(tz=None):
     """
 Returns the current simulation datetime.
@@ -168,7 +159,6 @@ dt : datetime
     The current simulation datetime converted to ``tz``.
     """
 
-
 def get_environment(field='platform'):
     """Query the execution environment.
 
@@ -208,7 +198,6 @@ def get_environment(field='platform'):
         Raised when ``field`` is not a valid option.
     """
 
-
 def get_order(order_id):
     """Lookup an order based on the order id returned from one of the
     order functions.
@@ -224,12 +213,10 @@ def get_order(order_id):
         The order object.
     """
 
-
 def history(bar_count, frequency, field, ffill=True):
     """DEPRECATED: use ``data.history`` instead.
     """
 
-
 def order(asset, amount, limit_price=None, stop_price=None, style=None):
     """Place an order.
 
@@ -271,9 +258,7 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
-
+def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
-def order_percent(asset, percent, limit_price=None, stop_price=None,
-                  style=None):
     """Place an order in the specified asset corresponding to the given
     percent of the current portfolio value.
 
@@ -308,7 +293,6 @@ def order_percent(asset, percent, limit_price=None, stop_price=None,
     :func:`catalyst.api.order_value`
     """
 
-
 def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     """Place an order to adjust a position to a target number of shares. If
     the position doesn't already exist, this is equivalent to placing a new
@@ -360,9 +344,7 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_target_value`
     """
 
-
+def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
-def order_target_percent(asset, target, limit_price=None, stop_price=None,
-                         style=None):
     """Place an order to adjust a position to a target percent of the
     current portfolio value. If the position doesn't already exist, this is
     equivalent to placing a new order. If the position does exist, this is
@@ -414,9 +396,7 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None,
     :func:`catalyst.api.order_target_value`
     """
 
-
+def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
-def order_target_value(asset, target, limit_price=None, stop_price=None,
-                       style=None):
     """Place an order to adjust a position to a target value. If
     the position doesn't already exist, this is equivalent to placing a new
     order. If the position does exist, this is equivalent to placing an
@@ -468,7 +448,6 @@ def order_target_value(asset, target, limit_price=None, stop_price=None,
     :func:`catalyst.api.order_target_percent`
     """
 
-
 def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     """Place an order by desired value rather than desired number of
     shares.
@@ -509,7 +488,6 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
-
 def pipeline_output(name):
     """Get the results of the pipeline that was attached with the name:
     ``name``.
@@ -536,7 +514,6 @@ def pipeline_output(name):
     :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
     """
 
-
 def record(*args, **kwargs):
     """Track and record values each day.
 
@@ -552,9 +529,7 @@ def record(*args, **kwargs):
     :func:`~catalyst.run_algorithm`.
     """
 
-
+def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
-def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
-                      calendar=None):
     """Schedules a function to be called according to some timed rules.
 
     Parameters
@@ -574,7 +549,6 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
     :class:`catalyst.api.time_rules`
     """
 
-
 def set_asset_restrictions(restrictions, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -588,7 +562,6 @@ def set_asset_restrictions(restrictions, on_error='fail'):
     catalyst.finance.asset_restrictions.Restrictions
     """
 
-
 def set_benchmark(benchmark):
     """Set the benchmark asset.
 
@@ -603,7 +576,6 @@ def set_benchmark(benchmark):
     automatically reinvested.
     """
 
-
 def set_cancel_policy(cancel_policy):
     """Sets the order cancellation policy for the simulation.
 
@@ -618,7 +590,6 @@ def set_cancel_policy(cancel_policy):
     :class:`catalyst.api.NeverCancel`
     """
 
-
 def set_commission(commission):
     """Sets the commission model for the simulation.
 
@@ -634,7 +605,6 @@ def set_commission(commission):
     :class:`catalyst.finance.commission.PerDollar`
     """
 
-
 def set_do_not_order_list(restricted_list, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -644,13 +614,11 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
         The assets that cannot be ordered.
     """
 
-
 def set_long_only(on_error='fail'):
     """Set a rule specifying that this algorithm cannot take short
     positions.
     """
 
-
 def set_max_leverage(max_leverage):
     """Set a limit on the maximum leverage of the algorithm.
 
@@ -661,7 +629,6 @@ def set_max_leverage(max_leverage):
         be no maximum.
     """
 
-
 def set_max_order_count(max_count, on_error='fail'):
     """Set a limit on the number of orders that can be placed in a single
     day.
@@ -672,9 +639,7 @@ def set_max_order_count(max_count, on_error='fail'):
         The maximum number of orders that can be placed on any single day.
     """
 
-
+def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
-def set_max_order_size(asset=None, max_shares=None, max_notional=None,
-                       on_error='fail'):
     """Set a limit on the number of shares and/or dollar value of any single
     order placed for sid.  Limits are treated as absolute values and are
     enforced at the time that the algo attempts to place an order for sid.
@@ -693,9 +658,7 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None,
         The maximum value that can be ordered at one time.
     """
 
-
+def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
-def set_max_position_size(asset=None, max_shares=None, max_notional=None,
-                          on_error='fail'):
     """Set a limit on the number of shares and/or dollar value held for the
     given sid. Limits are treated as absolute values and are enforced at
     the time that the algo attempts to place an order for sid. This means
@@ -718,7 +681,6 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None,
         The maximum value to hold for an asset.
     """
 
-
 def set_slippage(slippage):
     """Set the slippage model for the simulation.
 
@@ -732,7 +694,6 @@ def set_slippage(slippage):
     :class:`catalyst.finance.slippage.SlippageModel`
     """
 
-
 def set_symbol_lookup_date(dt):
     """Set the date for which symbols will be resolved to their assets
     (symbols may map to different firms or underlying assets at
@@ -744,7 +705,6 @@ def set_symbol_lookup_date(dt):
         The new symbol lookup date.
     """
 
-
 def sid(sid):
     """Lookup an Asset by its unique asset identifier.
 
@@ -764,7 +724,6 @@ def sid(sid):
         When a requested ``sid`` does not map to any asset.
     """
 
-
 def symbol(symbol_str):
     """Lookup an Equity by its ticker symbol.
 
@@ -789,7 +748,6 @@ def symbol(symbol_str):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
-
 def symbols(*args):
     """Lookup multuple Equities as a list.
 
@@ -815,18 +773,3 @@ def symbols(*args):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
-
-def get_dataset(ds_name, start=None, end=None):
-    """
-    Lookup a data source from the marketplace
-
-    Parameters
-    ----------
-    ds_name: str
-    start: pd.Timestamp
-    end: pd.Timestamp
-
-    Returns
-    -------
-
-    """

catalyst/assets/_assets.pyx

@@ -17,38 +17,34 @@
 """
 Cythonized Asset object.
 """
-
-import hashlib
-
 cimport cython
 from cpython.number cimport PyNumber_Index
 from cpython.object cimport (
-Py_EQ,
+    Py_EQ,
-Py_NE,
+    Py_NE,
-Py_GE,
+    Py_GE,
-Py_LE,
+    Py_LE,
-Py_GT,
+    Py_GT,
-Py_LT,
+    Py_LT,
 )
 from cpython cimport bool
 
-import pandas as pd
-from datetime import timedelta
 import numpy as np
 from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
-from catalyst.exchange.utils.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
-from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
+
 
 # IMPORTANT NOTE: You must change this template if you change
 # Asset.__reduce__, or else we'll attempt to unpickle an old version of this
 # class
 CACHE_FILE_TEMPLATE = '/tmp/.%s-%s.v7.cache'
 
+
 cdef class Asset:
+
     cdef readonly int sid
     # Cached hash of self.sid
     cdef int sid_hash
@@ -63,7 +59,6 @@ cdef class Asset:
 
     cdef readonly object exchange
     cdef readonly object exchange_full
-    cdef readonly object min_trade_size
 
     _kwargnames = frozenset({
         'sid',
@@ -75,20 +70,18 @@ cdef class Asset:
         'auto_close_date',
         'exchange',
         'exchange_full',
-        'min_trade_size',
     })
 
     def __init__(self,
-                 int sid,  # sid is required
+                 int sid, # sid is required
-                 object exchange,  # exchange is required
+                 object exchange, # exchange is required
                  object symbol="",
                  object asset_name="",
                  object start_date=None,
                  object end_date=None,
                  object first_traded=None,
                  object auto_close_date=None,
-                 object exchange_full=None,
+                 object exchange_full=None):
-                 object min_trade_size=None):
 
         self.sid = sid
         self.sid_hash = hash(sid)
@@ -101,7 +94,6 @@ cdef class Asset:
         self.end_date = end_date
         self.first_traded = first_traded
         self.auto_close_date = auto_close_date
-        self.min_trade_size = min_trade_size
 
     def __int__(self):
         return self.sid
@@ -156,8 +148,7 @@ cdef class Asset:
 
     def __repr__(self):
         attrs = ('symbol', 'asset_name', 'exchange',
-                 'start_date', 'end_date', 'first_traded', 'auto_close_date',
+                 'start_date', 'end_date', 'first_traded', 'auto_close_date')
-                 'min_trade_size')
         tuples = ((attr, repr(getattr(self, attr, None)))
                   for attr in attrs)
         strings = ('%s=%s' % (t[0], t[1]) for t in tuples)
@@ -179,8 +170,7 @@ cdef class Asset:
                                  self.end_date,
                                  self.first_traded,
                                  self.auto_close_date,
-                                 self.exchange_full,
+                                 self.exchange_full))
-                                 self.min_trade_size))
 
     cpdef to_dict(self):
         """
@@ -196,7 +186,6 @@ cdef class Asset:
             'auto_close_date': self.auto_close_date,
             'exchange': self.exchange,
             'exchange_full': self.exchange_full,
-            'min_trade_size': self.min_trade_size
         }
 
     @classmethod
@@ -241,11 +230,13 @@ cdef class Asset:
         calendar = get_calendar(self.exchange)
         return calendar.is_open_on_minute(dt_minute)
 
+
 cdef class Equity(Asset):
+
     def __repr__(self):
         attrs = ('symbol', 'asset_name', 'exchange',
                  'start_date', 'end_date', 'first_traded', 'auto_close_date',
-                 'exchange_full', 'min_trade_size')
+                 'exchange_full')
         tuples = ((attr, repr(getattr(self, attr, None)))
                   for attr in attrs)
         strings = ('%s=%s' % (t[0], t[1]) for t in tuples)
@@ -259,8 +250,8 @@ cdef class Equity(Asset):
         """
         def __get__(self):
             warnings.warn("The security_start_date property will soon be "
-                          "retired. Please use the start_date property instead.",
+            "retired. Please use the start_date property instead.",
-                          DeprecationWarning)
+            DeprecationWarning)
             return self.start_date
 
     property security_end_date:
@@ -270,8 +261,8 @@ cdef class Equity(Asset):
         """
         def __get__(self):
             warnings.warn("The security_end_date property will soon be "
-                          "retired. Please use the end_date property instead.",
+            "retired. Please use the end_date property instead.",
-                          DeprecationWarning)
+            DeprecationWarning)
             return self.end_date
 
     property security_name:
@@ -281,11 +272,13 @@ cdef class Equity(Asset):
         """
         def __get__(self):
             warnings.warn("The security_name property will soon be "
-                          "retired. Please use the asset_name property instead.",
+            "retired. Please use the asset_name property instead.",
-                          DeprecationWarning)
+            DeprecationWarning)
             return self.asset_name
 
+
 cdef class Future(Asset):
+
     cdef readonly object root_symbol
     cdef readonly object notice_date
     cdef readonly object expiration_date
@@ -310,8 +303,8 @@ cdef class Future(Asset):
     })
 
     def __init__(self,
-                 int sid,  # sid is required
+                 int sid, # sid is required
-                 object exchange,  # exchange is required
+                 object exchange, # exchange is required
                  object symbol="",
                  object root_symbol="",
                  object asset_name="",
@@ -395,258 +388,6 @@ cdef class Future(Asset):
         super_dict['multiplier'] = self.multiplier
         return super_dict
 
-cdef class TradingPair(Asset):
-    cdef readonly float leverage
-    cdef readonly object quote_currency
-    cdef readonly object base_currency
-    cdef readonly object end_daily
-    cdef readonly object end_minute
-    cdef readonly object exchange_symbol
-    cdef readonly float maker
-    cdef readonly float taker
-    cdef readonly int trading_state
-    cdef readonly object data_source
-    cdef readonly float max_trade_size
-    cdef readonly float lot
-    cdef readonly int decimals
-
-    _kwargnames = frozenset({
-        'sid',
-        'symbol',
-        'asset_name',
-        'start_date',
-        'end_date',
-        'first_traded',
-        'auto_close_date',
-        'exchange',
-        'exchange_full',
-        'leverage',
-        'quote_currency',
-        'base_currency',
-        'end_daily',
-        'end_minute',
-        'exchange_symbol',
-        'min_trade_size',
-        'max_trade_size',
-        'lot',
-        'maker',
-        'taker',
-        'trading_state',
-        'data_source',
-        'decimals'
-    })
-    def __init__(self,
-                 object symbol,
-                 object exchange,
-                 object start_date=None,
-                 object asset_name=None,
-                 int sid=0,
-                 float leverage=1.0,
-                 object end_daily=None,
-                 object end_minute=None,
-                 object end_date=None,
-                 object exchange_symbol=None,
-                 object first_traded=None,
-                 object auto_close_date=None,
-                 object exchange_full=None,
-                 float min_trade_size=0.0001,
-                 float max_trade_size=1000000,
-                 float maker=0.0015,
-                 float taker=0.0025,
-                 float lot=0,
-                 int decimals = 8,
-                 int trading_state=0,
-                 object data_source='catalyst'):
-        """
-        Replicates the Asset constructor with some built-in conventions
-        and adds properties for leverage and fees.
-
-        Symbol
-        ------
-        Catalyst defines its own set of "universal" symbols to reference
-        trading pairs across exchanges. This is required because exchanges
-        are not adhering to a universal symbolism. For example, Bitfinex
-        uses the BTC symbol for Bitcon while Kraken uses XBT. In addition,
-        pairs are sometimes presented differently. For example, Bitfinex
-        puts the market currency before the base currency without a
-        separator, Bittrex puts the base currency first and uses a dash
-        seperator.
-
-        Here is the Catalyst convention: [Market Currency]_[Base Currency]
-        For example: btc_usd, eth_btc, neo_eth, ltc_eur.
-
-        The symbol for each currency (e.g. btc, eth, ltc) is generally
-        aligned with the Bittrex exchange.
-
-        Sid
-        ---
-        The sid of each asset is calculated based on a numeric hash of the
-        universal symbol. This simple approach avoids maintaining a mapping
-        of sids.
-
-        Leverage
-        --------
-        In contrast with equities, crypto exchanges generally assign
-        leverage values to specific trading pairs. Pairs with the
-        highest volume and market cap generally benefit from high leverage.
-        New currencies from ICO generally cannot be leveraged.
-
-        Leverage allows you to open a larger position with a smaller amount
-        of funds. For example, if you open a $5,000 position in BTC/USD
-        with 5:1 leverage, only one-fifth of this amount, or $1000, will be
-        tied to the position from your balance. Your remaining balance will
-        be available for opening more positions. If you open this same
-        position with 2:1 leverage, $2,500 of your balance will be tied to
-        the position. If you open with 1:1 leverage, $5,000 of your balance
-        will be tied to the position.
-
-        Fees
-        ----
-        Exchanges generally charge a taker (taking from the order book) or
-        maker (adding to the order book) fee.
-
-        :param symbol:
-        :param exchange:
-        :param start_date:
-        :param asset_name:
-        :param sid:
-        :param leverage:
-        :param end_daily
-        :param end_minute
-        :param end_date:
-        :param exchange_symbol:
-        :param first_traded:
-        :param auto_close_date:
-        :param exchange_full:
-        :param min_trade_size:
-        :param max_trade_size:
-        :param maker:
-        :param taker:
-        :param data_source
-        :param decimals
-        :param lot
-        """
-
-        symbol = symbol.lower()
-        try:
-            self.base_currency, self.quote_currency = symbol.split('_')
-        except Exception as e:
-            raise InvalidSymbolError(symbol=symbol, error=e)
-
-        if sid == 0 or sid is None:
-            try:
-                sid = get_sid(symbol)
-            except Exception as e:
-                raise SidHashError(symbol=symbol)
-
-        if asset_name is None:
-            asset_name = ' / '.join(symbol.split('_')).upper()
-
-        if start_date is None:
-            start_date = pd.to_datetime('2009-1-1', utc=True)
-
-        if end_date is None:
-            end_date = pd.Timestamp.utcnow() + timedelta(days=365)
-
-        if lot == 0 and min_trade_size > 0:
-            lot = min_trade_size
-
-        super().__init__(
-            sid,
-            exchange,
-            symbol=symbol,
-            asset_name=asset_name,
-            start_date=start_date,
-            end_date=end_date,
-            first_traded=first_traded,
-            auto_close_date=auto_close_date,
-            exchange_full=exchange_full,
-            min_trade_size=min_trade_size,
-        )
-
-        self.maker = maker
-        self.taker = taker
-        self.leverage = leverage
-        self.end_daily = end_daily
-        self.end_minute = end_minute
-        self.exchange_symbol = exchange_symbol
-        self.trading_state = trading_state
-        self.data_source = data_source
-        self.max_trade_size = max_trade_size
-        self.lot = lot
-        self.decimals = decimals
-
-    def __repr__(self):
-        return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
-               'Introduced On: {start_date}, ' \
-               'Base Currency: {base_currency}, ' \
-               'Quote Currency: {quote_currency}, ' \
-               'Exchange Leverage: {leverage}, ' \
-               'Minimum Trade Size: {min_trade_size} ' \
-               'Last daily ingestion: {end_daily} ' \
-               'Last minutely ingestion: {end_minute}'.format(
-            symbol=self.symbol,
-            sid=self.sid,
-            exchange=self.exchange,
-            start_date=self.start_date,
-            quote_currency=self.quote_currency,
-            base_currency=self.base_currency,
-            leverage=self.leverage,
-            min_trade_size=self.min_trade_size,
-            end_daily=self.end_daily,
-            end_minute=self.end_minute
-        )
-
-    cpdef to_dict(self):
-        """
-        Convert to a python dict.
-        """
-        #TODO: missing fields
-        super_dict = super(TradingPair, self).to_dict()
-        super_dict['end_daily'] = self.end_daily
-        super_dict['end_minute'] = self.end_minute
-        super_dict['leverage'] = self.leverage
-        super_dict['min_trade_size'] = self.min_trade_size
-        return super_dict
-
-    def is_exchange_open(self, dt_minute):
-        """
-        Parameters
-        ----------
-        dt_minute: pd.Timestamp (UTC, tz-aware)
-            The minute to check.
-
-        Returns
-        -------
-        boolean: whether the asset's exchange is open at the given minute.
-        """
-        #TODO: make more dymanic to catch holds
-        return True
-
-    cpdef __reduce__(self):
-        """
-        Function used by pickle to determine how to serialize/deserialize this
-        class.  Should return a tuple whose first element is self.__class__,
-        and whose second element is a tuple of all the attributes that should
-        be serialized/deserialized during pickling.
-        """
-        #TODO: make sure that all fields set there
-        return (self.__class__, (self.symbol,
-                                 self.exchange,
-                                 self.start_date,
-                                 self.asset_name,
-                                 self.sid,
-                                 self.leverage,
-                                 self.end_date,
-                                 self.first_traded,
-                                 self.auto_close_date,
-                                 self.exchange_full,
-                                 self.min_trade_size,
-                                 self.max_trade_size,
-                                 self.lot,
-                                 self.decimals,
-                                 self.taker,
-                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/assets/asset_db_schema.py

@@ -39,8 +39,7 @@ equities = sa.Table(
     sa.Column('first_traded', sa.Integer),
     sa.Column('auto_close_date', sa.Integer),
     sa.Column('exchange', sa.Text),
-    sa.Column('exchange_full', sa.Text),
+    sa.Column('exchange_full', sa.Text)
-    sa.Column('min_trade_size', sa.Float)
 )
 
 equity_symbol_mappings = sa.Table(

catalyst/assets/asset_writer.py

@@ -73,7 +73,6 @@ _equities_defaults = {
     'exchange': None,
     # optional, something like "New York Stock Exchange"
     'exchange_full': None,
-    'min_trade_size': 1
 }
 
 # Default values for the futures DataFrame
@@ -391,8 +390,6 @@ class AssetDBWriter(object):
                   The date on which to close any positions in this asset.
               exchange : str
                   The exchange where this asset is traded.
-              min_trade_size: float, optional
-                  The minimum denomination this asset can be traded.
 
             The index of this dataframe should contain the sids.
         futures : pd.DataFrame, optional

catalyst/assets/assets.py

@@ -76,9 +76,7 @@ from catalyst.utils.numpy_utils import as_column
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.sqlite_utils import group_into_chunks, coerce_string_to_eng
 
-from catalyst.constants import LOG_LEVEL
+log = Logger('assets.py')
-
-log = Logger('assets.py', level=LOG_LEVEL)
 
 # A set of fields that need to be converted to strings before building an
 # Asset to avoid unicode fields

catalyst/constants.py

@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import logbook
-
-''' You can override the LOG level from your environment.
-    For example, if you want to see the DEBUG messages, run:
-    $ export CATALYST_LOG_LEVEL=10
-'''
-LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
-
-SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
-              '{exchange}/symbols.json'
-
-DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
-DATE_FORMAT = '%Y-%m-%d'
-
-try:
-    ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-except Exception as e:
-    print('unable to get catalyst path: {}'.format(e))
-
-AUTO_INGEST = False
-
-AUTH_SERVER = 'https://data.enigma.co'
-
-# TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
-
-
-MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
-                       'catalyst/master/catalyst/marketplace/' \
-                       'contract_marketplace_address.txt'
-
-MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
-                           'catalyst/master/catalyst/marketplace/' \
-                           'contract_marketplace_abi.json'
-
-# TODO: switch to mainnet
-ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-                  'master/catalyst/marketplace/' \
-                  'contract_enigma_address.txt'
-
-ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
-                      'catalyst/master/catalyst/marketplace/' \
-                      'contract_enigma_abi.json'

catalyst/curate/poloniex.py

@@ -1,47 +1,35 @@
-import csv
+import json, time, csv
-import json
-import os
-import shutil
-import time
 from datetime import datetime
-
-import logbook
 import pandas as pd
+import os
+import time
 import requests
+import logbook
 
-from catalyst.exchange.utils.exchange_utils import \
+DT_START        = time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())
-    get_exchange_symbols_filename
+CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
-
+CONN_RETRIES    = 2
-DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END = pd.to_datetime('today').value // 10 ** 9
-CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
-CONN_RETRIES = 2
 
 logbook.StderrHandler().push_application()
 log = logbook.Logger(__name__)
 
-
 class PoloniexCurator(object):
-    '''
+    """
     OHLCV data feed generator for crypto data. Based on Poloniex market data
-    '''
+    """
 
-    _api_path = 'https://poloniex.com/public?'
+    _api_path       = 'https://poloniex.com/public?'
-    currency_pairs = []
+    currency_pairs  = []
 
     def __init__(self):
         if not os.path.exists(CSV_OUT_FOLDER):
             try:
                 os.makedirs(CSV_OUT_FOLDER)
             except Exception as e:
-                log.error('Failed to create data folder: {}'.format(
+                log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
-                            CSV_OUT_FOLDER))
                 log.exception(e)
 
     def get_currency_pairs(self):
-        '''
-        Retrieves and returns all currency pairs from the exchange
-        '''
         url = self._api_path + 'command=returnTicker'
 
         try:
@@ -52,331 +40,105 @@ class PoloniexCurator(object):
             return None
 
         data = response.json()
-        self.currency_pairs = []
+        self.currency_pairs  = []
         for ticker in data:
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
 
-        log.debug('Currency pairs retrieved successfully: {}'.format(
+        log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
-                    len(self.currency_pairs)
-                    ))
 
-    def _retrieve_tradeID_date(self, row):
+    def _get_start_date(self, csv_fn):
-        '''
+        ''' Function returns latest appended date, if the file has been previously written
-        Helper function that reads tradeID and date fields from CSV readline
+            the last line is an empty one, so we have to read the second to last line 
-        '''
-        tId = int(row.split(',')[0])
-        d = pd.to_datetime(row.split(',')[1],
-                           infer_datetime_format=True).value // 10 ** 9
-        return tId, d
-
-    def retrieve_trade_history(self, currencyPair, start=DT_START,
-                               end=DT_END, temp=None):
-        '''
-        Retrieves TradeHistory from exchange for a given currencyPair
-        between start and end dates. If no start date is provided, uses
-        a system-wide one (beginning of time for cryptotrading).
-        If no end date is provided, 'now' is used.
-
-        Stores results in CSV file on disk.
-
-        This function is called recursively to work around the
-        limitations imposed by the provider API.
-        '''
-        csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
-
-        '''
-        Check what data we already have on disk, reading first and last
-        lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
             with open(csv_fn, 'ab+') as f: 
-                f.seek(0, os.SEEK_END)
+                f.seek(0, os.SEEK_END)              # First check file is not zero size
-                if(f.tell() > 2):                   # Check file size is not 0
+                if(f.tell() > 2):
-                    f.seek(0)                       # Go to start to read
+                    f.seek(-2, os.SEEK_END)         # Jump to the second last byte.
-                    last_tradeID, end_file = self._retrieve_tradeID_date(
-                                                    f.readline())
-                    f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
                     while f.read(1) != b"\n":       # Until EOL is found...
-                        # ...jump back the read byte plus one more.
+                        f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
-                        f.seek(-2, os.SEEK_CUR)
+                    lastrow = f.readline() 
-                    first_tradeID, start_file = self._retrieve_tradeID_date(
+                    return int(lastrow.split(',')[0]) + 300
-                                                    f.readline())
-
-                    if(end_file + 3600 * 6 > DT_END
-                        and (first_tradeID == 1
-                             or (currencyPair == 'BTC_HUC'
-                                 and first_tradeID == 2)
-                             or (currencyPair == 'BTC_RIC'
-                                 and first_tradeID == 2)
-                             or (currencyPair == 'BTC_XCP'
-                                 and first_tradeID == 2)
-                             or (currencyPair == 'BTC_NAV'
-                                 and first_tradeID == 4569)
-                             or (currencyPair == 'BTC_POT'
-                                 and first_tradeID == 23511))):
-                        return
 
         except Exception as e:
-            log.error('Error opening file: {}'.format(csv_fn))
+            log.error('Error opening file: %s' % csv_fn)
             log.exception(e)
 
-        '''
+        return DT_START
-        Poloniex API limits querying TradeHistory to intervals smaller
-        than 1 month, so we make sure that start date is never more than
-        1 month apart from end date
-        '''
-        if(end - start > 2419200):   # 60s/min * 60min/hr * 24hr/day * 28days
-            newstart = end - 2419200
-        else:
-            newstart = start
 
-        log.debug('{}: Retrieving from {} to {}\t {} - {}'.format(
+    def get_data(self, currencyPair, start, end=9999999999, period=300):
-                    currencyPair, str(newstart), str(end),
+        url = self._api_path + 'command=returnChartData&currencyPair=' + currencyPair + '&start=' + str(start) + '&end=' + str(end) + '&period=' + str(period)
-                    time.ctime(newstart), time.ctime(end)))
 
-        url = '{path}command=returnTradeHistory&currencyPair={pair}' \
+        try:
-              '&start={start}&end={end}'.format(
+            response = requests.get(url)
-                    path=self._api_path,
+        except Exception as e:
-                    pair=currencyPair,
+            log.error('Failed to retrieve candlestick chart data for %s' % currencyPair)
-                    start=str(newstart),
+            log.exception(e)
-                    end=str(end)
-                )
-
-        attempts = 0
-        success = 0
-        while attempts < CONN_RETRIES:
-            try:
-                response = requests.get(url)
-            except Exception as e:
-                log.error('Failed to retrieve trade history data'
-                          'for {}'.format(currencyPair))
-                log.exception(e)
-                attempts += 1
-            else:
-                try:
-                    if(isinstance(response.json(), dict)
-                       and response.json()['error']):
-                        log.error('Failed to to retrieve trade history data '
-                                  'for {}: {}'.format(
-                                    currencyPair,
-                                    response.json()['error']
-                                    ))
-                        attempts += 1
-                except Exception as e:
-                    log.exception(e)
-                    attempts += 1
-                else:
-                    success = 1
-                    break
-
-        if not success:
             return None
 
-        '''
+        return response.json()
-        If we get to transactionId == 1, and we already have that on
-        disk, we got to the end of TradeHistory for this coin.
-        '''
-        if('first_tradeID' in locals()
-                and response.json()[-1]['tradeID'] == first_tradeID):
-            return
 
-        '''
+    '''
-            There are primarily two scenarios:
+    Pulls latest data for a single pair
-                a) There is newer data available that we need to add at
+    '''
-                   the beginning of the file. We'll retrieve all what we
+    def append_data_single_pair(self, currencyPair, repeat=0):
-                   need until we get to what we already have, writing it
+        log.debug('Getting data for %s' % currencyPair)
-                   to a temporary file; and we will write that at the
+        csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv'
-                   beginning of our existing file.
+        start  = self._get_start_date(csv_fn)
-                b) We are going back in time, appending at the end of
+        # Only fetch data if more than 5min have passed since last fetch
-                   our existing TradeHistory until the first transaction
+        if (time.time() > start):
-                   for this currencyPair
+            data   = self.get_data(currencyPair, start)
-        '''
+            if data is not None:
-        try:
+                try: 
-            if(temp is not None
+                    with open(csv_fn, 'ab') as csvfile:
-                    or ('end_file' in locals() and end_file + 3600 < end)):
+                        csvwriter = csv.writer(csvfile)
-                if (temp is None):
+                        for item in data:
-                    temp = os.tmpfile()
+                            if item['date'] == 0:
-                tempcsv = csv.writer(temp)
+                                continue
-                for item in response.json():
+                            csvwriter.writerow([
-                    if(item['tradeID'] <= last_tradeID):
+                                item['date'],
-                        continue
+                                item['open'],
-                    tempcsv.writerow([
+                                item['high'],
-                        item['tradeID'],
+                                item['low'],
-                        item['date'],
+                                item['close'],
-                        item['type'],
+                                item['volume'],
-                        item['rate'],
+                            ])
-                        item['amount'],
+                except Exception as e:
-                        item['total'],
+                    log.error('Error opening %s' % csv_fn)
-                        item['globalTradeID'],
+                    log.exception(e)
-                    ])
+            elif (repeat < CONN_RETRIES):
-                if(response.json()[-1]['tradeID'] > last_tradeID):
+                    log.debug('Retrying: attemt %d' % (repeat+1) )
-                    end = pd.to_datetime(response.json()[-1]['date'],
+                    self.append_data_single_pair(currencyPair, repeat + 1)    
-                                         infer_datetime_format=True
-                                         ).value // 10**9
-                    self.retrieve_trade_history(currencyPair, start,
-                                                end, temp=temp)
-                else:
-                    with open(csv_fn, 'rb+') as f:
-                        shutil.copyfileobj(f, temp)
-                        f.seek(0)
-                        temp.seek(0)
-                        shutil.copyfileobj(temp, f)
-                    temp.close()
-                    end = start_file
-            else:
-                with open(csv_fn, 'ab') as csvfile:
-                    csvwriter = csv.writer(csvfile)
-                    for item in response.json():
-                        if('first_tradeID' in locals()
-                                and item['tradeID'] >= first_tradeID):
-                            continue
-                        csvwriter.writerow([
-                            item['tradeID'],
-                            item['date'],
-                            item['type'],
-                            item['rate'],
-                            item['amount'],
-                            item['total'],
-                            item['globalTradeID']
-                        ])
-                end = pd.to_datetime(response.json()[-1]['date'],
-                                     infer_datetime_format=True).value//10**9
 
-        except Exception as e:
+    '''
-            log.error('Error opening {}'.format(csv_fn))
+    Pulls latest data for all currency pairs
-            log.exception(e)
+    '''
+    def append_data(self):
+        for currencyPair in self.currency_pairs:
+            self.append_data_single_pair(currencyPair)
+            # Rate limit is 6 calls per second, sleep 1sec/6 to be safe
+            time.sleep(0.17)
 
-        '''
+    '''
-            If we got here, we aren't done yet. Call recursively with
+    Returns a data frame for all pairs, or for the requests currency pair.
-            'end' times that go sequentially back in time.
+    Makes sure data is up to date
-        '''
+    '''
-        self.retrieve_trade_history(currencyPair, start, end)
+    def to_dataframe(self, start, end, currencyPair=None):
+        csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv'
+        last_date = self._get_start_date(csv_fn)
+        if last_date + 300 < end or not os.path.exists(csv_fn):
+            # get latest data
+            self.append_data_single_pair(currencyPair)
 
-    def generate_ohlcv(self, df):
+        # CSV holds the latest snapshot
-        '''
+        df = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
-        Generates OHLCV dataframe from a dataframe containing all TradeHistory
+        df['date']=pd.to_datetime(df['date'],unit='s')
-        by resampling with 1-minute period
-        '''
-        df.set_index('date', inplace=True)             # Index by date
-        vol = df['total'].to_frame('volume')           # set Vol aside
-        df.drop('total', axis=1, inplace=True)         # Drop volume data
-        ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Rename cols
-        closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
-        vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin
-        ohlcv = pd.concat([ohlc, vol], axis=1)         # Concat OHLC + Vol
-        return ohlcv
-
-    def write_ohlcv_file(self, currencyPair):
-        '''
-        Generates OHLCV data file with 1minute bars from TradeHistory on disk
-        '''
-        csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
-        csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if(os.path.getmtime(csv_1min) > time.time() - 7200):
-            log.debug(currencyPair+': 1min data file already up to date. '
-                      'Delete the file if you want to rebuild it.')
-        else:
-            df = pd.read_csv(csv_trades,
-                             names=['tradeID',
-                                    'date',
-                                    'type',
-                                    'rate',
-                                    'amount',
-                                    'total',
-                                    'globalTradeID'],
-                             dtype={'tradeID': int,
-                                    'date': str,
-                                    'type': str,
-                                    'rate': float,
-                                    'amount': float,
-                                    'total': float,
-                                    'globalTradeID': int}
-                             )
-            df.drop(['tradeID', 'type', 'amount', 'globalTradeID'],
-                    axis=1, inplace=True)
-            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
-            ohlcv = self.generate_ohlcv(df)
-            try:
-                with open(csv_1min, 'w') as csvfile:
-                    csvwriter = csv.writer(csvfile)
-                    for item in ohlcv.itertuples():
-                        if item.Index == 0:
-                            continue
-                        csvwriter.writerow([
-                            item.Index.value // 10 ** 9,
-                            item.open,
-                            item.high,
-                            item.low,
-                            item.close,
-                            item.volume,
-                        ])
-            except Exception as e:
-                log.error('Error opening {}'.format(csv_1min))
-                log.exception(e)
-            log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
-
-    def onemin_to_dataframe(self, currencyPair, start, end):
-        '''
-        Returns a data frame for a given currencyPair from data on disk
-        '''
-        csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df = pd.read_csv(csv_fn, names=['date',
-                                        'open',
-                                        'high',
-                                        'low',
-                                        'close',
-                                        'volume'])
-        df['date'] = pd.to_datetime(df['date'], unit='s')
         df.set_index('date', inplace=True)
-        return df[start:end]
-
-    def generate_symbols_json(self, filename=None):
-        '''
-        Generates a symbols.json file with corresponding start_date
-        for each currencyPair
-        '''
-        symbol_map = {}
-
-        if(filename is None):
-            filename = get_exchange_symbols_filename('poloniex')
-
-        with open(filename, 'w') as symbols:
-            for currencyPair in self.currency_pairs:
-                start = None
-                csv_fn = '{}crypto_trades-{}.csv'.format(
-                                                    CSV_OUT_FOLDER,
-                                                    currencyPair)
-                with open(csv_fn, 'r') as f:
-                    f.seek(0, os.SEEK_END)
-                    if(f.tell() > 2):               # Check file size is not 0
-                        f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
-                        while f.read(1) != b"\n":   # Until EOL is found...
-                            # ...jump back the read byte plus one more.
-                            f.seek(-2, os.SEEK_CUR)
-                        start = pd.to_datetime(f.readline().split(',')[1],
-                                               infer_datetime_format=True)
-
-                if(start is None):
-                    start = time.gmtime()
-                base, market = currencyPair.lower().split('_')
-                symbol = '{market}_{base}'.format(market=market, base=base)
-                symbol_map[currencyPair] = dict(
-                    symbol=symbol,
-                    start_date=start.strftime("%Y-%m-%d")
-                )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2,
-                      separators=(',', ':'))
 
+        return df[datetime.fromtimestamp(start):datetime.fromtimestamp(end-1)]
 
 if __name__ == '__main__':
     pc = PoloniexCurator()
     pc.get_currency_pairs()
-    # pc.generate_symbols_json()
+    pc.append_data()
-
-    for currencyPair in pc.currency_pairs:
-        pc.retrieve_trade_history(currencyPair)
-        log.debug('{} up to date.'.format(currencyPair))
-        pc.write_ohlcv_file(currencyPair)

catalyst/data/_equities.pyx

@@ -215,13 +215,13 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in ['open', 'high', 'low', 'close', 'volume']:
+        if column_name in ['open', 'high', 'low', 'close']:
             where_nan = (outbuf == 0)
-            outbuf_as_float = outbuf.astype(float64) * .000000001
+            outbuf_as_float = outbuf.astype(float64) * .000001
             outbuf_as_float[where_nan] = NAN
             results.append(outbuf_as_float)
-        elif column_name in ['volume']:
+        elif column_name != 'volume':
-            results.append(outbuf.astype(float64) * .000000001)
+            results.append(outbuf.astype(uint32))
         else:
             results.append(outbuf)
     return results

catalyst/data/_minute_bar_internal.pyx

@@ -35,6 +35,17 @@ def minute_value(ndarray[long_t, ndim=1] market_opens,
 
     return market_opens[q] + r
 
+@cython.cdivision(True)
+def five_minute_value(ndarray[long_t, ndim=1] market_opens,
+                      Py_ssize_t pos,
+                      short five_minutes_per_day):
+
+    cdef short q, r
+    q = cython.cdiv(pos, five_minutes_per_day)
+    r = cython.cmod(pos, five_minutes_per_day)
+
+    return market_opens[q] + r
+
 def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
                             ndarray[long_t, ndim=1] market_closes,
                             long_t minute_val,
@@ -88,6 +99,26 @@ def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
 
     return (market_open_loc * minutes_per_day) + delta
 
+def find_position_of_five_minute(ndarray[long_t, ndim=1] market_opens,
+                                 ndarray[long_t, ndim=1] market_closes,
+                                 long_t five_minute_val,
+                                 short five_minutes_per_day,
+                                 bool forward_fill):
+
+    cdef Py_ssize_t market_open_loc, market_open, delta
+
+    market_open_loc = \
+        searchsorted(market_opens, five_minute_val, side='right') - 1
+    market_open = market_opens[market_open_loc]
+    market_close = market_closes[market_open_loc]
+
+    if not forward_fill and ((five_minute_val - market_open) >= five_minutes_per_day):
+        raise ValueError("Given five minutes is not between an open and a close")
+
+    delta = int_min(five_minute_val - market_open, market_close - market_open)
+
+    return (market_open_loc * five_minutes_per_day) + delta
+
 def find_last_traded_position_internal(
         ndarray[long_t, ndim=1] market_opens,
         ndarray[long_t, ndim=1] market_closes,
@@ -158,3 +189,50 @@ def find_last_traded_position_internal(
     # found a trade event
     return -1
 
+def find_last_traded_five_minute_position_internal(
+        ndarray[long_t, ndim=1] market_opens,
+        ndarray[long_t, ndim=1] market_closes,
+        long_t end_five_minute,
+        long_t start_five_minute,
+        volumes,
+        short five_minutes_per_day):
+    cdef Py_ssize_t minute_pos, current_minute, q
+
+    five_minute_pos = int_min(
+        find_position_of_five_minute(
+            market_opens,
+            market_closes,
+            end_five_minute,
+            five_minutes_per_day,
+            True,
+        ),
+        len(volumes) - 1,
+    )
+
+    while five_minute_pos >= 0:
+        current_five_minute = five_minute_value(
+            market_opens, five_minute_pos, five_minutes_per_day
+        )
+
+        q = cython.cdiv(five_minute_pos, five_minutes_per_day)
+        if current_five_minute > market_closes[q]:
+            five_minute_pos = find_position_of_five_minute(
+                market_opens,
+                market_closes,
+                market_closes[q],
+                five_minutes_per_day,
+                False,
+            )
+            continue
+
+        if current_five_minute < start_five_minute:
+            return -1
+
+        if volumes[five_minute_pos] != 0:
+            return five_minute_pos
+
+        five_minute_pos -= 1
+
+    # we've gone to the beginning of this asset's range, and still haven't
+    # found a trade event
+    return -1

catalyst/data/bundles/__init__.py

@@ -1,5 +1,6 @@
 # These imports are necessary to force module-scope register calls to happen.
 from . import quandl  # noqa
+from . import poloniex
 from .core import (
     UnknownBundle,
     bundles,

catalyst/data/bundles/base.py

@@ -13,9 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from itertools import count
 import tarfile
-from time import sleep
+from time import time, sleep
 
 from abc import abstractmethod, abstractproperty
 import logbook
@@ -29,14 +30,11 @@ from catalyst.utils.cli import (
 )
 from catalyst.utils.memoize import lazyval
 
-from catalyst.constants import LOG_LEVEL
-
 logbook.StderrHandler().push_application()
-log = logbook.Logger(__name__, level=LOG_LEVEL)
+log = logbook.Logger(__name__)
 
 DEFAULT_RETRIES = 5
 
-
 class BaseBundle(object):
     def __init__(self, asset_filter=[]):
         self._asset_filter = asset_filter
@@ -62,6 +60,10 @@ class BaseBundle(object):
     def minutes_per_day(self):
         raise NotImplementedError()
 
+    @lazyval
+    def five_minutes_per_day(self):
+        raise NotImplementedError()
+
     @lazyval
     def frequencies(self):
         raise NotImplementedError()
@@ -113,6 +115,7 @@ class BaseBundle(object):
                environ,
                asset_db_writer,
                minute_bar_writer,
+               five_minute_bar_writer,
                daily_bar_writer,
                adjustment_writer,
                calendar,
@@ -128,7 +131,7 @@ class BaseBundle(object):
             retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
 
             if is_compile:
-                # User has instructed local compilation & ingestion of bundle.
+                # User has instructed local compilation and ingestion of bundle.
                 # Fetch raw metadata for all symbols.
                 raw_metadata = self._fetch_metadata_frame(
                     api_key,
@@ -157,9 +160,9 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # Post-process metadata using cached symbol frames, and write
+                # Post-process metadata using cached symbol frames, and write to
-                # to disk.  This metadata must be written before any attempt
+                # disk.  This metadata must be written before any attempt to write
-                # to write minute data.
+                # either minute or 5-minute data.
                 metadata = self._post_process_metadata(
                     raw_metadata,
                     cache,
@@ -167,6 +170,26 @@ class BaseBundle(object):
                 )
                 asset_db_writer.write(metadata)
 
+                # Compile 5-minute symbol data if bundle supports 5-minute mode and
+                # persist the dataset to disk.
+                '''
+                if '5-minute' in self.frequencies:
+                    five_minute_bar_writer.write(
+                        self._fetch_symbol_iter(
+                            api_key,
+                            cache,
+                            symbol_map,
+                            calendar,
+                            start_session,
+                            end_session,
+                            '5-minute',
+                            retries,
+                        ),
+                        length=len(symbol_map),
+                        show_progress=show_progress,
+                    )
+                '''
+
                 # Compile minute symbol data if bundle supports minute mode and
                 # persist the dataset to disk.
                 if 'minute' in self.frequencies:
@@ -184,11 +207,10 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # For legacy purposes, this call is required to ensure the
+                # For legacy purposes, this call is required to ensure the database
-                # database contains an appropriately initialized file
+                # contains an appropriately initialized file structure.  We don't
-                # structure.  We don't forsee a usecase for adjustments at
+                # forsee a usecase for adjustments at this time, but may later
-                # this time, but may later choose to expose this functionality
+                # choose to expose this functionality in the future.
-                # in the future.
                 adjustment_writer.write(
                     splits=(
                         pd.concat(self.splits, ignore_index=True)
@@ -233,11 +255,11 @@ class BaseBundle(object):
             tar.extractall(output_dir)
 
     def _fetch_metadata_frame(self,
-                              api_key,
+                             api_key,
-                              cache,
+                             cache,
-                              retries=DEFAULT_RETRIES,
+                             retries=DEFAULT_RETRIES,
-                              environ=None,
+                             environ=None,
-                              show_progress=False):
+                             show_progress=False):
         
         # Setup raw metadata iterator to fetch pages if necessary.
         raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
@@ -270,10 +292,10 @@ class BaseBundle(object):
                             page_number,
                         )
                         break
-                    except ValueError:
+                    except ValueError as e:
                         raw = pd.DataFrame([])
                         break
-                    except Exception:
+                    except Exception as e:
                         log.exception(
                             'Failed to load metadata from {}. '
                             'Retrying.'.format(self.name)
@@ -284,6 +306,7 @@ class BaseBundle(object):
                         'attempts.'.format(page_number, retries)
                     )
 
+
             if raw.empty:
                 # Empty DataFrame signals completion.
                 break
@@ -318,16 +341,16 @@ class BaseBundle(object):
             show_percent=False,
         ) as symbols_map:
             for asset_id, symbol in symbols_map:
-                # Attempt to load data from disk, the cache should have an
+                # Attempt to load data from disk, the cache should have an entry
-                # entry for each symbol at this point of the execution. If one
+                # for each symbol at this point of the execution. If one does
-                # does not exist, we should fail.
+                # not exist, we should fail.
                 key = '{sym}.daily.frame'.format(sym=symbol)
                 try:
                     raw_data = cache[key]
                 except KeyError:
                     raise ValueError(
-                      'Unable to find cached data for symbol:'
+                      'Unable to find cached data for symbol: {0}'.format(symbol)
-                      ' {0}'.format(symbol))
+                    )
 
                 # Perform and require post-processing of metadata.
                 final_symbol_metadata = self.post_process_symbol_metadata(
@@ -363,8 +386,8 @@ class BaseBundle(object):
             # returns the cached data unaltered.  The `should_sleep` flag
             # indicates that an API call was attempted, and that we should be
             # ensure aren't exceeding our rate limit before proceeding to the
-            # next symbol. If the raw_data is updated, it is cached before
+            # next symbol. If the raw_data is updated, it is cached before being
-            # being returned.
+            # returned.
             raw_data, should_sleep = self._maybe_update_symbol_frame(
                 start_time,
                 api_key,
@@ -468,6 +491,7 @@ class BaseBundle(object):
                     data_frequency,
                 )
                 raw_data.index = pd.to_datetime(raw_data.index, utc=True)
+                raw_data.index = raw_data.index.tz_localize('UTC')
 
                 # Filter incoming data to fit start and end sessions.
                 raw_data = raw_data[
@@ -481,7 +505,7 @@ class BaseBundle(object):
 
                 return raw_data
 
-            except Exception:
+            except Exception as e:
                 log.exception(
                     'Exception raised fetching {name} data. Retrying.'
                     .format(name=self.name)

catalyst/data/bundles/base_pricing.py

@@ -16,7 +16,6 @@
 from catalyst.data.bundles.base import BaseBundle
 from catalyst.utils.memoize import lazyval
 
-
 class BasePricingBundle(BaseBundle):
     @lazyval
     def md_dtypes(self):
@@ -25,7 +24,6 @@ class BasePricingBundle(BaseBundle):
             ('start_date', 'datetime64[ns]'),
             ('end_date', 'datetime64[ns]'),
             ('ac_date', 'datetime64[ns]'),
-            ('min_trade_size', 'float'),
         ]
 
     @lazyval
@@ -39,7 +37,6 @@ class BasePricingBundle(BaseBundle):
             ('volume', 'float64'),
         ]
 
-
 class BaseCryptoPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):
@@ -49,6 +46,10 @@ class BaseCryptoPricingBundle(BasePricingBundle):
     def minutes_per_day(self):
         return 1440
 
+    @lazyval
+    def five_minutes_per_day(self):
+        return 288
+
     @property
     def splits(self):
         return []
@@ -57,7 +58,6 @@ class BaseCryptoPricingBundle(BasePricingBundle):
     def dividends(self):
         return []
 
-
 class BaseEquityPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):
@@ -67,6 +67,10 @@ class BaseEquityPricingBundle(BasePricingBundle):
     def minutes_per_day(self):
         return 390
 
+    @lazyval
+    def five_minutes_per_day(self):
+        return 78
+
     @property
     def splits(self):
         return self._splits

catalyst/data/bundles/core.py

@@ -17,6 +17,10 @@ from ..us_equity_pricing import (
     SQLiteAdjustmentReader,
     SQLiteAdjustmentWriter,
 )
+from ..five_minute_bars import (
+    BcolzFiveMinuteBarReader,
+    BcolzFiveMinuteBarWriter,
+)
 from ..minute_bars import (
     BcolzMinuteBarReader,
     BcolzMinuteBarWriter,
@@ -37,7 +41,6 @@ from catalyst.utils.cli import maybe_show_progress
 
 ONE_MEGABYTE = 1024 * 1024
 
-
 def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
     return pth.data_path(
         asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -51,6 +54,11 @@ def minute_path(bundle_name, timestr, environ=None):
         environ=environ,
     )
 
+def five_minute_path(bundle_name, timestr, environ=None):
+    return pth.data_path(
+        five_minute_relative(bundle_name, timestr, environ),
+        environ=environ,
+    )
 
 def daily_path(bundle_name, timestr, environ=None):
     return pth.data_path(
@@ -84,6 +92,8 @@ def cache_relative(bundle_name, timestr, environ=None):
 def daily_relative(bundle_name, timestr, environ=None):
     return bundle_name, timestr, 'daily_equities.bcolz'
 
+def five_minute_relative(bundle_name, timestr, environ=None):
+    return bundle_name, timestr, 'five_minute.bcolz'
 
 def minute_relative(bundle_name, timestr, environ=None):
     return bundle_name, timestr, 'minute_equities.bcolz'
@@ -136,7 +146,6 @@ def ingestions_for_bundle(bundle, environ=None):
         reverse=True,
     )
 
-
 def download_with_progress(url, chunk_size, **progress_kwargs):
     """
     Download streaming data from a URL, printing progress information to the
@@ -197,13 +206,14 @@ RegisteredBundle = namedtuple(
      'start_session',
      'end_session',
      'minutes_per_day',
+     'five_minutes_per_day',
      'ingest',
      'create_writers']
 )
 
 BundleData = namedtuple(
     'BundleData',
-    'asset_finder minute_bar_reader daily_bar_reader '
+    'asset_finder minute_bar_reader five_minute_bar_reader daily_bar_reader '
     'adjustment_reader',
 )
 
@@ -293,6 +303,7 @@ def _make_bundle_core():
             bundle.ingest,
             calendar_name=bundle.calendar_name,
             minutes_per_day=bundle.minutes_per_day,
+            five_minutes_per_day=bundle.five_minutes_per_day,
             start_session=start_session,
             end_session=end_session,
             create_writers=create_writers,
@@ -305,6 +316,7 @@ def _make_bundle_core():
                  start_session=None,
                  end_session=None,
                  minutes_per_day=1440,
+                 five_minutes_per_day=288,
                  create_writers=True):
         """Register a data bundle ingest function.
 
@@ -385,6 +397,7 @@ def _make_bundle_core():
             start_session=start_session,
             end_session=end_session,
             minutes_per_day=minutes_per_day,
+            five_minutes_per_day=five_minutes_per_day,
             ingest=f,
             create_writers=create_writers,
         )
@@ -483,6 +496,16 @@ def _make_bundle_core():
                 # that it can compute the adjustment ratios for the dividends.
                 daily_bar_writer.write(())
 
+                five_minute_bar_writer = BcolzFiveMinuteBarWriter(
+                    wd.ensure_dir(*five_minute_relative(
+                        name, timestr, environ=environ)
+                    ),
+                    calendar,
+                    start_session,
+                    end_session,
+                    five_minutes_per_day=bundle.five_minutes_per_day,
+                )
+
                 minute_bar_writer = BcolzMinuteBarWriter(
                     wd.ensure_dir(*minute_relative(
                         name, timestr, environ=environ)
@@ -509,6 +532,7 @@ def _make_bundle_core():
                 )
             else:
                 daily_bar_writer = None
+                five_minute_bar_writer = None
                 minute_bar_writer = None
                 asset_db_writer = None
                 adjustment_db_writer = None
@@ -520,6 +544,7 @@ def _make_bundle_core():
                 environ,
                 asset_db_writer,
                 minute_bar_writer,
+                five_minute_bar_writer,
                 daily_bar_writer,
                 adjustment_db_writer,
                 calendar,
@@ -606,6 +631,9 @@ def _make_bundle_core():
             minute_bar_reader=BcolzMinuteBarReader(
                 minute_path(name, timestr, environ=environ),
             ),
+            five_minute_bar_reader=BcolzFiveMinuteBarReader(
+                five_minute_path(name, timestr, environ=environ),
+            ),
             daily_bar_reader=BcolzDailyBarReader(
                 daily_path(name, timestr, environ=environ),
             ),
@@ -707,5 +735,4 @@ def _make_bundle_core():
     )
 
 
-bundles, register_bundle, register, unregister, ingest, load, clean = \
+bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
-    _make_bundle_core()

catalyst/data/bundles/poloniex.py

@@ -13,18 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from datetime import datetime
-from six.moves.urllib.parse import urlencode
 
 import pandas as pd
 
+from six.moves.urllib.parse import urlencode
+
 from catalyst.data.bundles.core import register_bundle
 from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
 from catalyst.utils.memoize import lazyval
 
-from catalyst.curate.poloniex import PoloniexCurator
-
-
 class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def name(self):
@@ -38,14 +36,14 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     def frequencies(self):
         return set((
             'daily',
-            'minute',
+            #'5-minute',
         ))
 
     @lazyval
     def tar_url(self):
         return (
-            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
+            'https://www.dropbox.com/s/9naqffawnq8o4r2/'
-            'poloniex/poloniex-bundle.tar.gz'
+            'poloniex-bundle.tar?dl=1'
         )
 
     @lazyval
@@ -66,25 +64,24 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         raw = raw.sort_index().reset_index()
         raw.rename(
-            columns={'index': 'symbol'},
+            columns={'index':'symbol'},
             inplace=True,
         )
 
         raw = raw[raw['isFrozen'] == 0]
+
         return raw
 
     def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
         start_date = sym_data.index[0]
         end_date = sym_data.index[-1]
         ac_date = end_date + pd.Timedelta(days=1)
-        min_trade_size = 0.00000001
 
         return (
             sym_md.symbol,
             start_date,
             end_date,
             ac_date,
-            min_trade_size,
         )
 
     def fetch_raw_symbol_frame(self,
@@ -94,31 +91,18 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                                start_date,
                                end_date,
                                frequency):
+        raw = pd.read_json(
+            self._format_data_url(
+                api_key,
+                symbol,
+                start_date,
+                end_date,
+                frequency,
+            ),
+            orient='records',
+        )
+        raw.set_index('date', inplace=True)
 
-        # TODO: replace this with direct exchange call
-        # The end date and frequency should be used to
-        # calculate the number of bars
-        if(frequency == 'minute'):
-            pc = PoloniexCurator()
-            raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
-
-        else:
-            raw = pd.read_json(
-                self._format_data_url(
-                    api_key,
-                    symbol,
-                    start_date,
-                    end_date,
-                    frequency,
-                ),
-                orient='records',
-            )
-            raw.set_index('date', inplace=True)
-
-        # BcolzDailyBarReader introduces a 1/1000 factor in the way
-        # pricing is stored on disk, which we compensate here to get
-        # the right pricing amounts
-        # ref: data/us_equity_pricing.py
         scale = 1
         raw.loc[:, 'open'] /= scale
         raw.loc[:, 'high'] /= scale
@@ -139,6 +123,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         return self._format_polo_query(query_params)
 
+
     def _format_data_url(self,
                          api_key,
                          symbol,
@@ -147,6 +132,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                          data_frequency):
         period_map = {
             'daily': 86400,
+#            '5-minute': 300,
         }
 
         try:
@@ -165,12 +151,10 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         return self._format_polo_query(query_params)
     
     def _format_polo_query(self, query_params):
-        # TODO: got against the exchange object
         return 'https://poloniex.com/public?{query}'.format(
             query=urlencode(query_params),
         )
 
-
 ''' 
 As a second parameter, you can pass an array of currency pairs 
 that will be processed as an asset_filter to only process that 
@@ -180,7 +164,4 @@ register_bundle(PoloniexBundle, ['USDT_BTC',])
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-if 'ingest' in sys.argv and '-c' in sys.argv:
+register_bundle(PoloniexBundle)
-    register_bundle(PoloniexBundle)
-else:
-    register_bundle(PoloniexBundle, create_writers=False)

catalyst/data/bundles/quandl.py

@@ -16,6 +16,7 @@
 from datetime import datetime
 
 import pandas as pd
+
 from six.moves.urllib.parse import urlencode
 
 from catalyst.data.bundles.core import register_bundle
@@ -25,16 +26,23 @@ from catalyst.utils.memoize import lazyval
 """
 Module for building a complete daily dataset from Quandl's WIKI dataset.
 """
+from itertools import count
+import tarfile
+from time import time, sleep
+from datetime import datetime
+
 from logbook import Logger
+import pandas as pd
+from six.moves.urllib.parse import urlencode
 
-from catalyst.constants import LOG_LEVEL
 from catalyst.utils.calendars import register_calendar_alias
+from catalyst.utils.cli import maybe_show_progress
 
+from . import core as bundles
 
-log = Logger(__name__, level=LOG_LEVEL)
+log = Logger(__name__)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
-
 class QuandlBundle(BaseEquityPricingBundle):
     @lazyval
     def name(self):
@@ -99,8 +107,8 @@ class QuandlBundle(BaseEquityPricingBundle):
         # Filter out invalid symbols
         raw = raw[~raw.symbol.isin(self._excluded_symbols)]
 
-        # cut out all the other stuff in the name column. We need to
+        # cut out all the other stuff in the name column
-        # escape the paren because it is actually splitting on a regex
+        # we need to escape the paren because it is actually splitting on a regex
         raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
 
         return raw
@@ -165,6 +173,7 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['sid'] = asset_id
         self.splits.append(df)
 
+
     def _update_dividends(self, asset_id, raw_data):
         divs = raw_data.ex_dividend
         df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -175,6 +184,7 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
         self.dividends.append(df)
 
+
     def _format_metadata_url(self, api_key, page_number):
         """Build the query RL for the quandl WIKI metadata.
         """
@@ -188,10 +198,10 @@ class QuandlBundle(BaseEquityPricingBundle):
             query_params = [('api_key', api_key)] + query_params
 
         return (
-            'https://www.quandl.com/api/v3/datasets.csv?'
+            'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
-            + urlencode(query_params)
         )
 
+
     def _format_wiki_url(self,
                          api_key,
                          symbol,
@@ -217,6 +227,5 @@ class QuandlBundle(BaseEquityPricingBundle):
             )
         )
 
-
 register_calendar_alias('QUANDL', 'NYSE')
 register_bundle(QuandlBundle)

catalyst/data/data_portal.py

@@ -42,6 +42,7 @@ from catalyst.assets.roll_finder import (
 )
 from catalyst.data.dispatch_bar_reader import (
     AssetDispatchMinuteBarReader,
+    AssetDispatchFiveMinuteBarReader,
     AssetDispatchSessionBarReader
 )
 from catalyst.data.resample import (
@@ -68,9 +69,7 @@ from catalyst.errors import (
     HistoryWindowStartsBeforeData,
 )
 
-from catalyst.constants import LOG_LEVEL
+log = Logger('DataPortal')
-
-log = Logger('DataPortal', level=LOG_LEVEL)
 
 BASE_FIELDS = frozenset([
     "open",
@@ -121,6 +120,10 @@ class DataPortal(object):
         daily data backtests or daily history calls in a minute backetest.
         If a daily bar reader is not provided but a minute bar reader is,
         the minutes will be rolled up to serve the daily requests.
+    five_minute_reader : BcolzFiveMinuteBarReader, optional
+        The five minute bar reader for equities. This will be used to service
+        5-minute data backtests or five-minute history calls.  This can be used
+        to serve daily calls if no daily bar reader is provided.
     minute_reader : BcolzMinuteBarReader, optional
         The minute bar reader for equities. This will be used to service
         minute data backtests or minute history calls. This can be used
@@ -147,6 +150,7 @@ class DataPortal(object):
                  trading_calendar,
                  first_trading_day,
                  daily_reader=None,
+                 five_minute_reader=None,
                  minute_reader=None,
                  future_daily_reader=None,
                  future_minute_reader=None,
@@ -198,6 +202,7 @@ class DataPortal(object):
                 reader.last_available_dt
                 for reader in [
                     minute_reader,
+                    five_minute_reader,
                     future_minute_reader,
                 ]
                 if reader is not None
@@ -209,6 +214,8 @@ class DataPortal(object):
 
         aligned_minute_reader = self._ensure_reader_aligned(
             minute_reader)
+        aligned_five_minute_reader = self._ensure_reader_aligned(
+            five_minute_reader)
         aligned_session_reader = self._ensure_reader_aligned(
             daily_reader)
         aligned_future_minute_reader = self._ensure_reader_aligned(
@@ -222,10 +229,13 @@ class DataPortal(object):
         }
 
         aligned_minute_readers = {}
+        aligned_five_minute_readers = {}
         aligned_session_readers = {}
 
         if aligned_minute_reader is not None:
             aligned_minute_readers[Equity] = aligned_minute_reader
+        if aligned_five_minute_reader is not None:
+            aligned_five_minute_readers[Equity] = aligned_five_minute_reader
         if aligned_session_reader is not None:
             aligned_session_readers[Equity] = aligned_session_reader
 
@@ -257,6 +267,13 @@ class DataPortal(object):
             self._last_available_minute,
         )
 
+        _dispatch_five_minute_reader = AssetDispatchFiveMinuteBarReader(
+            self.trading_calendar,
+            self.asset_finder,
+            aligned_five_minute_readers,
+            self._last_available_minute,
+        )
+
         _dispatch_session_reader = AssetDispatchSessionBarReader(
             self.trading_calendar,
             self.asset_finder,
@@ -266,6 +283,7 @@ class DataPortal(object):
 
         self._pricing_readers = {
             'minute': _dispatch_minute_reader,
+            '5-minute': _dispatch_five_minute_reader,
             'daily': _dispatch_session_reader,
         }
 
@@ -656,11 +674,11 @@ class DataPortal(object):
         return spot_value
 
     def _get_minutely_spot_value(self,
-                                 asset,
+                                asset,
-                                 column,
+                                column,
-                                 dt,
+                                dt,
-                                 data_frequency,
+                                data_frequency,
-                                 ffill=False):
+                                ffill=False):
 
         reader = self._get_pricing_reader(data_frequency)
 
@@ -701,6 +719,17 @@ class DataPortal(object):
             spot_value=result
         )
 
+
+    def _get_five_minute_spot_value(self, asset, column, dt, ffill=False):
+        return self._get_minutely_spot_value(
+            asset,
+            column,
+            dt,
+            ffill, 
+            '5-minute',
+        )
+        
+
     def _get_minute_spot_value(self, asset, column, dt, ffill=False):
         return self._get_minutely_spot_value(
             asset,

catalyst/data/dispatch_bar_reader.py

@@ -18,7 +18,6 @@ from numpy import (
     full,
     nan,
     int64,
-    float64,
     zeros
 )
 from six import iteritems, with_metaclass
@@ -71,9 +70,7 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         return self._dt_window_size(start_dt, end_dt), num_sids
 
     def _make_raw_array_out(self, field, shape):
-        if field == 'volume':
+        if field != 'volume' and field != 'sid':
-            out = zeros(shape, dtype=float64)
-        elif field != 'sid':
             out = full(shape, nan)
         else:
             out = zeros(shape, dtype=int64)
@@ -88,11 +85,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         if self._last_available_dt is not None:
             return self._last_available_dt
         else:
-            return min(r.last_available_dt for r in list(self._readers.values()))
+            return min(r.last_available_dt for r in self._readers.values())
 
     @lazyval
     def first_trading_day(self):
-        return max(r.first_trading_day for r in list(self._readers.values()))
+        return max(r.first_trading_day for r in self._readers.values())
 
     def get_value(self, sid, dt, field):
         asset = self._asset_finder.retrieve_asset(sid)
@@ -133,13 +130,17 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
 
         return results
 
-
 class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):
         return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
 
 
+class AssetDispatchFiveMinuteBarReader(AssetDispatchBarReader):
+
+    def _dt_window_size(self, start_dt, end_dt):
+        return len(self.trading_calendar.five_minutes_in_range(start_dt, end_dt))
+
 class AssetDispatchSessionBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):

catalyst/data/history_loader.py

@@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype
 from catalyst.utils.pandas_utils import find_in_sorted_index
 
 # Default number of decimal places used for rounding asset prices.
-DEFAULT_ASSET_PRICE_DECIMALS = 9
+DEFAULT_ASSET_PRICE_DECIMALS = 3
 
 
 class HistoryCompatibleUSEquityAdjustmentReader(object):

catalyst/data/loader.py

@@ -17,31 +17,36 @@ from collections import OrderedDict
 
 import logbook
 import pandas as pd
-import pytz
+import numpy as np
 from pandas_datareader.data import DataReader
+import datetime
+import time
+import pytz
 from six import iteritems
 from six.moves.urllib_error import HTTPError
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.utils.calendars import get_calendar
-from . import treasuries, treasuries_can
 from .benchmarks import get_benchmark_returns
-from ..utils.deprecate import deprecated
+from . import treasuries, treasuries_can
 from ..utils.paths import (
     cache_root,
     data_root,
 )
+from ..utils.deprecate import deprecated
 
-logger = logbook.Logger('Loader', level=LOG_LEVEL)
+from catalyst.data.bundles.poloniex import PoloniexBundle
+from catalyst.utils.calendars import get_calendar
+
+
+logger = logbook.Logger('Loader')
 
 # Mapping from index symbol to appropriate bond data
 INDEX_MAPPING = {
     'SPY':
-        (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
+    (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
     '^GSPTSE':
-        (treasuries_can, 'treasury_curves_can.csv', 'bankofcanada.ca'),
+    (treasuries_can, 'treasury_curves_can.csv', 'bankofcanada.ca'),
     '^FTSE':  # use US treasuries until UK bonds implemented
-        (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
+    (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
 }
 
 ONE_HOUR = pd.Timedelta(hours=1)
@@ -89,27 +94,19 @@ def has_data_for_dates(series_or_df, first_date, last_date):
     if not isinstance(dts, pd.DatetimeIndex):
         raise TypeError("Expected a DatetimeIndex, but got %s." % type(dts))
     first, last = dts[[0, -1]].tz_localize(None)
-    return (first <= first_date.tz_localize(None)) and (
+    return (first <= first_date.tz_localize(None)) and (last >= last_date.tz_localize(None))
-        last >= last_date.tz_localize(None))
 
-
+def load_crypto_market_data(trading_day=None,
-def load_crypto_market_data(trading_day=None, trading_days=None,
+                            trading_days=None,
-                            bm_symbol=None, bundle=None, bundle_data=None,
+                            bm_symbol='USDT_BTC',
-                            environ=None, exchange=None, start_dt=None,
+                            environ=None):
-                            end_dt=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
+    if trading_days is None:
+        trading_days = get_calendar('OPEN').all_sessions
 
-    # TODO: consider making configurable
+    first_date = trading_days[0]
-    bm_symbol = 'btc_usd'
+    now = pd.Timestamp.utcnow()
-    # if trading_days is None:
-    #    trading_days = get_calendar('OPEN').schedule
-
-    # if start_dt is None:
-    start_dt = get_calendar('OPEN').first_trading_session
-
-    if end_dt is None:
-        end_dt = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
     # **two** full trading days prior to the most recently completed trading
@@ -125,59 +122,30 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
 
     # We'll attempt to download new data if the latest entry in our cache is
     # before this date.
-    '''
+    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
-    if(bundle_data):
-        # If we are using the bundle to retrieve the cryptobenchmark, find
-        # the last date for which there is trading data in the bundle
-        asset = bundle_data.asset_finder.lookup_symbol(
-                    symbol=bm_symbol,as_of_date=None)
-        ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
-        last_date = pd.to_datetime(
-                    bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
-    else:
-        last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
-    '''
-    last_date = trading_days[trading_days.get_loc(end_dt, method='ffill') - 1]
 
-    if exchange is None:
+    br = ensure_crypto_benchmark_data(
-        # This is exceptional, since placing the import at the module scope
+        bm_symbol,
-        #  breaks things and it's only needed here
+        first_date,
-        from catalyst.exchange.utils.factory import get_exchange
+        last_date,
-        exchange = get_exchange(
+        now,
-            exchange_name='bitfinex', base_currency='usd'
+        # We need the trading_day to figure out the close prior to the first
-        )
+        # date so that we can compute returns for the first date.
-        exchange.init()
+        trading_day,
-
+        environ,
-    benchmark_asset = exchange.get_asset(bm_symbol)
+    )
-
+    # Override first_date for treasury data since we have it for many more years
-    # exchange.get_history_window() already ensures that we have the right data
+    # and is independent of crypto data
-    # for the right dates
+    first_date_treasury = pd.Timestamp('1990-01-01', tz='UTC') 
-    br = exchange.get_history_window_with_bundle(
-        assets=[benchmark_asset],
-        end_dt=last_date,
-        bar_count=pd.Timedelta(last_date - start_dt).days,
-        frequency='1d',
-        field='close',
-        data_frequency='daily',
-        force_auto_ingest=True)
-    br.columns = ['close']
-    br = br.pct_change(1).iloc[1:]
-    br.loc[start_dt] = 0
-    br = br.sort_index()
-
-    # Override first_date for treasury data since we have it for many more
-    # years and is independent of crypto data
-    first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
     tc = ensure_treasury_data(
         bm_symbol,
         first_date_treasury,
         last_date,
-        end_dt,
+        now,
         environ,
     )
-    benchmark_returns = br[br.index.slice_indexer(start_dt, last_date)]
+    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
-    treasury_curves = tc[
+    treasury_curves = tc[tc.index.slice_indexer(first_date_treasury, last_date)]
-        tc.index.slice_indexer(first_date_treasury, last_date)]
     return benchmark_returns, treasury_curves
 
 
@@ -272,16 +240,15 @@ def ensure_crypto_benchmark_data(symbol,
                                  last_date,
                                  now,
                                  trading_day,
-                                 bundle,
-                                 bundle_data,
                                  environ=None):
+
     filename = get_benchmark_filename(symbol)
 
     logger.info(
         ('Loading benchmark data for {symbol!r} '
-         'from {first_date} to {last_date}'),
+            'from {first_date} to {last_date}'),
         symbol=symbol,
-        first_date=first_date,
+        first_date=first_date - trading_day,
         last_date=last_date
     )
 
@@ -294,66 +261,34 @@ def ensure_crypto_benchmark_data(symbol,
         environ,
     )
 
+
     if data is not None:
         return data
 
     # If no cached data was found or it was missing any dates then download the
     # necessary data.
+    logger.info(
+        ('Downloading benchmark data for {symbol!r} '
+            'from {first_date} to {last_date}'),
+        symbol=symbol,
+        first_date=first_date - trading_day,
+        last_date=last_date
+    )
 
-    if (bundle == 'poloniex'):
+    # Load benchmark symbol from Poloniex API
-        '''
+    try:
-        If we're using the Poloniex bundle, we'll get the benchmark from the
+        bundle = PoloniexBundle()
-        bundle instead of downloading it from Poloniex every time we need it.
+        bench_raw = bundle._fetch_symbol_frame(
-        Poloniex has a captcha for API queries originating from outside the US
+            None,
-        that prevents users abroad from getting Catalyst to work
+            symbol,
-        '''
+            get_calendar(bundle.calendar_name),
-        logger.info(
+            first_date - trading_day,
-            ('Retrieving benchmark data from bundle for {symbol!r}'
+            last_date,
-             ' from {first_date} to {last_date}'),
+            'daily',
-            symbol=symbol, first_date=first_date, last_date=last_date)
+        )
-
+    except (OSError, IOError, HTTPError):
-        asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
+        logger.exception('Failed to fetch new crypto benchmark returns')
-                                                       as_of_date=None)
+        raise
-        fields = ['day', 'close']
-        raw = bundle_data.daily_bar_reader.load_raw_arrays(
-            columns=fields,
-            start_date=first_date - trading_day,
-            end_date=last_date,
-            assets=[asset, ])
-        bench_raw = pd.concat([pd.DataFrame(raw[0], columns=['date']),
-                               pd.DataFrame(raw[1], columns=['close'])],
-                              axis=1)
-        bench_raw['date'] = pd.to_datetime(bench_raw['date'], unit='s')
-        bench_raw.set_index('date', inplace=True)
-        bench_raw.sort_index(inplace=True)
-        bench_raw = bench_raw[
-                    pd.to_datetime(first_date - trading_day):pd.to_datetime(
-                        last_date)]
-
-    else:
-        # This is how it used to be: downloading the benchmark everytime.
-        # Leaving this code here to be repurposed in the future for
-        # other bundles.
-        logger.info(
-            ('Downloading benchmark data for {symbol!r}'
-             ' from {first_date} to {last_date}'),
-            symbol=symbol, first_date=first_date, last_date=last_date)
-
-        raise DeprecationWarning('poloniex bundle deprecated')
-        # Load benchmark symbol from Poloniex API
-        # try:
-        #     bundle = PoloniexBundle()
-        #     bench_raw = bundle._fetch_symbol_frame(
-        #         None,
-        #         symbol,
-        #         get_calendar(bundle.calendar_name),
-        #         first_date - trading_day,
-        #         last_date,
-        #         'daily',
-        #     )
-        # except (OSError, IOError, HTTPError):
-        #     logger.exception('Failed to fetch new crypto benchmark returns')
-        #     raise
 
     # select close column and compute percent change between days
     daily_close = bench_raw[['close']]
@@ -412,7 +347,68 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
     # necessary data.
     logger.info(
         ('Downloading benchmark data for {symbol!r} '
-         'from {first_date} to {last_date}'),
+            'from {first_date} to {last_date}'),
+        symbol=symbol,
+        first_date=first_date - trading_day,
+        last_date=last_date
+    )
+
+    try:
+        data = get_benchmark_returns(
+            symbol,
+            first_date - trading_day,
+            last_date,
+        )
+        data.to_csv(get_data_filepath(filename, environ))
+    except (OSError, IOError, HTTPError):
+        logger.exception('Failed to cache the new benchmark returns')
+        raise
+    if not has_data_for_dates(data, first_date, last_date):
+        logger.warn("Still don't have expected data after redownload!")
+    return data
+
+
+def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
+                          environ=None):
+    """
+    Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
+
+    Parameters
+    ----------
+    symbol : str
+        The symbol for the benchmark to load.
+    first_date : pd.Timestamp
+        First required date for the cache.
+    last_date : pd.Timestamp
+        Last required date for the cache.
+    now : pd.Timestamp
+        The current time.  This is used to prevent repeated attempts to
+        re-download data that isn't available due to scheduling quirks or other
+        failures.
+    trading_day : pd.CustomBusinessDay
+        A trading day delta.  Used to find the day before first_date so we can
+        get the close of the day prior to first_date.
+
+    We attempt to download data unless we already have data stored at the data
+    cache for `symbol` whose first entry is before or on `first_date` and whose
+    last entry is on or after `last_date`.
+
+    If we perform a download and the cache criteria are not satisfied, we wait
+    at least one hour before attempting a redownload.  This is determined by
+    comparing the current time to the result of os.path.getmtime on the cache
+    path.
+    """
+    filename = get_benchmark_filename(symbol)
+    data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
+                             environ)
+    if data is not None:
+        return data
+
+    # If no cached data was found or it was missing any dates then download the
+    # necessary data.
+    logger.info(
+        ('Downloading benchmark data for {symbol!r} '
+            'from {first_date} to {last_date}'),
         symbol=symbol,
         first_date=first_date - trading_day,
         last_date=last_date
@@ -496,8 +492,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name,
             data = pd.DataFrame.from_csv(path)
             if data.empty:
                 raise ValueError("File is empty.")
-            data.index = pd.to_datetime(data.index, infer_datetime_format=True,
+            data.index = pd.to_datetime(data.index, infer_datetime_format=True, errors='coerce' ).tz_localize('UTC')
-                                        errors='coerce').tz_localize('UTC')
             if has_data_for_dates(data, first_date, last_date):
                 return data
 
@@ -523,7 +518,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name,
             )
 
     logger.info(
-        "Cache at {path} does not have data from {start} to {end}.",
+        "Cache at {path} does not have data from {start} to {end}.\n",
         start=first_date,
         end=last_date,
         path=path,

catalyst/data/minute_bars.py

@@ -39,21 +39,20 @@ from catalyst.data._minute_bar_internal import (
 from catalyst.gens.sim_engine import NANOS_IN_MINUTE
 
 from catalyst.data.bar_reader import BarReader, NoDataOnDate
-from catalyst.data.us_equity_pricing import check_uint64_safe
+from catalyst.data.us_equity_pricing import check_uint32_safe
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
-from catalyst.constants import LOG_LEVEL
 
-logger = logbook.Logger('MinuteBars', level=LOG_LEVEL)
+logger = logbook.Logger('MinuteBars')
 
 US_EQUITIES_MINUTES_PER_DAY = 390
 FUTURES_MINUTES_PER_DAY = 1440
 
 DEFAULT_EXPECTEDLEN = US_EQUITIES_MINUTES_PER_DAY * 252 * 15
 
-OHLC_RATIO = 100000000
+OHLC_RATIO = 1000
 
 
 class BcolzMinuteOverlappingData(Exception):
@@ -115,15 +114,15 @@ def _sid_subdir_path(sid):
 
 
 def convert_cols(cols, scale_factor, sid, invalid_data_behavior):
-    """Adapt OHLCV columns into uint64 columns.
+    """Adapt OHLCV columns into uint32 columns.
 
     Parameters
     ----------
     cols : dict
         A dict mapping each column name (open, high, low, close, volume)
-        to a float column to convert to uint64.
+        to a float column to convert to uint32.
     scale_factor : int
-        Factor to use to scale float values before converting to uint64.
+        Factor to use to scale float values before converting to uint32.
     sid : int
         Sid of the relevant asset, for logging.
     invalid_data_behavior : str
@@ -136,7 +135,6 @@ def convert_cols(cols, scale_factor, sid, invalid_data_behavior):
     scaled_highs = np.nan_to_num(cols['high']) * scale_factor
     scaled_lows = np.nan_to_num(cols['low']) * scale_factor
     scaled_closes = np.nan_to_num(cols['close']) * scale_factor
-    scaled_volumes = np.nan_to_num(cols['volume']) * scale_factor
 
     exclude_mask = np.zeros_like(scaled_opens, dtype=bool)
 
@@ -145,12 +143,11 @@ def convert_cols(cols, scale_factor, sid, invalid_data_behavior):
         ('high', scaled_highs),
         ('low', scaled_lows),
         ('close', scaled_closes),
-        ('volume', scaled_volumes),
     ]:
         max_val = scaled_col.max()
 
         try:
-            check_uint64_safe(max_val, col_name)
+            check_uint32_safe(max_val, col_name)
         except ValueError:
             if invalid_data_behavior == 'raise':
                 raise
@@ -158,20 +155,20 @@ def convert_cols(cols, scale_factor, sid, invalid_data_behavior):
             if invalid_data_behavior == 'warn':
                 logger.warn(
                     'Values for sid={}, col={} contain some too large for '
-                    'uint64 (max={}), filtering them out',
+                    'uint32 (max={}), filtering them out',
                     sid, col_name, max_val,
                 )
 
             # We want to exclude all rows that have an unsafe value in
             # this column.
-            exclude_mask &= (scaled_col >= np.iinfo(np.uint64).max)
+            exclude_mask &= (scaled_col >= np.iinfo(np.uint32).max)
 
     # Convert all cols to uint32.
-    opens = scaled_opens.astype(np.uint64)
+    opens = scaled_opens.astype(np.uint32)
-    highs = scaled_highs.astype(np.uint64)
+    highs = scaled_highs.astype(np.uint32)
-    lows = scaled_lows.astype(np.uint64)
+    lows = scaled_lows.astype(np.uint32)
-    closes = scaled_closes.astype(np.uint64)
+    closes = scaled_closes.astype(np.uint32)
-    volumes = scaled_volumes.astype(np.uint64)
+    volumes = cols['volume'].astype(np.uint32)
 
     # Exclude rows with unsafe values by setting to zero.
     opens[exclude_mask] = 0
@@ -263,14 +260,14 @@ class BcolzMinuteBarMetadata(object):
             )
 
     def __init__(
-            self,
+        self,
-            default_ohlc_ratio,
+        default_ohlc_ratio,
-            ohlc_ratios_per_sid,
+        ohlc_ratios_per_sid,
-            calendar,
+        calendar,
-            start_session,
+        start_session,
-            end_session,
+        end_session,
-            minutes_per_day,
+        minutes_per_day,
-            version=FORMAT_VERSION,
+        version=FORMAT_VERSION,
     ):
         self.calendar = calendar
         self.start_session = start_session
@@ -291,7 +288,7 @@ class BcolzMinuteBarMetadata(object):
         ohlc_ratio : int
             The default ratio by which to multiply the pricing data to
             convert the floats from floats to an integer to fit within
-            the np.uint64. If ohlc_ratios_per_sid is None or does not
+            the np.uint32. If ohlc_ratios_per_sid is None or does not
             contain a mapping for a given sid, this ratio is used.
         ohlc_ratios_per_sid : dict
              A dict mapping each sid in the output to the factor by
@@ -341,10 +338,12 @@ class BcolzMinuteBarMetadata(object):
             'end_session': str(self.end_session.date()),
             # Write these values for backwards compatibility
             'first_trading_day': str(self.start_session.date()),
-            'market_opens': (market_opens.values.astype('datetime64[m]').
+            'market_opens': (
-                             astype(np.int64).tolist()),
+                market_opens.values.astype('datetime64[m]').
-            'market_closes': (market_closes.values.astype('datetime64[m]').
+                astype(np.int64).tolist()),
-                              astype(np.int64).tolist()),
+            'market_closes': (
+                market_closes.values.astype('datetime64[m]').
+                astype(np.int64).tolist()),
         }
         with open(self.metadata_path(rootdir), 'w+') as fp:
             json.dump(metadata, fp)
@@ -373,13 +372,13 @@ class BcolzMinuteBarWriter(object):
         The last trading session in the data set.
     default_ohlc_ratio : int, optional
         The default ratio by which to multiply the pricing data to
-        convert from floats to integers that fit within np.uint64. If
+        convert from floats to integers that fit within np.uint32. If
         ohlc_ratios_per_sid is None or does not contain a mapping for a
-        given sid, this ratio is used. Default is OHLC_RATIO (10^8).
+        given sid, this ratio is used. Default is OHLC_RATIO (1000).
     ohlc_ratios_per_sid : dict, optional
         A dict mapping each sid in the output to the ratio by which to
         multiply the pricing data to convert the floats from floats to
-        an integer to fit within the np.uint64.
+        an integer to fit within the np.uint32.
     expectedlen : int, optional
         The expected length of the dataset, used when creating the initial
         bcolz ctable.
@@ -402,9 +401,11 @@ class BcolzMinuteBarWriter(object):
     Each individual asset's data is stored as a bcolz table with a column for
     each pricing field: (open, high, low, close, volume)
 
-    The open, high, low, close and volume columns are integers which are 10^8 times
+    The open, high, low, and close columns are integers which are 1000 times
     the quoted price, so that the data can represented and stored as an
-    np.uint64, supporting market prices quoted up to the 1/10^8-th place.
+    np.uint32, supporting market prices quoted up to the thousands place.
+
+    volume is a np.uint32 with no mutation of the tens place.
 
     The 'index' for each individual asset are a repeating period of minutes of
     length `minutes_per_day` starting from each market open.
@@ -572,7 +573,7 @@ class BcolzMinuteBarWriter(object):
         if not os.path.exists(sid_containing_dirname):
             # Other sids may have already created the containing directory.
             os.makedirs(sid_containing_dirname)
-        initial_array = np.empty(0, np.uint64)
+        initial_array = np.empty(0, np.uint32)
         table = ctable(
             rootdir=path,
             columns=[
@@ -609,7 +610,7 @@ class BcolzMinuteBarWriter(object):
         minute_offset = len(table) % self._minutes_per_day
         num_to_prepend = numdays * self._minutes_per_day - minute_offset
 
-        prepend_array = np.zeros(num_to_prepend, np.uint64)
+        prepend_array = np.zeros(num_to_prepend, np.uint32)
         # Fill all OHLCV with zeros.
         table.append([prepend_array] * 5)
         table.flush()
@@ -814,11 +815,11 @@ class BcolzMinuteBarWriter(object):
 
         minutes_count = all_minutes_in_window.size
 
-        open_col = np.zeros(minutes_count, dtype=np.uint64)
+        open_col = np.zeros(minutes_count, dtype=np.uint32)
-        high_col = np.zeros(minutes_count, dtype=np.uint64)
+        high_col = np.zeros(minutes_count, dtype=np.uint32)
-        low_col = np.zeros(minutes_count, dtype=np.uint64)
+        low_col = np.zeros(minutes_count, dtype=np.uint32)
-        close_col = np.zeros(minutes_count, dtype=np.uint64)
+        close_col = np.zeros(minutes_count, dtype=np.uint32)
-        vol_col = np.zeros(minutes_count, dtype=np.uint64)
+        vol_col = np.zeros(minutes_count, dtype=np.uint32)
 
         dt_ixs = np.searchsorted(all_minutes_in_window.values,
                                  dts.astype('datetime64[ns]'))
@@ -913,10 +914,10 @@ class BcolzMinuteBarReader(MinuteBarReader):
         )
         self._schedule = self.calendar.schedule[slicer]
         self._market_opens = self._schedule.market_open
-        self._market_open_values = self._market_opens.values. \
+        self._market_open_values = self._market_opens.values.\
             astype('datetime64[m]').astype(np.int64)
         self._market_closes = self._schedule.market_close
-        self._market_close_values = self._market_closes.values. \
+        self._market_close_values = self._market_closes.values.\
             astype('datetime64[m]').astype(np.int64)
 
         self._default_ohlc_inverse = 1.0 / metadata.default_ohlc_ratio
@@ -1124,8 +1125,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
             else:
                 return np.nan
 
-        # if field != 'volume':
+        if field != 'volume':
-        value *= self._ohlc_ratio_inverse_for_sid(sid)
+            value *= self._ohlc_ratio_inverse_for_sid(sid)
         return value
 
     def get_last_traded_dt(self, asset, dt):
@@ -1247,25 +1248,25 @@ class BcolzMinuteBarReader(MinuteBarReader):
             if field != 'volume':
                 out = np.full(shape, np.nan)
             else:
-                out = np.zeros(shape, dtype=np.float64)
+                out = np.zeros(shape, dtype=np.uint32)
 
             for i, sid in enumerate(sids):
                 carray = self._open_minute_file(field, sid)
                 values = carray[start_idx:end_idx + 1]
                 if indices_to_exclude is not None:
                     for excl_start, excl_stop in indices_to_exclude[::-1]:
-                        excl_slice = np.s_[excl_start - start_idx:excl_stop
+                        excl_slice = np.s_[
-                                           - start_idx + 1]
+                            excl_start - start_idx:excl_stop - start_idx + 1]
                         values = np.delete(values, excl_slice)
 
                 where = values != 0
                 # first slice down to len(where) because we might not have
                 # written data for all the minutes requested
-                # if field != 'volume':
+                if field != 'volume':
-                out[:len(where), i][where] = (
+                    out[:len(where), i][where] = (
-                    values[where] * self._ohlc_ratio_inverse_for_sid(sid))
+                        values[where] * self._ohlc_ratio_inverse_for_sid(sid))
-                # else:
+                else:
-                #    out[:len(where), i][where] = values[where]
+                    out[:len(where), i][where] = values[where]
 
             results.append(out)
         return results
@@ -1318,8 +1319,9 @@ class H5MinuteBarUpdateWriter(object):
 
     def __init__(self, path, complevel=None, complib=None):
         self._complevel = complevel if complevel \
-                            is not None else self._COMPLEVEL
+            is not None else self._COMPLEVEL
-        self._complib = complib if complib is not None else self._COMPLIB
+        self._complib = complib if complib \
+            is not None else self._COMPLIB
         self._path = path
 
     def write(self, frames):
@@ -1351,7 +1353,6 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
     path : str
         The path of the HDF5 file from which to source data.
     """
-
     def __init__(self, path):
         self._panel = pd.read_hdf(path)
 

catalyst/data/resample.py

@@ -156,10 +156,7 @@ class DailyHistoryAggregator(object):
             cache = self._caches[field] = (session, market_open, {})
 
         _, market_open, entries = cache
-        try:
+        market_open = market_open.tz_localize('UTC')
-            market_open = market_open.tz_localize('UTC')
-        except TypeError:
-            market_open = market_open.tz_convert('UTC')
         if dt != market_open:
             prev_dt = dt_value - self._one_min
         else:

catalyst/data/us_equity_pricing.py

@@ -11,9 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from __future__ import division  # Python2 req for division of ints yield float
-
 from errno import ENOENT
 from functools import partial
 from os import remove
@@ -83,9 +80,8 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-from catalyst.constants import LOG_LEVEL
 
-logger = logbook.Logger('UsEquityPricing', level=LOG_LEVEL)
+logger = logbook.Logger('UsEquityPricing')
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
 OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])
@@ -120,9 +116,6 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
-# Provides 9 decimals resolution. Also affects _equities.pyx L220
-PRICE_ADJUSTMENT_FACTOR = 1000000000
-
 
 def check_uint32_safe(value, colname):
     if value >= UINT32_MAX:
@@ -131,7 +124,6 @@ def check_uint32_safe(value, colname):
             "for uint32" % (value, colname)
         )
 
-
 def check_uint64_safe(value, colname):
     if value >= UINT64_MAX:
         raise ValueError(
@@ -324,8 +316,8 @@ class BcolzDailyBarWriter(object):
         # Maps column name -> output carray.
         columns = {
             k: carray(array([], dtype=uint64))
-            if k in OHLCV
+               if k in OHLCV
-            else carray(array([], dtype=uint32))
+               else carray(array([], dtype=uint32))
             for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
         }
 
@@ -441,13 +433,11 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)]
+        processed = (raw_data[list(OHLC)] * 1000000).astype('uint64')
-                     * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
-        processed['volume'] = (raw_data.volume
+        processed['volume'] = raw_data.volume.astype('uint64')
-                               * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         return ctable.fromdataframe(processed)
 
 
@@ -500,8 +490,9 @@ class BcolzDailyBarReader(SessionBarReader):
 
     The data in these columns is interpreted as follows:
 
-    - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
+    - Price columns ('open', 'high', 'low', 'close') are interpreted as 1000 *
-      as 10^9 * as-traded dollar value.
+      as-traded dollar value.
+    - Volume is interpreted as as-traded volume.
     - Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
     - Id is the asset id of the row.
 
@@ -528,6 +519,7 @@ class BcolzDailyBarReader(SessionBarReader):
         # Need to test keeping the entire array in memory for the course of a
         # process first.
         self._spot_cols = {}
+        self.PRICE_ADJUSTMENT_FACTOR = 0.001
         self._read_all_threshold = read_all_threshold
 
     @lazyval
@@ -767,10 +759,13 @@ class BcolzDailyBarReader(SessionBarReader):
         """
         ix = self.sid_day_index(sid, dt)
         price = self._spot_col(field)[ix]
-        if field != 'volume' and price == 0:
+        if field != 'volume':
-            return nan
+            if price == 0:
+                return nan
+            else:
+                return price * 0.001
         else:
-            return price / PRICE_ADJUSTMENT_FACTOR
+            return price
 
 
 class PanelBarReader(SessionBarReader):

catalyst/examples/README.rst

@@ -1,3 +0,0 @@
-An overview of most of the trading strategies in this folder can be found in the 
-`Examples Algorithms <https://enigmampc.github.io/catalyst/example-algos.html>`_
-section of our documentation website.

catalyst/examples/arbitrage_with_interface.py

@@ -1,282 +0,0 @@
-from logbook import Logger
-
-from catalyst.api import (
-    record,
-    order,
-    symbol,
-    get_open_orders
-)
-from catalyst.exchange.utils.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'arbitrage_eth_btc'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing arbitrage algorithm')
-
-    # The context contains a new "exchanges" attribute which is a dictionary
-    # of exchange objects by exchange name. This allow easy access to the
-    # exchanges.
-    context.buying_exchange = context.exchanges['poloniex']
-    context.selling_exchange = context.exchanges['bitfinex']
-
-    context.trading_pair_symbol = 'eth_btc'
-    context.trading_pairs = dict()
-
-    # Note the second parameter of the symbol() method
-    # Passing the exchange name here returns a TradingPair object including
-    # the exchange information. This allow all other operations using
-    # the TradingPair to target the correct exchange.
-    context.trading_pairs[context.buying_exchange] = \
-        symbol('eth_btc', context.buying_exchange.name)
-
-    context.trading_pairs[context.selling_exchange] = \
-        symbol(context.trading_pair_symbol, context.selling_exchange.name)
-
-    context.entry_points = [
-        dict(gap=0.03, amount=0.05),
-        dict(gap=0.04, amount=0.1),
-        dict(gap=0.05, amount=0.5),
-    ]
-    context.exit_points = [
-        dict(gap=-0.02, amount=0.5),
-    ]
-
-    context.SLIPPAGE_ALLOWED = 0.02
-    pass
-
-
-def place_orders(context, amount, buying_price, selling_price, action):
-    """
-    This method will always place two orders of the same amount to keep
-    the currency position the same as it moves between the two exchanges.
-
-    :param context: TradingAlgorithm
-    :param amount: float
-        The trading pair amount to trade on both exchanges.
-    :param buying_price: float
-        The current trading pair price on the buying exchange.
-    :param selling_price: float
-        The current trading pair price on the selling exchange.
-    :param action: string
-        "enter": buys on the buying exchange and sells on the selling exchange
-        "exit": buys on the selling exchange and sells on the buying exchange
-
-    :return:
-    """
-    if action == 'enter':
-        enter_exchange = context.buying_exchange
-        entry_price = buying_price
-
-        exit_exchange = context.selling_exchange
-        exit_price = selling_price
-
-    elif action == 'exit':
-        enter_exchange = context.selling_exchange
-        entry_price = selling_price
-
-        exit_exchange = context.buying_exchange
-        exit_price = buying_price
-
-    else:
-        raise ValueError('invalid order action')
-
-    quote_currency = enter_exchange.quote_currency
-    quote_currency_amount = enter_exchange.portfolio.cash
-
-    exit_balances = exit_exchange.get_balances()
-    exit_currency = context.trading_pairs[
-        context.selling_exchange].quote_currency
-
-    if exit_currency in exit_balances:
-        quote_currency_amount = exit_balances[exit_currency]
-    else:
-        log.warn(
-            'the selling exchange {exchange_name} does not hold '
-            'currency {currency}'.format(
-                exchange_name=exit_exchange.name,
-                currency=exit_currency
-            )
-        )
-        return
-
-    if quote_currency_amount < (amount * entry_price):
-        adj_amount = quote_currency_amount / entry_price
-        log.warn(
-            'not enough {quote_currency} ({quote_currency_amount}) to buy '
-            '{amount}, adjusting the amount to {adj_amount}'.format(
-                quote_currency=quote_currency,
-                quote_currency_amount=quote_currency_amount,
-                amount=amount,
-                adj_amount=adj_amount
-            )
-        )
-        amount = adj_amount
-
-    elif quote_currency_amount < amount:
-        log.warn(
-            'not enough {currency} ({currency_amount}) to sell '
-            '{amount}, aborting'.format(
-                currency=exit_currency,
-                currency_amount=quote_currency_amount,
-                amount=amount
-            )
-        )
-        return
-
-    adj_buy_price = entry_price * (1 + context.SLIPPAGE_ALLOWED)
-    log.info(
-        'buying {amount} {trading_pair} on {exchange_name} with price '
-        'limit {limit_price}'.format(
-            amount=amount,
-            trading_pair=context.trading_pair_symbol,
-            exchange_name=enter_exchange.name,
-            limit_price=adj_buy_price
-        )
-    )
-    order(
-        asset=context.trading_pairs[enter_exchange],
-        amount=amount,
-        limit_price=adj_buy_price
-    )
-
-    adj_sell_price = exit_price * (1 - context.SLIPPAGE_ALLOWED)
-    log.info(
-        'selling {amount} {trading_pair} on {exchange_name} with price '
-        'limit {limit_price}'.format(
-            amount=-amount,
-            trading_pair=context.trading_pair_symbol,
-            exchange_name=exit_exchange.name,
-            limit_price=adj_sell_price
-        )
-    )
-    order(
-        asset=context.trading_pairs[exit_exchange],
-        amount=-amount,
-        limit_price=adj_sell_price
-    )
-    pass
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-
-    buying_price = data.current(
-        context.trading_pairs[context.buying_exchange], 'price')
-
-    log.info('price on buying exchange {exchange}: {price}'.format(
-        exchange=context.buying_exchange.name.upper(),
-        price=buying_price,
-    ))
-
-    selling_price = data.current(
-        context.trading_pairs[context.selling_exchange], 'price')
-
-    log.info('price on selling exchange {exchange}: {price}'.format(
-        exchange=context.selling_exchange.name.upper(),
-        price=selling_price,
-    ))
-
-    # If for example,
-    #   selling price = 50
-    #   buying price = 25
-    #   expected gap = 1
-
-    # If follows that,
-    #   selling price - buying price / buying price
-    #   50 - 25 / 25 = 1
-    gap = (selling_price - buying_price) / buying_price
-    log.info(
-        'the price gap: {gap} ({gap_percent}%)'.format(
-            gap=gap,
-            gap_percent=gap * 100
-        )
-    )
-    record(buying_price=buying_price, selling_price=selling_price, gap=gap)
-
-    # Waiting for orders to close before initiating new ones
-    for exchange in context.trading_pairs:
-        asset = context.trading_pairs[exchange]
-
-        orders = get_open_orders(asset)
-        if orders:
-            log.info(
-                'found {order_count} open orders on {exchange_name} '
-                'skipping bar until all open orders execute'.format(
-                    order_count=len(orders),
-                    exchange_name=exchange.name
-                )
-            )
-            return
-
-    # Consider the least ambitious entry point first
-    # Override of wider gap is found
-    entry_points = sorted(
-        context.entry_points,
-        key=lambda point: point['gap'],
-    )
-
-    buy_amount = None
-    for entry_point in entry_points:
-        if gap > entry_point['gap']:
-            buy_amount = entry_point['amount']
-
-    if buy_amount:
-        log.info('found buy trigger for amount: {}'.format(buy_amount))
-        place_orders(
-            context=context,
-            amount=buy_amount,
-            buying_price=buying_price,
-            selling_price=selling_price,
-            action='enter'
-        )
-
-    else:
-        # Consider the narrowest exit gap first
-        # Override of wider gap is found
-        exit_points = sorted(
-            context.exit_points,
-            key=lambda point: point['gap'],
-            reverse=True
-        )
-
-        sell_amount = None
-        for exit_point in exit_points:
-            if gap < exit_point['gap']:
-                sell_amount = exit_point['amount']
-
-        if sell_amount:
-            log.info('found sell trigger for amount: {}'.format(sell_amount))
-            place_orders(
-                context=context,
-                amount=sell_amount,
-                buying_price=buying_price,
-                selling_price=selling_price,
-                action='exit'
-            )
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-    pass
-
-
-if __name__ == '__main__':
-    # The execution mode: backtest or live
-    MODE = 'live'
-    if MODE == 'live':
-        run_algorithm(
-            capital_base=0.1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex,bitfinex',
-            live=True,
-            algo_namespace=algo_namespace,
-            base_currency='btc',
-            live_graph=False,
-            simulate_orders=True,
-            stats_output=None,
-        )

catalyst/examples/buy_and_hodl.py

@@ -14,28 +14,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import pandas as pd
-import matplotlib.pyplot as plt
 
-from catalyst import run_algorithm
+from catalyst.api import (
-from catalyst.api import (order_target_value, symbol, record,
+    order_target_value,
-                          cancel_order, get_open_orders, )
+    symbol,
+    record,
+    cancel_order,
+    get_open_orders,
+)
 
 
 def initialize(context):
-    context.ASSET_NAME = 'btc_usdt'
+    context.ASSET_NAME = 'USDT_BTC'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
+    # For all trading pairs in the poloniex bundle, the default denomination
+    # currently supported by Catalyst is 1/1000th of a full coin. Use this
+    # constant to scale the price of up to that of a full coin if desired.
+    context.TICK_SIZE = 1000.0
+
     context.is_buying = True
     context.asset = symbol(context.ASSET_NAME)
 
     context.i = 0
 
-
 def handle_data(context, data):
     context.i += 1
 
+    print 'i:', context.i
+
     starting_cash = context.portfolio.starting_cash
     target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
     reserve_value = context.RESERVE_RATIO * starting_cash
@@ -51,56 +59,53 @@ def handle_data(context, data):
         context.is_buying = False
 
     # Retrieve current asset price from pricing data
-    price = data.current(context.asset, 'price')
+    price = data[context.asset].price
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
-        print('buying')
         # Place order to make position in asset equal to target_hodl_value
         order_target_value(
             context.asset,
             target_hodl_value,
-            limit_price=price * 1.1,
+            limit_price=price*1.1,
+            stop_price=price*0.9,
         )
 
     record(
         price=price,
-        volume=data.current(context.asset, 'volume'),
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,
     )
 
-
 def analyze(context=None, results=None):
-
+    import matplotlib.pyplot as plt
     # Plot the portfolio and asset data.
-    ax1 = plt.subplot(611)
+    ax1 = plt.subplot(511)
     results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio\nValue\n(USD)')
+    ax1.set_ylabel('Portfolio Value (USD)')
 
-    ax2 = plt.subplot(612, sharex=ax1)
+    ax2 = plt.subplot(512, sharex=ax1)
-    ax2.set_ylabel('{asset}\n(USD)'.format(asset=context.ASSET_NAME))
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    results[['price']].plot(ax=ax2)
+    (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     buys = trans.ix[
         [t[0]['amount'] > 0 for t in trans.transactions]
     ]
-    ax2.scatter(
+    ax2.plot(
-        buys.index.to_pydatetime(),
+        buys.index,
-        results.price[buys.index],
+        context.TICK_SIZE * results.price[buys.index],
-        marker='^',
+        '^',
-        s=100,
+        markersize=10,
-        c='g',
+        color='g',
-        label=''
     )
 
-    ax3 = plt.subplot(613, sharex=ax1)
+    ax3 = plt.subplot(513, sharex=ax1)
     results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
     ax3.set_ylabel('Leverage ')
 
-    ax4 = plt.subplot(614, sharex=ax1)
+    ax4 = plt.subplot(514, sharex=ax1)
     results[['starting_cash', 'cash']].plot(ax=ax4)
     ax4.set_ylabel('Cash (USD)')
 
@@ -114,35 +119,16 @@ def analyze(context=None, results=None):
         'benchmark_period_return',
     ]]
 
-    ax5 = plt.subplot(615, sharex=ax1)
+    ax5 = plt.subplot(515, sharex=ax1)
     results[[
         'treasury',
         'algorithm',
         'benchmark',
     ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent\nChange')
+    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(616, sharex=ax1)
-    results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume')
 
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
     plt.show()
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        algo_namespace='buy_and_hodl',
-        base_currency='usdt',
-        start=pd.to_datetime('2015-03-01', utc=True),
-        end=pd.to_datetime('2017-10-31', utc=True),
-    )

catalyst/examples/buy_and_hold_test.py

@@ -0,0 +1,79 @@
+from catalyst.utils.run_algo import run_algorithm
+from datetime import datetime
+import pytz
+
+from catalyst.api import (
+    order_target_value,
+    symbol,
+    record,
+    cancel_order,
+    get_open_orders,
+)
+
+
+def initialize(context):
+    context.ASSET_NAME = 'USDT_BTC'
+    context.TARGET_HODL_RATIO = 0.8
+    context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
+
+    # For all trading pairs in the poloniex bundle, the default denomination
+    # currently supported by Catalyst is 1/1000th of a full coin. Use this
+    # constant to scale the price of up to that of a full coin if desired.
+    context.TICK_SIZE = 1000.0
+
+    context.is_buying = True
+    context.asset = symbol(context.ASSET_NAME)
+
+    context.i = 0
+
+
+def handle_data(context, data):
+    context.i += 1
+
+    print 'i:', context.i
+
+    starting_cash = context.portfolio.starting_cash
+    target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
+    reserve_value = context.RESERVE_RATIO * starting_cash
+
+    # Cancel any outstanding orders
+    orders = get_open_orders(context.asset) or []
+    for order in orders:
+        cancel_order(order)
+
+    # Stop buying after passing the reserve threshold
+    cash = context.portfolio.cash
+    if cash <= reserve_value:
+        context.is_buying = False
+
+    # Retrieve current asset price from pricing data
+    price = data[context.asset].price
+
+    # Check if still buying and could (approximately) afford another purchase
+    if context.is_buying and cash > price:
+        # Place order to make position in asset equal to target_hodl_value
+        order_target_value(
+            context.asset,
+            target_hodl_value,
+            limit_price=price * 1.1,
+            stop_price=price * 0.9,
+        )
+
+    record(
+        price=price,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+    )
+
+
+start = datetime(2015, 3, 1, 0, 0, 0, 0, pytz.utc)
+end = datetime(2017, 6, 28, 0, 0, 0, 0, pytz.utc)
+run_algorithm(
+    initialize=initialize,
+    handle_data=handle_data,
+    start=start,
+    end=end,
+    capital_base=100000,
+    bundle='poloniex'
+)

catalyst/examples/buy_btc_simple.py

@@ -1,49 +0,0 @@
-'''
-    This is a very simple example referenced in the beginner's tutorial:
-    https://enigmampc.github.io/catalyst/beginner-tutorial.html
-
-    Run this example, by executing the following from your terminal:
-      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
-      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
-        --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    If you want to run this code using another exchange, make sure that
-    the asset is available on that exchange. For example, if you were to run
-    it for exchange Poloniex, you would need to edit the following line:
-
-        context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
-
-    and specify exchange poloniex as follows:
-    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
-    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
-        --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    To see which assets are available on each exchange, visit:
-    https://www.enigma.co/catalyst/status
-'''
-from catalyst import run_algorithm
-from catalyst.api import order, record, symbol
-import pandas as pd
-
-
-def initialize(context):
-    context.asset = symbol('btc_usdt')
-
-
-def handle_data(context, data):
-    order(context.asset, 1)
-    record(btc=data.current(context.asset, 'price'))
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        exchange_name='poloniex',
-        algo_namespace='buy_and_hodl',
-        base_currency='usdt',
-        start=pd.to_datetime('2015-03-01', utc=True),
-        end=pd.to_datetime('2017-10-31', utc=True),
-    )

catalyst/examples/buy_the_dip_live.py

@@ -1,5 +1,4 @@
 import talib
-import pandas as pd
 from logbook import Logger
 
 from catalyst.api import (
@@ -9,58 +8,61 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'buy_the_dip_live'
-log = Logger('buy low sell high')
+log = Logger(algo_namespace)
 
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'btc_usdt'
+    context.ASSET_NAME = 'XRP_USD'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 30
+    context.TARGET_POSITIONS = 5000
     context.PROFIT_TARGET = 0.1
     context.SLIPPAGE_ALLOWED = 0.02
 
+    context.retry_check_open_orders = 10
+    context.retry_update_portfolio = 10
+    context.retry_order = 5
+
     context.errors = []
     pass
 
 
 def _handle_data(context, data):
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='1D'
+        frequency='15m'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 1
+        buy_increment = 50
     elif rsi <= 40:
-        buy_increment = 0.5
+        buy_increment = 20
     elif rsi <= 70:
-        buy_increment = 0.2
+        buy_increment = 5
     else:
-        buy_increment = 0.1
+        buy_increment = None
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     record(
         price=price,
         rsi=rsi,
     )
 
-    orders = context.blotter.open_orders
+    orders = get_open_orders(context.asset)
     if orders:
         log.info('skipping bar until all open orders execute')
         return
@@ -84,8 +86,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif (position.amount > 0
+        elif position.amount > 0 and \
-              and price > cost_basis * (1 + context.PROFIT_TARGET)):
+                        price > cost_basis * (1 + context.PROFIT_TARGET):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -138,34 +140,16 @@ def handle_data(context, data):
 
 
 def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
+    log.info('the full stats:\n{}'.format(stats.head()))
     pass
 
 
-if __name__ == '__main__':
+run_algorithm(
-    live = True
+    initialize=initialize,
-    if live:
+    handle_data=handle_data,
-        run_algorithm(
+    analyze=analyze,
-            capital_base=1000,
+    exchange_name='bitfinex',
-            initialize=initialize,
+    live=True,
-            handle_data=handle_data,
+    algo_namespace=algo_namespace,
-            analyze=analyze,
+    base_currency='usd'
-            exchange_name='bittrex',
+)
-            live=True,
-            algo_namespace=algo_namespace,
-            base_currency='btc',
-            simulate_orders=True,
-        )
-    else:
-        run_algorithm(
-            capital_base=10000,
-            data_frequency='daily',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            algo_namespace='buy_and_hodl',
-            base_currency='usdt',
-            start=pd.to_datetime('2015-03-01', utc=True),
-            end=pd.to_datetime('2017-10-31', utc=True),
-        )

catalyst/examples/dual_moving_average.py

@@ -1,164 +0,0 @@
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-from logbook import Logger
-
-from catalyst import run_algorithm
-from catalyst.api import (record, symbol, order_target_percent,)
-from catalyst.exchange.utils.stats_utils import extract_transactions
-
-NAMESPACE = 'dual_moving_average'
-log = Logger(NAMESPACE)
-
-
-def initialize(context):
-    context.i = 0
-    context.asset = symbol('ltc_usd')
-    context.base_price = None
-
-
-def handle_data(context, data):
-    # define the windows for the moving averages
-    short_window = 50
-    long_window = 200
-
-    # Skip as many bars as long_window to properly compute the average
-    context.i += 1
-    if context.i < long_window:
-        return
-
-    # Compute moving averages calling data.history() for each
-    # moving average with the appropriate parameters. We choose to use
-    # minute bars for this simulation -> freq="1m"
-    # Returns a pandas dataframe.
-    short_data = data.history(context.asset,
-                              'price',
-                              bar_count=short_window,
-                              frequency="1T",
-                              )
-    short_mavg = short_data.mean()
-    long_data = data.history(context.asset,
-                             'price',
-                             bar_count=long_window,
-                             frequency="1T",
-                             )
-    long_mavg = long_data.mean()
-
-    # Let's keep the price of our asset in a more handy variable
-    price = data.current(context.asset, 'price')
-
-    # If base_price is not set, we use the current value. This is the
-    # price at the first bar which we reference to calculate price_change.
-    if context.base_price is None:
-        context.base_price = price
-    price_change = (price - context.base_price) / context.base_price
-
-    # Save values for later inspection
-    record(price=price,
-           cash=context.portfolio.cash,
-           price_change=price_change,
-           short_mavg=short_mavg,
-           long_mavg=long_mavg)
-
-    # Since we are using limit orders, some orders may not execute immediately
-    # we wait until all orders are executed before considering more trades.
-    orders = context.blotter.open_orders
-    if len(orders) > 0:
-        return
-
-    # Exit if we cannot trade
-    if not data.can_trade(context.asset):
-        return
-
-    # We check what's our position on our portfolio and trade accordingly
-    pos_amount = context.portfolio.positions[context.asset].amount
-
-    # Trading logic
-    if short_mavg > long_mavg and pos_amount == 0:
-        # we buy 100% of our portfolio for this asset
-        order_target_percent(context.asset, 1)
-    elif short_mavg < long_mavg and pos_amount > 0:
-        # we sell all our positions for this asset
-        order_target_percent(context.asset, 0)
-
-
-def analyze(context, perf):
-    # Get the base_currency that was passed as a parameter to the simulation
-    exchange = list(context.exchanges.values())[0]
-    base_currency = exchange.base_currency.upper()
-
-    # First chart: Plot portfolio value using base_currency
-    ax1 = plt.subplot(411)
-    perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-    ax1.legend_.remove()
-    ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-    start, end = ax1.get_ylim()
-    ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
-
-    # Second chart: Plot asset price, moving averages and buys/sells
-    ax2 = plt.subplot(412, sharex=ax1)
-    perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
-        ax=ax2,
-        label='Price')
-    ax2.legend_.remove()
-    ax2.set_ylabel('{asset}\n({base})'.format(
-        asset=context.asset.symbol,
-        base=base_currency
-    ))
-    start, end = ax2.get_ylim()
-    ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
-
-    transaction_df = extract_transactions(perf)
-    if not transaction_df.empty:
-        buy_df = transaction_df[transaction_df['amount'] > 0]
-        sell_df = transaction_df[transaction_df['amount'] < 0]
-        ax2.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'price'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax2.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'price'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-
-    # Third chart: Compare percentage change between our portfolio
-    # and the price of the asset
-    ax3 = plt.subplot(413, sharex=ax1)
-    perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-    ax3.legend_.remove()
-    ax3.set_ylabel('Percent Change')
-    start, end = ax3.get_ylim()
-    ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
-
-    # Fourth chart: Plot our cash
-    ax4 = plt.subplot(414, sharex=ax1)
-    perf.cash.plot(ax=ax4)
-    ax4.set_ylabel('Cash\n({})'.format(base_currency))
-    start, end = ax4.get_ylim()
-    ax4.yaxis.set_ticks(np.arange(0, end, end / 5))
-
-    plt.show()
-
-
-if __name__ == '__main__':
-    
-    run_algorithm(
-            capital_base=1000,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='bitfinex',
-            algo_namespace=NAMESPACE,
-            base_currency='usd',
-            start=pd.to_datetime('2017-9-22', utc=True),
-            end=pd.to_datetime('2017-9-23', utc=True),
-        )

catalyst/examples/dual_vwap.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Enigma MPC, Inc.
+# Copyright 2014 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from catalyst.api import (
+    order_target_percent,
+    record,
+    symbol,
+    get_open_orders,
+    set_max_leverage,
+    schedule_function,
+    date_rules,
+    attach_pipeline,
+    pipeline_output,
+)
+
+from catalyst.pipeline import Pipeline
+from catalyst.pipeline.data import CryptoPricing
+from catalyst.pipeline.factors.crypto import VWAP
+
+
+def initialize(context):
+    context.ASSET_NAME = 'USDT_BTC'
+    context.TARGET_INVESTMENT_RATIO = 0.8
+    context.SHORT_WINDOW = 30
+    context.LONG_WINDOW = 100
+
+    # For all trading pairs in the poloniex bundle, the default denomination
+    # currently supported by Catalyst is 1/1000th of a full coin. Use this
+    # constant to scale the price of up to that of a full coin if desired.
+    context.TICK_SIZE = 1000.0
+
+    context.i = 0
+    context.asset = symbol(context.ASSET_NAME)
+
+    set_max_leverage(1.0)
+
+    attach_pipeline(make_pipeline(context), 'vwap_pipeline')
+
+    schedule_function(
+        rebalance,
+        time_rules=times_rules.every_minute(),
+    )
+
+
+def before_trading_start(context, data):
+    context.pipeline_data = pipeline_output('vwap_pipeline')
+
+def make_pipeline(context):
+    return Pipeline(
+        columns={
+            'price': CryptoPricing.open.latest,
+            'volume': CryptoPricing.volume.latest,
+            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
+            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
+        }
+    )
+
+def rebalance(context, data):
+    context.i += 1
+
+    # skip first LONG_WINDOW bars to fill windows
+    if context.i < context.LONG_WINDOW:
+        return
+
+    # get pipeline data for asset of interest
+    pipeline_data = context.pipeline_data
+    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
+
+    # retrieve long and short moving averages from pipeline
+    short_mavg = pipeline_data.short_mavg
+    long_mavg = pipeline_data.long_mavg
+    price = pipeline_data.price
+    volume = pipeline_data.volume
+
+    # check that order has not already been placed
+    open_orders = get_open_orders()
+    if context.asset not in open_orders:
+        # check that the asset of interest can currently be traded
+        if data.can_trade(context.asset):
+            # adjust portfolio based on comparison of long and short vwap
+            if short_mavg > long_mavg:
+                order_target_percent(
+                    context.asset,
+                    context.TARGET_INVESTMENT_RATIO,
+                )
+            elif short_mavg < long_mavg:
+                order_target_percent(
+                    context.asset,
+                    0.0,
+                )
+
+    record(
+        price=price,
+        cash=context.portfolio.cash,
+        leverage=context.account.leverage,
+        short_mavg=short_mavg,
+        long_mavg=long_mavg,
+        volume=volume,
+    )
+    
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results[['portfolio_value']].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio value (USD)')
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
+    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
+
+    trans = results.ix[[t != [] for t in results.transactions]]
+    amounts = [t[0]['amount'] for t in trans.transactions]
+
+    buys = trans.ix[
+        [t[0]['amount'] > 0 for t in trans.transactions]
+    ]
+    sells = trans.ix[
+        [t[0]['amount'] < 0 for t in trans.transactions]
+    ]
+
+    ax2.plot(
+        buys.index,
+        context.TICK_SIZE * results.price[buys.index],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        context.TICK_SIZE * results.price[sells.index],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Leverage (USD)')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results[['cash']].plot(ax=ax4)
+    ax4.set_ylabel('Cash (USD)')
+
+    results[[
+        'treasury',
+        'algorithm',
+        'benchmark',
+    ]] = results[[
+        'treasury_period_return',
+        'algorithm_period_return',
+        'benchmark_period_return',
+    ]]
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results[[
+        'treasury',
+        'algorithm',
+        'benchmark',
+    ]].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results[['volume']].plot(ax=ax6)
+    ax6.set_ylabel('Volume (mBTC/day)')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()

catalyst/examples/marketplace/github-research.py

@@ -1,70 +0,0 @@
-import pandas as pd
-import matplotlib.pyplot as plt
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, get_dataset
-
-START = '2017-01-01'
-END = '2017-12-31'
-
-
-def initialize(context):
-    pass
-
-
-def handle_data(context, data):
-    context.github = get_dataset('github')
-    context.github.sort_index(level=0, inplace=True)
-
-    context.zec = data.history(symbol('zec_usdt'),
-                               ['price', ],
-                               bar_count=365,
-                               frequency="1d")
-    context.xmr = data.history(symbol('xmr_usdt'),
-                               ['price', ],
-                               bar_count=365,
-                               frequency="1d")
-
-
-def analyze(context=None, results=None):
-    ax1 = plt.subplot(211)
-    idx = pd.IndexSlice
-    df = context.github.loc[START:END].loc[
-            idx[:, [b'ZEC']], ['commits']].reset_index(
-                level='symbol', drop=True)
-    df.plot(ax=ax1, color='blue')
-    ax1.legend(loc=2)
-    ax1.set_title('Zcash')
-    ax2 = ax1.twinx()
-    context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
-    ax2.legend(loc=1)
-
-    ax3 = plt.subplot(212)
-    idx = pd.IndexSlice
-    df = context.github.loc[START:END].loc[
-            idx[:, [b'XMR']], ['commits']].reset_index(
-                level='symbol', drop=True)
-    df.plot(ax=ax3, color='blue')
-    ax3.legend(loc=2)
-    ax3.set_title('Monero')
-    ax4 = ax3.twinx()
-    context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
-    ax4.legend(loc=1)
-
-    plt.show()
-
-
-if __name__ == '__main__':
-    run_algorithm(
-            capital_base=1000,
-            data_frequency='daily',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            algo_namespace='algo-github',
-            base_currency='usdt',
-            live=False,
-            start=pd.to_datetime(END, utc=True),
-            end=pd.to_datetime(END, utc=True),
-        )

catalyst/examples/marketplace/mean_reversion_by_marketcap.py

@@ -1,237 +0,0 @@
-# For this example, we're going to write a simple momentum script.  When the
-# stock goes up quickly, we're going to buy; when it goes down quickly, we're
-# going to sell.  Hopefully we'll ride the waves.
-import os
-import tempfile
-import time
-
-import pandas as pd
-import talib
-from logbook import Logger
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, record, order_target_percent, get_dataset
-from catalyst.exchange.utils.stats_utils import set_print_settings, \
-    get_pretty_stats
-# We give a name to the algorithm which Catalyst will use to persist its state.
-# In this example, Catalyst will create the `.catalyst/data/live_algos`
-# directory. If we stop and start the algorithm, Catalyst will resume its
-# state using the files included in the folder.
-from catalyst.utils.paths import ensure_directory
-
-NAMESPACE = 'mean_reversion_simple'
-log = Logger(NAMESPACE)
-
-
-# To run an algorithm in Catalyst, you need two functions: initialize and
-# handle_data.
-
-def initialize(context):
-    # This initialize function sets any data or variables that you'll use in
-    # your algorithm.  For instance, you'll want to define the trading pair (or
-    # trading pairs) you want to backtest.  You'll also want to define any
-    # parameters or values you're going to use.
-
-    # In our example, we're looking at Neo in Ether.
-    df = get_dataset('testmarketcap2')  # type: pd.DataFrame
-
-    # Picking a specific date in our DataFrame
-    first_dt = df.index.get_level_values(0)[0]
-    # Since we use a MultiIndex with date / symbol, picking a date will
-    # result in a new DataFrame for the selected date with a single
-    # symbol index
-    df = df.xs(first_dt, level=0)
-    # Keep only the top coins by market cap
-    df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
-
-    set_print_settings()
-
-    df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
-    print('the marketplace data:\n{}'.format(df))
-
-    # Pick the 5 assets with the lowest market cap for trading
-    quote_currency = 'eth'
-    exchange = context.exchanges[next(iter(context.exchanges))]
-    symbols = [a.symbol for a in exchange.assets
-               if a.start_date < context.datetime]
-    context.assets = []
-    for currency, price in df['market_cap_usd'].iteritems():
-        if len(context.assets) >= 5:
-            break
-
-        s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
-        if s in symbols:
-            context.assets.append(symbol(s))
-
-    context.base_price = None
-    context.current_day = None
-
-    context.RSI_OVERSOLD = 55
-    context.RSI_OVERBOUGHT = 60
-    context.CANDLE_SIZE = '5T'
-
-    context.start_time = time.time()
-
-
-def handle_data(context, data):
-    # This handle_data function is where the real work is done.  Our data is
-    # minute-level tick data, and each minute is called a frame.  This function
-    # runs on each frame of the data.
-
-    # We flag the first period of each day.
-    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-    # would only execute once. This method works with minute and daily
-    # frequencies.
-    today = data.current_dt.floor('1D')
-    if today != context.current_day:
-        context.traded_today = dict()
-        context.current_day = today
-
-    # Preparing dictionaries for asset-level data points
-    volumes = dict()
-    rsis = dict()
-    price_values = dict()
-    cash = context.portfolio.cash
-
-    for asset in context.assets:
-        # We're computing the volume-weighted-average-price of the security
-        # defined above, in the context.assets variable.  For this example,
-        #  we're using three bars on the 15 min bars.
-
-        # The frequency attribute determine the bar size. We use this
-        # convention for the frequency alias:
-        # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-        prices = data.history(
-            asset,
-            fields='close',
-            bar_count=50,
-            frequency=context.CANDLE_SIZE
-        )
-
-        # Ta-lib calculates various technical indicator based on price and
-        # volume arrays.
-
-        # In this example, we are comp
-        rsi = talib.RSI(prices.values, timeperiod=14)
-
-        # We need a variable for the current price of the security to compare
-        #  to the average. Since we are requesting two fields, data.current()
-        # returns a DataFrame with
-        current = data.current(asset, fields=['close', 'volume'])
-        price = current['close']
-
-        # If base_price is not set, we use the current value. This is the
-        # price at the first bar which we reference to calculate price_change.
-        # if asset not in context.base_price:
-        #     context.base_price[asset] = price
-        #
-        # base_price = context.base_price[asset]
-        # price_change = (price - base_price) / base_price
-
-        # Tracking the relevant data
-        volumes[asset] = current['volume']
-        rsis[asset] = rsi[-1]
-        price_values[asset] = price
-        # price_changes[asset] = price_change
-
-        # We are trying to avoid over-trading by limiting our trades to
-        # one per day.
-        if asset in context.traded_today:
-            continue
-
-        # Exit if we cannot trade
-        if not data.can_trade(asset):
-            continue
-
-        # Another powerful built-in feature of the Catalyst backtester is the
-        # portfolio object.  The portfolio object tracks your positions, cash,
-        # cost basis of specific holdings, and more.  In this line, we
-        # calculate how long or short our position is at this minute.
-        pos_amount = context.portfolio.positions[asset].amount
-
-        if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-            log.info(
-                '{}: buying - price: {}, rsi: {}'.format(
-                    data.current_dt, price, rsi[-1]
-                )
-            )
-            # Set a style for limit orders,
-            limit_price = price * 1.005
-            target = 1.0 / len(context.assets)
-            order_target_percent(
-                asset, target, limit_price=limit_price
-            )
-            context.traded_today[asset] = True
-
-        elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-            log.info(
-                '{}: selling - price: {}, rsi: {}'.format(
-                    data.current_dt, price, rsi[-1]
-                )
-            )
-            limit_price = price * 0.995
-            order_target_percent(
-                asset, 0, limit_price=limit_price
-            )
-            context.traded_today[asset] = True
-
-    # Now that we've collected all current data for this frame, we use
-    # the record() method to save it. This data will be available as
-    # a parameter of the analyze() function for further analysis.
-    record(
-        current_price=price_values,
-        volume=volumes,
-        rsi=rsis,
-        cash=cash,
-    )
-
-
-def analyze(context=None, perf=None):
-    stats = get_pretty_stats(perf)
-    print('the algo stats:\n{}'.format(stats))
-    pass
-
-
-if __name__ == '__main__':
-    # The execution mode: backtest or live
-    live = False
-
-    if live:
-        run_algorithm(
-            capital_base=0.1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='btc',
-            live_graph=False,
-            simulate_orders=False,
-            stats_output=None,
-        )
-
-    else:
-        folder = os.path.join(
-            tempfile.gettempdir(), 'catalyst', NAMESPACE
-        )
-        ensure_directory(folder)
-
-        timestr = time.strftime('%Y%m%d-%H%M%S')
-        out = os.path.join(folder, '{}.p'.format(timestr))
-        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
-        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
-        #   --data-frequency minute --capital-base 10000
-        run_algorithm(
-            capital_base=100,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            start=pd.to_datetime('2017-10-01', utc=True),
-            end=pd.to_datetime('2017-10-15', utc=True),
-        )
-        log.info('saved perf stats: {}'.format(out))

catalyst/examples/mean_reversion_simple.py

@@ -1,289 +0,0 @@
-# For this example, we're going to write a simple momentum script.  When the
-# stock goes up quickly, we're going to buy; when it goes down quickly, we're
-# going to sell.  Hopefully we'll ride the waves.
-import os
-import tempfile
-import time
-
-import numpy as np
-import pandas as pd
-import talib
-from logbook import Logger
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.utils.stats_utils import extract_transactions
-# We give a name to the algorithm which Catalyst will use to persist its state.
-# In this example, Catalyst will create the `.catalyst/data/live_algos`
-# directory. If we stop and start the algorithm, Catalyst will resume its
-# state using the files included in the folder.
-from catalyst.utils.paths import ensure_directory
-
-NAMESPACE = 'mean_reversion_simple'
-log = Logger(NAMESPACE)
-
-
-# To run an algorithm in Catalyst, you need two functions: initialize and
-# handle_data.
-
-def initialize(context):
-    # This initialize function sets any data or variables that you'll use in
-    # your algorithm.  For instance, you'll want to define the trading pair (or
-    # trading pairs) you want to backtest.  You'll also want to define any
-    # parameters or values you're going to use.
-
-    # In our example, we're looking at Neo in Ether.
-    context.market = symbol('bnb_eth')
-    context.base_price = None
-    context.current_day = None
-
-    context.RSI_OVERSOLD = 60
-    context.RSI_OVERBOUGHT = 70
-    context.CANDLE_SIZE = '15T'
-
-    context.start_time = time.time()
-
-    context.set_commission(maker=0.001, taker=0.002)
-    context.set_slippage(spread=0.001)
-
-
-def handle_data(context, data):
-    # This handle_data function is where the real work is done.  Our data is
-    # minute-level tick data, and each minute is called a frame.  This function
-    # runs on each frame of the data.
-
-    # We flag the first period of each day.
-    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-    # would only execute once. This method works with minute and daily
-    # frequencies.
-    today = data.current_dt.floor('1D')
-    if today != context.current_day:
-        context.traded_today = False
-        context.current_day = today
-
-    # We're computing the volume-weighted-average-price of the security
-    # defined above, in the context.market variable.  For this example, we're
-    # using three bars on the 15 min bars.
-
-    # The frequency attribute determine the bar size. We use this convention
-    # for the frequency alias:
-    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-    prices = data.history(
-        context.market,
-        fields='close',
-        bar_count=50,
-        frequency=context.CANDLE_SIZE
-    )
-
-    # Ta-lib calculates various technical indicator based on price and
-    # volume arrays.
-
-    # In this example, we are comp
-    rsi = talib.RSI(prices.values, timeperiod=14)
-
-    # We need a variable for the current price of the security to compare to
-    # the average. Since we are requesting two fields, data.current()
-    # returns a DataFrame with
-    current = data.current(context.market, fields=['close', 'volume'])
-    price = current['close']
-
-    # If base_price is not set, we use the current value. This is the
-    # price at the first bar which we reference to calculate price_change.
-    if context.base_price is None:
-        context.base_price = price
-
-    price_change = (price - context.base_price) / context.base_price
-    cash = context.portfolio.cash
-
-    # Now that we've collected all current data for this frame, we use
-    # the record() method to save it. This data will be available as
-    # a parameter of the analyze() function for further analysis.
-
-    record(
-        volume=current['volume'],
-        price=price,
-        price_change=price_change,
-        rsi=rsi[-1],
-        cash=cash
-    )
-    # We are trying to avoid over-trading by limiting our trades to
-    # one per day.
-    if context.traded_today:
-        return
-
-    # TODO: retest with open orders
-    # Since we are using limit orders, some orders may not execute immediately
-    # we wait until all orders are executed before considering more trades.
-    orders = context.blotter.open_orders
-    if len(orders) > 0:
-        log.info('exiting because orders are open: {}'.format(orders))
-        return
-
-    # Exit if we cannot trade
-    if not data.can_trade(context.market):
-        return
-
-    # Another powerful built-in feature of the Catalyst backtester is the
-    # portfolio object.  The portfolio object tracks your positions, cash,
-    # cost basis of specific holdings, and more.  In this line, we calculate
-    # how long or short our position is at this minute.
-    pos_amount = context.portfolio.positions[context.market].amount
-
-    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-        log.info(
-            '{}: buying - price: {}, rsi: {}'.format(
-                data.current_dt, price, rsi[-1]
-            )
-        )
-        # Set a style for limit orders,
-        limit_price = price * 1.005
-        order_target_percent(
-            context.market, 1, limit_price=limit_price
-        )
-        context.traded_today = True
-
-    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-        log.info(
-            '{}: selling - price: {}, rsi: {}'.format(
-                data.current_dt, price, rsi[-1]
-            )
-        )
-        limit_price = price * 0.995
-        order_target_percent(
-            context.market, 0, limit_price=limit_price
-        )
-        context.traded_today = True
-
-
-def analyze(context=None, perf=None):
-    end = time.time()
-    log.info('elapsed time: {}'.format(end - context.start_time))
-
-    import matplotlib.pyplot as plt
-    # The base currency of the algo exchange
-    base_currency = list(context.exchanges.values())[0].base_currency.upper()
-
-    # Plot the portfolio value over time.
-    ax1 = plt.subplot(611)
-    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
-
-    # Plot the price increase or decrease over time.
-    ax2 = plt.subplot(612, sharex=ax1)
-    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-    ax2.set_ylabel('{asset}\n({base})'.format(
-        asset=context.market.symbol, base=base_currency
-    ))
-
-    transaction_df = extract_transactions(perf)
-    if not transaction_df.empty:
-        buy_df = transaction_df[transaction_df['amount'] > 0]
-        sell_df = transaction_df[transaction_df['amount'] < 0]
-        ax2.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index.floor('1 min'), 'price'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax2.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index.floor('1 min'), 'price'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-
-    ax4 = plt.subplot(613, sharex=ax1)
-    perf.loc[:, 'cash'].plot(
-        ax=ax4, label='Base Currency ({})'.format(base_currency)
-    )
-    ax4.set_ylabel('Cash\n({})'.format(base_currency))
-
-    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-    ax5 = plt.subplot(614, sharex=ax1)
-    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent\nChange')
-
-    ax6 = plt.subplot(615, sharex=ax1)
-    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-    ax6.set_ylabel('RSI')
-    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
-    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
-
-    if not transaction_df.empty:
-        ax6.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax6.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-    plt.legend(loc=3)
-    start, end = ax6.get_ylim()
-    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-    pass
-
-
-if __name__ == '__main__':
-    # The execution mode: backtest or live
-    live = True
-
-    if live:
-        run_algorithm(
-            capital_base=0.1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='binance',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            live_graph=False,
-            simulate_orders=False,
-            stats_output=None,
-            # auth_aliases=dict(poloniex='auth2')
-        )
-
-    else:
-        folder = os.path.join(
-            tempfile.gettempdir(), 'catalyst', NAMESPACE
-        )
-        ensure_directory(folder)
-
-        timestr = time.strftime('%Y%m%d-%H%M%S')
-        out = os.path.join(folder, '{}.p'.format(timestr))
-        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
-        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
-        #   --data-frequency minute --capital-base 10000
-        run_algorithm(
-            capital_base=0.035,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='bitfinex',
-            algo_namespace=NAMESPACE,
-            base_currency='btc',
-            start=pd.to_datetime('2017-10-01', utc=True),
-            end=pd.to_datetime('2017-11-10', utc=True),
-            output=out
-        )
-        log.info('saved perf stats: {}'.format(out))

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -1,288 +0,0 @@
-# For this example, we're going to write a simple momentum script.  When the
-# stock goes up quickly, we're going to buy; when it goes down quickly, we're
-# going to sell.  Hopefully we'll ride the waves.
-import os
-import tempfile
-import time
-
-import numpy as np
-import pandas as pd
-import talib
-from logbook import Logger
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.utils.stats_utils import extract_transactions
-# We give a name to the algorithm which Catalyst will use to persist its state.
-# In this example, Catalyst will create the `.catalyst/data/live_algos`
-# directory. If we stop and start the algorithm, Catalyst will resume its
-# state using the files included in the folder.
-from catalyst.utils.paths import ensure_directory
-
-NAMESPACE = 'mean_reversion_simple'
-log = Logger(NAMESPACE)
-
-
-# To run an algorithm in Catalyst, you need two functions: initialize and
-# handle_data.
-
-def initialize(context):
-    # This initialize function sets any data or variables that you'll use in
-    # your algorithm.  For instance, you'll want to define the trading pair (or
-    # trading pairs) you want to backtest.  You'll also want to define any
-    # parameters or values you're going to use.
-
-    # In our example, we're looking at Neo in Ether.
-    context.market = symbol('eth_btc')
-    context.base_price = None
-    context.current_day = None
-
-    context.RSI_OVERSOLD = 50
-    context.RSI_OVERBOUGHT = 60
-    context.CANDLE_SIZE = '5T'
-
-    context.start_time = time.time()
-
-    context.set_commission(maker=0.001, taker=0.002)
-    # context.set_slippage(spread=0.001)
-
-
-def handle_data(context, data):
-    # This handle_data function is where the real work is done.  Our data is
-    # minute-level tick data, and each minute is called a frame.  This function
-    # runs on each frame of the data.
-
-    # We flag the first period of each day.
-    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-    # would only execute once. This method works with minute and daily
-    # frequencies.
-    today = data.current_dt.floor('1D')
-    if today != context.current_day:
-        context.traded_today = False
-        context.current_day = today
-
-    # We're computing the volume-weighted-average-price of the security
-    # defined above, in the context.market variable.  For this example, we're
-    # using three bars on the 15 min bars.
-
-    # The frequency attribute determine the bar size. We use this convention
-    # for the frequency alias:
-    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-    prices = data.history(
-        context.market,
-        fields='close',
-        bar_count=50,
-        frequency=context.CANDLE_SIZE
-    )
-
-    # Ta-lib calculates various technical indicator based on price and
-    # volume arrays.
-
-    # In this example, we are comp
-    rsi = talib.RSI(prices.values, timeperiod=14)
-
-    # We need a variable for the current price of the security to compare to
-    # the average. Since we are requesting two fields, data.current()
-    # returns a DataFrame with
-    current = data.current(context.market, fields=['close', 'volume'])
-    price = current['close']
-
-    # If base_price is not set, we use the current value. This is the
-    # price at the first bar which we reference to calculate price_change.
-    if context.base_price is None:
-        context.base_price = price
-
-    price_change = (price - context.base_price) / context.base_price
-    cash = context.portfolio.cash
-
-    # Now that we've collected all current data for this frame, we use
-    # the record() method to save it. This data will be available as
-    # a parameter of the analyze() function for further analysis.
-
-    record(
-        volume=current['volume'],
-        price=price,
-        price_change=price_change,
-        rsi=rsi[-1],
-        cash=cash
-    )
-    # We are trying to avoid over-trading by limiting our trades to
-    # one per day.
-    if context.traded_today:
-        return
-
-    # TODO: retest with open orders
-    # Since we are using limit orders, some orders may not execute immediately
-    # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.market)
-    if len(orders) > 0:
-        log.info('exiting because orders are open: {}'.format(orders))
-        return
-
-    # Exit if we cannot trade
-    if not data.can_trade(context.market):
-        return
-
-    # Another powerful built-in feature of the Catalyst backtester is the
-    # portfolio object.  The portfolio object tracks your positions, cash,
-    # cost basis of specific holdings, and more.  In this line, we calculate
-    # how long or short our position is at this minute.
-    pos_amount = context.portfolio.positions[context.market].amount
-
-    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-        log.info(
-            '{}: buying - price: {}, rsi: {}'.format(
-                data.current_dt, price, rsi[-1]
-            )
-        )
-        # Set a style for limit orders,
-        limit_price = price * 1.005
-        order_target_percent(
-            context.market, 1, limit_price=limit_price
-        )
-        context.traded_today = True
-
-    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-        log.info(
-            '{}: selling - price: {}, rsi: {}'.format(
-                data.current_dt, price, rsi[-1]
-            )
-        )
-        limit_price = price * 0.995
-        order_target_percent(
-            context.market, 0, limit_price=limit_price
-        )
-        context.traded_today = True
-
-
-def analyze(context=None, perf=None):
-    end = time.time()
-    log.info('elapsed time: {}'.format(end - context.start_time))
-
-    import matplotlib.pyplot as plt
-    # The base currency of the algo exchange
-    base_currency = list(context.exchanges.values())[0].base_currency.upper()
-
-    # Plot the portfolio value over time.
-    ax1 = plt.subplot(611)
-    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
-
-    # Plot the price increase or decrease over time.
-    ax2 = plt.subplot(612, sharex=ax1)
-    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-    ax2.set_ylabel('{asset}\n({base})'.format(
-        asset=context.market.symbol, base=base_currency
-    ))
-
-    transaction_df = extract_transactions(perf)
-    if not transaction_df.empty:
-        buy_df = transaction_df[transaction_df['amount'] > 0]
-        sell_df = transaction_df[transaction_df['amount'] < 0]
-        ax2.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index.floor('1 min'), 'price'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax2.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index.floor('1 min'), 'price'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-
-    ax4 = plt.subplot(613, sharex=ax1)
-    perf.loc[:, 'cash'].plot(
-        ax=ax4, label='Base Currency ({})'.format(base_currency)
-    )
-    ax4.set_ylabel('Cash\n({})'.format(base_currency))
-
-    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-    ax5 = plt.subplot(614, sharex=ax1)
-    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent\nChange')
-
-    ax6 = plt.subplot(615, sharex=ax1)
-    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-    ax6.set_ylabel('RSI')
-    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
-    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
-
-    if not transaction_df.empty:
-        ax6.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax6.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-    plt.legend(loc=3)
-    start, end = ax6.get_ylim()
-    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-    pass
-
-
-if __name__ == '__main__':
-    # The execution mode: backtest or live
-    live = False
-
-    if live:
-        run_algorithm(
-            capital_base=0.025,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='btc',
-            live_graph=False,
-            simulate_orders=False,
-            stats_output=None,
-        )
-
-    else:
-        folder = os.path.join(
-            tempfile.gettempdir(), 'catalyst', NAMESPACE
-        )
-        ensure_directory(folder)
-
-        timestr = time.strftime('%Y%m%d-%H%M%S')
-        out = os.path.join(folder, '{}.p'.format(timestr))
-        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
-        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
-        #   --data-frequency minute --capital-base 10000
-        run_algorithm(
-            capital_base=0.1,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='bitfinex',
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            start=pd.to_datetime('2017-10-01', utc=True),
-            end=pd.to_datetime('2017-11-10', utc=True),
-            output=out
-        )
-        log.info('saved perf stats: {}'.format(out))

catalyst/examples/portfolio_optimization.py

@@ -1,150 +0,0 @@
-'''Use this code to execute a portfolio optimization model. This code
-   will select the portfolio with the maximum Sharpe Ratio. The parameters
-   are set to use 180 days of historical data and rebalance every 30 days.
-
-   This is the code used in the following article:
-   https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
-
-   You can run this code using the Python interpreter:
-
-   $ python portfolio_optimization.py
-'''
-
-from __future__ import division
-import os
-import pytz
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from datetime import datetime
-
-from catalyst.api import record, symbols, order_target_percent
-from catalyst.utils.run_algo import run_algorithm
-
-np.set_printoptions(threshold='nan', suppress=True)
-
-
-def initialize(context):
-    # Portfolio assets list
-    context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
-                             'xmr_usdt')
-    context.nassets = len(context.assets)
-    # Set the time window that will be used to compute expected return
-    # and asset correlations
-    context.window = 180
-    # Set the number of days between each portfolio rebalancing
-    context.rebalance_period = 30
-    context.i = 0
-
-
-def handle_data(context, data):
-    # Only rebalance at the beggining of the algorithm execution and
-    # every multiple of the rebalance period
-    if context.i == 0 or context.i % context.rebalance_period == 0:
-        n = context.window
-        prices = data.history(context.assets, fields='price',
-                              bar_count=n + 1, frequency='1d')
-        pr = np.asmatrix(prices)
-        t_prices = prices.iloc[1:n + 1]
-        t_val = t_prices.values
-        tminus_prices = prices.iloc[0:n]
-        tminus_val = tminus_prices.values
-        # Compute daily returns (r)
-        r = np.asmatrix(t_val / tminus_val - 1)
-        # Compute the expected returns of each asset with the average
-        # daily return for the selected time window
-        m = np.asmatrix(np.mean(r, axis=0))
-        # ###
-        stds = np.std(r, axis=0)
-        # Compute excess returns matrix (xr)
-        xr = r - m
-        # Matrix algebra to get variance-covariance matrix
-        cov_m = np.dot(np.transpose(xr), xr) / n
-        # Compute asset correlation matrix (informative only)
-        corr_m = cov_m / np.dot(np.transpose(stds), stds)
-
-        # Define portfolio optimization parameters
-        n_portfolios = 50000
-        results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in xrange(n_portfolios):
-            weights = np.random.random(context.nassets)
-            weights /= np.sum(weights)
-            w = np.asmatrix(weights)
-            p_r = np.sum(np.dot(w, np.transpose(m))) * 365
-            p_std = np.sqrt(np.dot(np.dot(w, cov_m),
-                                   np.transpose(w))) * np.sqrt(365)
-
-            # store results in results array
-            results_array[0, p] = p_r
-            results_array[1, p] = p_std
-            # store Sharpe Ratio (return / volatility) - risk free rate element
-            # excluded for simplicity
-            results_array[2, p] = results_array[0, p] / results_array[1, p]
-            i = 0
-            for iw in weights:
-                results_array[3 + i, p] = weights[i]
-                i += 1
-
-        # convert results array to Pandas DataFrame
-        results_frame = pd.DataFrame(np.transpose(results_array),
-                                     columns=['r', 'stdev', 'sharpe']
-                                             + context.assets)
-        # locate position of portfolio with highest Sharpe Ratio
-        max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
-        # locate positon of portfolio with minimum standard deviation
-        # min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
-
-        # order optimal weights for each asset
-        for asset in context.assets:
-            if data.can_trade(asset):
-                order_target_percent(asset, max_sharpe_port[asset])
-
-        # create scatter plot coloured by Sharpe Ratio
-        plt.scatter(results_frame.stdev,
-                    results_frame.r,
-                    c=results_frame.sharpe,
-                    cmap='RdYlGn')
-        plt.xlabel('Volatility')
-        plt.ylabel('Returns')
-        plt.colorbar()
-        # plot red star to highlight position of portfolio
-        # with highest Sharpe Ratio
-        plt.scatter(max_sharpe_port[1],
-                    max_sharpe_port[0],
-                    marker='o',
-                    color='b',
-                    s=200)
-        # plot green star to highlight position of minimum variance portfolio
-        plt.show()
-        print(max_sharpe_port)
-        record(pr=pr,
-               r=r,
-               m=m,
-               stds=stds,
-               max_sharpe_port=max_sharpe_port,
-               corr_m=corr_m)
-    context.i += 1
-
-
-def analyze(context=None, results=None):
-    # Form DataFrame with selected data
-    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
-                    'portfolio_value']]
-
-    # Save results in CSV file
-    filename = os.path.splitext(os.path.basename(__file__))[0]
-    data.to_csv(filename + '.csv')
-
-
-if __name__ == '__main__':
-    # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
-    start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
-    end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
-    results = run_algorithm(initialize=initialize,
-                            handle_data=handle_data,
-                            analyze=analyze,
-                            start=start,
-                            end=end,
-                            exchange_name='poloniex',
-                            capital_base=100000,
-                            base_currency='usdt', )

catalyst/examples/rsi_profit_target.py

@@ -1,265 +0,0 @@
-from datetime import timedelta
-
-import pandas as pd
-import numpy as np
-import talib
-from logbook import Logger
-
-from catalyst.api import (
-    order,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'rsi'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.asset = symbol('eth_btc')
-    context.base_price = None
-
-    context.MAX_HOLDINGS = 0.2
-    context.RSI_OVERSOLD = 30
-    context.RSI_OVERSOLD_BBANDS = 45
-    context.RSI_OVERBOUGHT_BBANDS = 55
-    context.SLIPPAGE_ALLOWED = 0.03
-
-    context.TARGET = 0.15
-    context.STOP_LOSS = 0.1
-    context.STOP = 0.03
-    context.position = None
-
-    context.last_bar = None
-
-    context.errors = []
-    pass
-
-
-def _handle_buy_sell_decision(context, data, signal, price):
-    orders = get_open_orders(context.asset)
-    if len(orders) > 0:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    positions = context.portfolio.positions
-    if context.position is None and context.asset in positions:
-        position = positions[context.asset]
-        context.position = dict(
-            cost_basis=position['cost_basis'],
-            amount=position['amount'],
-            stop=None
-        )
-
-    # action = None
-    if context.position is not None:
-        cost_basis = context.position['cost_basis']
-        amount = context.position['amount']
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=amount,
-                cost_basis=cost_basis
-            )
-        )
-        stop = context.position['stop']
-
-        target = cost_basis * (1 + context.TARGET)
-        if price >= target:
-            context.position['cost_basis'] = price
-            context.position['stop'] = context.STOP
-
-        stop_target = context.STOP_LOSS if stop is None else context.STOP
-        if price < cost_basis * (1 - stop_target):
-            log.info('executing stop loss')
-            order(
-                asset=context.asset,
-                amount=-amount,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-            # action = 0
-            context.position = None
-
-    else:
-        if signal == 'long':
-            log.info('opening position')
-            buy_amount = context.MAX_HOLDINGS / price
-            order(
-                asset=context.asset,
-                amount=buy_amount,
-                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
-            )
-            context.position = dict(
-                cost_basis=price,
-                amount=buy_amount,
-                stop=None
-            )
-            # action = 0
-
-
-def _handle_data_rsi_only(context, data):
-    price = data.current(context.asset, 'close')
-    log.info('got price {price}'.format(price=price))
-
-    if price is np.nan:
-        log.warn('no pricing data')
-        return
-
-    if context.base_price is None:
-        context.base_price = price
-
-    try:
-        prices = data.history(
-            context.asset,
-            fields='price',
-            bar_count=20,
-            frequency='30T'
-        )
-    except Exception as e:
-        log.warn('historical data not available: '.format(e))
-        return
-
-    rsi = talib.RSI(prices.values, timeperiod=16)[-1]
-    log.info('got rsi {}'.format(rsi))
-
-    signal = None
-    if rsi < context.RSI_OVERSOLD:
-        signal = 'long'
-
-    # Making sure that the price is still current
-    price = data.current(context.asset, 'close')
-    cash = context.portfolio.cash
-    log.info(
-        'base currency available: {cash}, cap: {cap}'.format(
-            cash=cash,
-            cap=context.MAX_HOLDINGS
-        )
-    )
-    volume = data.current(context.asset, 'volume')
-    price_change = (price - context.base_price) / context.base_price
-    record(
-        price=price,
-        price_change=price_change,
-        rsi=rsi,
-        volume=volume,
-        cash=cash,
-        starting_cash=context.portfolio.starting_cash,
-        leverage=context.account.leverage,
-    )
-
-    _handle_buy_sell_decision(context, data, signal, price)
-
-
-def handle_data(context, data):
-    dt = data.current_dt
-
-    if context.last_bar is None or (
-            context.last_bar + timedelta(minutes=15)) <= dt:
-        context.last_bar = dt
-    else:
-        return
-
-    log.info('BAR {}'.format(dt))
-    try:
-        _handle_data_rsi_only(context, data)
-    except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    base_currency = list(context.exchanges.values())[0].base_currency.upper()
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(611)
-    results.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
-
-    ax2 = plt.subplot(612, sharex=ax1)
-    results.loc[:, 'price'].plot(ax=ax2)
-    ax2.set_ylabel('{asset} ({base})'.format(
-        asset=context.asset.symbol, base=base_currency
-    ))
-
-    trans = results.loc[[t != [] for t in results.transactions], :]
-    buys = trans.loc[[t[0]['amount'] > 0 for t in trans.transactions], :]
-    sells = trans.loc[[t[0]['amount'] < 0 for t in trans.transactions], :]
-    # buys = results.loc[results['action'] == 1, :]
-    # sells = results.loc[results['action'] == 0, :]
-
-    ax2.plot(
-        buys.index,
-        results.loc[buys.index, 'price'],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        results.loc[sells.index, 'price'],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(613, sharex=ax1)
-    results.loc[:, ['alpha', 'beta']].plot(ax=ax3)
-    ax3.set_ylabel('Alpha / Beta ')
-
-    ax4 = plt.subplot(614, sharex=ax1)
-    results.loc[:, ['starting_cash', 'cash']].plot(ax=ax4)
-    ax4.set_ylabel('Base Currency ({})'.format(base_currency))
-
-    results['algorithm'] = results.loc[:, 'algorithm_period_return']
-
-    ax5 = plt.subplot(615, sharex=ax1)
-    results.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(616, sharex=ax1)
-    results.loc[:, 'rsi'].plot(ax=ax6)
-    ax6.set_ylabel('RSI')
-
-    ax6.plot(
-        buys.index,
-        results.loc[buys.index, 'rsi'],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax6.plot(
-        sells.index,
-        results.loc[sells.index, 'rsi'],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-    pass
-
-
-if __name__ == '__main__':
-    # Backtest
-    run_algorithm(
-        capital_base=0.5,
-        data_frequency='minute',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        algo_namespace=algo_namespace,
-        base_currency='btc',
-        start=pd.to_datetime('2017-9-1', utc=True),
-        end=pd.to_datetime('2017-10-1', utc=True),
-    )

catalyst/examples/simple_loop.py

@@ -1,144 +0,0 @@
-import pandas as pd
-import talib
-from logbook import Logger, INFO
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, record
-from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
-    extract_transactions
-
-log = Logger('simple_loop', level=INFO)
-
-
-def initialize(context):
-    log.info('initializing')
-    context.asset = symbol('eth_btc')
-    context.base_price = None
-
-
-def handle_data(context, data):
-    log.info('handling bar: {}'.format(data.current_dt))
-
-    price = data.current(context.asset, 'close')
-    log.info('got price {price}'.format(price=price))
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='2H'
-    )
-    last_traded = prices.index[-1]
-    log.info('last candle date: {}'.format(last_traded))
-
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # If base_price is not set, we use the current value. This is the
-    # price at the first bar which we reference to calculate price_change.
-    if context.base_price is None:
-        context.base_price = price
-
-    price_change = (price - context.base_price) / context.base_price
-    cash = context.portfolio.cash
-
-    # Now that we've collected all current data for this frame, we use
-    # the record() method to save it. This data will be available as
-    # a parameter of the analyze() function for further analysis.
-    record(
-        price=price,
-        price_change=price_change,
-        cash=cash
-    )
-
-
-def analyze(context, perf):
-    import matplotlib.pyplot as plt
-    log.info('the stats: {}'.format(get_pretty_stats(perf)))
-
-    # The base currency of the algo exchange
-    base_currency = list(context.exchanges.values())[0].base_currency.upper()
-
-    # Plot the portfolio value over time.
-    ax1 = plt.subplot(611)
-    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
-
-    # Plot the price increase or decrease over time.
-    ax2 = plt.subplot(612, sharex=ax1)
-    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-    ax2.set_ylabel('{asset} ({base})'.format(
-        asset=context.asset.symbol, base=base_currency
-    ))
-
-    transaction_df = extract_transactions(perf)
-    if not transaction_df.empty:
-        buy_df = transaction_df[transaction_df['amount'] > 0]
-        sell_df = transaction_df[transaction_df['amount'] < 0]
-        ax2.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'price'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax2.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'price'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-
-    ax4 = plt.subplot(613, sharex=ax1)
-    perf.loc[:, 'cash'].plot(
-        ax=ax4, label='Base Currency ({})'.format(base_currency)
-    )
-    ax4.set_ylabel('Cash ({})'.format(base_currency))
-
-    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-    ax5 = plt.subplot(614, sharex=ax1)
-    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-    pass
-
-
-if __name__ == '__main__':
-    mode = 'live'
-
-    if mode == 'backtest':
-        run_algorithm(
-            capital_base=1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=None,
-            exchange_name='poloniex',
-            algo_namespace='simple_loop',
-            base_currency='eth',
-            data_frequency='minute',
-            start=pd.to_datetime('2017-9-1', utc=True),
-            end=pd.to_datetime('2017-12-1', utc=True),
-        )
-    else:
-        run_algorithm(
-            capital_base=1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=None,
-            exchange_name='binance',
-            live=True,
-            algo_namespace='simple_loop',
-            base_currency='eth',
-            live_graph=False,
-            simulate_orders=True
-        )

catalyst/examples/simple_universe.py

@@ -1,171 +0,0 @@
-"""
-Requires Catalyst version 0.3.0 or above
-Tested on Catalyst version 0.3.3
-
-This example aims to provide an easy way for users to learn how to
-collect data from any given exchange and select a subset of the available
-currency pairs for trading. You simply need to specify the exchange and
-the market (base_currency) that you want to focus on. You will then see
-how to create a universe of assets, and filter it based the market you
-desire.
-
-The example prints out the closing price of all the pairs for a given
-market in a given exchange every 30 minutes. The example also contains
-the OHLCV data with minute-resolution for the past seven days which
-could be used to create indicators. Use this code as the backbone to
-create your own trading strategy.
-
-The lookback_date variable is used to ensure data for a coin existed on
-the lookback period specified.
-
-To run, execute the following two commands in a terminal (inside catalyst
-environment). The first one retrieves all the pricing data needed for this
-script to run (only needs to be run once), and the second one executes this
-script with the parameters specified in the run_algorithm() call at the end
-of the file:
-
-catalyst ingest-exchange -x bitfinex -f minute
-
-python simple_universe.py
-
-"""
-from datetime import timedelta
-
-import numpy as np
-import pandas as pd
-
-from catalyst import run_algorithm
-from catalyst.api import (symbols, )
-from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
-
-
-def initialize(context):
-    context.i = -1  # minute counter
-    context.exchange = list(context.exchanges.values())[0].name.lower()
-    context.base_currency = list(context.exchanges.values())[0].base_currency.lower()
-
-
-def handle_data(context, data):
-    context.i += 1
-    lookback_days = 7  # 7 days
-
-    # current date & time in each iteration formatted into a string
-    now = data.current_dt
-    date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-    lookback_date = now - timedelta(days=lookback_days)
-    # keep only the date as a string, discard the time
-    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
-
-    one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
-    # update universe everyday at midnight
-    if not context.i % one_day_in_minutes:
-        context.universe = universe(context, lookback_date, date)
-
-    # get data every 30 minutes
-    minutes = 30
-
-    # get lookback_days of history data: that is 'lookback' number of bins
-    lookback = int(one_day_in_minutes / minutes * lookback_days)
-    if not context.i % minutes and context.universe:
-        # we iterate for every pair in the current universe
-        for coin in context.coins:
-            pair = str(coin.symbol)
-
-            # Get 30 minute interval OHLCV data. This is the standard data
-            # required for candlestick or indicators/signals. Return Pandas
-            # DataFrames. 30T means 30-minute re-sampling of one minute data.
-            # Adjust it to your desired time interval as needed.
-            opened = fill(data.history(coin,
-                                       'open',
-                                       bar_count=lookback,
-                                       frequency='30T')).values
-            high = fill(data.history(coin,
-                                     'high',
-                                     bar_count=lookback,
-                                     frequency='30T')).values
-            low = fill(data.history(coin,
-                                    'low',
-                                    bar_count=lookback,
-                                    frequency='30T')).values
-            close = fill(data.history(coin,
-                                      'price',
-                                      bar_count=lookback,
-                                      frequency='30T')).values
-            volume = fill(data.history(coin,
-                                       'volume',
-                                       bar_count=lookback,
-                                       frequency='30T')).values
-
-            # close[-1] is the last value in the set, which is the equivalent
-            # to current price (as in the most recent value)
-            # displays the minute price for each pair every 30 minutes
-            print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
-                  '\tV:{v}'.format(
-                    now=now,
-                    pair=pair,
-                    o=opened[-1],
-                    h=high[-1],
-                    l=low[-1],
-                    c=close[-1],
-                    v=volume[-1],
-                  ))
-
-            # -------------------------------------------------------------
-            # --------------- Insert Your Strategy Here -------------------
-            # -------------------------------------------------------------
-
-
-def analyze(context=None, results=None):
-    pass
-
-
-# Get the universe for a given exchange and a given base_currency market
-# Example: Poloniex BTC Market
-def universe(context, lookback_date, current_date):
-    # get all the pairs for the given exchange
-    json_symbols = get_exchange_symbols(context.exchange)
-    # convert into a DataFrame for easier processing
-    df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
-    df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
-                                   axis=1)
-    df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
-                                     axis=1)
-
-    # Filter all the pairs to get only the ones for a given base_currency
-    df = df[df['base_currency'] == context.base_currency]
-
-    # Filter all pairs to ensure that pair existed in the current date range
-    df = df[df.start_date < lookback_date]
-    df = df[df.end_daily >= current_date]
-    context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
-
-    return df.symbol.tolist()
-
-
-# Replace all NA, NAN or infinite values with its nearest value
-def fill(series):
-    if isinstance(series, pd.Series):
-        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-    elif isinstance(series, np.ndarray):
-        return pd.Series(series).replace(
-                     [np.inf, -np.inf], np.nan
-                    ).ffill().bfill().values
-    else:
-        return series
-
-
-if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-11-10', utc=True)
-    end_date = pd.to_datetime('2017-11-13', utc=True)
-
-    performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=100.0,  # amount of base_currency
-                                initialize=initialize,
-                                handle_data=handle_data,
-                                analyze=analyze,
-                                exchange_name='poloniex',
-                                data_frequency='minute',
-                                base_currency='btc',
-                                live=False,
-                                live_graph=False,
-                                algo_namespace='simple_universe')

catalyst/examples/talib_simple.py

@@ -1,366 +0,0 @@
-# Run Command
-# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
-#   -f talib_simple.py -x poloniex
-#
-# Description
-# Simple TALib Example showing how to use various indicators
-# in you strategy. Based loosly on
-# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
-
-import os
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import talib as ta
-from logbook import Logger
-from matplotlib.dates import date2num
-from matplotlib.finance import candlestick_ohlc
-
-from catalyst import run_algorithm
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-)
-from catalyst.exchange.utils.stats_utils import get_pretty_stats
-
-algo_namespace = 'talib_sample'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('Starting TALib Simple Example')
-
-    context.ASSET_NAME = 'BTC_USDT'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.ORDER_SIZE = 10
-    context.SLIPPAGE_ALLOWED = 0.05
-
-    context.swallow_errors = True
-    context.errors = []
-
-    # Bars to look at per iteration should be bigger than SMA_SLOW
-    context.BARS = 365
-    context.COUNT = 0
-
-    # Technical Analysis Settings
-    context.SMA_FAST = 50
-    context.SMA_SLOW = 100
-    context.RSI_PERIOD = 14
-    context.RSI_OVER_BOUGHT = 80
-    context.RSI_OVER_SOLD = 20
-    context.RSI_AVG_PERIOD = 15
-    context.MACD_FAST = 12
-    context.MACD_SLOW = 26
-    context.MACD_SIGNAL = 9
-    context.STOCH_K = 14
-    context.STOCH_D = 3
-    context.STOCH_OVER_BOUGHT = 80
-    context.STOCH_OVER_SOLD = 20
-
-    pass
-
-
-def _handle_data(context, data):
-    # Get price, open, high, low, close
-    prices = data.history(
-        context.asset,
-        bar_count=context.BARS,
-        fields=['price', 'open', 'high', 'low', 'close'],
-        frequency='1d')
-
-    # Create a analysis data frame
-    analysis = pd.DataFrame(index=prices.index)
-
-    # SMA FAST
-    analysis['sma_f'] = ta.SMA(prices.close.as_matrix(), context.SMA_FAST)
-    # SMA SLOW
-    analysis['sma_s'] = ta.SMA(prices.close.as_matrix(), context.SMA_SLOW)
-
-    # Relative Strength Index
-    analysis['rsi'] = ta.RSI(prices.close.as_matrix(), context.RSI_PERIOD)
-    # RSI SMA
-    analysis['sma_r'] = ta.SMA(analysis.rsi.as_matrix(),
-                               context.RSI_AVG_PERIOD)
-
-    # MACD, MACD Signal, MACD Histogram
-    analysis['macd'], analysis['macdSignal'], analysis['macdHist'] = ta.MACD(
-        prices.close.as_matrix(), fastperiod=context.MACD_FAST,
-        slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
-
-    # Stochastics %K %D
-    # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
-    # %D = 3-day SMA of %K
-    analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
-        prices.high.as_matrix(), prices.low.as_matrix(),
-        prices.close.as_matrix(), slowk_period=context.STOCH_K,
-        slowd_period=context.STOCH_D)
-
-    # SMA FAST over SLOW Crossover
-    analysis['sma_test'] = np.where(analysis.sma_f > analysis.sma_s, 1, 0)
-
-    # MACD over Signal Crossover
-    analysis['macd_test'] = np.where((analysis.macd > analysis.macdSignal), 1,
-                                     0)
-
-    # Stochastics OVER BOUGHT & Decreasing
-    analysis['stoch_over_bought'] = np.where(
-        (analysis.stoch_k > context.STOCH_OVER_BOUGHT) & (
-            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
-
-    # Stochastics OVER SOLD & Increasing
-    analysis['stoch_over_sold'] = np.where(
-        (analysis.stoch_k < context.STOCH_OVER_SOLD) & (
-            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
-
-    # RSI OVER BOUGHT & Decreasing
-    analysis['rsi_over_bought'] = np.where(
-        (analysis.rsi > context.RSI_OVER_BOUGHT) & (
-            analysis.rsi < analysis.rsi.shift(1)), 1, 0)
-
-    # RSI OVER SOLD & Increasing
-    analysis['rsi_over_sold'] = np.where(
-        (analysis.rsi < context.RSI_OVER_SOLD) & (
-            analysis.rsi > analysis.rsi.shift(1)), 1, 0)
-
-    # Save the prices and analysis to send to analyze
-    context.prices = prices
-    context.analysis = analysis
-    context.price = data.current(context.asset, 'price')
-
-    makeOrders(context, analysis)
-
-    # Log the values of this bar
-    logAnalysis(analysis)
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    try:
-        _handle_data(context, data)
-    except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, results):
-    # Save results in CSV file
-    filename = os.path.splitext(os.path.basename('talib_simple'))[0]
-    results.to_csv(filename + '.csv')
-
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(results)))
-    chart(context, context.prices, context.analysis, results)
-    pass
-
-
-def makeOrders(context, analysis):
-    if context.asset in context.portfolio.positions:
-
-        # Current position
-        position = context.portfolio.positions[context.asset]
-
-        if (position == 0):
-            log.info('Position Zero')
-            return
-
-        # Cost Basis
-        cost_basis = position.cost_basis
-
-        log.info(
-            'Holdings: {amount} @ {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        # Sell when holding and got sell singnal
-        if isSell(context, analysis):
-            profit = (context.price * position.amount) - (
-                cost_basis * position.amount)
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=context.price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-            log.info(
-                'Sold {amount} @ {price} Profit: {profit}'.format(
-                    amount=position.amount,
-                    price=context.price,
-                    profit=profit
-                )
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        # Buy when not holding and got buy signal
-        if isBuy(context, analysis):
-            order(
-                asset=context.asset,
-                amount=context.ORDER_SIZE,
-                limit_price=context.price * (1 + context.SLIPPAGE_ALLOWED)
-            )
-            log.info(
-                'Bought {amount} @ {price}'.format(
-                    amount=context.ORDER_SIZE,
-                    price=context.price
-                )
-            )
-
-
-def isBuy(context, analysis):
-    # Bullish SMA Crossover
-    if (getLast(analysis, 'sma_test') == 1):
-        # Bullish MACD
-        if (getLast(analysis, 'macd_test') == 1):
-            return True
-
-    # # Bullish Stochastics
-    # if(getLast(analysis, 'stoch_over_sold') == 1):
-    #     return True
-
-    # # Bullish RSI
-    # if(getLast(analysis, 'rsi_over_sold') == 1):
-    #     return True
-
-    return False
-
-
-def isSell(context, analysis):
-    # Bearish SMA Crossover
-    if (getLast(analysis, 'sma_test') == 0):
-        # Bearish MACD
-        if (getLast(analysis, 'macd_test') == 0):
-            return True
-
-    # # Bearish Stochastics
-    # if(getLast(analysis, 'stoch_over_bought') == 0):
-    #     return True
-
-    # # Bearish RSI
-    # if(getLast(analysis, 'rsi_over_bought') == 0):
-    #     return True
-
-    return False
-
-
-def chart(context, prices, analysis, results):
-    results.portfolio_value.plot()
-
-    # Data for matplotlib finance plot
-    dates = date2num(prices.index.to_pydatetime())
-
-    # Create the Open High Low Close Tuple
-    prices_ohlc = [tuple([dates[i],
-                          prices.open[i],
-                          prices.high[i],
-                          prices.low[i],
-                          prices.close[i]]) for i in range(len(dates))]
-
-    fig = plt.figure(figsize=(14, 18))
-
-    # Draw the candle sticks
-    ax1 = fig.add_subplot(411)
-    ax1.set_ylabel(context.ASSET_NAME, size=20)
-    candlestick_ohlc(ax1, prices_ohlc, width=0.4, colorup='g', colordown='r')
-
-    # Draw Moving Averages
-    analysis.sma_f.plot(ax=ax1, c='r')
-    analysis.sma_s.plot(ax=ax1, c='g')
-
-    # RSI
-    ax2 = fig.add_subplot(412)
-    ax2.set_ylabel('RSI', size=12)
-    analysis.rsi.plot(ax=ax2, c='g',
-                      label='Period: ' + str(context.RSI_PERIOD))
-    analysis.sma_r.plot(ax=ax2, c='r',
-                        label='MA: ' + str(context.RSI_AVG_PERIOD))
-    ax2.axhline(y=30, c='b')
-    ax2.axhline(y=50, c='black')
-    ax2.axhline(y=70, c='b')
-    ax2.set_ylim([0, 100])
-    handles, labels = ax2.get_legend_handles_labels()
-    ax2.legend(handles, labels)
-
-    # Draw MACD computed with Talib
-    ax3 = fig.add_subplot(413)
-    ax3.set_ylabel('MACD: ' + str(context.MACD_FAST) + ', ' + str(
-        context.MACD_SLOW) + ', ' + str(context.MACD_SIGNAL), size=12)
-    analysis.macd.plot(ax=ax3, color='b', label='Macd')
-    analysis.macdSignal.plot(ax=ax3, color='g', label='Signal')
-    analysis.macdHist.plot(ax=ax3, color='r', label='Hist')
-    ax3.axhline(0, lw=2, color='0')
-    handles, labels = ax3.get_legend_handles_labels()
-    ax3.legend(handles, labels)
-
-    # Stochastic plot
-    ax4 = fig.add_subplot(414)
-    ax4.set_ylabel('Stoch (k,d)', size=12)
-    analysis.stoch_k.plot(ax=ax4, label='stoch_k:' + str(context.STOCH_K),
-                          color='r')
-    analysis.stoch_d.plot(ax=ax4, label='stoch_d:' + str(context.STOCH_D),
-                          color='g')
-    handles, labels = ax4.get_legend_handles_labels()
-    ax4.legend(handles, labels)
-    ax4.axhline(y=20, c='b')
-    ax4.axhline(y=50, c='black')
-    ax4.axhline(y=80, c='b')
-
-    plt.show()
-
-
-def logAnalysis(analysis):
-    # Log only the last value in the array
-    log.info('- sma_f:          {:.2f}'.format(getLast(analysis, 'sma_f')))
-    log.info('- sma_s:          {:.2f}'.format(getLast(analysis, 'sma_s')))
-
-    log.info('- rsi:            {:.2f}'.format(getLast(analysis, 'rsi')))
-    log.info('- sma_r:          {:.2f}'.format(getLast(analysis, 'sma_r')))
-
-    log.info('- macd:           {:.2f}'.format(getLast(analysis, 'macd')))
-    log.info(
-        '- macdSignal:     {:.2f}'.format(getLast(analysis, 'macdSignal')))
-    log.info('- macdHist:       {:.2f}'.format(getLast(analysis, 'macdHist')))
-
-    log.info('- stoch_k:        {:.2f}'.format(getLast(analysis, 'stoch_k')))
-    log.info('- stoch_d:        {:.2f}'.format(getLast(analysis, 'stoch_d')))
-
-    log.info('- sma_test:       {}'.format(getLast(analysis, 'sma_test')))
-    log.info('- macd_test:      {}'.format(getLast(analysis, 'macd_test')))
-
-    log.info('- stoch_over_bought:   {}'.format(
-        getLast(analysis, 'stoch_over_bought')))
-    log.info(
-        '- stoch_over_sold:   {}'.format(getLast(analysis, 'stoch_over_sold')))
-
-    log.info('- rsi_over_bought:       {}'.format(
-        getLast(analysis, 'rsi_over_bought')))
-    log.info(
-        '- rsi_over_sold:       {}'.format(getLast(analysis, 'rsi_over_sold')))
-
-
-def getLast(arr, name):
-    return arr[name][arr[name].index[-1]]
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        base_currency='usdt',
-        start=pd.to_datetime('2016-11-1', utc=True),
-        end=pd.to_datetime('2017-11-10', utc=True),
-    )

catalyst/exchange/algorithm_exchange.py

@@ -0,0 +1,437 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import signal
+import sys
+import pickle
+from datetime import timedelta
+from time import sleep
+from os import listdir
+from os.path import isfile, join
+
+import logbook
+import pandas as pd
+
+import catalyst.protocol as zp
+from catalyst.algorithm import TradingAlgorithm
+from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
+    BcolzMinuteBarReader
+from catalyst.errors import OrderInBeforeTradingStart
+from catalyst.exchange.exchange_clock import ExchangeClock
+from catalyst.exchange.exchange_errors import (
+    ExchangeRequestError,
+    ExchangePortfolioDataError,
+    ExchangeTransactionError
+)
+from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
+    save_algo_object, get_algo_object, get_algo_folder
+from catalyst.finance.performance.period import calc_period_stats
+from catalyst.gens.tradesimulation import AlgorithmSimulator
+from catalyst.utils.api_support import (
+    api_method,
+    disallowed_in_before_trading_start)
+from catalyst.utils.input_validation import error_keywords
+
+log = logbook.Logger("ExchangeTradingAlgorithm")
+
+
+class ExchangeAlgorithmExecutor(AlgorithmSimulator):
+    def __init__(self, *args, **kwargs):
+        super(self.__class__, self).__init__(*args, **kwargs)
+
+
+class ExchangeTradingAlgorithm(TradingAlgorithm):
+    def __init__(self, *args, **kwargs):
+        self.exchange = kwargs.pop('exchange', None)
+        self.algo_namespace = kwargs.pop('algo_namespace', None)
+        self.orders = {}
+        self.is_running = True
+
+        self.retry_check_open_orders = 5
+        self.retry_update_portfolio = 5
+        self.retry_get_open_orders = 5
+        self.retry_order = 2
+        self.retry_delay = 5
+
+        super(self.__class__, self).__init__(*args, **kwargs)
+        self._create_minute_writer()
+
+        signal.signal(signal.SIGINT, self.signal_handler)
+
+        log.info('exchange trading algorithm successfully initialized')
+
+    def _create_minute_writer(self):
+        root = get_exchange_minute_writer_root(self.exchange.name)
+        filename = os.path.join(root, 'metadata.json')
+
+        if os.path.isfile(filename):
+            writer = BcolzMinuteBarWriter.open(
+                root, self.sim_params.end_session)
+        else:
+            writer = BcolzMinuteBarWriter(
+                rootdir=root,
+                calendar=self.trading_calendar,
+                minutes_per_day=1440,
+                start_session=self.sim_params.start_session,
+                end_session=self.sim_params.end_session,
+                write_metadata=True
+            )
+
+        self.exchange.minute_writer = writer
+        self.exchange.minute_reader = BcolzMinuteBarReader(root)
+
+    def signal_handler(self, signal, frame):
+        self.is_running = False
+
+        log.info('You pressed Ctrl+C!')
+
+        stats = None
+        try:
+            algo_folder = get_algo_folder(self.algo_namespace)
+            folder = join(algo_folder, 'daily_perf')
+            files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+            daily_perf_list = []
+            for item in files:
+                filename = join(folder, item)
+                with open(filename, 'rb') as handle:
+                    daily_perf_list.append(pickle.load(handle))
+
+            stats = pd.DataFrame(daily_perf_list)
+            stats.set_index('period_close', drop=True, inplace=True)
+
+        except Exception as e:
+            log.warn('Unable to compute daily stats: {}'.format(e))
+
+        self.analyze(stats)
+        sys.exit(0)
+
+    def _create_clock(self):
+
+        # The calendar's execution times are the minutes over which we actually
+        # want to run the clock. Typically the execution times simply adhere to
+        # the market open and close times. In the case of the futures calendar,
+        # for example, we only want to simulate over a subset of the full 24
+        # hour calendar, so the execution times dictate a market open time of
+        # 6:31am US/Eastern and a close of 5:00pm US/Eastern.
+
+        # In our case, we are trading around the clock, so the market close
+        # corresponds to the last minute of the day.
+
+        # This method is taken from TradingAlgorithm.
+        # The clock has been replaced to use RealtimeClock
+        # TODO: should we apply a time skew? not sure to understand the utility.
+        return ExchangeClock(
+            self.sim_params.sessions,
+            time_skew=self.exchange.time_skew
+        )
+
+    def _create_generator(self, sim_params):
+        if self.perf_tracker is None:
+            self.perf_tracker = get_algo_object(
+                algo_name=self.algo_namespace,
+                key='perf_tracker'
+            )
+
+        # Call the simulation trading algorithm for side-effects:
+        # it creates the perf tracker
+        TradingAlgorithm._create_generator(self, sim_params)
+        self.trading_client = ExchangeAlgorithmExecutor(
+            self,
+            sim_params,
+            self.data_portal,
+            self._create_clock(),
+            self._create_benchmark_source(),
+            self.restrictions,
+            universe_func=self._calculate_universe
+        )
+
+        return self.trading_client.transform()
+
+    def updated_portfolio(self):
+        """
+        We skip the entire performance tracker business and update the
+        portfolio directly.
+        :return:
+        """
+        return self.exchange.portfolio
+
+    def updated_account(self):
+        return self.exchange.account
+
+    def _update_portfolio(self, attempt_index=0):
+        try:
+            self.exchange.update_portfolio()
+
+            # Applying the updated last_sales_price to the positions
+            # in the performance tracker. This seems a bit redundant
+            # but it will make sense when we have multiple exchange portfolios
+            # feeding into the same performance tracker.
+            tracker = self.perf_tracker.todays_performance.position_tracker
+            for asset in self.exchange.portfolio.positions:
+                position = self.exchange.portfolio.positions[asset]
+                tracker.update_position(
+                    asset=asset,
+                    last_sale_date=position.last_sale_date,
+                    last_sale_price=position.last_sale_price
+                )
+        except ExchangeRequestError as e:
+            log.warn(
+                'update portfolio attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_update_portfolio:
+                sleep(self.retry_delay)
+                self._update_portfolio(attempt_index + 1)
+            else:
+                raise ExchangePortfolioDataError(
+                    data_type='update-portfolio',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def _check_open_orders(self, attempt_index=0):
+        try:
+            return self.exchange.check_open_orders()
+        except ExchangeRequestError as e:
+            log.warn(
+                'check open orders attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_check_open_orders:
+                sleep(self.retry_delay)
+                return self._check_open_orders(attempt_index + 1)
+            else:
+                raise ExchangePortfolioDataError(
+                    data_type='order-status',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def prepare_period_stats(self, start_dt, end_dt):
+        """
+        Creates a dictionary representing the state of the tracker.
+
+
+        I rewrote this in an attempt to better control the stats.
+        I don't want things to happen magically through complex logic
+        pertaining to backtesting.
+
+        """
+        tracker = self.perf_tracker
+        period = tracker.todays_performance
+
+        pos_stats = period.position_tracker.stats()
+        period_stats = calc_period_stats(pos_stats, period.ending_cash)
+
+        stats = dict(
+            period_start=tracker.period_start,
+            period_end=tracker.period_end,
+            capital_base=tracker.capital_base,
+            progress=tracker.progress,
+            ending_value=period.ending_value,
+            ending_exposure=period.ending_exposure,
+            capital_used=period.cash_flow,
+            starting_value=period.starting_value,
+            starting_exposure=period.starting_exposure,
+            starting_cash=period.starting_cash,
+            ending_cash=period.ending_cash,
+            portfolio_value=period.ending_cash + period.ending_value,
+            pnl=period.pnl,
+            returns=period.returns,
+            period_open=period.period_open,
+            period_close=period.period_close,
+            gross_leverage=period_stats.gross_leverage,
+            net_leverage=period_stats.net_leverage,
+            short_exposure=pos_stats.short_exposure,
+            long_exposure=pos_stats.long_exposure,
+            short_value=pos_stats.short_value,
+            long_value=pos_stats.long_value,
+            longs_count=pos_stats.longs_count,
+            shorts_count=pos_stats.shorts_count,
+        )
+
+        # Merging cumulative risk
+        stats.update(tracker.cumulative_risk_metrics.to_dict())
+
+        # Merging latest recorded variables
+        stats.update(self.recorded_vars)
+
+        stats['positions'] = period.position_tracker.get_positions_list()
+
+        # we want the key to be absent, not just empty
+        # Only include transactions for given dt
+        stats['transactions'] = dict()
+        for date in period.processed_transactions:
+            if start_dt <= date < end_dt:
+                stats['transactions'][date] = \
+                    period.processed_transactions[date]
+
+        stats['orders'] = dict()
+        for date in period.orders_by_modified:
+            if start_dt <= date < end_dt:
+                stats['orders'][date] = \
+                    period.orders_by_modified[date]
+
+        return stats
+
+    def handle_data(self, data):
+        if not self.is_running:
+            return
+
+        self._update_portfolio()
+
+        transactions = self._check_open_orders()
+        for transaction in transactions:
+            self.perf_tracker.process_transaction(transaction)
+
+        if self._handle_data:
+            self._handle_data(self, data)
+
+        # Unlike trading controls which remain constant unless placing an
+        # order, account controls can change each bar. Thus, must check
+        # every bar no matter if the algorithm places an order or not.
+        self.validate_account_controls()
+
+        try:
+            # Since the clock runs 24/7, I trying to disable the daily
+            # Performance tracker and keep only minute and cumulative
+            self.perf_tracker.update_performance()
+
+            # TODO: save for future use?
+            minute_stats = self.prepare_period_stats(
+                data.current_dt, data.current_dt + timedelta(minutes=1))
+            log.debug('the minute performance:\n{}'.format(minute_stats))
+
+            today = pd.to_datetime('today', utc=True)
+            daily_stats = self.prepare_period_stats(
+                start_dt=today,
+                end_dt=pd.Timestamp.utcnow()
+            )
+            save_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                obj=daily_stats,
+                rel_path='daily_perf'
+            )
+
+        except Exception as e:
+            log.warn('unable to calculate performance: {}'.format(e))
+
+        try:
+            save_algo_object(
+                algo_name=self.algo_namespace,
+                key='perf_tracker',
+                obj=self.perf_tracker
+            )
+        except Exception as e:
+            log.warn('unable to save minute perfs to disk: {}'.format(e))
+
+        try:
+            save_algo_object(
+                algo_name=self.algo_namespace,
+                key='portfolio_{}'.format(self.exchange.name),
+                obj=self.exchange.portfolio
+            )
+        except Exception as e:
+            log.warn('unable to save portfolio to disk: {}'.format(e))
+
+    def _order(self,
+               asset,
+               amount,
+               limit_price=None,
+               stop_price=None,
+               style=None,
+               attempt_index=0):
+        try:
+            return self.exchange.order(asset, amount, limit_price,
+                                       stop_price,
+                                       style)
+        except ExchangeRequestError as e:
+            log.warn(
+                'order attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_order:
+                sleep(self.retry_delay)
+                return self._order(
+                    asset, amount, limit_price, stop_price, style,
+                    attempt_index + 1)
+            else:
+                raise ExchangeTransactionError(
+                    transaction_type='order',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    @api_method
+    @disallowed_in_before_trading_start(OrderInBeforeTradingStart())
+    def order(self,
+              asset,
+              amount,
+              limit_price=None,
+              stop_price=None,
+              style=None):
+        amount, style = self._calculate_order(asset, amount,
+                                              limit_price, stop_price,
+                                              style)
+
+        order_id = self._order(asset, amount, limit_price, stop_price, style)
+        order = self.portfolio.open_orders[order_id]
+
+        self.perf_tracker.process_order(order)
+        return order
+
+    def round_order(self, amount):
+        """
+        We need fractions with cryptocurrencies
+
+        :param amount:
+        :return:
+        """
+        return amount
+
+    @api_method
+    def batch_market_order(self, share_counts):
+        raise NotImplementedError()
+
+    def _get_open_orders(self, asset=None, attempt_index=0):
+        try:
+            return self.exchange.get_open_orders(asset)
+        except ExchangeRequestError as e:
+            log.warn(
+                'open orders attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_get_open_orders:
+                sleep(self.retry_delay)
+                return self._get_open_orders(asset, attempt_index + 1)
+            else:
+                raise ExchangePortfolioDataError(
+                    data_type='open-orders',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    @error_keywords(sid='Keyword argument `sid` is no longer supported for '
+                        'get_open_orders. Use `asset` instead.')
+    @api_method
+    def get_open_orders(self, asset=None):
+        return self._get_open_orders(asset)
+
+    @api_method
+    def get_order(self, order_id):
+        return self.exchange.get_order(order_id)
+
+    @api_method
+    def cancel_order(self, order_param):
+        order_id = order_param
+        if isinstance(order_param, zp.Order):
+            order_id = order_param.id
+        self.exchange.cancel_order(order_id)

catalyst/exchange/asset_finder_exchange.py

@@ -0,0 +1,91 @@
+from logbook import Logger
+
+log = Logger('AssetFinderExchange')
+
+
+class AssetFinderExchange(object):
+    def __init__(self, exchange):
+        self.exchange = exchange
+        self._asset_cache = {}
+
+    @property
+    def sids(self):
+        """
+        This seems to be used to pre-fetch assets.
+        I don't think that we need this for live-trading.
+        Leaving the list empty.
+        """
+        return list()
+
+    def retrieve_all(self, sids, default_none=False):
+        """
+        Retrieve all assets in `sids`.
+
+        Parameters
+        ----------
+        sids : iterable of int
+            Assets to retrieve.
+        default_none : bool
+            If True, return None for failed lookups.
+            If False, raise `SidsNotFound`.
+
+        Returns
+        -------
+        assets : list[Asset or None]
+            A list of the same length as `sids` containing Assets (or Nones)
+            corresponding to the requested sids.
+
+        Raises
+        ------
+        SidsNotFound
+            When a requested sid is not found and default_none=False.
+        """
+        for sid in sids:
+            if sid in self._asset_cache:
+                log.info('got asset from cache: {}'.format(sid))
+            else:
+                log.info('fetching asset: {}'.format(sid))
+        return list()
+
+    def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
+        """Lookup an asset by symbol.
+
+        Parameters
+        ----------
+        symbol : str
+            The ticker symbol to resolve.
+        as_of_date : datetime or None
+            Look up the last owner of this symbol as of this datetime.
+            If ``as_of_date`` is None, then this can only resolve the equity
+            if exactly one equity has ever owned the ticker.
+        fuzzy : bool, optional
+            Should fuzzy symbol matching be used? Fuzzy symbol matching
+            attempts to resolve differences in representations for
+            shareclasses. For example, some people may represent the ``A``
+            shareclass of ``BRK`` as ``BRK.A``, where others could write
+            ``BRK_A``.
+
+        Returns
+        -------
+        equity : Asset
+            The equity that held ``symbol`` on the given ``as_of_date``, or the
+            only equity to hold ``symbol`` if ``as_of_date`` is None.
+
+        Raises
+        ------
+        SymbolNotFound
+            Raised when no equity has ever held the given symbol.
+        MultipleSymbolsFound
+            Raised when no ``as_of_date`` is given and more than one equity
+            has held ``symbol``. This is also raised when ``fuzzy=True`` and
+            there are multiple candidates for the given ``symbol`` on the
+            ``as_of_date``.
+        """
+        log.info('looking up symbol: {}'.format(symbol))
+
+        if symbol in self._asset_cache:
+            return self._asset_cache[symbol]
+        else:
+            asset = self.exchange.get_asset(symbol)
+            self._asset_cache[symbol] = asset
+            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -0,0 +1,647 @@
+import base64
+import numpy as np
+import hashlib
+import hmac
+import json
+import re
+import time
+
+import pandas as pd
+import pytz
+import requests
+import six
+from catalyst.assets._assets import Asset
+from logbook import Logger
+
+# from websocket import create_connection
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_errors import (
+    ExchangeRequestError,
+    InvalidHistoryFrequencyError
+)
+from catalyst.finance.execution import (MarketOrder,
+                                        LimitOrder,
+                                        StopOrder,
+                                        StopLimitOrder)
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.protocol import Account
+
+# Trying to account for REST api instability
+# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
+requests.adapters.DEFAULT_RETRIES = 20
+
+BITFINEX_URL = 'https://api.bitfinex.com'
+
+log = Logger('Bitfinex')
+warning_logger = Logger('AlgoWarning')
+
+
+class Bitfinex(Exchange):
+    def __init__(self, key, secret, base_currency, portfolio=None):
+        self.url = BITFINEX_URL
+        self.key = key
+        self.secret = secret
+        self.id = 'b'
+        self.name = 'bitfinex'
+        self.assets = {}
+        self.load_assets()
+        self.base_currency = base_currency
+        self._portfolio = portfolio
+        self.minute_writer = None
+        self.minute_reader = None
+
+    def _request(self, operation, data, version='v1'):
+        payload_object = {
+            'request': '/{}/{}'.format(version, operation),
+            'nonce': '{0:f}'.format(time.time() * 1000000),
+            # convert to string
+            'options': {}
+        }
+
+        if data is None:
+            payload_dict = payload_object
+        else:
+            payload_dict = payload_object.copy()
+            payload_dict.update(data)
+
+        payload_json = json.dumps(payload_dict)
+        if six.PY3:
+            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
+        else:
+            payload = base64.b64encode(payload_json)
+
+        m = hmac.new(self.secret, payload, hashlib.sha384)
+        m = m.hexdigest()
+
+        # headers
+        headers = {
+            'X-BFX-APIKEY': self.key,
+            'X-BFX-PAYLOAD': payload,
+            'X-BFX-SIGNATURE': m
+        }
+
+        if data is None:
+            request = requests.get(
+                '{url}/{version}/{operation}'.format(
+                    url=self.url,
+                    version=version,
+                    operation=operation
+                ), data={},
+                headers=headers)
+        else:
+            request = requests.post(
+                '{url}/{version}/{operation}'.format(
+                    url=self.url,
+                    version=version,
+                    operation=operation
+                ),
+                headers=headers)
+
+        return request
+
+    def _get_v2_symbol(self, asset):
+        pair = asset.symbol.split('_')
+        symbol = 't' + pair[0].upper() + pair[1].upper()
+        return symbol
+
+    def _get_v2_symbols(self, assets):
+        """
+        Workaround to support Bitfinex v2
+        TODO: Might require a separate asset dictionary
+
+        :param assets:
+        :return:
+        """
+
+        v2_symbols = []
+        for asset in assets:
+            v2_symbols.append(self._get_v2_symbol(asset))
+
+        return v2_symbols
+
+    def _create_order(self, order_status):
+        """
+        Create a Catalyst order object from a Bitfinex order dictionary
+        :param order_status:
+        :return: Order
+        """
+        if order_status['is_cancelled']:
+            status = ORDER_STATUS.CANCELLED
+        elif not order_status['is_live']:
+            log.info('found executed order {}'.format(order_status))
+            status = ORDER_STATUS.FILLED
+        else:
+            status = ORDER_STATUS.OPEN
+
+        amount = float(order_status['original_amount'])
+        filled = float(order_status['executed_amount'])
+        is_buy = (amount > 0)
+
+        price = float(order_status['price'])
+        order_type = order_status['type']
+
+        stop_price = None
+        limit_price = None
+
+        # TODO: is this comprehensive enough?
+        if order_type.endswith('limit'):
+            limit_price = price
+        elif order_type.endswith('stop'):
+            stop_price = price
+
+        executed_price = float(order_status['avg_execution_price'])
+
+        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
+        commission = None
+
+        # TODO: zipline likes rounded dates to match statistics, is this ok?
+        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
+        date = pytz.utc.localize(date)
+        order = Order(
+            dt=date,
+            asset=self.assets[order_status['symbol']],
+            amount=amount,
+            stop=stop_price,
+            limit=limit_price,
+            filled=filled,
+            id=order_status['id'],
+            commission=commission
+        )
+        order.status = status
+
+        return order, executed_price
+
+    def update_portfolio(self):
+        """
+        Update the portfolio cash and position balances based on the
+        latest ticker prices.
+
+        :return:
+        """
+        try:
+            response = self._request('balances', None)
+            balances = response.json()
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'message' in balances:
+            raise ExchangeRequestError(
+                error='unable to fetch balance {}'.format(balances['message'])
+            )
+
+        base_position = None
+        for position in balances:
+            if not base_position and position['type'] == 'exchange' \
+                    and position['currency'] == self.base_currency:
+                base_position = position
+
+        if position is None:
+            raise ValueError(
+                error='Base currency %s not found in portfolio' % self.base_currency
+            )
+
+        portfolio = self._portfolio
+        portfolio.cash = float(base_position['available'])
+        if portfolio.starting_cash is None:
+            portfolio.starting_cash = portfolio.cash
+
+        if portfolio.positions:
+            assets = portfolio.positions.keys()
+            tickers = self.tickers(assets)
+            portfolio.positions_value = 0.0
+            for ticker in tickers:
+                # TODO: convert if the position is not in the base currency
+                position = portfolio.positions[ticker['asset']]
+                position.last_sale_price = ticker['last_price']
+                position.last_sale_date = ticker['timestamp']
+
+                portfolio.positions_value += \
+                    position.amount * position.last_sale_price
+                portfolio.portfolio_value = \
+                    portfolio.positions_value + portfolio.cash
+
+    @property
+    def portfolio(self):
+        """
+        Return the Portfolio
+
+        :return:
+        """
+        # if self._portfolio is None:
+        #     portfolio = ExchangePortfolio(
+        #         start_date=pd.Timestamp.utcnow()
+        #     )
+        #     self.store.portfolio = portfolio
+        #     self.update_portfolio()
+        #
+        #     portfolio.starting_cash = portfolio.cash
+        # else:
+        #     portfolio = self.store.portfolio
+
+        return self._portfolio
+
+    @property
+    def account(self):
+        account = Account()
+
+        account.settled_cash = None
+        account.accrued_interest = None
+        account.buying_power = None
+        account.equity_with_loan = None
+        account.total_positions_value = None
+        account.total_positions_exposure = None
+        account.regt_equity = None
+        account.regt_margin = None
+        account.initial_margin_requirement = None
+        account.maintenance_margin_requirement = None
+        account.available_funds = None
+        account.excess_liquidity = None
+        account.cushion = None
+        account.day_trades_remaining = None
+        account.leverage = None
+        account.net_leverage = None
+        account.net_liquidation = None
+
+        return account
+
+    @property
+    def positions(self):
+        return self.portfolio.positions
+
+    @property
+    def time_skew(self):
+        # TODO: research the time skew conditions
+        return pd.Timedelta('0s')
+
+    def subscribe_to_market_data(self, symbol):
+        pass
+
+    def get_candles(self, data_frequency, assets, bar_count=None):
+        """
+        Retrieve OHLVC candles from Bitfinex
+
+        :param data_frequency:
+        :param assets:
+        :param bar_count:
+        :return:
+
+        Available Frequencies
+        ---------------------
+        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
+         '1M'
+        """
+
+        # TODO: use BcolzMinuteBarReader to read from cache
+        freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
+        if freq_match:
+            number = int(freq_match.group(1))
+            unit = freq_match.group(2)
+
+            if unit == 'd':
+                converted_unit = 'D'
+            else:
+                converted_unit = unit
+
+            frequency = '{}{}'.format(number, converted_unit)
+            allowed_frequencies = ['1m', '5m', '15m', '30m', '1h', '3h', '6h',
+                                   '12h', '1D', '7D', '14D', '1M']
+
+            if frequency not in allowed_frequencies:
+                raise InvalidHistoryFrequencyError(
+                    frequency=data_frequency
+                )
+        elif data_frequency == 'minute':
+            frequency = '1m'
+        elif data_frequency == 'daily':
+            frequency = '1D'
+        else:
+            raise InvalidHistoryFrequencyError(
+                frequency=data_frequency
+            )
+
+        # Making sure that assets are iterable
+        asset_list = [assets] if isinstance(assets, Asset) else assets
+        ohlc_list = dict()
+        for asset in asset_list:
+            symbol = self._get_v2_symbol(asset)
+            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
+                url=self.url,
+                frequency=frequency,
+                symbol=symbol
+            )
+
+            if bar_count:
+                is_list = True
+                url += '/hist?limit={}'.format(int(bar_count))
+            else:
+                is_list = False
+                url += '/last'
+
+            try:
+                response = requests.get(url)
+            except Exception as e:
+                raise ExchangeRequestError(error=e)
+
+            if 'error' in response.content:
+                raise ExchangeRequestError(
+                    error='Unable to retrieve candles: {}'.format(
+                        response.content)
+                )
+
+            candles = response.json()
+
+            def ohlc_from_candle(candle):
+                ohlc = dict(
+                    open=np.float64(candle[1]),
+                    high=np.float64(candle[3]),
+                    low=np.float64(candle[4]),
+                    close=np.float64(candle[2]),
+                    volume=np.float64(candle[5]),
+                    price=np.float64(candle[2]),
+                    last_traded=pd.Timestamp.utcfromtimestamp(
+                        candle[0] / 1000.0),
+                    minute_dt=pd.Timestamp.utcnow().floor('1 min')
+                )
+                return ohlc
+
+            if is_list:
+                ohlc_bars = []
+                # We can to list candles from old to new
+                for candle in reversed(candles):
+                    ohlc = ohlc_from_candle(candle)
+                    ohlc_bars.append(ohlc)
+
+                ohlc_list[asset] = ohlc_bars
+
+            else:
+                ohlc = ohlc_from_candle(candles)
+                ohlc_list[asset] = ohlc
+
+        return ohlc_list[assets] \
+            if isinstance(assets, Asset) else ohlc_list
+
+    def order(self, asset, amount, limit_price, stop_price, style):
+        """Place an order.
+
+        Parameters
+        ----------
+        asset : Asset
+            The asset that this order is for.
+        amount : int
+            The amount of shares to order. If ``amount`` is positive, this is
+            the number of shares to buy or cover. If ``amount`` is negative,
+            this is the number of shares to sell or short.
+        limit_price : float, optional
+            The limit price for the order.
+        stop_price : float, optional
+            The stop price for the order.
+        style : ExecutionStyle, optional
+            The execution style for the order.
+
+        Returns
+        -------
+        order_id : str or None
+            The unique identifier for this order, or None if no order was
+            placed.
+
+        Notes
+        -----
+        The ``limit_price`` and ``stop_price`` arguments provide shorthands for
+        passing common execution styles. Passing ``limit_price=N`` is
+        equivalent to ``style=LimitOrder(N)``. Similarly, passing
+        ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing
+        ``limit_price=N`` and ``stop_price=M`` is equivalent to
+        ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style``
+        and ``limit_price`` or ``stop_price``.
+
+        Bitfinex Order Types
+        --------------------
+        LIMIT, MARKET, STOP, TRAILING STOP,
+        EXCHANGE MARKET, EXCHANGE LIMIT, EXCHANGE STOP,
+        EXCHANGE TRAILING STOP, FOK, EXCHANGE FOK.
+
+        See Also
+        --------
+        :class:`catalyst.finance.execution.ExecutionStyle`
+        :func:`catalyst.api.order_value`
+        :func:`catalyst.api.order_percent`
+        """
+        if amount == 0:
+            log.warn('skipping order amount of 0')
+            return None
+
+        base_currency = asset.symbol.split('_')[1]
+        if base_currency.lower() != self.base_currency.lower():
+            raise NotImplementedError(
+                'Currency pairs must share their base with the exchange.'
+            )
+
+        is_buy = (amount > 0)
+
+        if isinstance(style, MarketOrder):
+            order_type = 'market'
+        elif isinstance(style, LimitOrder):
+            order_type = 'limit'
+            price = limit_price
+        elif isinstance(style, StopOrder):
+            order_type = 'stop'
+            price = stop_price
+        elif isinstance(style, StopLimitOrder):
+            log.warn('using limit order instead of stop/limit')
+            # TODO: Not sure how to do this with the api. Investigate.
+            order_type = 'limit'
+            price = limit_price
+        else:
+            raise NotImplementedError('%s orders not available' % style)
+
+        log.debug(
+            'ordering {amount} {symbol} for {price}'.format(
+                amount=amount,
+                symbol=asset.symbol,
+                price=price
+            )
+        )
+
+        exchange_symbol = self.get_symbol(asset)
+        req = dict(
+            symbol=exchange_symbol,
+            amount=str(float(abs(amount))),
+            price=str(float(price)),
+            side='buy' if is_buy else 'sell',
+            type='exchange ' + order_type,  # TODO: support margin trades
+            exchange=self.name,
+            is_hidden=False,
+            is_postonly=False,
+            use_all_available=0,
+            ocoorder=False,
+            buy_price_oco=0,
+            sell_price_oco=0
+        )
+
+        date = pd.Timestamp.utcnow()
+        try:
+            response = self._request('order/new', req)
+            exchange_order = response.json()
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'message' in exchange_order:
+            raise ExchangeRequestError(
+                error='unable to create Bitfinex order {}'.format(
+                    exchange_order['message'])
+            )
+
+        order_id = exchange_order['id']
+        order = Order(
+            dt=date,
+            asset=asset,
+            amount=amount,
+            stop=style.get_stop_price(is_buy),
+            limit=style.get_limit_price(is_buy),
+            id=order_id
+        )
+        # TODO: is this required?
+        order.broker_order_id = order_id
+
+        self.portfolio.create_order(order)
+
+        return order_id
+
+    def get_open_orders(self, asset=None):
+        """Retrieve all of the current open orders.
+
+        Parameters
+        ----------
+        asset : Asset
+            If passed and not None, return only the open orders for the given
+            asset instead of all open orders.
+
+        Returns
+        -------
+        open_orders : dict[list[Order]] or list[Order]
+            If no asset is passed this will return a dict mapping Assets
+            to a list containing all the open orders for the asset.
+            If an asset is passed then this will return a list of the open
+            orders for this asset.
+        """
+        try:
+            response = self._request('orders', None)
+            order_statuses = response.json()
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'message' in order_statuses:
+            raise ExchangeRequestError(
+                error='Unable to retrieve open orders: {}'.format(
+                    order_statuses['message'])
+            )
+
+        orders = list()
+        for order_status in order_statuses:
+            order, = self._create_order(order_status)
+            if asset is None or asset == order.sid:
+                orders.append(order)
+
+        return orders
+
+    def get_order(self, order_id):
+        """Lookup an order based on the order id returned from one of the
+        order functions.
+
+        Parameters
+        ----------
+        order_id : str
+            The unique identifier for the order.
+
+        Returns
+        -------
+        order : Order
+            The order object.
+        """
+        try:
+            response = self._request(
+                'order/status', {'order_id': int(order_id)})
+            order_status = response.json()
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'message' in order_status:
+            raise ExchangeRequestError(
+                error='Unable to retrieve order status: {}'.format(
+                    order_status['message'])
+            )
+        return self._create_order(order_status)
+
+    def cancel_order(self, order_param):
+        """Cancel an open order.
+
+        Parameters
+        ----------
+        order_param : str or Order
+            The order_id or order object to cancel.
+        """
+        order_id = order_param.id \
+            if isinstance(order_param, Order) else order_param
+
+        try:
+            response = self._request('order/cancel', {'order_id': order_id})
+            status = response.json()
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'message' in status:
+            raise ExchangeRequestError(
+                error='Unable to cancel order: {} {}'.format(
+                    order_id, status['message'])
+            )
+
+    def tickers(self, assets):
+        """
+        Fetch ticket data for assets
+        https://docs.bitfinex.com/v2/reference#rest-public-tickers
+
+        :param assets:
+        :return:
+        """
+        symbols = self._get_v2_symbols(assets)
+        log.debug('fetching tickers {}'.format(symbols))
+
+        try:
+            response = requests.get(
+                '{url}/v2/tickers?symbols={symbols}'.format(
+                    url=self.url,
+                    symbols=','.join(symbols),
+                )
+            )
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        if 'error' in response.content:
+            raise ExchangeRequestError(
+                error='Unable to retrieve tickers: {}'.format(
+                    response.content)
+            )
+
+        tickers = response.json()
+
+        formatted_tickers = []
+        for index, ticker in enumerate(tickers):
+            if not len(ticker) == 11:
+                raise ExchangeRequestError(
+                    error='Invalid ticker in response: {}'.format(ticker)
+                )
+
+            tick = dict(
+                asset=assets[index],
+                timestamp=pd.Timestamp.utcnow(),
+                bid=ticker[1],
+                ask=ticker[3],
+                last_price=ticker[7],
+                low=ticker[10],
+                high=ticker[9],
+                volume=ticker[8],
+            )
+            formatted_tickers.append(tick)
+
+        log.debug('got tickers {}'.format(formatted_tickers))
+        return formatted_tickers

catalyst/exchange/data_portal_exchange.py

@@ -0,0 +1,121 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import sleep
+
+from logbook import Logger
+
+from catalyst.data.data_portal import DataPortal
+from catalyst.exchange.exchange_errors import (
+    ExchangeRequestError,
+    ExchangeBarDataError
+)
+
+log = Logger('DataPortalExchange')
+
+
+class DataPortalExchange(DataPortal):
+    def __init__(self, exchange, *args, **kwargs):
+        self.exchange = exchange
+
+        # TODO: put somewhere accessible by each algo
+        self.retry_get_history_window = 5
+        self.retry_get_spot_value = 5
+        self.retry_delay = 5
+
+        super(DataPortalExchange, self).__init__(*args, **kwargs)
+
+    def _get_history_window(self,
+                            assets,
+                            end_dt,
+                            bar_count,
+                            frequency,
+                            field,
+                            data_frequency,
+                            ffill=True,
+                            attempt_index=0):
+        try:
+            return self.exchange.get_history_window(
+                assets,
+                end_dt,
+                bar_count,
+                frequency,
+                field,
+                data_frequency,
+                ffill)
+        except ExchangeRequestError as e:
+            log.warn(
+                'get history attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_get_history_window:
+                sleep(self.retry_delay)
+                return self._get_history_window(assets,
+                                                end_dt,
+                                                bar_count,
+                                                frequency,
+                                                field,
+                                                data_frequency,
+                                                ffill,
+                                                attempt_index + 1)
+            else:
+                raise ExchangeBarDataError(
+                    data_type='history',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def get_history_window(self,
+                           assets,
+                           end_dt,
+                           bar_count,
+                           frequency,
+                           field,
+                           data_frequency,
+                           ffill=True):
+        return self._get_history_window(assets,
+                                        end_dt,
+                                        bar_count,
+                                        frequency,
+                                        field,
+                                        data_frequency,
+                                        ffill)
+
+    def _get_spot_value(self, assets, field, dt, data_frequency,
+                        attempt_index=0):
+        try:
+            return self.exchange.get_spot_value(assets, field, dt,
+                                                data_frequency)
+        except ExchangeRequestError as e:
+            log.warn(
+                'get spot value attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_get_spot_value:
+                sleep(self.retry_delay)
+                return self._get_spot_value(assets, field, dt, data_frequency,
+                                            attempt_index + 1)
+            else:
+                raise ExchangeBarDataError(
+                    data_type='spot',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def get_spot_value(self, assets, field, dt, data_frequency):
+        return self._get_spot_value(assets, field, dt, data_frequency)
+
+    def get_adjusted_value(self, asset, field, dt,
+                           perspective_dt,
+                           data_frequency,
+                           spot_value=None):
+        # TODO: does this pertain to cryptocurrencies?
+        raise NotImplementedError("get_adjusted_value is not implemented yet!")

catalyst/exchange/exchange_asset_finder.py

@@ -1,179 +0,0 @@
-import pandas as pd
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.utils.factory import find_exchanges
-from logbook import Logger
-
-log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
-
-
-class ExchangeAssetFinder(object):
-    def __init__(self, exchanges):
-        self.exchanges = exchanges
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        all_sids = []
-        for exchange_name in self.exchanges:
-            # This is what initializes each exchanges at the beginning
-            # of an algo
-            exchange = self.exchanges[exchange_name]
-            exchange.init()
-
-            all_sids += [asset.sid for asset in exchange.assets]
-
-        sids = list(set(all_sids))
-        return sids
-
-    def retrieve_asset(self, sid, default_none=False):
-        """
-        Retrieve the first Asset found for a given sid.
-        """
-        asset = None
-        for exchange_name in self.exchanges:
-            if asset is not None:
-                break
-
-            exchange = self.exchanges[exchange_name]
-            assets = [asset for asset in exchange.assets if asset.sid == sid]
-            if assets:
-                asset = assets[0]
-
-        return asset
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        assets = []
-        for exchange_name in self.exchanges:
-            exchange = self.exchanges[exchange_name]
-            xas = [asset for asset in exchange.assets if asset.sid in sids]
-            assets += xas
-
-        return assets
-
-    def lookup_symbol(self, symbol, exchange, data_frequency=None,
-                      as_of_date=None, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
-
-        return exchange.get_asset(symbol, data_frequency)
-
-    def lifetimes(self, dates, include_start_date):
-        """
-        Compute a DataFrame representing asset lifetimes for the specified date
-        range.
-
-        Parameters
-        ----------
-        dates : pd.DatetimeIndex
-            The dates for which to compute lifetimes.
-        include_start_date : bool
-            Whether or not to count the asset as alive on its start_date.
-
-            This is useful in a backtesting context where `lifetimes` is being
-            used to signify "do I have data for this asset as of the morning of
-            this date?"  For many financial metrics, (e.g. daily close), data
-            isn't available for an asset until the end of the asset's first
-            day.
-
-        Returns
-        -------
-        lifetimes : pd.DataFrame
-            A frame of dtype bool with `dates` as index and an Int64Index of
-            assets as columns.  The value at `lifetimes.loc[date, asset]` will
-            be True iff `asset` existed on `date`.  If `include_start_date` is
-            False, then lifetimes.loc[date, asset] will be false when date ==
-            asset.start_date.
-
-        See Also
-        --------
-        numpy.putmask
-        catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
-        """
-        exchanges = find_exchanges(features=['minuteBundle'])
-        if not exchanges:
-            raise ValueError('exchange with minute bundles not found')
-
-        # TODO: find a way to support multiple exchanges
-        exchange = exchanges[0]
-        # Using a single exchange for now because are not unique for the
-        # same asset in different exchanges. I'd like to avoid binding
-        # pipeline to a single exchange.
-        exchange.init()
-
-        data = []
-        for dt in dates:
-            exists = []
-
-            for asset in exchange.assets:
-                if include_start_date:
-                    condition = (asset.start_date <= dt < asset.end_minute)
-
-                else:
-                    condition = (asset.start_date < dt < asset.end_minute)
-
-                exists.append(condition)
-
-            data.append(exists)
-
-        sids = [asset.sid for asset in exchange.assets]
-        df = pd.DataFrame(data, index=dates, columns=exchange.assets)
-
-        return df

catalyst/exchange/exchange_bcolz.py

@@ -1,98 +0,0 @@
-import numpy as np
-
-from catalyst import get_calendar
-from catalyst.data.minute_bars import BcolzMinuteBarReader, \
-    BcolzMinuteBarWriter
-
-
-class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
-    def __init__(self, *args, **kwargs):
-        self._data_frequency = kwargs.pop('data_frequency', None)
-        kwargs.pop('minutes_per_day', None)
-        kwargs.pop('calendar', None)
-
-        end_session = kwargs.pop('end_session', None)
-        if end_session is not None:
-            end_session = end_session.floor('1d')
-
-        minutes_per_day = 1440 if self._data_frequency == 'minute' else 1
-        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000)
-        calendar = get_calendar('OPEN')
-
-        super(BcolzExchangeBarWriter, self) \
-            .__init__(*args, **dict(kwargs,
-                                    minutes_per_day=minutes_per_day,
-                                    default_ohlc_ratio=default_ohlc_ratio,
-                                    calendar=calendar,
-                                    end_session=end_session
-                                    ))
-
-
-class BcolzExchangeBarReader(BcolzMinuteBarReader):
-    def __init__(self, *args, **kwargs):
-        self._data_frequency = kwargs.pop('data_frequency', None)
-
-        super(BcolzExchangeBarReader, self).__init__(*args, **kwargs)
-
-    @property
-    def data_frequency(self):
-        return self._data_frequency
-
-    def load_raw_arrays(self, fields, start_dt, end_dt, sids):
-        """
-        Parameters
-        ----------
-        fields : list of str
-           'open', 'high', 'low', 'close', or 'volume'
-        start_dt: Timestamp
-           Beginning of the window range.
-        end_dt: Timestamp
-           End of the window range.
-        sids : list of int
-           The asset identifiers in the window.
-
-        Returns
-        -------
-        list of np.ndarray
-            A list with an entry per field of ndarrays with shape
-            (minutes in range, sids) with a dtype of float64, containing the
-            values for the respective field over start and end dt range.
-        """
-        start_idx = self._find_position_of_minute(start_dt)
-        end_idx = self._find_position_of_minute(end_dt)
-
-        periods = self.calendar.minutes_in_range(start_dt, end_dt) \
-            if self.data_frequency == 'minute' \
-            else self.calendar.sessions_in_range(start_dt, end_dt)
-
-        num_days = len(periods)
-        shape = num_days, len(sids)
-
-        all_fields = fields[:]
-        if len(all_fields) == 1 and all_fields[0] == 'volume':
-            all_fields.insert(0, 'close')
-
-        mask = None
-        data = []
-        for field in all_fields:
-            if field != 'volume':
-                out = np.full(shape, np.nan)
-            else:
-                out = np.zeros(shape, dtype=np.float64)
-
-            for i, sid in enumerate(sids):
-                carray = self._open_minute_file(field, sid)
-                a = carray[start_idx:end_idx + 1]
-
-                if mask is None:
-                    mask = a != 0
-
-                inverse_ratio = self._ohlc_ratio_inverse_for_sid(sid)
-                out[:len(mask), i][mask] = (
-                    a[mask] * inverse_ratio
-                )
-
-            if field in fields:
-                data.append(out)
-
-        return data

catalyst/exchange/exchange_blotter.py

@@ -1,277 +0,0 @@
-import numpy as np
-import pandas as pd
-from logbook import Logger
-from redo import retry
-
-from catalyst.assets._assets import TradingPair
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import ExchangeRequestError
-from catalyst.finance.blotter import Blotter
-from catalyst.finance.commission import CommissionModel
-from catalyst.finance.order import ORDER_STATUS
-from catalyst.finance.slippage import SlippageModel
-from catalyst.finance.transaction import create_transaction, Transaction
-from catalyst.utils.input_validation import expect_types
-
-log = Logger('exchange_blotter', level=LOG_LEVEL)
-
-
-class TradingPairFeeSchedule(CommissionModel):
-    """
-    Calculates a commission for a transaction based on a per percentage fee.
-
-    Parameters
-    ----------
-    maker : float, optional
-        The percentage maker fee.
-
-    taker: float, optional
-        The percentage taker fee.
-    """
-
-    def __init__(self, maker=None, taker=None):
-        self.maker = maker
-        self.taker = taker
-
-    def __repr__(self):
-        return (
-            '{class_name}(maker={maker}, '
-            'taker={taker})'.format(
-                class_name=self.__class__.__name__,
-                maker=self.maker,
-                taker=self.taker,
-            )
-        )
-
-    def get_maker_taker(self, asset):
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
-        return maker, taker
-
-    def calculate(self, order, transaction):
-        """
-        Calculate the final fee based on the order parameters.
-
-        :param order: Order
-        :param transaction: Transaction
-
-        :return float:
-            The total commission.
-        """
-        cost = abs(transaction.amount) * transaction.price
-
-        asset = order.asset
-        maker, taker = self.get_maker_taker(asset)
-
-        multiplier = taker
-        if order.limit is not None:
-            multiplier = maker \
-                if ((order.amount > 0 and order.limit < transaction.price)
-                    or (order.amount < 0 and order.limit > transaction.price)) \
-                and order.limit_reached else taker
-
-        fee = cost * multiplier
-        return fee
-
-
-class TradingPairFixedSlippage(SlippageModel):
-    """
-    Model slippage as a fixed spread.
-
-    Parameters
-    ----------
-    spread : float, optional
-        spread / 2 will be added to buys and subtracted from sells.
-    """
-
-    def __init__(self, spread=0.0001):
-        super(TradingPairFixedSlippage, self).__init__()
-        self.spread = spread
-
-    def __repr__(self):
-        return '{class_name}(spread={spread})'.format(
-            class_name=self.__class__.__name__, spread=self.spread,
-        )
-
-    def simulate(self, data, asset, orders_for_asset):
-        self._volume_for_bar = 0
-        price = data.current(asset, 'close')
-
-        dt = data.current_dt
-        for order in orders_for_asset:
-            if order.open_amount == 0:
-                continue
-
-            order.check_triggers(price, dt)
-            if not order.triggered:
-                log.info(
-                    'order has not reached the trigger at current '
-                    'price {}'.format(price)
-                )
-                continue
-
-            execution_price, execution_volume = self.process_order(data, order)
-            if execution_price is not None:
-                transaction = create_transaction(
-                    order, dt, execution_price, execution_volume
-                )
-
-                self._volume_for_bar += abs(transaction.amount)
-                yield order, transaction
-
-    def process_order(self, data, order):
-        price = data.current(order.asset, 'close')
-
-        if order.amount > 0:
-            # Buy order
-            adj_price = price * (1 + self.spread)
-        else:
-            # Sell order
-            adj_price = price * (1 - self.spread)
-
-        log.debug('added slippage to price: {} => {}'.format(price, adj_price))
-
-        return adj_price, order.amount
-
-
-class ExchangeBlotter(Blotter):
-    def __init__(self, *args, **kwargs):
-        self.simulate_orders = kwargs.pop('simulate_orders', False)
-        self.attempts = kwargs.pop('attempts', False)
-
-        self.exchanges = kwargs.pop('exchanges', None)
-        if not self.exchanges:
-            raise ValueError(
-                'ExchangeBlotter must have an `exchanges` attribute.'
-            )
-
-        super(ExchangeBlotter, self).__init__(*args, **kwargs)
-
-        # Using the equity models for now
-        # We may be able to define more sophisticated models based on the fee
-        # structure of each exchange.
-        self.slippage_models = {
-            TradingPair: TradingPairFixedSlippage()
-        }
-        self.commission_models = {
-            TradingPair: TradingPairFeeSchedule()
-        }
-
-    def exchange_order(self, asset, amount, style=None):
-        exchange = self.exchanges[asset.exchange]
-        return exchange.order(
-            asset, amount, style
-        )
-
-    @expect_types(asset=TradingPair)
-    def order(self, asset, amount, style, order_id=None):
-        log.debug('ordering {} {}'.format(amount, asset.symbol))
-        if amount == 0:
-            log.warn('skipping 0 amount orders')
-            return None
-
-        if self.simulate_orders:
-            return super(ExchangeBlotter, self).order(
-                asset, amount, style, order_id
-            )
-
-        else:
-            order = retry(
-                action=self.exchange_order,
-                attempts=self.attempts['order_attempts'],
-                sleeptime=self.attempts['retry_sleeptime'],
-                retry_exceptions=(ExchangeRequestError,),
-                cleanup=lambda: log.warn('Ordering again.'),
-                args=(asset, amount, style),
-            )
-
-            self.open_orders[order.asset].append(order)
-            self.orders[order.id] = order
-            self.new_orders.append(order)
-
-            return order.id
-
-    def check_open_orders(self):
-        """
-        Loop through the list of open orders in the Portfolio object.
-        For each executed order found, create a transaction and apply to the
-        Portfolio.
-
-        Returns
-        -------
-        list[Transaction]
-
-        """
-        for asset in self.open_orders:
-            exchange = self.exchanges[asset.exchange]
-
-            for order in self.open_orders[asset]:
-                log.debug('found open order: {}'.format(order.id))
-
-                transactions = exchange.process_order(order)
-                # This is a temporary measure, we should really update all
-                # trades, not just when the order gets filled. I just think
-                # that this is safer until we have a robust way to track
-                # the trades already processed by the algo. We can't loose
-                # them if the algo shuts down.
-                if transactions and order.status == ORDER_STATUS.FILLED:
-                    avg_price = np.average(
-                        a=[t.price for t in transactions],
-                        weights=[t.amount for t in transactions],
-                    )
-                    ostatus = 'filled' if order.open_amount == 0 else 'partial'
-                    log.info(
-                        '{} order {} / {}: {}, avg price: {}'.format(
-                            ostatus,
-                            order.id,
-                            asset.symbol,
-                            order.filled,
-                            avg_price,
-                        )
-                    )
-                    for transaction in transactions:
-                        yield order, transaction
-
-                elif order.status == ORDER_STATUS.CANCELLED:
-                    yield order, None
-
-                else:
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        '{exchange} order {order_id} for {symbol} still open '
-                        'after {delta}'.format(
-                            exchange=exchange.name,
-                            order_id=order.id,
-                            delta=delta,
-                            symbol=order.asset.symbol,
-                        )
-                    )
-
-    def get_exchange_transactions(self):
-        closed_orders = []
-        transactions = []
-        commissions = []
-
-        for order, txn in self.check_open_orders():
-            order.dt = txn.dt
-            transactions.append(txn)
-
-            if not order.open:
-                closed_orders.append(order)
-
-        return transactions, commissions, closed_orders
-
-    def get_transactions(self, bar_data):
-        if self.simulate_orders:
-            return super(ExchangeBlotter, self).get_transactions(bar_data)
-
-        else:
-            return retry(
-                action=self.get_exchange_transactions,
-                attempts=self.attempts['get_transactions_attempts'],
-                sleeptime=self.attempts['retry_sleeptime'],
-                retry_exceptions=(ExchangeRequestError,),
-                cleanup=lambda: log.warn(
-                    'Fetching exchange transactions again.'
-                )
-            )

catalyst/exchange/exchange_clock.py

@@ -14,24 +14,24 @@
 from time import sleep
 
 import pandas as pd
-from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START
+    SESSION_START,
+    MINUTE_END,
+    SESSION_END
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock', level=LOG_LEVEL)
+log = Logger('ExchangeClock')
 
 
-class SimpleClock(object):
+class ExchangeClock(object):
     """Realtime clock for live trading.
 
     This class is a drop-in replacement for
     :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
 
-    This is a stripped down version because crypto exchanges run
+    This is a stripped down version because crypto exchanges run around the clock.
-    around the clock.
 
     The :param:`time_skew` parameter represents the time difference between
     the Broker and the live trading machine's clock.

catalyst/exchange/exchange_data_portal.py

@@ -1,373 +0,0 @@
-import abc
-
-import numpy as np
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from catalyst.constants import LOG_LEVEL, AUTO_INGEST
-from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    PricingDataNotLoadedError)
-from catalyst.exchange.utils.exchange_utils import resample_history_df, \
-    group_assets_by_exchange
-from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
-from logbook import Logger
-from redo import retry
-
-log = Logger('DataPortalExchange', level=LOG_LEVEL)
-
-
-class DataPortalExchangeBase(DataPortal):
-    def __init__(self, *args, **kwargs):
-        self.attempts = dict(
-            get_spot_value_attempts=5,
-            get_history_window_attempts=5,
-            retry_sleeptime=5,
-        )
-
-        super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
-
-    def _get_history_window(self,
-                            assets,
-                            end_dt,
-                            bar_count,
-                            frequency,
-                            field,
-                            data_frequency,
-                            ffill=True):
-        exchange_assets = group_assets_by_exchange(assets)
-        if len(exchange_assets) > 1:
-            df_list = []
-            for exchange_name in exchange_assets:
-                assets = exchange_assets[exchange_name]
-
-                df_exchange = self.get_exchange_history_window(
-                    exchange_name,
-                    assets,
-                    end_dt,
-                    bar_count,
-                    frequency,
-                    field,
-                    data_frequency,
-                    ffill)
-
-                df_list.append(df_exchange)
-
-            # Merging the values values of each exchange
-            return pd.concat(df_list)
-
-        else:
-            exchange_name = list(exchange_assets.keys())[0]
-            return self.get_exchange_history_window(
-                exchange_name,
-                assets,
-                end_dt,
-                bar_count,
-                frequency,
-                field,
-                data_frequency,
-                ffill)
-
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency=None,
-                           ffill=True):
-
-        if field == 'price':
-            field = 'close'
-
-        return retry(
-            action=self._get_history_window,
-            attempts=self.attempts['get_history_window_attempts'],
-            sleeptime=self.attempts['retry_sleeptime'],
-            retry_exceptions=(ExchangeRequestError,),
-            cleanup=lambda: log.warn('fetching history again.'),
-            args=(assets,
-                  end_dt,
-                  bar_count,
-                  frequency,
-                  field,
-                  data_frequency,
-                  ffill))
-
-    @abc.abstractmethod
-    def get_exchange_history_window(self,
-                                    exchange_name,
-                                    assets,
-                                    end_dt,
-                                    bar_count,
-                                    frequency,
-                                    field,
-                                    data_frequency,
-                                    ffill=True):
-        pass
-
-    def _get_spot_value(self, assets, field, dt, data_frequency):
-        if isinstance(assets, TradingPair):
-            spot_values = self.get_exchange_spot_value(
-                assets.exchange, [assets], field, dt, data_frequency)
-
-            if not spot_values:
-                return np.nan
-
-            return spot_values[0]
-
-        else:
-            exchange_assets = dict()
-            for asset in assets:
-                if asset.exchange not in exchange_assets:
-                    exchange_assets[asset.exchange] = list()
-
-                exchange_assets[asset.exchange].append(asset)
-
-            if len(list(exchange_assets.keys())) == 1:
-                exchange_name = list(exchange_assets.keys())[0]
-                return self.get_exchange_spot_value(
-                    exchange_name, assets, field, dt, data_frequency)
-
-            else:
-                spot_values = []
-                for exchange_name in exchange_assets:
-                    assets = exchange_assets[exchange_name]
-                    exchange_spot_values = self.get_exchange_spot_value(
-                        exchange_name,
-                        assets,
-                        field,
-                        dt,
-                        data_frequency
-                    )
-                    if len(assets) == 1:
-                        spot_values.append(exchange_spot_values)
-                    else:
-                        spot_values += exchange_spot_values
-
-                return spot_values
-
-    def get_spot_value(self, assets, field, dt, data_frequency):
-        if field == 'price':
-            field = 'close'
-
-        return retry(
-            action=self._get_spot_value,
-            attempts=self.attempts['get_spot_value_attempts'],
-            sleeptime=self.attempts['retry_sleeptime'],
-            retry_exceptions=(ExchangeRequestError,),
-            cleanup=lambda: log.warn('fetching spot value again.'),
-            args=(assets, field, dt, data_frequency))
-
-    @abc.abstractmethod
-    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
-                                data_frequency):
-        return
-
-    def get_adjusted_value(self, asset, field, dt,
-                           perspective_dt,
-                           data_frequency,
-                           spot_value=None):
-        # TODO: does this pertain to cryptocurrencies?
-        log.warn('get_adjusted_value is not implemented yet!')
-        return spot_value
-
-
-class DataPortalExchangeLive(DataPortalExchangeBase):
-    def __init__(self, *args, **kwargs):
-        self.exchanges = kwargs.pop('exchanges', None)
-        super(DataPortalExchangeLive, self).__init__(*args, **kwargs)
-
-    def get_exchange_history_window(self,
-                                    exchange_name,
-                                    assets,
-                                    end_dt,
-                                    bar_count,
-                                    frequency,
-                                    field,
-                                    data_frequency,
-                                    ffill=True):
-        """
-        Fetching price history window from the exchange.
-
-        Parameters
-        ----------
-        exchange_name: Exchange
-        assets: list[TradingPair]
-        end_dt: datetime
-        bar_count: int
-        frequency: str
-        field: str
-        data_frequency: str
-        ffill: bool
-
-        Returns
-        -------
-        DataFrame
-
-        """
-        exchange = self.exchanges[exchange_name]
-
-        df = exchange.get_history_window(
-            assets,
-            end_dt,
-            bar_count,
-            frequency,
-            field,
-            data_frequency,
-            False)
-        return df
-
-    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
-                                data_frequency):
-        """
-        A spot value for the exchange.
-
-        Parameters
-        ----------
-        exchange_name: str
-        assets: list[TradingPair]
-        field: str
-        dt: datetime
-        data_frequency: str
-
-        Returns
-        -------
-        float
-
-        """
-        exchange = self.exchanges[exchange_name]
-        exchange_spot_values = exchange.get_spot_value(
-            assets, field, dt, data_frequency)
-
-        return exchange_spot_values
-
-
-class DataPortalExchangeBacktest(DataPortalExchangeBase):
-    def __init__(self, *args, **kwargs):
-        self.exchange_names = kwargs.pop('exchange_names', None)
-
-        super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs)
-
-        self.exchange_bundles = dict()
-        self.history_loaders = dict()
-        self.minute_history_loaders = dict()
-
-        for name in self.exchange_names:
-            self.exchange_bundles[name] = ExchangeBundle(name)
-
-    def _get_first_trading_day(self, assets):
-        first_date = None
-        for asset in assets:
-            if first_date is None or asset.start_date > first_date:
-                first_date = asset.start_date
-        return first_date
-
-    def get_exchange_history_window(self,
-                                    exchange_name,
-                                    assets,
-                                    end_dt,
-                                    bar_count,
-                                    frequency,
-                                    field,
-                                    data_frequency,
-                                    ffill=True):
-        """
-        Fetching price history window from the exchange bundle.
-
-        Parameters
-        ----------
-        exchange: Exchange
-        assets: list[TradingPair]
-        end_dt: datetime
-        bar_count: int
-        frequency: str
-        field: str
-        data_frequency: str
-        ffill: bool
-
-        Returns
-        -------
-        DataFrame
-
-        """
-        # TODO: verify that the exchange supports the timeframe
-        bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
-
-        freq, candle_size, unit, adj_data_frequency = get_frequency(
-            frequency, data_frequency
-        )
-        adj_bar_count = candle_size * bar_count
-
-        if data_frequency == 'minute' and adj_data_frequency == 'daily':
-            end_dt = end_dt.floor('1D')
-
-        series = bundle.get_history_window_series_and_load(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=adj_bar_count,
-            field=field,
-            data_frequency=adj_data_frequency,
-            algo_end_dt=self._last_available_session,
-        )
-
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-        df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
-        return df
-
-    def get_exchange_spot_value(self,
-                                exchange_name,
-                                assets,
-                                field,
-                                dt,
-                                data_frequency
-                                ):
-        """
-        A spot value for the exchange bundle. Try to ingest data if not in
-        the bundle.
-
-        Parameters
-        ----------
-        exchange_name: str
-        assets: list[TradingPair]
-        field: str
-        dt: datetime
-        data_frequency: str
-
-        Returns
-        -------
-        float
-
-        """
-        bundle = self.exchange_bundles[exchange_name]
-        if data_frequency == 'daily':
-            dt = dt.floor('1D')
-        else:
-            dt = dt.floor('1 min')
-
-        if AUTO_INGEST:
-            try:
-                return bundle.get_spot_values(
-                    assets, field, dt, data_frequency
-                )
-            except PricingDataNotLoadedError:
-                log.info(
-                    'pricing data for {symbol} not found on {dt}'
-                    ', updating the bundles.'.format(
-                        symbol=[asset.symbol for asset in assets],
-                        dt=dt
-                    )
-                )
-                bundle.ingest_assets(
-                    assets=assets,
-                    start_dt=self._first_trading_day,
-                    end_dt=self._last_available_session,
-                    data_frequency=data_frequency,
-                    show_progress=True
-                )
-                return bundle.get_spot_values(
-                    assets, field, dt, data_frequency, True
-                )
-        else:
-            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -1,24 +1,6 @@
-import sys
-import traceback
-
 from catalyst.errors import ZiplineError
 
 
-def silent_except_hook(exctype, excvalue, exctraceback):
-    if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange,
-                   ExchangeAuthEmpty]:
-        fn = traceback.extract_tb(exctraceback)[-1][0]
-        ln = traceback.extract_tb(exctraceback)[-1][1]
-        print("Error traceback: {1} (line {2})\n"
-              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
-    else:
-        sys.__excepthook__(exctype, excvalue, exctraceback)
-
-
-sys.excepthook = silent_except_hook
-
-
 class ExchangeRequestError(ZiplineError):
     msg = (
         'Request failed: {error}'
@@ -52,13 +34,6 @@ class ExchangeTransactionError(ZiplineError):
     ).strip()
 
 
-class ExchangeNotFoundError(ZiplineError):
-    msg = (
-        'Exchange {exchange_name} not found. Please specify exchanges '
-        'supported by Catalyst and verify spelling for accuracy.'
-    ).strip()
-
-
 class ExchangeAuthNotFound(ZiplineError):
     msg = (
         'Please create an auth.json file containing the api token and key for '
@@ -66,13 +41,6 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
-class ExchangeAuthEmpty(ZiplineError):
-    msg = (
-        'Please enter your API token key and secret for exchange {exchange} '
-        'in the following file: {filename}'
-    ).strip()
-
-
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '
@@ -86,239 +54,7 @@ class AlgoPickleNotFound(ZiplineError):
     ).strip()
 
 
-class InvalidHistoryFrequencyAlias(ZiplineError):
-    msg = (
-        'Invalid frequency alias {freq}. Valid suffixes are M (minute) '
-        'and D (day). For example, these aliases would be valid '
-        '1M, 5M, 1D.'
-    ).strip()
-
-
 class InvalidHistoryFrequencyError(ZiplineError):
     msg = (
-        'Frequency {frequency} not supported by the exchange.'
+        'History frequency {frequency} not supported by the exchange.'
-    ).strip()
-
-
-class UnsupportedHistoryFrequencyError(ZiplineError):
-    msg = (
-        '{exchange} does not support candle frequency {freq}, please choose '
-        'from: {freqs}.'
-    ).strip()
-
-
-class InvalidHistoryTimeframeError(ZiplineError):
-    msg = (
-        'CCXT timeframe {timeframe} not supported by the exchange.'
-    ).strip()
-
-
-class MismatchingFrequencyError(ZiplineError):
-    msg = (
-        'Bar aggregate frequency {frequency} not compatible with '
-        'data frequency {data_frequency}.'
-    ).strip()
-
-
-class InvalidSymbolError(ZiplineError):
-    msg = (
-        'Invalid trading pair symbol: {symbol}. '
-        'Catalyst symbols must follow this convention: '
-        '[Market Currency]_[Base Currency]. For example: eth_usd, btc_usd, '
-        'neo_eth, ubq_btc. Error details: {error}'
-    ).strip()
-
-
-class InvalidOrderStyle(ZiplineError):
-    msg = (
-        'Order style {style} not supported by exchange {exchange}.'
-    ).strip()
-
-
-class CreateOrderError(ZiplineError):
-    msg = (
-        'Unable to create order on exchange {exchange} {error}.'
-    ).strip()
-
-
-class OrderNotFound(ZiplineError):
-    msg = (
-        'Order {order_id} not found on exchange {exchange}.'
-    ).strip()
-
-
-class OrphanOrderError(ZiplineError):
-    msg = (
-        'Order {order_id} found in exchange {exchange} but not tracked by '
-        'the algorithm.'
-    ).strip()
-
-
-class OrphanOrderReverseError(ZiplineError):
-    msg = (
-        'Order {order_id} tracked by algorithm, but not found in exchange '
-        '{exchange}.'
-    ).strip()
-
-
-class OrderCancelError(ZiplineError):
-    msg = (
-        'Unable to cancel order {order_id} on exchange {exchange} {error}.'
-    ).strip()
-
-
-class SidHashError(ZiplineError):
-    msg = (
-        'Unable to hash sid from symbol {symbol}.'
-    ).strip()
-
-
-class BaseCurrencyNotFoundError(ZiplineError):
-    msg = (
-        'Algorithm base currency {base_currency} not found in account '
-        'balances on {exchange}: {balances}'
-    ).strip()
-
-
-class MismatchingBaseCurrencies(ZiplineError):
-    msg = (
-        'Unable to trade with base currency {base_currency} when the '
-        'algorithm uses {algo_currency}.'
-    ).strip()
-
-
-class MismatchingBaseCurrenciesExchanges(ZiplineError):
-    msg = (
-        'Unable to trade with base currency {base_currency} when the '
-        'exchange {exchange_name} users {exchange_currency}.'
-    ).strip()
-
-
-class SymbolNotFoundOnExchange(ZiplineError):
-    """
-    Raised when a symbol() call contains a non-existent symbol.
-    """
-    msg = ('Symbol {symbol} not found on exchange {exchange}. '
-           'Choose from: {supported_symbols}').strip()
-
-
-class BundleNotFoundError(ZiplineError):
-    msg = ('Unable to find bundle data for exchange {exchange} and '
-           'data frequency {data_frequency}.'
-           'Please ingest some price data.'
-           'See `catalyst ingest-exchange --help` for details.').strip()
-
-
-class TempBundleNotFoundError(ZiplineError):
-    msg = ('Temporary bundle not found in: {path}.').strip()
-
-
-class EmptyValuesInBundleError(ZiplineError):
-    msg = ('{name} with end minute {end_minute} has empty rows '
-           'in ranges: {dates}').strip()
-
-
-class PricingDataBeforeTradingError(ZiplineError):
-    msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
-           'starts on {first_trading_day}, but you are either trying to trade '
-           'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
-           ).strip()
-
-
-class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Missing data for {exchange} {symbols} in date range '
-           '[{start_dt} - {end_dt}]'
-           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
-           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
-           'for details.').strip()
-
-
-class PricingDataValueError(ZiplineError):
-    msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
-           '[{start_dt} - {end_dt}]: {error}').strip()
-
-
-class DataCorruptionError(ZiplineError):
-    msg = (
-        'Unable to validate data for {exchange} {symbols} in date range '
-        '[{start_dt} - {end_dt}]. The data is either corrupted or '
-        'unavailable. Please try deleting this bundle:'
-        '\n`catalyst clean-exchange -x {exchange}\n'
-        'Then, ingest the data again. Please contact the Catalyst team if '
-        'the issue persists.'
-    ).strip()
-
-
-class ApiCandlesError(ZiplineError):
-    msg = (
-        'Unable to fetch candles from the remote API: {error}.'
-    ).strip()
-
-
-class NoDataAvailableOnExchange(ZiplineError):
-    msg = (
-        'Requested data for trading pair {symbol} is not available on '
-        'exchange {exchange} '
-        'in `{data_frequency}` frequency at this time. '
-        'Check `http://enigma.co/catalyst/status` for market coverage.'
-    ).strip()
-
-
-class NoValueForField(ZiplineError):
-    msg = (
-        'Value not found for field: {field}.'
-    ).strip()
-
-
-class OrderTypeNotSupported(ZiplineError):
-    msg = (
-        'Order type `{order_type}` not currency supported by Catalyst. '
-        'Please use `limit` or `market` orders only.'
-    ).strip()
-
-
-class NotEnoughCapitalError(ZiplineError):
-    msg = (
-        'Not enough capital on exchange {exchange} for trading. Each '
-        'exchange should contain at least as much {base_currency} '
-        'as the specified `capital_base`. The current balance {balance} is '
-        'lower than the `capital_base`: {capital_base}'
-    ).strip()
-
-
-class NotEnoughCashError(ZiplineError):
-    msg = (
-        'Total {currency} amount on {exchange} is lower than the cash '
-        'reserved for this algo: {free} < {cash}. While trades can be made on '
-        'the exchange accounts outside of the algo, exchange must have enough '
-        'free {currency} to cover the algo cash.'
-    ).strip()
-
-
-class LastCandleTooEarlyError(ZiplineError):
-    msg = (
-        'The trade date of the last candle {last_traded} is before the '
-        'specified end date minus one candle {end_dt}. Please verify how '
-        '{exchange} calculates the start date of OHLCV candles.'
-    ).strip()
-
-
-class TickerNotFoundError(ZiplineError):
-    msg = (
-        'Unable to fetch ticker for {symbol} on {exchange}.'
-    ).strip()
-
-
-class BalanceNotFoundError(ZiplineError):
-    msg = (
-        '{currency} not found in account balance on {exchange}: {balances}.'
-    ).strip()
-
-
-class BalanceTooLowError(ZiplineError):
-    msg = (
-        'Balance for {currency} on {exchange} too low: {free} < {amount}. '
-        'Positions have likely been sold outside of this algorithm. Please '
-        'add positions to hold a free amount greater than {amount}, or clean '
-        'the state of this algo and restart.'
     ).strip()

catalyst/exchange/exchange_execution.py

@@ -1,67 +0,0 @@
-from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
-
-
-class ExchangeLimitOrder(LimitOrder):
-    def get_limit_price(self, is_buy):
-        """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers.
-
-        Parameters
-        ----------
-        is_buy: bool
-
-        Returns
-        -------
-        float
-
-        """
-        return self.limit_price
-
-
-class ExchangeStopOrder(StopOrder):
-    def get_stop_price(self, is_buy):
-        """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers.
-
-        Parameters
-        ----------
-        is_buy: bool
-
-        Returns
-        -------
-        float
-
-        """
-        return self.stop_price
-
-
-class ExchangeStopLimitOrder(StopLimitOrder):
-    def get_limit_price(self, is_buy):
-        """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers.
-
-        Parameters
-        ----------
-        is_buy: bool
-
-        Returns
-        -------
-        float
-
-        """
-        return self.limit_price
-
-    def get_stop_price(self, is_buy):
-        """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers.
-
-        Parameters
-        ----------
-        is_buy: bool
-
-        Returns
-        -------
-        float
-
-        """
-        return self.stop_price

catalyst/exchange/exchange_portfolio.py

@@ -1,16 +1,15 @@
 import numpy as np
-from catalyst.constants import LOG_LEVEL
-from catalyst.protocol import Portfolio, Positions, Position
 from logbook import Logger
 
-log = Logger('ExchangePortfolio', level=LOG_LEVEL)
+from catalyst.protocol import Portfolio, Positions, Position
+
+log = Logger('ExchangePortfolio')
 
 
 class ExchangePortfolio(Portfolio):
     """
     Since the goal is to support multiple exchanges, it makes sense to
-    include additional stats in the portfolio object. This fills the role
+    include additional stats in the portfolio object.
-    of Blotter and Portfolio in live mode.
 
     Instead of relying on the performance tracker, each exchange portfolio
     tracks its own holding. This offers a separation between tracking an
@@ -29,23 +28,12 @@ class ExchangePortfolio(Portfolio):
         self.positions_value = 0.0
         self.open_orders = dict()
 
+    def calculate_pnl(self):
+        log.debug('calculating pnl')
+
     def create_order(self, order):
-        """
-        Create an open order and store in memory.
-
-        Parameters
-        ----------
-        order: Order
-
-        """
         log.debug('creating order {}'.format(order.id))
-
+        self.open_orders[order.id] = order
-        open_orders = self.open_orders[order.asset] \
-            if order.asset is self.open_orders else []
-
-        open_orders.append(order)
-
-        self.open_orders[order.asset] = open_orders
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
@@ -57,40 +45,16 @@ class ExchangePortfolio(Portfolio):
         order_position.amount += order.amount
         log.debug('open order added to portfolio')
 
-    def _remove_open_order(self, order):
-        try:
-            open_orders = self.open_orders[order.asset]
-            if order in open_orders:
-                open_orders.remove(order)
-
-        except Exception:
-            raise ValueError(
-                'unable to clear order not found in open order list.'
-            )
-
     def execute_order(self, order, transaction):
-        """
-        Update the open orders and positions to apply an executed order.
-
-        Unlike with backtesting, we do not need to add slippage and fees.
-        The executed price includes transaction fees.
-
-        Parameters
-        ----------
-        order: Order
-        transaction: Transaction
-
-        """
         log.debug('executing order {}'.format(order.id))
-        self._remove_open_order(order)
+        del self.open_orders[order.id]
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
 
         if order_position is None:
             raise ValueError(
-                'Trying to execute order for a position not held:'
+                'Trying to execute order for a position not held: %s' % order.id
-                ' {}'.format(order.id)
             )
 
         self.capital_used += order.amount * transaction.price
@@ -107,16 +71,8 @@ class ExchangePortfolio(Portfolio):
         log.debug('updated portfolio with executed order')
 
     def remove_order(self, order):
-        """
-        Removing an open order.
-
-        Parameters
-        ----------
-        order: Order
-
-        """
         log.info('removing cancelled order {}'.format(order.id))
-        self._remove_open_order(order)
+        del self.open_orders[order.id]
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None

catalyst/exchange/exchange_pricing_loader.py

@@ -1,177 +0,0 @@
-# Copyright 2015 Quantopian, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from catalyst.constants import LOG_LEVEL
-from catalyst.data.us_equity_pricing import BcolzDailyBarReader
-from catalyst.errors import NoFurtherDataError
-from catalyst.exchange.utils.factory import get_exchange
-from catalyst.lib.adjusted_array import AdjustedArray
-from catalyst.pipeline.data import DataSet, Column
-from catalyst.pipeline.loaders.base import PipelineLoader
-from catalyst.utils.calendars import get_calendar
-from catalyst.utils.numpy_utils import float64_dtype
-from logbook import Logger
-from numpy import (
-    iinfo,
-    uint32,
-)
-
-UINT32_MAX = iinfo(uint32).max
-
-log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
-
-
-class TradingPairPricing(DataSet):
-    """
-    Dataset representing daily trading prices and volumes.
-    """
-    open = Column(float64_dtype)
-    high = Column(float64_dtype)
-    low = Column(float64_dtype)
-    close = Column(float64_dtype)
-    volume = Column(float64_dtype)
-
-
-class ExchangePricingLoader(PipelineLoader):
-    """
-    PipelineLoader for Crypto Pricing data
-
-    Delegates loading of baselines and adjustments.
-    """
-
-    def __init__(self, data_frequency):
-
-        cal = get_calendar('OPEN')
-
-        if data_frequency == 'daily':
-            reader = None
-            all_sessions = cal.all_sessions
-
-        elif data_frequency == 'minute':
-            reader = None
-            all_sessions = cal.all_minutes
-
-        else:
-            raise ValueError(
-                'Invalid data frequency: {}'.format(data_frequency)
-            )
-
-        self.data_frequency = data_frequency
-        self.raw_price_loader = reader
-        self._columns = TradingPairPricing.columns
-        self._all_sessions = all_sessions
-
-    @classmethod
-    def from_files(cls, pricing_path):
-        """
-        Create a loader from a bcolz equity pricing dir and a SQLite
-        adjustments path.
-
-        Parameters
-        ----------
-        pricing_path : str
-            Path to a bcolz directory written by a BcolzDailyBarWriter.
-        """
-        return cls(
-            BcolzDailyBarReader(pricing_path),
-        )
-
-    def load_adjusted_array(self, columns, dates, assets, mask):
-        # load_adjusted_array is called with dates on which the user's algo
-        # will be shown data, which means we need to return the data that would
-        # be known at the start of each date.  We assume that the latest data
-        # known on day N is the data from day (N - 1), so we shift all query
-        # dates back by a day.
-        start_date, end_date = _shift_dates(
-            self._all_sessions, dates[0], dates[-1], shift=1,
-        )
-        colnames = [c.name for c in columns]
-
-        if len(assets) == 0:
-            raise ValueError(
-                'Pipeline cannot load data with eligible assets.'
-            )
-
-        exchange_names = []
-        for asset in assets:
-            if asset.exchange not in exchange_names:
-                exchange_names.append(asset.exchange)
-
-        exchange = get_exchange(exchange_names[0])
-        reader = exchange.bundle.get_reader(self.data_frequency)
-
-        raw_arrays = reader.load_raw_arrays(
-            colnames,
-            start_date,
-            end_date,
-            assets,
-        )
-
-        out = {}
-        for c, c_raw in zip(columns, raw_arrays):
-            out[c] = AdjustedArray(
-                c_raw.astype(c.dtype),
-                mask,
-                {},
-                c.missing_value,
-            )
-        return out
-
-    @property
-    def columns(self):
-        return self._columns
-
-
-def _shift_dates(dates, start_date, end_date, shift):
-    try:
-        start = dates.get_loc(start_date)
-    except KeyError:
-        if start_date < dates[0]:
-            raise NoFurtherDataError(
-                msg=(
-                    "Pipeline Query requested data starting on {query_start}, "
-                    "but first known date is {calendar_start}"
-                ).format(
-                    query_start=str(start_date),
-                    calendar_start=str(dates[0]),
-                )
-            )
-        else:
-            raise ValueError("Query start %s not in calendar" % start_date)
-
-    # Make sure that shifting doesn't push us out of the calendar.
-    if start < shift:
-        raise NoFurtherDataError(
-            msg=(
-                "Pipeline Query requested data from {shift}"
-                " days before {query_start}, but first known date is only "
-                "{start} days earlier."
-            ).format(shift=shift, query_start=start_date, start=start),
-        )
-
-    try:
-        end = dates.get_loc(end_date)
-    except KeyError:
-        if end_date > dates[-1]:
-            raise NoFurtherDataError(
-                msg=(
-                    "Pipeline Query requesting data up to {query_end}, "
-                    "but last known date is {calendar_end}"
-                ).format(
-                    query_end=end_date,
-                    calendar_end=dates[-1],
-                )
-            )
-        else:
-            raise ValueError("Query end %s not in calendar" % end_date)
-    return dates[start - shift], dates[end - shift]

catalyst/exchange/exchange_utils.py

@@ -0,0 +1,133 @@
+import json
+import os
+import pickle
+import urllib
+from datetime import date, datetime
+
+from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
+    ExchangeSymbolsNotFound
+from catalyst.utils.paths import data_root, ensure_directory
+
+SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
+              'live-trading/catalyst/exchange/symbols/{exchange}.json'
+
+
+def get_exchange_folder(exchange_name, environ=None):
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    exchange_folder = os.path.join(root, 'exchanges', exchange_name)
+    ensure_directory(exchange_folder)
+
+    return exchange_folder
+
+
+def download_exchange_symbols(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'symbols.json')
+
+    url = SYMBOLS_URL.format(exchange=exchange_name)
+    response = urllib.urlretrieve(url=url, filename=filename)
+    return response
+
+
+def get_exchange_symbols(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'symbols.json')
+
+    if not os.path.isfile(filename):
+        download_exchange_symbols(exchange_name, environ)
+
+    if os.path.isfile(filename):
+        with open(filename) as data_file:
+            data = json.load(data_file)
+            return data
+    else:
+        raise ExchangeSymbolsNotFound(
+            exchange=exchange_name,
+            filename=filename
+        )
+
+
+def get_exchange_auth(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'auth.json')
+
+    if os.path.isfile(filename):
+        with open(filename) as data_file:
+            data = json.load(data_file)
+            return data
+    else:
+        raise ExchangeAuthNotFound(
+            exchange=exchange_name,
+            filename=filename
+        )
+
+
+def get_algo_folder(algo_name, environ=None):
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    algo_folder = os.path.join(root, 'live_algos', algo_name)
+    ensure_directory(algo_folder)
+
+    return algo_folder
+
+
+def get_algo_object(algo_name, key, environ=None, rel_path=None):
+    folder = get_algo_folder(algo_name, environ)
+
+    if rel_path is not None:
+        folder = os.path.join(folder, rel_path)
+
+    filename = os.path.join(folder, key + '.p')
+
+    if os.path.isfile(filename):
+        try:
+            with open(filename, 'rb') as handle:
+                return pickle.load(handle)
+        except Exception as e:
+            return None
+    else:
+        return None
+
+
+def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+    folder = get_algo_folder(algo_name, environ)
+
+    if rel_path is not None:
+        folder = os.path.join(folder, rel_path)
+        ensure_directory(folder)
+
+    filename = os.path.join(folder, key + '.p')
+
+    with open(filename, 'wb') as handle:
+        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def append_algo_object(algo_name, key, obj, environ=None):
+    algo_folder = get_algo_folder(algo_name, environ)
+    filename = os.path.join(algo_folder, key + '.p')
+
+    mode = 'a+b' if os.path.isfile(filename) else 'wb'
+    with open(filename, mode) as handle:
+        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def get_exchange_minute_writer_root(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+
+    minute_data_folder = os.path.join(exchange_folder, 'minute_data')
+    ensure_directory(minute_data_folder)
+
+    return minute_data_folder
+
+
+def perf_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializable" % type(obj))

catalyst/exchange/live_graph_clock.py

@@ -1,74 +0,0 @@
-import pandas as pd
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.utils.stats_utils import prepare_stats
-from catalyst.gens.sim_engine import (
-    BAR,
-    SESSION_START
-)
-from logbook import Logger
-
-log = Logger('LiveGraphClock', level=LOG_LEVEL)
-
-
-class LiveGraphClock(object):
-    """Realtime clock for live trading.
-
-    This class is a drop-in replacement for
-    :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
-
-    This mixes the clock with a live graph.
-
-    Notes
-    -----
-    This seemingly awkward approach allows us to run the program using a single
-    thread. This is important because Matplotlib does not play nice with
-    multi-threaded environments. Zipline probably does not either.
-
-
-    Matplotlib has a pause() method which is a wrapper around time.sleep()
-    used in the SimpleClock. The key difference is that users
-    can still interact with the chart during the pause cycles. This is
-    what enables us to keep a single thread. This is also why we are not using
-    the 'animate' callback of Matplotlib. We need to direct access to the
-    __iter__ method in order to yield events to Zipline.
-
-    The :param:`time_skew` parameter represents the time difference between
-    the exchange and the live trading machine's clock. It's not used currently.
-    """
-
-    def __init__(self, sessions, context, callback=None,
-                 time_skew=pd.Timedelta('0s')):
-
-        self.sessions = sessions
-        self.time_skew = time_skew
-        self._last_emit = None
-        self._before_trading_start_bar_yielded = True
-        self.context = context
-        self.callback = callback
-
-    def __iter__(self):
-        from matplotlib import pyplot as plt
-        yield pd.Timestamp.utcnow(), SESSION_START
-
-        while True:
-            current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1T')
-
-            if self._last_emit is None or current_minute > self._last_emit:
-                log.debug('emitting minutely bar: {}'.format(current_minute))
-
-                self._last_emit = current_minute
-                yield current_minute, BAR
-
-                recorded_cols = list(self.context.recorded_vars.keys())
-                df, _ = prepare_stats(
-                    self.context.frame_stats, recorded_cols=recorded_cols
-                )
-                self.callback(self.context, df)
-
-            else:
-                # I can't use the "animate" reactive approach here because
-                # I need to yield from the main loop.
-
-                # Workaround: https://stackoverflow.com/a/33050617/814633
-                plt.pause(1)

catalyst/exchange/symbols/bitfinex.json

@@ -0,0 +1,110 @@
+{
+  "btcusd": {
+    "symbol": "btc_usd",
+    "start_date": "2010-01-01"
+  },
+  "ltcusd": {
+    "symbol": "ltc_usd",
+    "start_date": "2010-01-01"
+  },
+  "ltcbtc": {
+    "symbol": "ltc_btc",
+    "start_date": "2010-01-01"
+  },
+  "ethusd": {
+    "symbol": "eth_usd",
+    "start_date": "2010-01-01"
+  },
+  "ethbtc": {
+    "symbol": "eth_btc",
+    "start_date": "2010-01-01"
+  },
+  "etcbtc": {
+    "symbol": "etc_btc",
+    "start_date": "2010-01-01"
+  },
+  "etcusd": {
+    "symbol": "etc_usd",
+    "start_date": "2010-01-01"
+  },
+  "rrtusd": {
+    "symbol": "rrt_usd",
+    "start_date": "2010-01-01"
+  },
+  "rrtbtc": {
+    "symbol": "rrt_btc",
+    "start_date": "2010-01-01"
+  },
+  "zecusd": {
+    "symbol": "zec_usd",
+    "start_date": "2010-01-01"
+  },
+  "zecbtc": {
+    "symbol": "zec_btc",
+    "start_date": "2010-01-01"
+  },
+  "xmrusd": {
+    "symbol": "xmr_usd",
+    "start_date": "2010-01-01"
+  },
+  "xmrbtc": {
+    "symbol": "xmr_btc",
+    "start_date": "2010-01-01"
+  },
+  "dshusd": {
+    "symbol": "dsh_usd",
+    "start_date": "2010-01-01"
+  },
+  "dshbtc": {
+    "symbol": "dsh_btc",
+    "start_date": "2010-01-01"
+  },
+  "bccbtc": {
+    "symbol": "bcc_btc",
+    "start_date": "2010-01-01"
+  },
+  "bcubtc": {
+    "symbol": "bcu_btc",
+    "start_date": "2010-01-01"
+  },
+  "bccusd": {
+    "symbol": "bcc_usd",
+    "start_date": "2010-01-01"
+  },
+  "bcuusd": {
+    "symbol": "bcu_usd",
+    "start_date": "2010-01-01"
+  },
+  "xrpusd": {
+    "symbol": "xrp_usd",
+    "start_date": "2010-01-01"
+  },
+  "xrpbtc": {
+    "symbol": "xrp_btc",
+    "start_date": "2010-01-01"
+  },
+  "iotusd": {
+    "symbol": "iot_usd",
+    "start_date": "2010-01-01"
+  },
+  "iotbtc": {
+    "symbol": "iot_btc",
+    "start_date": "2010-01-01"
+  },
+  "ioteth": {
+    "symbol": "iot_eth",
+    "start_date": "2010-01-01"
+  },
+  "eosusd": {
+    "symbol": "eos_usd",
+    "start_date": "2010-01-01"
+  },
+  "eosbtc": {
+    "symbol": "eos_btc",
+    "start_date": "2010-01-01"
+  },
+  "eoseth": {
+    "symbol": "eos_eth",
+    "start_date": "2010-01-01"
+  }
+}

catalyst/exchange/utils/bundle_utils.py

@@ -1,158 +0,0 @@
-import os
-import tarfile
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-
-from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
-import os
-import tarfile
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-
-from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
-
-EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
-API_URL = 'http://data.enigma.co/api/v1'
-
-
-def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
-    """
-    Download and extract a bcolz bundle.
-
-    Parameters
-    ----------
-    exchange_name: str
-    symbol: str
-    data_frequency: str
-    period: str
-
-    Returns
-    -------
-    str
-        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-
-    """
-    root = get_exchange_bundles_folder(exchange_name)
-    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
-        exchange=exchange_name,
-        frequency=data_frequency,
-        symbol=symbol,
-        period=period
-    )
-    path = os.path.join(root, name)
-
-    if not os.path.isdir(path):
-        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
-              'exchange-{exchange}/{name}.tar.gz'.format(
-            exchange=exchange_name,
-            name=name)
-
-        bytes = download_without_progress(url)
-        with tarfile.open('r', fileobj=bytes) as tar:
-            tar.extractall(path)
-
-    return path
-
-
-def get_df_from_arrays(arrays, periods):
-    """
-    A DataFrame from the specified OHCLV arrays.
-
-    Parameters
-    ----------
-    arrays: Object
-    periods: DateTimeIndex
-
-    Returns
-    -------
-    DataFrame
-
-    """
-    ohlcv = dict()
-    for index, field in enumerate(
-            ['open', 'high', 'low', 'close', 'volume']):
-        ohlcv[field] = arrays[index].flatten()
-
-    df = pd.DataFrame(
-        data=ohlcv,
-        index=periods
-    )
-    return df
-
-
-def range_in_bundle(asset, start_dt, end_dt, reader):
-    """
-    Evaluate whether price data of an asset is included has been ingested in
-    the exchange bundle for the given date range.
-
-    Parameters
-    ----------
-    asset: TradingPair
-    start_dt: datetime
-    end_dt: datetime
-    reader: BcolzBarMinuteReader
-
-    Returns
-    -------
-    bool
-
-    """
-    has_data = True
-    dates = [start_dt, end_dt]
-
-    while dates and has_data:
-        try:
-            dt = dates.pop(0)
-            close = reader.get_value(asset.sid, dt, 'close')
-
-            if np.isnan(close):
-                has_data = False
-
-        except Exception:
-            has_data = False
-
-    return has_data
-
-
-def get_assets(exchange, include_symbols, exclude_symbols):
-    """
-    Get assets from an exchange, including or excluding the specified
-    symbols.
-
-    Parameters
-    ----------
-    exchange: Exchange
-    include_symbols: str
-    exclude_symbols: str
-
-    Returns
-    -------
-    list[TradingPair]
-
-    """
-    if include_symbols is not None:
-        include_symbols_list = include_symbols.split(',')
-
-        return exchange.get_assets(include_symbols_list)
-
-    else:
-        all_assets = exchange.get_assets()
-
-        if exclude_symbols is not None:
-            exclude_symbols_list = exclude_symbols.split(',')
-
-            assets = []
-            for asset in all_assets:
-                if asset.symbol not in exclude_symbols_list:
-                    assets.append(asset)
-
-            return assets
-
-        else:
-            return all_assets

catalyst/exchange/utils/datetime_utils.py

@@ -1,328 +0,0 @@
-import calendar
-import re
-from datetime import datetime, timedelta, date
-
-import pandas as pd
-import pytz
-
-from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-    InvalidHistoryFrequencyAlias
-
-
-def get_date_from_ms(ms):
-    """
-    The date from the number of miliseconds from the epoch.
-
-    Parameters
-    ----------
-    ms: int
-
-    Returns
-    -------
-    datetime
-
-    """
-    return datetime.fromtimestamp(ms / 1000.0)
-
-
-def get_seconds_from_date(date):
-    """
-    The number of seconds from the epoch.
-
-    Parameters
-    ----------
-    date: datetime
-
-    Returns
-    -------
-    int
-
-    """
-    epoch = datetime.utcfromtimestamp(0)
-    epoch = epoch.replace(tzinfo=pytz.UTC)
-
-    return int((date - epoch).total_seconds())
-
-
-def get_delta(periods, data_frequency):
-    """
-    Get a time delta based on the specified data frequency.
-
-    Parameters
-    ----------
-    periods: int
-    data_frequency: str
-
-    Returns
-    -------
-    timedelta
-
-    """
-    return timedelta(minutes=periods) \
-        if data_frequency == 'minute' else timedelta(days=periods)
-
-
-def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
-    """
-    Get a date range for the specified parameters.
-
-    Parameters
-    ----------
-    start_dt: datetime
-    end_dt: datetime
-    freq: str
-
-    Returns
-    -------
-    DateTimeIndex
-
-    """
-    if freq == 'minute':
-        freq = 'T'
-
-    elif freq == 'daily':
-        freq = 'D'
-
-    if start_dt is not None and end_dt is not None and periods is None:
-
-        return pd.date_range(start_dt, end_dt, freq=freq)
-
-    elif periods is not None and (start_dt is not None or end_dt is not None):
-        _, unit_periods, unit, _ = get_frequency(freq)
-        adj_periods = periods * unit_periods
-
-        # TODO: standardize time aliases to avoid any mapping
-        unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
-        delta = pd.Timedelta(adj_periods, unit)
-
-        if start_dt is not None:
-            return pd.date_range(
-                start=start_dt,
-                end=start_dt + delta,
-                freq=freq,
-                closed='left',
-            )
-
-        else:
-            return pd.date_range(
-                start=end_dt - delta,
-                end=end_dt,
-                freq=freq,
-            )
-
-    else:
-        raise ValueError(
-            'Choose only two parameters between start_dt, end_dt '
-            'and periods.'
-        )
-
-
-def get_periods(start_dt, end_dt, freq):
-    """
-    The number of periods in the specified range.
-
-    Parameters
-    ----------
-    start_dt: datetime
-    end_dt: datetime
-    freq: str
-
-    Returns
-    -------
-    int
-
-    """
-    return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
-
-
-def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
-    """
-    The start date based on specified end date and data frequency.
-
-    Parameters
-    ----------
-    end_dt: datetime
-    bar_count: int
-    data_frequency: str
-    include_first
-
-    Returns
-    -------
-    datetime
-
-    """
-    periods = bar_count
-    if periods > 1:
-        delta = get_delta(periods, data_frequency)
-        start_dt = end_dt - delta
-
-        if not include_first:
-            start_dt += get_delta(1, data_frequency)
-    else:
-        start_dt = end_dt
-
-    return start_dt
-
-
-def get_period_label(dt, data_frequency):
-    """
-    The period label for the specified date and frequency.
-
-    Parameters
-    ----------
-    dt: datetime
-    data_frequency: str
-
-    Returns
-    -------
-    str
-
-    """
-    if data_frequency == 'minute':
-        return '{}-{:02d}'.format(dt.year, dt.month)
-    else:
-        return '{}'.format(dt.year)
-
-
-def get_month_start_end(dt, first_day=None, last_day=None):
-    """
-    The first and last day of the month for the specified date.
-
-    Parameters
-    ----------
-    dt: datetime
-    first_day: datetime
-    last_day: datetime
-
-    Returns
-    -------
-    datetime, datetime
-
-    """
-    month_range = calendar.monthrange(dt.year, dt.month)
-
-    if first_day:
-        month_start = first_day
-    else:
-        month_start = pd.to_datetime(datetime(
-            dt.year, dt.month, 1, 0, 0, 0, 0
-        ), utc=True)
-
-    if last_day:
-        month_end = last_day
-    else:
-        month_end = pd.to_datetime(datetime(
-            dt.year, dt.month, month_range[1], 23, 59, 0, 0
-        ), utc=True)
-
-        if month_end > pd.Timestamp.utcnow():
-            month_end = pd.Timestamp.utcnow().floor('1D')
-
-    return month_start, month_end
-
-
-def get_year_start_end(dt, first_day=None, last_day=None):
-    """
-    The first and last day of the year for the specified date.
-
-    Parameters
-    ----------
-
-    dt: datetime
-    first_day: datetime
-    last_day: datetime
-
-    Returns
-    -------
-    datetime, datetime
-
-    """
-    year_start = first_day if first_day \
-        else pd.to_datetime(date(dt.year, 1, 1), utc=True)
-    year_end = last_day if last_day \
-        else pd.to_datetime(date(dt.year, 12, 31), utc=True)
-
-    if year_end > pd.Timestamp.utcnow():
-        year_end = pd.Timestamp.utcnow().floor('1D')
-
-    return year_start, year_end
-
-
-def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
-    """
-    Get the frequency parameters.
-
-    Notes
-    -----
-    We're trying to use Pandas convention for frequency aliases.
-
-    Parameters
-    ----------
-    freq: str
-    data_frequency: str
-
-    Returns
-    -------
-    str, int, str, str
-
-    """
-    if data_frequency is None:
-        data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
-
-    if freq == 'minute':
-        unit = 'T'
-        candle_size = 1
-
-    elif freq == 'daily':
-        unit = 'D'
-        candle_size = 1
-
-    else:
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
-                else 1
-            unit = freq_match.group(2)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-    # TODO: some exchanges support H and W frequencies but not bundles
-    # Find a way to pass-through these parameters to exchanges
-    # but resample from minute or daily in backtest mode
-    # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
-    # Pandas offet aliases (used by Catalyst) and the CCXT timeframes
-    if unit.lower() == 'd':
-        unit = 'D'
-        alias = '{}D'.format(candle_size)
-
-        if data_frequency == 'minute':
-            data_frequency = 'daily'
-
-    elif unit.lower() == 'm' or unit == 'T':
-        unit = 'T'
-        alias = '{}T'.format(candle_size)
-        data_frequency = 'minute'
-
-    elif unit.lower() == 'h':
-        if 'H' in supported_freqs:
-            unit = 'H'
-            alias = '{}H'.format(candle_size)
-
-        else:
-            candle_size = candle_size * 60
-            alias = '{}T'.format(candle_size)
-            data_frequency = 'minute'
-
-    else:
-        raise InvalidHistoryFrequencyAlias(freq=freq)
-
-    return alias, candle_size, unit, data_frequency
-
-
-def from_ms_timestamp(ms):
-    return pd.to_datetime(ms, unit='ms', utc=True)
-
-
-def get_epoch():
-    return pd.to_datetime('1970-1-1', utc=True)

catalyst/exchange/utils/exchange_utils.py

@@ -1,753 +0,0 @@
-import hashlib
-import json
-import os
-import pickle
-import shutil
-from datetime import date, datetime
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from six import string_types
-from six.moves.urllib import request
-
-from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
-from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
-    ExchangeJSONDecoder
-from catalyst.utils.paths import data_root, ensure_directory, \
-    last_modified_time
-
-
-def get_sid(symbol):
-    """
-    Create a sid by hashing the symbol of a currency pair.
-
-    Parameters
-    ----------
-    symbol: str
-
-    Returns
-    -------
-    int
-        The resulting sid.
-
-    """
-    sid = int(
-        hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
-    ) % 10 ** 6
-    return sid
-
-
-def get_exchange_folder(exchange_name, environ=None):
-    """
-    The root path of an exchange folder.
-
-    Parameters
-    ----------
-    exchange_name: str
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    if not environ:
-        environ = os.environ
-
-    root = data_root(environ)
-    exchange_folder = os.path.join(root, 'exchanges', exchange_name)
-    ensure_directory(exchange_folder)
-
-    return exchange_folder
-
-
-def is_blacklist(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'blacklist.txt')
-
-    return os.path.exists(filename)
-
-
-def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
-    """
-    The absolute path of the exchange's symbol.json file.
-
-    Parameters
-    ----------
-    exchange_name:
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    name = 'symbols.json' if not is_local else 'symbols_local.json'
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    return os.path.join(exchange_folder, name)
-
-
-def download_exchange_symbols(exchange_name, environ=None):
-    """
-    Downloads the exchange's symbols.json from the repository.
-
-    Parameters
-    ----------
-    exchange_name: str
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    filename = get_exchange_symbols_filename(exchange_name)
-    url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = request.urlretrieve(url=url, filename=filename)
-    return response
-
-
-def get_exchange_symbols(exchange_name, is_local=False, environ=None):
-    """
-    The de-serialized content of the exchange's symbols.json.
-
-    Parameters
-    ----------
-    exchange_name: str
-    is_local: bool
-    environ:
-
-    Returns
-    -------
-    Object
-
-    """
-    filename = get_exchange_symbols_filename(exchange_name, is_local)
-
-    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-            pd.Timestamp('now', tz='UTC') - last_modified_time(
-            filename)).days > 1):
-        try:
-            download_exchange_symbols(exchange_name, environ)
-        except Exception:
-            pass
-
-    if os.path.isfile(filename):
-        with open(filename) as data_file:
-            try:
-                data = json.load(data_file, cls=ExchangeJSONDecoder)
-                return data
-
-            except ValueError:
-                return dict()
-    else:
-        raise ExchangeSymbolsNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
-
-
-def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
-    """
-    Save assets into an exchange_symbols file.
-
-    Parameters
-    ----------
-    exchange_name: str
-    assets: list[dict[str, object]]
-    is_local: bool
-    environ
-
-    Returns
-    -------
-
-    """
-    asset_dicts = dict()
-    for symbol in assets:
-        asset_dicts[symbol] = assets[symbol].to_dict()
-
-    filename = get_exchange_symbols_filename(
-        exchange_name, is_local, environ
-    )
-    with open(filename, 'wt') as handle:
-        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
-
-
-def get_symbols_string(assets):
-    """
-    A concatenated string of symbols from a list of assets.
-
-    Parameters
-    ----------
-    assets: list[TradingPair]
-
-    Returns
-    -------
-    str
-
-    """
-    array = [assets] if isinstance(assets, TradingPair) else assets
-    return ', '.join([asset.symbol for asset in array])
-
-
-def get_exchange_auth(exchange_name, alias=None, environ=None):
-    """
-    The de-serialized contend of the exchange's auth.json file.
-
-    Parameters
-    ----------
-    exchange_name: str
-    environ:
-
-    Returns
-    -------
-    Object
-
-    """
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    name = 'auth' if alias is None else alias
-    filename = os.path.join(exchange_folder, '{}.json'.format(name))
-
-    if os.path.isfile(filename):
-        with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
-    else:
-        data = dict(name=exchange_name, key='', secret='')
-        with open(filename, 'w') as f:
-            json.dump(data, f, sort_keys=False, indent=2,
-                      separators=(',', ':'))
-            return data
-
-
-def delete_algo_folder(algo_name, environ=None):
-    """
-    Delete the folder containing the algo state.
-
-    Parameters
-    ----------
-    algo_name: str
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    folder = get_algo_folder(algo_name, environ)
-    shutil.rmtree(folder)
-
-
-def get_algo_folder(algo_name, environ=None):
-    """
-    The algorithm root folder of the algorithm.
-
-    Parameters
-    ----------
-    algo_name: str
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    if not environ:
-        environ = os.environ
-
-    root = data_root(environ)
-    algo_folder = os.path.join(root, 'live_algos', algo_name)
-    ensure_directory(algo_folder)
-
-    return algo_folder
-
-
-def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
-    """
-    The de-serialized object of the algo name and key.
-
-    Parameters
-    ----------
-    algo_name: str
-    key: str
-    environ:
-    rel_path: str
-    how: str
-
-    Returns
-    -------
-    Object
-
-    """
-    if algo_name is None:
-        return None
-
-    folder = get_algo_folder(algo_name, environ)
-
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-
-    name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
-    filename = os.path.join(folder, name)
-
-    if os.path.isfile(filename):
-        if how == 'pickle':
-            with open(filename, 'rb') as handle:
-                return pickle.load(handle)
-
-        else:
-            with open(filename) as data_file:
-                data = json.load(data_file, cls=ExchangeJSONDecoder)
-                return data
-
-    else:
-        return None
-
-
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
-                     how='pickle'):
-    """
-    Serialize and save an object by algo name and key.
-
-    Parameters
-    ----------
-    algo_name: str
-    key: str
-    obj: Object
-    environ:
-    rel_path: str
-    how: str
-
-    """
-    folder = get_algo_folder(algo_name, environ)
-
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-        ensure_directory(folder)
-
-    if how == 'json':
-        filename = os.path.join(folder, '{}.json'.format(key))
-        with open(filename, 'wt') as handle:
-            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
-
-    else:
-        filename = os.path.join(folder, '{}.p'.format(key))
-        with open(filename, 'wb') as handle:
-            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
-def get_algo_df(algo_name, key, environ=None, rel_path=None):
-    """
-    The de-serialized DataFrame of an algo name and key.
-
-    Parameters
-    ----------
-    algo_name: str
-    key: str
-    environ:
-    rel_path: str
-
-    Returns
-    -------
-    DataFrame
-
-    """
-    folder = get_algo_folder(algo_name, environ)
-
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-
-    filename = os.path.join(folder, key + '.csv')
-
-    if os.path.isfile(filename):
-        try:
-            with open(filename, 'rb') as handle:
-                return pd.read_csv(handle, index_col=0, parse_dates=True)
-        except IOError:
-            return pd.DataFrame()
-    else:
-        return pd.DataFrame()
-
-
-def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
-    """
-    Serialize to csv and save a DataFrame by algo name and key.
-
-    Parameters
-    ----------
-    algo_name: str
-    key: str
-    df: pd.DataFrame
-    environ:
-    rel_path: str
-
-    """
-    folder = get_algo_folder(algo_name, environ)
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-        ensure_directory(folder)
-
-    filename = os.path.join(folder, key + '.csv')
-
-    with open(filename, 'wt') as handle:
-        df.to_csv(handle, encoding='UTF_8')
-
-
-def clear_frame_stats_directory(algo_name):
-    """
-    remove the outdated directory
-    to avoid overloading the disk
-
-    Parameters
-    ----------
-    algo_name: str
-
-    Returns
-    -------
-    error: str
-
-    """
-    error = None
-    algo_folder = get_algo_folder(algo_name)
-    folder = os.path.join(algo_folder, 'frame_stats')
-    if os.path.exists(folder):
-        try:
-            shutil.rmtree(folder)
-        except OSError:
-            error = 'unable to remove {}, the analyze ' \
-                    'data will be inconsistent'.format(folder)
-    return error
-
-
-def remove_old_files(algo_name, today, rel_path, environ=None):
-    """
-    remove old files from a directory
-    to avoid overloading the disk
-
-    Parameters
-    ----------
-    algo_name: str
-    today: Timestamp
-    rel_path: str
-    environ:
-
-    Returns
-    -------
-    error: str
-
-    """
-
-    error = None
-    algo_folder = get_algo_folder(algo_name, environ)
-    folder = os.path.join(algo_folder, rel_path)
-    ensure_directory(folder)
-
-    # run on all files in the folder
-    for f in os.listdir(folder):
-        try:
-            file_path = os.path.join(folder, f)
-            creation_unix = os.path.getctime(file_path)
-            creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
-
-            # if the file is older than 30 days erase it
-            if today - pd.DateOffset(30) > creation_time:
-                os.unlink(file_path)
-        except OSError:
-            error = 'unable to erase files in {}'.format(folder)
-
-    return error
-
-
-def get_exchange_minute_writer_root(exchange_name, environ=None):
-    """
-    The minute writer folder for the exchange.
-
-    Parameters
-    ----------
-    exchange_name: str
-    environ:
-
-    Returns
-    -------
-    BcolzExchangeBarWriter
-
-    """
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-
-    minute_data_folder = os.path.join(exchange_folder, 'minute_data')
-    ensure_directory(minute_data_folder)
-
-    return minute_data_folder
-
-
-def get_exchange_bundles_folder(exchange_name, environ=None):
-    """
-    The temp folder for bundle downloads by algo name.
-
-    Parameters
-    ----------
-    exchange_name: str
-    environ:
-
-    Returns
-    -------
-    str
-
-    """
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-
-    temp_bundles = os.path.join(exchange_folder, 'temp_bundles')
-    ensure_directory(temp_bundles)
-
-    return temp_bundles
-
-
-def has_bundle(exchange_name, data_frequency, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-
-    folder_name = '{}_bundle'.format(data_frequency.lower())
-    folder = os.path.join(exchange_folder, folder_name)
-
-    return os.path.isdir(folder)
-
-
-def symbols_serial(obj):
-    """
-    JSON serializer for objects not serializable by default json code
-
-    Parameters
-    ----------
-    obj: Object
-
-    Returns
-    -------
-    str
-
-    """
-    if isinstance(obj, (datetime, date)):
-        return obj.floor('1D').strftime(DATE_FORMAT)
-
-    raise TypeError("Type %s not serializable" % type(obj))
-
-
-def perf_serial(obj):
-    """
-    JSON serializer for objects not serializable by default json code
-
-    Parameters
-    ----------
-    obj: Object
-
-    Returns
-    -------
-    str
-
-    """
-    if isinstance(obj, (datetime, date)):
-        return obj.isoformat()
-
-    raise TypeError("Type %s not serializable" % type(obj))
-
-
-def get_common_assets(exchanges):
-    """
-    The assets available in all specified exchanges.
-
-    Parameters
-    ----------
-    exchanges: list[Exchange]
-
-    Returns
-    -------
-    list[TradingPair]
-
-    """
-    symbols = []
-    for exchange_name in exchanges:
-        s = [asset.symbol for asset in exchanges[exchange_name].get_assets()]
-        symbols.append(s)
-
-    inter_symbols = set.intersection(*map(set, symbols))
-
-    assets = []
-    for symbol in inter_symbols:
-        for exchange_name in exchanges:
-            asset = exchanges[exchange_name].get_asset(symbol)
-            assets.append(asset)
-
-    return assets
-
-
-def resample_history_df(df, freq, field, start_dt=None):
-    """
-    Resample the OHCLV DataFrame using the specified frequency.
-
-    Parameters
-    ----------
-    df: DataFrame
-    freq: str
-    field: str
-
-    Returns
-    -------
-    DataFrame
-
-    """
-    if field == 'open':
-        agg = 'first'
-    elif field == 'high':
-        agg = 'max'
-    elif field == 'low':
-        agg = 'min'
-    elif field == 'close':
-        agg = 'last'
-    elif field == 'volume':
-        agg = 'sum'
-    else:
-        raise ValueError('Invalid field.')
-
-    resampled_df = df.resample(
-        freq, closed='left', label='left'
-    ).agg(agg)  # type: pd.DataFrame
-
-    # Because the samples are closed left, we get one more candle at
-    # the beginning then the requested number for bars. Removing this
-    # candle to avoid confusion.
-    if start_dt and not resampled_df.empty:
-        resampled_df = resampled_df[resampled_df.index >= start_dt]
-
-    return resampled_df
-
-
-def mixin_market_params(exchange_name, params, market):
-    """
-    Applies a CCXT market dict to parameters of TradingPair init.
-
-    Parameters
-    ----------
-    params: dict[Object]
-    market: dict[Object]
-
-    Returns
-    -------
-
-    """
-    # TODO: make this more externalized / configurable
-    if 'lot' in market:
-        params['min_trade_size'] = market['lot']
-        params['lot'] = market['lot']
-
-    if exchange_name == 'bitfinex':
-        params['maker'] = 0.001
-        params['taker'] = 0.002
-
-    elif 'maker' in market and 'taker' in market and \
-            market['maker'] is not None and market['taker'] is not None:
-
-        params['maker'] = market['maker']
-        params['taker'] = market['taker']
-
-    else:
-        # TODO: default commission, make configurable
-        params['maker'] = 0.0015
-        params['taker'] = 0.0025
-
-    info = market['info'] if 'info' in market else None
-    if info:
-        if 'minimum_order_size' in info:
-            params['min_trade_size'] = float(info['minimum_order_size'])
-
-            if 'lot' not in params:
-                params['lot'] = params['min_trade_size']
-
-
-def group_assets_by_exchange(assets):
-    exchange_assets = dict()
-    for asset in assets:
-        if asset.exchange not in exchange_assets:
-            exchange_assets[asset.exchange] = list()
-
-        exchange_assets[asset.exchange].append(asset)
-
-    return exchange_assets
-
-
-def get_catalyst_symbol(market_or_symbol):
-    """
-    The Catalyst symbol.
-
-    Parameters
-    ----------
-    market_or_symbol
-
-    Returns
-    -------
-
-    """
-    if isinstance(market_or_symbol, string_types):
-        parts = market_or_symbol.split('/')
-        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
-
-    else:
-        return '{}_{}'.format(
-            market_or_symbol['base'].lower(),
-            market_or_symbol['quote'].lower(),
-        )
-
-
-def save_asset_data(folder, df, decimals=8):
-    symbols = df.index.get_level_values('symbol')
-    for symbol in symbols:
-        symbol_df = df.loc[(symbols == symbol)]  # Type: pd.DataFrame
-
-        filename = os.path.join(folder, '{}.csv'.format(symbol))
-        if os.path.exists(filename):
-            print_headers = False
-
-        else:
-            print_headers = True
-
-        with open(filename, 'a') as f:
-            symbol_df.to_csv(
-                path_or_buf=f,
-                header=print_headers,
-                float_format='%.{}f'.format(decimals),
-            )
-
-
-def forward_fill_df_if_needed(df, periods):
-    df = df.reindex(periods)
-    # volume should always be 0 (if there were no trades in this interval)
-    df['volume'] = df['volume'].fillna(0.0)
-    # ie pull the last close into this close
-    df['close'] = df.fillna(method='pad')
-    # now copy the close that was pulled down from the last timestep
-    # into this row, across into o/h/l
-    df['open'] = df['open'].fillna(df['close'])
-    df['low'] = df['low'].fillna(df['close'])
-    df['high'] = df['high'].fillna(df['close'])
-    return df
-
-
-def transform_candles_to_df(candles):
-    return pd.DataFrame(candles).set_index('last_traded')
-
-
-def get_candles_df(candles, field, freq, bar_count, end_dt=None):
-    all_series = dict()
-
-    for asset in candles:
-        asset_df = transform_candles_to_df(candles[asset])
-        rounded_end_dt = end_dt.floor(freq)
-        periods = pd.date_range(end=rounded_end_dt,
-                                periods=bar_count,
-                                freq=freq)
-        asset_df = forward_fill_df_if_needed(asset_df, periods)
-
-        all_series[asset] = pd.Series(asset_df[field])
-
-    df = pd.DataFrame(all_series)
-    df.dropna(inplace=True)
-
-    return df

catalyst/exchange/utils/factory.py

@@ -1,99 +0,0 @@
-import os
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.ccxt.ccxt_exchange import CCXT
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
-from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
-    get_exchange_folder, is_blacklist
-from logbook import Logger
-
-log = Logger('factory', level=LOG_LEVEL)
-exchange_cache = dict()
-
-
-def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
-                 skip_init=False, auth_alias=None):
-    key = (exchange_name, base_currency)
-    if key in exchange_cache:
-        return exchange_cache[key]
-
-    exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
-
-    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
-    if must_authenticate and not has_auth:
-        raise ExchangeAuthEmpty(
-            exchange=exchange_name.title(),
-            filename=os.path.join(
-                get_exchange_folder(exchange_name), 'auth.json'
-            )
-        )
-
-    exchange = CCXT(
-        exchange_name=exchange_name,
-        key=exchange_auth['key'],
-        secret=exchange_auth['secret'],
-        password=exchange_auth['password'] if 'password'
-                                              in exchange_auth.keys() else '',
-        base_currency=base_currency,
-    )
-    exchange_cache[key] = exchange
-
-    if not skip_init:
-        exchange.init()
-
-    return exchange
-
-
-def get_exchanges(exchange_names):
-    exchanges = dict()
-    for exchange_name in exchange_names:
-        exchanges[exchange_name] = get_exchange(exchange_name)
-
-    return exchanges
-
-
-def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
-                   base_currency=None):
-    """
-    Find exchanges filtered by a list of feature.
-
-    Parameters
-    ----------
-    features: str
-        The list of features.
-
-    skip_blacklist: bool
-    is_authenticated: bool
-    base_currency: bool
-
-    Returns
-    -------
-    list[Exchange]
-
-    """
-    exchange_names = CCXT.find_exchanges(features, is_authenticated)
-
-    exchanges = []
-    for exchange_name in exchange_names:
-        if skip_blacklist and is_blacklist(exchange_name):
-            continue
-
-        exchange = get_exchange(
-            exchange_name=exchange_name,
-            skip_init=True,
-            base_currency=base_currency,
-        )
-
-        if features is not None:
-            if 'dailyBundle' in features \
-                    and not exchange.has_bundle('daily'):
-                continue
-
-            elif 'minuteBundle' in features \
-                    and not exchange.has_bundle('minute'):
-                continue
-
-        exchanges.append(exchange)
-
-    return exchanges

catalyst/exchange/utils/live_chart_utils.py

@@ -1,131 +0,0 @@
-import matplotlib.dates as mdates
-import pandas as pd
-
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
-
-fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
-
-
-def format_ax(ax):
-    """
-    Trying to assign reasonable parameters to the time axis.
-
-    Parameters
-    ----------
-    ax:
-
-    """
-    # TODO: room for improvement
-    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-    ax.xaxis.set_major_formatter(fmt)
-
-    locator = mdates.HourLocator(interval=4)
-    locator.MAXTICKS = 5000
-    ax.xaxis.set_minor_locator(locator)
-
-    datemin = pd.Timestamp.utcnow()
-    ax.set_xlim(datemin)
-
-    ax.grid(True)
-
-
-def set_legend(ax):
-    """
-    Set legend on the chart.
-
-    Parameters
-    ----------
-    ax
-
-    """
-    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-
-def draw_pnl(ax, df):
-    """
-    Draw p&l line on the chart.
-
-    """
-    ax.clear()
-    ax.set_title('Performance')
-    index = df.index.unique()
-    dt = index.get_level_values(level=0)
-    pnl = index.get_level_values(level=4)
-    ax.plot(
-        dt, pnl, '-',
-        color='green',
-        linewidth=1.0,
-        label='Performance'
-    )
-
-    def perc(val):
-        return '{:2f}'.format(val)
-
-    ax.format_ydata = perc
-
-    set_legend(ax)
-    format_ax(ax)
-
-
-def draw_custom_signals(ax, df):
-    """
-    Draw custom signals on the chart.
-
-    """
-    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-    ax.clear()
-    ax.set_title('Custom Signals')
-    for index, column in enumerate(df.columns.values.tolist()):
-        ax.plot(df.index, df[column], '-',
-                color=colors[index],
-                linewidth=1.0,
-                label=column
-                )
-
-    set_legend(ax)
-    format_ax(ax)
-
-
-def draw_exposure(ax, df, context):
-    """
-    Draw exposure line on the chart.
-
-    """
-    # TODO: list exchanges in graph
-    base_currency = None
-    positions = []
-    for exchange_name in context.exchanges:
-        exchange = context.exchanges[exchange_name]
-
-        if not base_currency:
-            base_currency = exchange.base_currency
-        elif base_currency != exchange.base_currency:
-            raise MismatchingBaseCurrenciesExchanges(
-                base_currency=base_currency,
-                exchange_name=exchange.name,
-                exchange_currency=exchange.base_currency
-            )
-
-        positions += exchange.portfolio.positions
-
-    ax.clear()
-    ax.set_title('Exposure')
-    ax.plot(df.index, df['base_currency'], '-',
-            color='green',
-            linewidth=1.0,
-            label='Base Currency: {}'.format(base_currency.upper())
-            )
-
-    symbols = []
-    for position in positions:
-        symbols.append(position.symbol)
-
-    ax.plot(df.index, df['long_exposure'], '-',
-            color='blue',
-            linewidth=1.0,
-            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-    set_legend(ax)
-    format_ax(ax)

catalyst/exchange/utils/serialization_utils.py

@@ -1,69 +0,0 @@
-import json
-import re
-from json import JSONEncoder
-
-import pandas as pd
-from catalyst.constants import DATE_TIME_FORMAT
-from six import string_types
-
-
-class ExchangeJSONEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, pd.Timestamp):
-            return obj.strftime(DATE_TIME_FORMAT)
-
-        # Let the base class default method raise the TypeError
-        return JSONEncoder.default(self, obj)
-
-
-class ExchangeJSONDecoder(json.JSONDecoder):
-    def __init__(self, *args, **kwargs):
-        json.JSONDecoder.__init__(
-            self, object_hook=self.object_hook, *args, **kwargs
-        )
-
-    def recursive_iter(self, obj):
-        if isinstance(obj, dict):
-            for key, value in obj.items():
-                match = isinstance(value, string_types) and re.search(
-                    r'(\d{4}-\d{2}-\d{2}).*', value
-                )
-                if match:
-                    try:
-                        obj[key] = pd.to_datetime(value, utc=True)
-                    except ValueError:
-                        pass
-
-        elif any(isinstance(obj, t) for t in (list, tuple)):
-            for item in obj:
-                self.recursive_iter(item)
-
-    def object_hook(self, obj):
-        self.recursive_iter(obj)
-        return obj
-
-
-def portfolio_to_dict(portfolio):
-    positions = []
-    for asset in portfolio.positions:
-        p = portfolio.positions[asset]  # Type: Position
-
-        position = dict(
-            symbol=asset.symbol,
-            exchange=asset.exchange,
-            amount=p.amount,
-            cost_basis=p.cost_basis,
-            last_sale_price=p.last_sale_price,
-            last_sale_date=p.last_sale_date,
-        )
-        positions.append(position)
-
-    portfolio_dict = vars(portfolio)
-    portfolio_dict['positions'] = positions
-
-    return portfolio_dict
-
-
-def portfolio_from_dict(self, portfolio_data):
-    from catalyst.protocol import Portfolio
-    return Portfolio()

catalyst/exchange/utils/stats_utils.py

@@ -1,486 +0,0 @@
-import copy
-import csv
-import json
-import numbers
-import os
-import time
-
-import numpy as np
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from catalyst.exchange.utils.exchange_utils import get_algo_folder
-from catalyst.utils.paths import data_root, ensure_directory
-from operator import itemgetter
-
-s3_conn = []
-mailgun = []
-
-
-def trend_direction(series):
-    if series[-1] is np.nan or series[-1] is np.nan:
-        return None
-
-    if series[-1] > series[-2]:
-        return 'up'
-    else:
-        return 'down'
-
-
-def crossover(source, target):
-    """
-    The `x`-series is defined as having crossed over `y`-series if the value
-    of `x` is greater than the value of `y` and the value of `x` was less than
-    the value of `y` on the bar immediately preceding the current bar.
-
-    Parameters
-    ----------
-    source: Series
-    target: Series
-
-    Returns
-    -------
-    bool
-
-    """
-    if isinstance(target, numbers.Number):
-        if source[-1] is np.nan or source[-2] is np.nan \
-            or target is np.nan:
-            return False
-
-        if source[-1] >= target > source[-2]:
-            return True
-        else:
-            return False
-
-    else:
-        if source[-1] is np.nan or source[-2] is np.nan \
-            or target[-1] is np.nan or target[-2] is np.nan:
-            return False
-
-        if source[-1] > target[-1] and source[-2] < target[-2]:
-            return True
-        else:
-            return False
-
-
-def crossunder(source, target):
-    """
-    The `x`-series is defined as having crossed under `y`-series if the value
-    of `x` is less than the value of `y` and the value of `x` was greater than
-    the value of `y` on the bar immediately preceding the current bar.
-
-    Parameters
-    ----------
-    source: Series
-    target: Series
-
-    Returns
-    -------
-    bool
-
-    """
-    if isinstance(target, numbers.Number):
-        if source[-1] is np.nan or source[-2] is np.nan \
-            or target is np.nan:
-            return False
-
-        if source[-1] < target <= source[-2]:
-            return True
-        else:
-            return False
-    else:
-        if source[-1] is np.nan or source[-2] is np.nan \
-            or target[-1] is np.nan or target[-2] is np.nan:
-            return False
-
-        if source[-1] < target[-1] and source[-2] >= target[-2]:
-            return True
-        else:
-            return False
-
-
-def vwap(df):
-    """
-    Volume-weighted average price (VWAP) is a ratio generally used by
-    institutional investors and mutual funds to make buys and sells so as not
-    to disturb the market prices with large orders. It is the average share
-    price of a stock weighted against its trading volume within a particular
-    time frame, generally one day.
-
-    Read more: Volume Weighted Average Price - VWAP
-    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-
-    Returns
-    -------
-
-    """
-    if 'close' not in df.columns or 'volume' not in df.columns:
-        raise ValueError('price data must include `volume` and `close`')
-
-    vol_sum = np.nansum(df['volume'].values)
-
-    try:
-        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
-    except ZeroDivisionError:
-        ret = np.nan
-
-    return ret
-
-
-def set_position_row(row, asset, asset_values=list()):
-    """
-    Apply the position data as individual columns.
-
-    Parameters
-    ----------
-    row: dict[str, Object]
-    asset: TradingPair
-    asset_values: list[str]
-        If a recorded_col contains a tuple which first value is an asset
-        matching a position, its value will be displayed with the
-        position and not in the index.
-
-    Returns
-    -------
-
-    """
-    asset_cols = ['symbol']
-    row['symbol'] = asset.symbol
-
-    position = next((p for p in row['positions'] if p['sid'] == asset), None)
-
-    columns = ['amount', 'cost_basis', 'last_sale_price']
-    for column in columns:
-        if position is not None:
-            row[column] = position[column]
-
-        else:
-            row[column] = 0
-
-        asset_cols.append(column)
-
-    values = asset_values[asset] if asset in asset_values else list()
-    for column in values:
-        row[column] = values[column]
-
-        asset_cols.append(column)
-
-    return asset_cols
-
-
-def prepare_stats(stats, recorded_cols=list()):
-    """
-    Prepare the stats DataFrame for user-friendly output.
-
-    Parameters
-    ----------
-    stats: list[Object]
-    recorded_cols: list[str]
-
-    Returns
-    -------
-
-    """
-    asset_cols = list()
-
-    stats = copy.deepcopy(stats)
-    # Using a copy since we are adding rows inside the loop.
-    for row_index, row_data in enumerate(list(stats)):
-        assets = [p['sid'] for p in row_data['positions']]
-
-        asset_values = dict()
-        if recorded_cols is not None:
-            for column in recorded_cols[:]:
-                value = row_data[column]
-                if isinstance(value, pd.Series):
-                    value = value.to_dict()
-
-                if type(value) is dict:
-                    for asset in value:
-                        if not isinstance(asset, TradingPair):
-                            break
-
-                        if asset not in assets:
-                            assets.append(asset)
-
-                        if asset not in asset_values:
-                            asset_values[asset] = dict()
-
-                        asset_values[asset][column] = value[asset]
-
-        if len(assets) == 1:
-            row = stats[row_index]
-            asset_cols = set_position_row(row, assets[0], asset_values)
-
-        elif len(assets) > 1:
-            for asset_index, asset in enumerate(assets):
-                if asset_index > 0:
-                    row = copy.deepcopy(row_data)
-                    stats.append(row)
-
-                else:
-                    row = stats[row_index]
-
-                asset_cols = set_position_row(row, assets[asset_index],
-                                              asset_values)
-
-    df = pd.DataFrame(stats)
-    df['orders'] = df['orders'].apply(lambda orders: len(orders))
-    df['transactions'] = df['transactions'].apply(
-        lambda transactions: len(transactions)
-    )
-    index_cols = [
-        'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
-        'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
-    ]
-
-    # Removing the asset specific entries
-    if recorded_cols is not None:
-        recorded_cols = [x for x in recorded_cols if x not in asset_cols]
-        for column in recorded_cols:
-            index_cols.append(column)
-
-    if asset_cols:
-        columns = asset_cols
-        df.set_index(index_cols, drop=True, inplace=True)
-
-    else:
-        columns = index_cols
-        columns.remove('period_close')
-        df.set_index('period_close', drop=False, inplace=True)
-
-    df.dropna(axis=1, how='all', inplace=True)
-    df.sort_index(axis=0, level=0, inplace=True)
-
-    return df, columns
-
-
-def set_print_settings():
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
-    """
-    Format and print the last few rows of a statistics DataFrame.
-    See the pyfolio project for the data structure.
-
-    Parameters
-    ----------
-    stats: list[Object]
-        An array of statistics for the period.
-
-    num_rows: int
-        The number of rows to display on the screen.
-
-    Returns
-    -------
-    str
-
-    """
-    if isinstance(stats, pd.DataFrame):
-        stats = list(stats.T.to_dict().values())
-        stats.sort(key=itemgetter('period_close'))
-
-    if len(stats) > num_rows:
-        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
-    else:
-        display_stats = stats
-
-    df, columns = prepare_stats(
-        display_stats, recorded_cols=recorded_cols
-    )
-    set_print_settings()
-    return df.to_string(columns=columns)
-
-
-def get_csv_stats(stats, recorded_cols=None):
-    """
-    Create a CSV buffer from the stats DataFrame.
-
-    Parameters
-    ----------
-    path: str
-    stats: list[Object]
-    recorded_cols: list[str]
-
-    Returns
-    -------
-
-    """
-    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
-
-    return df.to_csv(
-        None,
-        columns=columns,
-        # encoding='utf-8',
-        quoting=csv.QUOTE_NONNUMERIC
-    ).encode()
-
-
-def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
-                folder='catalyst/stats', bytes_to_write=None):
-    """
-    Uploads the performance stats to a S3 bucket.
-
-    Parameters
-    ----------
-    uri: str
-    stats: list[Object]
-    algo_namespace: str
-    recorded_cols: list[str]
-    folder: str
-    bytes_to_write: str
-        Option to reuse bytes instead of re-computing the csv
-
-    Returns
-    -------
-
-    """
-    if not s3_conn:
-        import boto3
-        s3_conn.append(boto3.resource('s3'))
-
-    s3 = s3_conn[0]
-
-    if bytes_to_write is None:
-        bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
-
-    now = pd.Timestamp.utcnow()
-    timestr = now.strftime('%Y%m%d')
-    pid = os.getpid()
-
-    parts = uri.split('//')
-    path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
-        folder=folder,
-        algo=algo_namespace,
-        time=timestr,
-        pid=pid,
-    )
-    obj = s3.Object(parts[1], path)
-    obj.put(Body=bytes_to_write)
-
-
-def email_error(algo_name, dt, e, environ=None):
-    import requests
-    import traceback
-
-    if not mailgun:
-        root = data_root(environ)
-        filename = os.path.join(root, 'mailgun.json')
-        if not os.path.exists(filename):
-            raise ValueError(
-                'mailgun.json not found in the catalyst data folder'
-            )
-
-        with open(filename) as data_file:
-            mailgun.append(json.load(data_file))
-
-    mg = mailgun[0]
-
-    return requests.post(
-        mg['url'],
-        auth=("api", mg['api']),
-        data={
-            "from": mg['from'],
-            "to": mg['to'],
-            "subject": 'Error: {}'.format(algo_name),
-            "text": '{}\n\n{}\n{}'.format(
-                dt, e, traceback.format_exc()
-            )})
-
-
-def stats_to_algo_folder(stats, algo_namespace,
-                         folder_name, recorded_cols=None):
-    """
-    Saves the performance stats to the algo local folder.
-
-    Parameters
-    ----------
-    stats: list[Object]
-    algo_namespace: str
-    folder_name: str
-    recorded_cols: list[str]
-
-    Returns
-    -------
-    str
-
-    """
-    bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
-
-    timestr = time.strftime('%Y%m%d')
-    folder = get_algo_folder(algo_namespace)
-
-    stats_folder = os.path.join(folder, folder_name)
-    ensure_directory(stats_folder)
-
-    filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
-
-    with open(filename, 'wb') as handle:
-        handle.write(bytes_to_write)
-
-    return bytes_to_write
-
-
-def df_to_string(df):
-    """
-    Create a formatted str representation of the DataFrame.
-
-    Parameters
-    ----------
-    df: DataFrame
-
-    Returns
-    -------
-    str
-
-    """
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-    return df.to_string()
-
-
-def extract_orders(perf):
-    order_list = perf.orders.values
-    all_orders = [t for sublist in order_list for t in sublist]
-    all_orders.sort(key=lambda o: o['dt'])
-
-    orders = pd.DataFrame(all_orders)
-    if not orders.empty:
-        orders.set_index('dt', inplace=True, drop=True)
-    return orders
-
-
-def extract_transactions(perf):
-    """
-    Compute indexes for buy and sell transactions
-
-    Parameters
-    ----------
-    perf: DataFrame
-        The algo performance DataFrame.
-
-    Returns
-    -------
-    DataFrame
-        A DataFrame of transactions.
-
-    """
-    trans_list = perf.transactions.values
-    all_trans = [t for sublist in trans_list for t in sublist]
-    all_trans.sort(key=lambda t: t['dt'])
-
-    transactions = pd.DataFrame(all_trans)
-    if not transactions.empty:
-        transactions.set_index('dt', inplace=True, drop=True)
-    return transactions

catalyst/exchange/utils/test_utils.py

@@ -1,82 +0,0 @@
-import os
-import random
-import tempfile
-
-from catalyst.assets._assets import TradingPair
-from catalyst.exchange.utils.exchange_utils import get_exchange_folder
-from catalyst.exchange.utils.factory import find_exchanges
-from catalyst.utils.paths import ensure_directory
-
-
-def handle_exchange_error(exchange, e):
-    try:
-        message = '{}: {}'.format(
-            e.__class__, e.message.decode('ascii', 'ignore')
-        )
-    except Exception:
-        message = 'unexpected error'
-
-    folder = get_exchange_folder(exchange.name)
-    filename = os.path.join(folder, 'blacklist.txt')
-    with open(filename, 'wt') as handle:
-        handle.write(message)
-
-
-def select_random_exchanges(population=3, features=None,
-                            is_authenticated=False, base_currency=None):
-    all_exchanges = find_exchanges(
-        features=features,
-        is_authenticated=is_authenticated,
-        base_currency=base_currency,
-    )
-
-    if population is not None:
-        if len(all_exchanges) < population:
-            population = len(all_exchanges)
-
-        exchanges = random.sample(all_exchanges, population)
-
-    else:
-        exchanges = all_exchanges
-
-    return exchanges
-
-
-def select_random_assets(all_assets, population=3):
-    assets = random.sample(all_assets, population)
-    return assets
-
-
-def output_df(df, assets, name=None):
-    """
-    Outputs a price DataFrame to a temp folder.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-    assets
-    name
-
-    Returns
-    -------
-
-    """
-    if isinstance(assets, TradingPair):
-        asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
-    else:
-        asset_folder = ','.join(
-            ['{}_{}'.format(a.exchange, a.symbol) for a in assets]
-        )
-
-    folder = os.path.join(
-        tempfile.gettempdir(), 'catalyst', asset_folder
-    )
-    ensure_directory(folder)
-
-    if name is None:
-        name = 'output'
-
-    path = os.path.join(folder, '{}.csv'.format(name))
-    df.to_csv(path)
-
-    return path, folder

catalyst/finance/blotter.py

@@ -34,9 +34,7 @@ from catalyst.finance.commission import (
 from catalyst.finance.cancel_policy import NeverCancel
 from catalyst.utils.input_validation import expect_types
 
-from catalyst.constants import LOG_LEVEL
+log = Logger('Blotter')
-
-log = Logger('Blotter', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/finance/controls.py

@@ -24,9 +24,7 @@ from catalyst.errors import (
     TradingControlViolation,
 )
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('TradingControl')
-
-log = logbook.Logger('TradingControl', level=LOG_LEVEL)
 
 
 class TradingControl(with_metaclass(abc.ABCMeta)):

catalyst/finance/execution.py

@@ -15,9 +15,14 @@
 
 import abc
 
-from numpy import isfinite
+from sys import float_info
+
 from six import with_metaclass
 
+import catalyst.utils.math_utils as zp_math
+
+from numpy import isfinite
+
 from catalyst.errors import BadOrderParameters
 
 
@@ -72,7 +77,6 @@ class LimitOrder(ExecutionStyle):
     Execution style representing an order to be executed at a price equal to or
     better than a specified limit price.
     """
-
     def __init__(self, limit_price, exchange=None):
         """
         Store the given price.
@@ -95,7 +99,6 @@ class StopOrder(ExecutionStyle):
     Execution style representing an order to be placed once the market price
     reaches a specified stop price.
     """
-
     def __init__(self, stop_price, exchange=None):
         """
         Store the given price.
@@ -118,7 +121,6 @@ class StopLimitOrder(ExecutionStyle):
     Execution style representing a limit order to be placed with a specified
     limit price once the market reaches a specified stop price.
     """
-
     def __init__(self, limit_price, stop_price, exchange=None):
         """
         Store the given prices
@@ -142,20 +144,31 @@ class StopLimitOrder(ExecutionStyle):
 def asymmetric_round_price_to_penny(price, prefer_round_down,
                                     diff=(0.0095 - .005)):
     """
-    Modified the original function because we do not want to round
+    Asymmetric rounding function for adjusting prices to two places in a way
-    prices on crypto exchange.
+    that "improves" the price.  For limit prices, this means preferring to
+    round down on buys and preferring to round up on sells.  For stop prices,
+    it means the reverse.
 
-    Parameters
+    If prefer_round_down == True:
-    ----------
+        When .05 below to .95 above a penny, use that penny.
-    price: float
+    If prefer_round_down == False:
-
+        When .95 below to .05 above a penny, use that penny.
-    Returns
-    -------
-    float
 
+    In math-speak:
+    If prefer_round_down: [<X-1>.0095, X.0195) -> round to X.01.
+    If not prefer_round_down: (<X-1>.0005, X.0105] -> round to X.01.
     """
-    # TODO: consider overriding outside of the original function
+    # Subtracting an epsilon from diff to enforce the open-ness of the upper
-    return price
+    # bound on buys and the lower bound on sells.  Using the actual system
+    # epsilon doesn't quite get there, so use a slightly less epsilon-ey value.
+    epsilon = float_info.epsilon * 10
+    diff = diff - epsilon
+
+    # relies on rounding half away from zero, unlike numpy's bankers' rounding
+    rounded = round(price - (diff if prefer_round_down else -diff), 2)
+    if zp_math.tolerant_equals(rounded, 0.0):
+        return 0.0
+    return rounded
 
 
 def check_stoplimit_prices(price, label):

catalyst/finance/performance/period.py

@@ -88,10 +88,7 @@ from six import itervalues, iteritems
 
 import catalyst.protocol as zp
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Performance')
-
-log = logbook.Logger('Performance', level=LOG_LEVEL)
-
 TRADE_TYPE = zp.DATASOURCE_TYPE.TRADE
 
 

catalyst/finance/performance/position.py

@@ -40,9 +40,7 @@ import logbook
 from catalyst.assets import Future, Asset
 from catalyst.utils.input_validation import expect_types
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Performance')
-
-log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class Position(object):

catalyst/finance/performance/position_tracker.py

@@ -32,9 +32,7 @@ from catalyst.assets import (
 )
 from . position import positiondict
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Performance')
-
-log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 PositionStats = namedtuple('PositionStats',

catalyst/finance/performance/tracker.py

@@ -70,9 +70,7 @@ import catalyst.finance.risk as risk
 
 from . position_tracker import PositionTracker
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Performance')
-
-log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class PerformanceTracker(object):
@@ -113,11 +111,27 @@ class PerformanceTracker(object):
                     self.treasury_curves,
                     self.trading_calendar
                 )
+        elif self.emission_rate == '5-minute':
+            self.all_benchmark_returns = pd.Series(
+                index=pd.date_range(
+                    self.sim_params.first_open,
+                    self.sim_params.last_close,
+                    freq='5min'
+                ),
+            )
+            self.cumulative_risk_metrics = \
+                risk.RiskMetricsCumulative(
+                    self.sim_params,
+                    self.treasury_curves,
+                    self.trading_calendar,
+                    create_first_day_stats=True,
+                )
         elif self.emission_rate == 'minute':
             self.all_benchmark_returns = pd.Series(index=pd.date_range(
                 self.sim_params.first_open, self.sim_params.last_close,
                 freq='Min')
             )
+
             self.cumulative_risk_metrics = \
                 risk.RiskMetricsCumulative(
                     self.sim_params,

catalyst/finance/risk/cumulative.py

@@ -22,25 +22,24 @@ from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
-from .risk import (
+from . risk import (
     check_entry,
     choose_treasury
 )
 
-from catalyst.patches.stats import (
+from empyrical import (
     alpha_beta_aligned,
     annual_volatility,
+    cum_returns,
     downside_risk,
     information_ratio,
     max_drawdown,
     sharpe_ratio,
     sortino_ratio,
-    cum_returns,
 )
-import warnings
-from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
+log = logbook.Logger('Risk Cumulative')
+
 
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
@@ -144,8 +143,6 @@ class RiskMetricsCumulative(object):
         self.num_trading_days = 0
 
     def update(self, dt, algorithm_returns, benchmark_returns, leverage):
-        warnings.filterwarnings('error')
-
         # Keep track of latest dt for use in to_dict and other methods
         # that report current state.
         self.latest_dt = dt
@@ -161,13 +158,9 @@ class RiskMetricsCumulative(object):
             if len(self.algorithm_returns) == 1:
                 self.algorithm_returns = np.append(0.0, self.algorithm_returns)
 
-        try:
+        self.algorithm_cumulative_returns[dt_loc] = cum_returns(
-            self.algorithm_cumulative_returns[dt_loc] = cum_returns(
+            self.algorithm_returns
-                self.algorithm_returns
+        )[-1]
-            )[-1]
-        except Exception as e:
-            log.debug('unable to calculate cum returns: {}'.format(e))
-            self.algorithm_cumulative_returns[dt_loc] = np.nan
 
         algo_cumulative_returns_to_date = \
             self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -196,15 +189,9 @@ class RiskMetricsCumulative(object):
             if len(self.benchmark_returns) == 1:
                 self.benchmark_returns = np.append(0.0, self.benchmark_returns)
 
-        try:
+        self.benchmark_cumulative_returns[dt_loc] = cum_returns(
-            self.benchmark_cumulative_returns[dt_loc] = cum_returns(
+            self.benchmark_returns
-                self.benchmark_returns
+        )[-1]
-            )[-1]
-        except Exception as e:
-            log.debug(
-                'unable to calculate benchmark cum returns: {}'.format(e)
-            )
-            self.benchmark_cumulative_returns[dt_loc] = np.nan
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -276,49 +263,24 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.sharpe[dt_loc] = sharpe_ratio(
             self.algorithm_returns,
         )
-
+        self.downside_risk[dt_loc] = downside_risk(
-        try:
+            self.algorithm_returns
-            self.downside_risk[dt_loc] = downside_risk(
+        )
-                self.algorithm_returns
+        self.sortino[dt_loc] = sortino_ratio(
-            )
+            self.algorithm_returns,
-        except Exception as e:
+            _downside_risk=self.downside_risk[dt_loc]
-            log.debug(
+        )
-                'unable to calculate downside risk returns: {}'.format(e)
-            )
-            self.downside_risk[dt_loc] = np.nan
-
-        try:
-            risk = self.downside_risk[dt_loc]
-            self.sortino[dt_loc] = sortino_ratio(
-                self.algorithm_returns,
-                _downside_risk=risk
-            )
-        except Exception as e:
-            log.debug(
-                'unable to calculate benchmark cum returns: {}'.format(e)
-            )
-            self.sortino[dt_loc] = np.nan
-
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
         )
-        try:
+        self.max_drawdown = max_drawdown(
-            self.max_drawdown = max_drawdown(
+            self.algorithm_returns
-                self.algorithm_returns
+        )
-            )
-        except Exception as e:
-            log.debug(
-                'unable to calculate max drawdown: {}'.format(e)
-            )
-            self.max_drawdown = np.nan
-
         self.max_drawdowns[dt_loc] = self.max_drawdown
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage
 
-        warnings.resetwarnings()
-
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.
@@ -330,18 +292,18 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         rval = {
             'trading_days': self.num_trading_days,
             'benchmark_volatility':
-                self.benchmark_volatility[dt_loc],
+            self.benchmark_volatility[dt_loc],
             'algo_volatility':
-                self.algorithm_volatility[dt_loc],
+            self.algorithm_volatility[dt_loc],
             'treasury_period_return': self.treasury_period_return,
             # Though the two following keys say period return,
             # they would be more accurately called the cumulative return.
             # However, the keys need to stay the same, for now, for backwards
             # compatibility with existing consumers.
             'algorithm_period_return':
-                self.algorithm_cumulative_returns[dt_loc],
+            self.algorithm_cumulative_returns[dt_loc],
             'benchmark_period_return':
-                self.benchmark_cumulative_returns[dt_loc],
+            self.benchmark_cumulative_returns[dt_loc],
             'beta': self.beta[dt_loc],
             'alpha': self.alpha[dt_loc],
             'sharpe': self.sharpe[dt_loc],

catalyst/finance/risk/period.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import functools
-import warnings
 
 import logbook
 
@@ -24,24 +23,20 @@ import numpy as np
 import pandas as pd
 
 from . import risk
-from .risk import check_entry
+from . risk import check_entry
 
 from empyrical import (
     alpha_beta_aligned,
     annual_volatility,
+    cum_returns,
     downside_risk,
     information_ratio,
+    max_drawdown,
     sharpe_ratio,
     sortino_ratio
 )
-from catalyst.patches.stats import (
-    max_drawdown,
-    cum_returns,
-)
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Risk Period')
-
-log = logbook.Logger('Risk Period', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(risk.choose_treasury,
                                     risk.select_treasury_duration)
@@ -81,20 +76,14 @@ class RiskMetricsPeriod(object):
         self.calculate_metrics()
 
     def calculate_metrics(self):
-        warnings.filterwarnings('error')
+        self.benchmark_period_returns = \
-
+            cum_returns(self.benchmark_returns).iloc[-1]
-        try:
-            self.benchmark_period_returns = \
-                cum_returns(self.benchmark_returns).iloc[-1]
-        except Exception:
-            # TODO: why is there an error
-            self.benchmark_period_returns = 0
 
         self.algorithm_period_returns = \
             cum_returns(self.algorithm_returns).iloc[-1]
 
         if not self.algorithm_returns.index.equals(
-                self.benchmark_returns.index
+            self.benchmark_returns.index
         ):
             message = "Mismatch between benchmark_returns ({bm_count}) and \
             algorithm_returns ({algo_count}) in range {start} : {end}"
@@ -137,17 +126,10 @@ class RiskMetricsPeriod(object):
         self.downside_risk = downside_risk(
             self.algorithm_returns.values
         )
-
+        self.sortino = sortino_ratio(
-        try:
+            self.algorithm_returns.values,
-            risk = self.downside_risk
+            _downside_risk=self.downside_risk,
-            self.sortino = sortino_ratio(
+        )
-                self.algorithm_returns.values,
-                _downside_risk=risk,
-            )
-        except Exception:
-            # TODO: what causes it to error out?
-            self.sortino = 0
-
         self.information = information_ratio(
             self.algorithm_returns.values,
             self.benchmark_returns.values,
@@ -156,13 +138,11 @@ class RiskMetricsPeriod(object):
             self.algorithm_returns.values,
             self.benchmark_returns.values,
         )
-        self.excess_return = self.algorithm_period_returns \
+        self.excess_return = self.algorithm_period_returns - \
-            - self.treasury_period_return
+            self.treasury_period_return
         self.max_drawdown = max_drawdown(self.algorithm_returns.values)
         self.max_leverage = self.calculate_max_leverage()
 
-        warnings.resetwarnings()
-
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/finance/risk/report.py

@@ -63,9 +63,7 @@ from dateutil.relativedelta import relativedelta
 
 from . period import RiskMetricsPeriod
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Risk Report')
-
-log = logbook.Logger('Risk Report', level=LOG_LEVEL)
 
 
 class RiskReport(object):

catalyst/finance/risk/risk.py

@@ -61,9 +61,7 @@ Risk Report
 import logbook
 import numpy as np
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Risk')
-
-log = logbook.Logger('Risk', level=LOG_LEVEL)
 
 
 TREASURY_DURATIONS = [
@@ -160,8 +158,7 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
                 )
                 break
 
-        # Supress warning for 'OPEN' calendar
+        if search_day and trading_calendar.name != 'OPEN':          # Supress warning for 'OPEN' calendar
-        if search_day and trading_calendar.name != 'OPEN':
             if (search_dist is None or search_dist > 1) and \
                     search_days[0] <= end_session <= search_days[-1]:
                 message = "No rate within 1 trading day of end date = \

catalyst/finance/slippage.py

@@ -205,22 +205,20 @@ class VolumeShareSlippage(SlippageModel):
     def process_order(self, data, order):
         volume = data.current(order.asset, "volume")
 
-        min_trade_size = order.asset.min_trade_size
-
         max_volume = self.volume_limit * volume
 
         # price impact accounts for the total volume of transactions
         # created against the current minute bar
         remaining_volume = max_volume - self.volume_for_bar
-        if remaining_volume < min_trade_size:
+        if remaining_volume < 1:
             # we can't fill any more transactions
             raise LiquidityExceeded()
 
         # the current order amount will be the min of the
         # volume available in the bar or the open amount.
-        cur_volume = min(remaining_volume, abs(order.open_amount))
+        cur_volume = int(min(remaining_volume, abs(order.open_amount)))
 
-        if cur_volume < min_trade_size:
+        if cur_volume < 1:
             return None, None
 
         # tally the current amount into our total amount ordered.

catalyst/finance/trading.py

@@ -26,9 +26,7 @@ from catalyst.data.loader import load_market_data
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.memoize import remember_last
 
-from catalyst.constants import LOG_LEVEL
+log = logbook.Logger('Trading')
-
-log = logbook.Logger('Trading', level=LOG_LEVEL)
 
 
 DEFAULT_CAPITAL_BASE = 1e5

catalyst/finance/transaction.py

@@ -65,10 +65,14 @@ def create_transaction(order, dt, price, amount):
     # floor the amount to protect against non-whole number orders
     # TODO: Investigate whether we can add a robust check in blotter
     # and/or tradesimulation, as well.
+    amount_magnitude = int(abs(amount))
+
+    if amount_magnitude < 1:
+        raise Exception("Transaction magnitude must be at least 1.")
 
     transaction = Transaction(
         asset=order.asset,
-        amount=amount,
+        amount=int(amount),
         dt=dt,
         price=price,
         order_id=order.id

catalyst/gens/sim_engine.pyx

@@ -20,7 +20,9 @@ cimport cython
 from cpython cimport bool
 
 cdef np.int64_t _nanos_in_minute = 60000000000
+cdef np.int64_t _nanos_in_five_minutes = 5 * _nanos_in_minute
 NANOS_IN_MINUTE = _nanos_in_minute
+NANOS_IN_FIVE_MINUTES = _nanos_in_five_minutes
 
 cpdef enum:
     BAR = 0
@@ -115,3 +117,24 @@ cdef class MinuteSimulationClock:
             yield minute, BAR
             if minute_emission:
                 yield minute, MINUTE_END
+
+cdef class FiveMinuteSimulationClock(MinuteSimulationClock):
+    @cython.boundscheck(False)
+    @cython.wraparound(False)
+    cdef dict calc_minutes_by_session(self):
+        cdef dict five_minutes_by_session
+        cdef int session_idx
+        cdef np.int64_t session_nano
+        cdef np.ndarray[np.int64_t, ndim=1] five_minutes_nanos
+
+        five_minutes_by_session = {}
+        for session_idx, session_nano in enumerate(self.sessions_nanos):
+            five_minutes_nanos = np.arange(
+                self.market_opens_nanos[session_idx],
+                self.market_closes_nanos[session_idx],
+                _nanos_in_five_minutes
+            )
+            five_minutes_by_session[session_nano] = pd.to_datetime(
+                five_minutes_nanos, utc=True, box=True
+            )
+        return five_minutes_by_session

catalyst/gens/tradesimulation.py

@@ -27,15 +27,14 @@ from catalyst.gens.sim_engine import (
     BEFORE_TRADING_START_BAR
 )
 
-from catalyst.constants import LOG_LEVEL
+log = Logger('Trade Simulation')
-
-log = Logger('Trade Simulation', level=LOG_LEVEL)
 
 
 class AlgorithmSimulator(object):
 
     EMISSION_TO_PERF_KEY_MAP = {
         'minute': 'minute_perf',
+        '5-minute': '5_minute_perf',
         'daily': 'daily_perf'
     }
 
@@ -203,7 +202,7 @@ class AlgorithmSimulator(object):
             stack.enter_context(self.processor)
             stack.enter_context(ZiplineAPI(self.algo))
 
-            if algo.data_frequency == 'minute':
+            if algo.data_frequency in set(('minute', '5-minute')):
                 def execute_order_cancellation_policy():
                     algo.blotter.execute_cancel_policy(SESSION_END)
 

catalyst/marketplace/contract_enigma_abi.json

@@ -1,302 +0,0 @@
-[
-    {
-      "constant": true,
-      "inputs": [],
-      "name": "name",
-      "outputs": [
-        {
-          "name": "",
-          "type": "string"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [
-        {
-          "name": "_spender",
-          "type": "address"
-        },
-        {
-          "name": "_value",
-          "type": "uint256"
-        }
-      ],
-      "name": "approve",
-      "outputs": [
-        {
-          "name": "",
-          "type": "bool"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [],
-      "name": "totalSupply",
-      "outputs": [
-        {
-          "name": "",
-          "type": "uint256"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [
-        {
-          "name": "_from",
-          "type": "address"
-        },
-        {
-          "name": "_to",
-          "type": "address"
-        },
-        {
-          "name": "_value",
-          "type": "uint256"
-        }
-      ],
-      "name": "transferFrom",
-      "outputs": [
-        {
-          "name": "",
-          "type": "bool"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [],
-      "name": "INITIAL_SUPPLY",
-      "outputs": [
-        {
-          "name": "",
-          "type": "uint256"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [],
-      "name": "decimals",
-      "outputs": [
-        {
-          "name": "",
-          "type": "uint8"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [
-        {
-          "name": "_spender",
-          "type": "address"
-        },
-        {
-          "name": "_subtractedValue",
-          "type": "uint256"
-        }
-      ],
-      "name": "decreaseApproval",
-      "outputs": [
-        {
-          "name": "success",
-          "type": "bool"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [],
-      "name": "getAfterApproveTest",
-      "outputs": [
-        {
-          "name": "",
-          "type": "uint256"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [
-        {
-          "name": "_owner",
-          "type": "address"
-        }
-      ],
-      "name": "balanceOf",
-      "outputs": [
-        {
-          "name": "balance",
-          "type": "uint256"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [],
-      "name": "symbol",
-      "outputs": [
-        {
-          "name": "",
-          "type": "string"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [
-        {
-          "name": "_to",
-          "type": "address"
-        },
-        {
-          "name": "_value",
-          "type": "uint256"
-        }
-      ],
-      "name": "transfer",
-      "outputs": [
-        {
-          "name": "",
-          "type": "bool"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": false,
-      "inputs": [
-        {
-          "name": "_spender",
-          "type": "address"
-        },
-        {
-          "name": "_addedValue",
-          "type": "uint256"
-        }
-      ],
-      "name": "increaseApproval",
-      "outputs": [
-        {
-          "name": "success",
-          "type": "bool"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "function"
-    },
-    {
-      "constant": true,
-      "inputs": [
-        {
-          "name": "_owner",
-          "type": "address"
-        },
-        {
-          "name": "_spender",
-          "type": "address"
-        }
-      ],
-      "name": "allowance",
-      "outputs": [
-        {
-          "name": "",
-          "type": "uint256"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "view",
-      "type": "function"
-    },
-    {
-      "inputs": [
-        {
-          "name": "testValue",
-          "type": "address"
-        }
-      ],
-      "payable": false,
-      "stateMutability": "nonpayable",
-      "type": "constructor"
-    },
-    {
-      "anonymous": false,
-      "inputs": [
-        {
-          "indexed": true,
-          "name": "owner",
-          "type": "address"
-        },
-        {
-          "indexed": true,
-          "name": "spender",
-          "type": "address"
-        },
-        {
-          "indexed": false,
-          "name": "value",
-          "type": "uint256"
-        }
-      ],
-      "name": "Approval",
-      "type": "event"
-    },
-    {
-      "anonymous": false,
-      "inputs": [
-        {
-          "indexed": true,
-          "name": "from",
-          "type": "address"
-        },
-        {
-          "indexed": true,
-          "name": "to",
-          "type": "address"
-        },
-        {
-          "indexed": false,
-          "name": "value",
-          "type": "uint256"
-        }
-      ],
-      "name": "Transfer",
-      "type": "event"
-    }
-  ]

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +0,0 @@
-0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +0,0 @@
-0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4

catalyst/marketplace/marketplace.py

@@ -1,710 +0,0 @@
-from __future__ import print_function
-
-import glob
-import json
-import os
-import re
-import shutil
-import sys
-import time
-
-import bcolz
-import logbook
-import pandas as pd
-import requests
-from requests_toolbelt import MultipartDecoder
-from requests_toolbelt.multipart.decoder import \
-    NonMultipartContentTypeException
-
-from catalyst.constants import (
-    LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
-    MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
-from catalyst.exchange.utils.stats_utils import set_print_settings
-from catalyst.marketplace.marketplace_errors import (
-    MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
-    MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
-    MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
-from catalyst.marketplace.utils.auth_utils import get_key_secret, \
-    get_signed_headers
-from catalyst.marketplace.utils.bundle_utils import merge_bundles
-from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
-    to_grains
-from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
-    get_data_source_folder, get_marketplace_folder, \
-    get_user_pubaddr, get_temp_bundles_folder, extract_bundle
-
-if sys.version_info.major < 3:
-    import urllib
-else:
-    import urllib.request as urllib
-
-log = logbook.Logger('Marketplace', level=LOG_LEVEL)
-
-
-class Marketplace:
-    def __init__(self):
-        global Web3
-        try:
-            from web3 import Web3, HTTPProvider
-        except ImportError:
-            raise MarketplaceRequiresPython3()
-
-        self.addresses = get_user_pubaddr()
-
-        if self.addresses[0]['pubAddr'] == '':
-            raise MarketplacePubAddressEmpty(
-                filename=os.path.join(
-                    get_marketplace_folder(), 'addresses.json')
-            )
-        self.default_account = self.addresses[0]['pubAddr']
-
-        self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
-
-        contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
-
-        self.mkt_contract_address = Web3.toChecksumAddress(
-            contract_url.readline().decode(
-                contract_url.info().get_content_charset()).strip())
-
-        abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
-        abi = json.load(abi_url)
-
-        self.mkt_contract = self.web3.eth.contract(
-            self.mkt_contract_address,
-            abi=abi,
-        )
-
-        contract_url = urllib.urlopen(ENIGMA_CONTRACT)
-
-        self.eng_contract_address = Web3.toChecksumAddress(
-            contract_url.readline().decode(
-                contract_url.info().get_content_charset()).strip())
-
-        abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
-        abi = json.load(abi_url)
-
-        self.eng_contract = self.web3.eth.contract(
-            self.eng_contract_address,
-            abi=abi,
-        )
-
-    # def get_data_sources_map(self):
-    #     return [
-    #         dict(
-    #             name='Marketcap',
-    #             desc='The marketcap value in USD.',
-    #             start_date=pd.to_datetime('2017-01-01'),
-    #             end_date=pd.to_datetime('2018-01-15'),
-    #             data_frequencies=['daily'],
-    #         ),
-    #         dict(
-    #             name='GitHub',
-    #             desc='The rate of development activity on GitHub.',
-    #             start_date=pd.to_datetime('2017-01-01'),
-    #             end_date=pd.to_datetime('2018-01-15'),
-    #             data_frequencies=['daily', 'hour'],
-    #         ),
-    #         dict(
-    #             name='Influencers',
-    #             desc='Tweets & related sentiments by selected influencers.',
-    #             start_date=pd.to_datetime('2017-01-01'),
-    #             end_date=pd.to_datetime('2018-01-15'),
-    #             data_frequencies=['daily', 'hour', 'minute'],
-    #         ),
-    #     ]
-
-    def to_text(self, hex):
-        return Web3.toText(hex).rstrip('\0')
-
-    def choose_pubaddr(self):
-        if len(self.addresses) == 1:
-            address = self.addresses[0]['pubAddr']
-            address_i = 0
-            print('Using {} for this transaction.'.format(address))
-        else:
-            while True:
-                for i in range(0, len(self.addresses)):
-                    print('{}\t{}\t{}'.format(
-                        i,
-                        self.addresses[i]['pubAddr'],
-                        self.addresses[i]['desc'])
-                    )
-                address_i = int(input('Choose your address associated with '
-                                      'this transaction: [default: 0] ') or 0)
-                if not (0 <= address_i < len(self.addresses)):
-                    print('Please choose a number between 0 and {}\n'.format(
-                        len(self.addresses) - 1))
-                else:
-                    address = Web3.toChecksumAddress(
-                        self.addresses[address_i]['pubAddr'])
-                    break
-
-        return address, address_i
-
-    def sign_transaction(self, from_address, tx):
-
-        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
-              'enter the following parameters:\n\n'
-              'From Address:\t\t{_from}\n'
-              '\n\tClick the "Generate Information" button\n\n'
-              'To Address:\t\t{to}\n'
-              'Value / Amount to Send:\t{value}\n'
-              'Gas Limit:\t\t{gas}\n'
-              'Gas Price:\t\t[Accept the default value]\n'
-              'Nonce:\t\t\t{nonce}\n'
-              'Data:\t\t\t{data}\n'.format(
-                _from=from_address,
-                to=tx['to'],
-                value=tx['value'],
-                gas=tx['gas'],
-                nonce=tx['nonce'],
-                data=tx['data'], )
-              )
-
-        signed_tx = input('Copy and Paste the "Signed Transaction" '
-                          'field here:\n')
-
-        if signed_tx.startswith('0x'):
-            signed_tx = signed_tx[2:]
-
-        return signed_tx
-
-    def check_transaction(self, tx_hash):
-
-        if 'ropsten' in ETH_REMOTE_NODE:
-            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
-                tx_hash)
-        else:
-            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
-
-        print('\nYou can check the outcome of your transaction here:\n'
-              '{}\n\n'.format(etherscan))
-
-    def list(self):
-
-        data_sources = self.mkt_contract.functions.getAllProviders().call()
-
-        data = []
-        for index, data_source in enumerate(data_sources):
-            if index > 0:
-                if 'test' not in Web3.toText(data_source).lower():
-                    data.append(
-                        dict(
-                            dataset=self.to_text(data_source)
-                        )
-                    )
-
-        df = pd.DataFrame(data)
-        set_print_settings()
-        if df.empty:
-            print('There are no datasets available yet.')
-        else:
-            print(df)
-
-    def subscribe(self, dataset):
-
-        dataset = dataset.lower()
-
-        address = self.choose_pubaddr()[0]
-        provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(dataset)
-        ).call()
-
-        if not provider_info[4]:
-            print('The requested "{}" dataset is not registered in '
-                  'the Data Marketplace.'.format(dataset))
-            return
-
-        grains = provider_info[1]
-        price = from_grains(grains)
-
-        subscribed = self.mkt_contract.functions.checkAddressSubscription(
-            address, Web3.toHex(dataset)
-        ).call()
-
-        if subscribed[5]:
-            print(
-                '\nYou are already subscribed to the "{}" dataset.\n'
-                'Your subscription started on {} UTC, and is valid until '
-                '{} UTC.'.format(
-                    dataset,
-                    pd.to_datetime(subscribed[3], unit='s', utc=True),
-                    pd.to_datetime(subscribed[4], unit='s', utc=True)
-                )
-            )
-            return
-
-        print('\nThe price for a monthly subscription to this dataset is'
-              ' {} ENG'.format(price))
-
-        print(
-            'Checking that the ENG balance in {} is greater than {} '
-            'ENG... '.format(address, price), end=''
-        )
-
-        wallet_address = address[2:]
-        balance = self.web3.eth.call({
-            'from': address,
-            'to': self.eng_contract_address,
-            'data': '0x70a08231000000000000000000000000{}'.format(
-                wallet_address
-            )
-        })
-
-        try:
-            balance = Web3.toInt(balance)  # web3 >= 4.0.0b7
-        except TypeError:
-            balance = Web3.toInt(hexstr=balance)  # web3 <= 4.0.0b6
-
-        if balance > grains:
-            print('OK.')
-        else:
-            print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
-                  'than the price of the dataset that you are trying to\n'
-                  'buy: {} ENG. Get enough ENG to cover the costs of the '
-                  'monthly\nsubscription for what you are trying to buy, '
-                  'and try again.'.format(
-                    address, from_grains(balance), price))
-            return
-
-        while True:
-            agree_pay = input('Please confirm that you agree to pay {} ENG '
-                              'for a monthly subscription to the dataset "{}" '
-                              'starting today. [default: Y] '.format(
-                                price, dataset)) or 'y'
-            if agree_pay.lower() not in ('y', 'n'):
-                print("Please answer Y or N.")
-            else:
-                if agree_pay.lower() == 'y':
-                    break
-                else:
-                    return
-
-        print('Ready to subscribe to dataset {}.\n'.format(dataset))
-        print('In order to execute the subscription, you will need to sign '
-              'two different transactions:\n'
-              '1. First transaction is to authorize the Marketplace contract '
-              'to spend {} ENG on your behalf.\n'
-              '2. Second transaction is the actual subscription for the '
-              'desired dataset'.format(price))
-
-        tx = self.eng_contract.functions.approve(
-            self.mkt_contract_address,
-            grains,
-        ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
-        )
-
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
-        try:
-            tx_hash = '0x{}'.format(
-                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
-            )
-            print(
-                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
-            )
-
-        except Exception as e:
-            print('Unable to subscribe to data source: {}'.format(e))
-            return
-
-        self.check_transaction(tx_hash)
-
-        print('Waiting for the first transaction to succeed...')
-
-        while True:
-            try:
-                if self.web3.eth.getTransactionReceipt(tx_hash).status:
-                    break
-                else:
-                    print('\nTransaction failed. Aborting...')
-                    return
-            except AttributeError:
-                pass
-            for i in range(0, 10):
-                print('.', end='', flush=True)
-                time.sleep(1)
-
-        print('\nFirst transaction successful!\n'
-              'Now processing second transaction.')
-
-        tx = self.mkt_contract.functions.subscribe(
-            Web3.toHex(dataset),
-        ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)})
-
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
-
-        try:
-            tx_hash = '0x{}'.format(bin_hex(
-                self.web3.eth.sendRawTransaction(signed_tx)))
-            print('\nThis is the TxHash for this transaction: '
-                  '{}'.format(tx_hash))
-
-        except Exception as e:
-            print('Unable to subscribe to data source: {}'.format(e))
-            return
-
-        self.check_transaction(tx_hash)
-
-        print('Waiting for the second transaction to succeed...')
-
-        while True:
-            try:
-                if self.web3.eth.getTransactionReceipt(tx_hash).status:
-                    break
-                else:
-                    print('\nTransaction failed. Aborting...')
-                    return
-            except AttributeError:
-                pass
-            for i in range(0, 10):
-                print('.', end='', flush=True)
-                time.sleep(1)
-
-        print('\nSecond transaction successful!\n'
-              'You have successfully subscribed to dataset {} with'
-              'address {}.\n'
-              'You can now ingest this dataset anytime during the '
-              'next month by running the following command:\n'
-              'catalyst marketplace ingest --dataset={}'.format(
-                dataset, address, dataset))
-
-    def process_temp_bundle(self, ds_name, path):
-        """
-        Merge the temp bundle into the main bundle for the specified
-        data source.
-
-        Parameters
-        ----------
-        ds_name
-        path
-
-        Returns
-        -------
-
-        """
-        tmp_bundle = extract_bundle(path)
-        bundle_folder = get_data_source_folder(ds_name)
-        if os.listdir(bundle_folder):
-            zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
-            ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
-            merge_bundles(zsource, ztarget)
-
-        else:
-            os.rename(tmp_bundle, bundle_folder)
-
-        pass
-
-    def ingest(self, ds_name, start=None, end=None, force_download=False):
-
-        # ds_name = ds_name.lower()
-
-        # TODO: catch error conditions
-        provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(ds_name)
-        ).call()
-
-        if not provider_info[4]:
-            print('The requested "{}" dataset is not registered in '
-                  'the Data Marketplace.'.format(ds_name))
-            return
-
-        address, address_i = self.choose_pubaddr()
-        fns = self.mkt_contract.functions
-        check_sub = fns.checkAddressSubscription(
-            address, Web3.toHex(ds_name)
-        ).call()
-
-        if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
-            print('You are not subscribed to dataset "{}" with address {}. '
-                  'Plese subscribe first.'.format(ds_name, address))
-            return
-
-        if not check_sub[5]:
-            print('Your subscription to dataset "{}" expired on {} UTC.'
-                  'Please renew your subscription by running:\n'
-                  'catalyst marketplace subscribe --dataset={}'.format(
-                    ds_name,
-                    pd.to_datetime(check_sub[4], unit='s', utc=True),
-                    ds_name)
-                  )
-
-        if 'key' in self.addresses[address_i]:
-            key = self.addresses[address_i]['key']
-            secret = self.addresses[address_i]['secret']
-        else:
-            key, secret = get_key_secret(address)
-
-        headers = get_signed_headers(ds_name, key, secret)
-        log.debug('Starting download of dataset for ingestion...')
-        r = requests.post(
-            '{}/marketplace/ingest'.format(AUTH_SERVER),
-            headers=headers,
-            stream=True,
-        )
-        if r.status_code == 200:
-            target_path = get_temp_bundles_folder()
-            try:
-                decoder = MultipartDecoder.from_response(r)
-                for part in decoder.parts:
-                    h = part.headers[b'Content-Disposition'].decode('utf-8')
-                    # Extracting the filename from the header
-                    name = re.search(r'filename="(.*)"', h).group(1)
-
-                    filename = os.path.join(target_path, name)
-                    with open(filename, 'wb') as f:
-                        # for chunk in part.content.iter_content(
-                        #         chunk_size=1024):
-                        #     if chunk: # filter out keep-alive new chunks
-                        #         f.write(chunk)
-                        f.write(part.content)
-
-                    self.process_temp_bundle(ds_name, filename)
-
-            except NonMultipartContentTypeException:
-                response = r.json()
-                raise MarketplaceHTTPRequest(
-                    request='ingest dataset',
-                    error=response,
-                )
-        else:
-            raise MarketplaceHTTPRequest(
-                request='ingest dataset',
-                error=r.status_code,
-            )
-
-        log.info('{} ingested successfully'.format(ds_name))
-
-    def get_dataset(self, ds_name, start=None, end=None):
-        ds_name = ds_name.lower()
-
-        # TODO: filter ctable by start and end date
-        bundle_folder = get_data_source_folder(ds_name)
-        z = bcolz.ctable(rootdir=bundle_folder, mode='r')
-
-        df = z.todataframe()  # type: pd.DataFrame
-        df.set_index(['date', 'symbol'], drop=True, inplace=True)
-
-        # TODO: implement the filter more carefully
-        # if start and end is None:
-        #     df = df.xs(start, level=0)
-
-        return df
-
-    def clean(self, data_source_name, data_frequency=None):
-        data_source_name = data_source_name.lower()
-
-        if data_frequency is None:
-            folder = get_data_source_folder(data_source_name)
-
-        else:
-            folder = get_bundle_folder(data_source_name, data_frequency)
-
-        shutil.rmtree(folder)
-        pass
-
-    def create_metadata(self, key, secret, ds_name, data_frequency, desc,
-                        has_history=True, has_live=True):
-        """
-
-        Returns
-        -------
-
-        """
-        headers = get_signed_headers(ds_name, key, secret)
-        r = requests.post(
-            '{}/marketplace/register'.format(AUTH_SERVER),
-            json=dict(
-                ds_name=ds_name,
-                desc=desc,
-                data_frequency=data_frequency,
-                has_history=has_history,
-                has_live=has_live,
-            ),
-            headers=headers,
-        )
-
-        if r.status_code != 200:
-            raise MarketplaceHTTPRequest(
-                request='register', error=r.status_code
-            )
-
-        if 'error' in r.json():
-            raise MarketplaceHTTPRequest(
-                request='upload file', error=r.json()['error']
-            )
-
-    def register(self):
-        while True:
-            desc = input('Enter the name of the dataset to register: ')
-            dataset = desc.lower()
-            provider_info = self.mkt_contract.functions.getDataProviderInfo(
-                Web3.toHex(dataset)
-            ).call()
-
-            if provider_info[4]:
-                print('There is already a dataset registered under '
-                      'the name "{}". Please choose a different '
-                      'name.'.format(dataset))
-            else:
-                break
-
-        price = int(
-            input(
-                'Enter the price for a monthly subscription to '
-                'this dataset in ENG: '
-            )
-        )
-        while True:
-            freq = input('Enter the data frequency [daily, hourly, minute]: ')
-            if freq.lower() not in ('daily', 'hourly', 'minute'):
-                print('Not a valid frequency.')
-            else:
-                break
-
-        while True:
-            reg_pub = input(
-                'Does it include historical data? [default: Y]: '
-            ) or 'y'
-            if reg_pub.lower() not in ('y', 'n'):
-                print('Please answer Y or N.')
-            else:
-                if reg_pub.lower() == 'y':
-                    has_history = True
-                else:
-                    has_history = False
-                break
-
-        while True:
-            reg_pub = input(
-                'Doest it include live data? [default: Y]: '
-            ) or 'y'
-            if reg_pub.lower() not in ('y', 'n'):
-                print('Please answer Y or N.')
-            else:
-                if reg_pub.lower() == 'y':
-                    has_live = True
-                else:
-                    has_live = False
-                break
-
-        address, address_i = self.choose_pubaddr()
-        if 'key' in self.addresses[address_i]:
-            key = self.addresses[address_i]['key']
-            secret = self.addresses[address_i]['secret']
-        else:
-            key, secret = get_key_secret(address)
-
-        grains = to_grains(price)
-
-        tx = self.mkt_contract.functions.register(
-            Web3.toHex(dataset),
-            grains,
-            address,
-        ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
-        )
-
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
-
-        try:
-            tx_hash = '0x{}'.format(
-                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
-            )
-            print(
-                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
-            )
-
-        except Exception as e:
-            print('Unable to register the requested dataset: {}'.format(e))
-            return
-
-        self.check_transaction(tx_hash)
-
-        print('Waiting for the transaction to succeed...')
-
-        while True:
-            try:
-                if self.web3.eth.getTransactionReceipt(tx_hash).status:
-                    break
-                else:
-                    print('\nTransaction failed. Aborting...')
-                    return
-            except AttributeError:
-                pass
-            for i in range(0, 10):
-                print('.', end='', flush=True)
-                time.sleep(1)
-
-        print('\nWarming up the {} dataset'.format(dataset))
-        self.create_metadata(
-            key=key,
-            secret=secret,
-            ds_name=dataset,
-            data_frequency=freq,
-            desc=desc,
-            has_history=has_history,
-            has_live=has_live,
-        )
-        print('\n{} registered successfully'.format(dataset))
-
-    def publish(self, dataset, datadir, watch):
-        dataset = dataset.lower()
-        provider_info = self.mkt_contract.functions.getDataProviderInfo(
-            Web3.toHex(dataset)
-        ).call()
-
-        if not provider_info[4]:
-            raise MarketplaceDatasetNotFound(dataset=dataset)
-
-        match = next(
-            (l for l in self.addresses if l['pubAddr'] == provider_info[0]),
-            None
-        )
-        if not match:
-            raise MarketplaceNoAddressMatch(
-                dataset=dataset,
-                address=provider_info[0])
-
-        print('Using address: {} to publish this dataset.'.format(
-            provider_info[0]))
-
-        if 'key' in match:
-            key = match['key']
-            secret = match['secret']
-        else:
-            key, secret = get_key_secret(provider_info[0])
-
-        headers = get_signed_headers(dataset, key, secret)
-        filenames = glob.glob(os.path.join(datadir, '*.csv'))
-
-        if not filenames:
-            raise MarketplaceNoCSVFiles(datadir=datadir)
-
-        files = []
-        for file in filenames:
-            files.append(('file', open(file, 'rb')))
-
-        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
-                          files=files,
-                          headers=headers)
-
-        if r.status_code != 200:
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.status_code)
-
-        if 'error' in r.json():
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.json()['error'])
-
-        print('Dataset {} uploaded successfully.'.format(dataset))