BLD: added a cmd for running on the cloud (WIP)

# Conflicts:
#	catalyst/examples/mean_reversion_simple.py
#	catalyst/exchange/ccxt/ccxt_exchange.py
#	catalyst/exchange/exchange.py
#	catalyst/exchange/utils/serialization_utils.py
#	etc/python2.7-environment.yml
#	etc/requirements.txt
#	tests/exchange/test_suites/test_suite_exchange.py

catalyst/__main__.py

@@ -506,197 +506,6 @@ def live(ctx,
     return perf
 
 
-@main.command(name='serve')
-@click.option(
-    '-f',
-    '--algofile',
-    default=None,
-    type=click.File('r'),
-    help='The file that contains the algorithm to run.',
-)
-@click.option(
-    '-t',
-    '--algotext',
-    help='The algorithm script to run.',
-)
-@click.option(
-    '-D',
-    '--define',
-    multiple=True,
-    help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be"
-         " any python expression. These are evaluated in order so they"
-         " may refer to previously defined names.",
-)
-@click.option(
-    '--data-frequency',
-    type=click.Choice({'daily', 'minute'}),
-    default='daily',
-    show_default=True,
-    help='The data frequency of the simulation.',
-)
-@click.option(
-    '--capital-base',
-    type=float,
-    show_default=True,
-    help='The starting capital for the simulation.',
-)
-@click.option(
-    '-b',
-    '--bundle',
-    default='poloniex',
-    metavar='BUNDLE-NAME',
-    show_default=True,
-    help='The data bundle to use for the simulation.',
-)
-@click.option(
-    '--bundle-timestamp',
-    type=Timestamp(),
-    default=pd.Timestamp.utcnow(),
-    show_default=False,
-    help='The date to lookup data on or before.\n'
-         '[default: <current-time>]'
-)
-@click.option(
-    '-s',
-    '--start',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The start date of the simulation.',
-)
-@click.option(
-    '-e',
-    '--end',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The end date of the simulation.',
-)
-@click.option(
-    '-o',
-    '--output',
-    default='-',
-    metavar='FILENAME',
-    show_default=True,
-    help="The location to write the perf data. If this is '-' the perf"
-         " will be written to stdout.",
-)
-@click.option(
-    '--print-algo/--no-print-algo',
-    is_flag=True,
-    default=False,
-    help='Print the algorithm to stdout.',
-)
-@ipython_only(click.option(
-    '--local-namespace/--no-local-namespace',
-    is_flag=True,
-    default=None,
-    help='Should the algorithm methods be resolved in the local namespace.'
-))
-@click.option(
-    '-x',
-    '--exchange-name',
-    help='The name of the targeted exchange.',
-)
-@click.option(
-    '-n',
-    '--algo-namespace',
-    help='A label assigned to the algorithm for data storage purposes.'
-)
-@click.option(
-    '-c',
-    '--base-currency',
-    help='The base currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
-@click.pass_context
-def run(ctx,
-        algofile,
-        algotext,
-        define,
-        data_frequency,
-        capital_base,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        exchange_name,
-        algo_namespace,
-        base_currency):
-    """Run a backtest for the given algorithm on the server.
-    """
-
-    if (algotext is not None) == (algofile is not None):
-        ctx.fail(
-            "must specify exactly one of '-f' / '--algofile' or"
-            " '-t' / '--algotext'",
-        )
-
-    # check that the start and end dates are passed correctly
-    if start is None and end is None:
-        # check both at the same time to avoid the case where a user
-        # does not pass either of these and then passes the first only
-        # to be told they need to pass the second argument also
-        ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'"
-            " in backtest mode",
-        )
-    if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'"
-                 " in backtest mode")
-    if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'"
-                 " in backtest mode")
-
-    if exchange_name is None:
-        ctx.fail("must specify an exchange name '-x'")
-
-    if base_currency is None:
-        ctx.fail("must specify a base currency with '-c' in backtest mode")
-
-    if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'")
-
-    click.echo('Running in backtesting mode.', sys.stdout)
-
-    perf = run_server(
-        initialize=None,
-        handle_data=None,
-        before_trading_start=None,
-        analyze=None,
-        algofile=algofile,
-        algotext=algotext,
-        defines=define,
-        data_frequency=data_frequency,
-        capital_base=capital_base,
-        data=None,
-        bundle=bundle,
-        bundle_timestamp=bundle_timestamp,
-        start=start,
-        end=end,
-        output=output,
-        print_algo=print_algo,
-        local_namespace=local_namespace,
-        environ=os.environ,
-        live=False,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency,
-        analyze_live=None,
-        live_graph=False,
-        simulate_orders=True,
-        auth_aliases=None,
-        stats_output=None,
-    )
-
-    if output == '-':
-        click.echo(str(perf), sys.stdout)
-    elif output != os.devnull:  # make the catalyst magic not write any data
-        perf.to_pickle(output)
-
-    return perf
-
-
 @main.command(name='serve-live')
 @click.option(
     '-f',
@@ -837,7 +646,7 @@ def serve_live(ctx,
         handle_data=None,
         before_trading_start=None,
         analyze=None,
-            algofile=algofile,
+        algofile=algofile,
         algotext=algotext,
         defines=define,
         data_frequency=None,

catalyst/constants.py

@@ -27,20 +27,20 @@ AUTH_SERVER = 'https://data.enigma.co'
 # TODO: switch to mainnet
 ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
 
-
+# TODO: move to MASTER branch on github
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
-                       'catalyst/master/catalyst/marketplace/' \
+                       'catalyst/develop/catalyst/marketplace/' \
                        'contract_marketplace_address.txt'
 
 MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
-                           'catalyst/master/catalyst/marketplace/' \
+                           'catalyst/develop/catalyst/marketplace/' \
                            'contract_marketplace_abi.json'
 
 # TODO: switch to mainnet
 ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-                  'master/catalyst/marketplace/' \
+                  'develop/catalyst/marketplace/' \
                   'contract_enigma_address.txt'
 
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
-                      'catalyst/master/catalyst/marketplace/' \
+                      'catalyst/develop/catalyst/marketplace/' \
                       'contract_enigma_abi.json'

catalyst/exchange/exchange_blotter.py

@@ -68,7 +68,7 @@ class TradingPairFeeSchedule(CommissionModel):
             multiplier = maker \
                 if ((order.amount > 0 and order.limit < transaction.price)
                     or (order.amount < 0 and order.limit > transaction.price)) \
-                and order.limit_reached else taker
+                   and order.limit_reached else taker
 
         fee = cost * multiplier
         return fee

catalyst/exchange/exchange_bundle.py

@@ -843,7 +843,6 @@ class ExchangeBundle:
         field: str
         data_frequency: str
         algo_end_dt: pd.Timestamp
-        force_auto_ingest: 
 
         Returns
         -------

catalyst/exchange/utils/exchange_utils.py

@@ -716,36 +716,25 @@ def save_asset_data(folder, df, decimals=8):
             )
 
 
-def forward_fill_df_if_needed(df, periods):
+def get_candles_df(candles, field, freq, bar_count, end_dt,
-    df = df.reindex(periods)
+                   previous_value=None):
-    # volume should always be 0 (if there were no trades in this interval)
-    df['volume'] = df['volume'].fillna(0.0)
-    # ie pull the last close into this close
-    df['close'] = df.fillna(method='pad')
-    # now copy the close that was pulled down from the last timestep
-    # into this row, across into o/h/l
-    df['open'] = df['open'].fillna(df['close'])
-    df['low'] = df['low'].fillna(df['close'])
-    df['high'] = df['high'].fillna(df['close'])
-    return df
-
-
-def transform_candles_to_df(candles):
-    return pd.DataFrame(candles).set_index('last_traded')
-
-
-def get_candles_df(candles, field, freq, bar_count, end_dt=None):
     all_series = dict()
-
     for asset in candles:
-        asset_df = transform_candles_to_df(candles[asset])
+        periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
-        rounded_end_dt = end_dt.floor(freq)
-        periods = pd.date_range(end=rounded_end_dt,
-                                periods=bar_count,
-                                freq=freq)
-        asset_df = forward_fill_df_if_needed(asset_df, periods)
 
-        all_series[asset] = pd.Series(asset_df[field])
+        dates = [candle['last_traded'] for candle in candles[asset]]
+        values = [candle[field] for candle in candles[asset]]
+        series = pd.Series(values, index=dates)
+
+        """
+        series = series.reindex(
+            periods,
+            method='ffill',
+            fill_value=previous_value,
+        )
+        series.sort_index(inplace=True)
+        """
+        all_series[asset] = series
 
     df = pd.DataFrame(all_series)
     df.dropna(inplace=True)

catalyst/marketplace/marketplace.py

@@ -23,7 +23,7 @@ from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.marketplace.marketplace_errors import (
     MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
     MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
-    MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
+    MarketplaceNoCSVFiles)
 from catalyst.marketplace.utils.auth_utils import get_key_secret, \
     get_signed_headers
 from catalyst.marketplace.utils.bundle_utils import merge_bundles
@@ -44,10 +44,7 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL)
 class Marketplace:
     def __init__(self):
         global Web3
-        try:
+        from web3 import Web3, HTTPProvider
-            from web3 import Web3, HTTPProvider
-        except ImportError:
-            raise MarketplaceRequiresPython3()
 
         self.addresses = get_user_pubaddr()
 
@@ -63,8 +60,7 @@ class Marketplace:
         contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
 
         self.mkt_contract_address = Web3.toChecksumAddress(
-            contract_url.readline().decode(
+            contract_url.readline().strip())
-                contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
         abi = json.load(abi_url)
@@ -77,8 +73,7 @@ class Marketplace:
         contract_url = urllib.urlopen(ENIGMA_CONTRACT)
 
         self.eng_contract_address = Web3.toChecksumAddress(
-            contract_url.readline().decode(
+            contract_url.readline().strip())
-                contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
         abi = json.load(abi_url)
@@ -153,13 +148,13 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-                _from=from_address,
+            _from=from_address,
-                to=tx['to'],
+            to=tx['to'],
-                value=tx['value'],
+            value=tx['value'],
-                gas=tx['gas'],
+            gas=tx['gas'],
-                nonce=tx['nonce'],
+            nonce=tx['nonce'],
-                data=tx['data'], )
+            data=tx['data'], )
-              )
+        )
 
         signed_tx = input('Copy and Paste the "Signed Transaction" '
                           'field here:\n')
@@ -264,14 +259,14 @@ class Marketplace:
                   'buy: {} ENG. Get enough ENG to cover the costs of the '
                   'monthly\nsubscription for what you are trying to buy, '
                   'and try again.'.format(
-                    address, from_grains(balance), price))
+                address, from_grains(balance), price))
             return
 
         while True:
             agree_pay = input('Please confirm that you agree to pay {} ENG '
                               'for a monthly subscription to the dataset "{}" '
                               'starting today. [default: Y] '.format(
-                                price, dataset)) or 'y'
+                price, dataset)) or 'y'
             if agree_pay.lower() not in ('y', 'n'):
                 print("Please answer Y or N.")
             else:
@@ -374,7 +369,7 @@ class Marketplace:
               'You can now ingest this dataset anytime during the '
               'next month by running the following command:\n'
               'catalyst marketplace ingest --dataset={}'.format(
-                dataset, address, dataset))
+            dataset, address, dataset))
 
     def process_temp_bundle(self, ds_name, path):
         """
@@ -431,10 +426,10 @@ class Marketplace:
             print('Your subscription to dataset "{}" expired on {} UTC.'
                   'Please renew your subscription by running:\n'
                   'catalyst marketplace subscribe --dataset={}'.format(
-                    ds_name,
+                ds_name,
-                    pd.to_datetime(check_sub[4], unit='s', utc=True),
+                pd.to_datetime(check_sub[4], unit='s', utc=True),
-                    ds_name)
+                ds_name)
-                  )
+            )
 
         if 'key' in self.addresses[address_i]:
             key = self.addresses[address_i]['key']
@@ -626,7 +621,7 @@ class Marketplace:
             )
 
         except Exception as e:
-            print('Unable to register the requested dataset: {}'.format(e))
+            print('Unable to subscribe to data source: {}'.format(e))
             return
 
         self.check_transaction(tx_hash)

catalyst/marketplace/marketplace_errors.py

@@ -9,8 +9,7 @@ def silent_except_hook(exctype, excvalue, exctraceback):
                    MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
                    MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
                    MarketplaceSubscriptionExpired, MarketplaceJSONError,
-                   MarketplaceWalletNotSupported, MarketplaceEmptySignature,
+                   MarketplaceWalletNotSupported, MarketplaceEmptySignature]:
-                   MarketplaceRequiresPython3]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
         print("Error traceback: {1} (line {2})\n"
@@ -87,11 +86,3 @@ class MarketplaceJSONError(ZiplineError):
         'The configuration file {file} is malformed. Please correct '
         'the following error:\n{error}'
     )
-
-
-class MarketplaceRequiresPython3(ZiplineError):
-    msg = (
-        '\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n'
-        'If you want to use the Data Marketplace, you need to reinstall '
-        'Catalyst\nwith Python3. See the documentation website for additional '
-        'information.')

catalyst/support/issue_236.py

@@ -1,32 +0,0 @@
-from catalyst.api import symbol
-from catalyst.utils.run_algo import run_algorithm
-
-coins = ['dash', 'btc', 'dash', 'etc', 'eth', 'ltc', 'nxt', 'rep', 'str', 'xmr', 'xrp', 'zec']
-symbols = None
-
-
-def initialize(context):
-    pass
-
-
-def _handle_data(context, data):
-    global symbols
-    if symbols is None: symbols = [symbol(c + '_usdt') for c in coins]
-
-    print'getting history for: %s' % [s.symbol for s in symbols]
-    history = data.history(symbols,
-                    ['close', 'volume'],
-                    bar_count=1, # EXCEPTION, Change to 2
-                    frequency='5T')
-    #print 'history: %s' % history.shape
-
-run_algorithm(initialize=initialize,
-              handle_data=_handle_data,
-            analyze=lambda _, results: True,
-            exchange_name='poloniex',
-            base_currency='usdt',
-            algo_namespace='issue-236',
-            live=True,
-            data_frequency='minute',
-            capital_base=3000,
-            simulate_orders=True)

catalyst/utils/run_server.py

@@ -1,103 +0,0 @@
-#!flask/bin/python
-import base64
-
-import requests
-import pandas as pd
-import json
-
-
-def convert_date(date):
-    """
-    when transferring dates by json,
-    converts it to str
-    :param date:
-    :return: str(date)
-    """
-    if isinstance(date, pd.Timestamp):
-        return date.__str__()
-
-
-def run_server(
-        initialize,
-        handle_data,
-        before_trading_start,
-        analyze,
-        algofile,
-        algotext,
-        defines,
-        data_frequency,
-        capital_base,
-        data,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        environ,
-        live,
-        exchange,
-        algo_namespace,
-        base_currency,
-        live_graph,
-        analyze_live,
-        simulate_orders,
-        auth_aliases,
-        stats_output,
-        ):
-
-    # address to send
-    url = 'http://sandbox.enigma.co/api/catalyst/serve'
-    # url = 'http://127.0.0.1:5000/api/catalyst/serve'
-
-    # argument preparation - encode the file for transfer
-    if algotext:
-        algotext = base64.b64encode(algotext)
-    else:
-        algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8')
-        algofile = None
-
-    json_file = {'arguments': {
-        'initialize': initialize,
-        'handle_data': handle_data,
-        'before_trading_start': before_trading_start,
-        'analyze': analyze,
-        'algotext': algotext,
-        'defines': defines,
-        'data_frequency': data_frequency,
-        'capital_base': capital_base,
-        'data': data,
-        'bundle': bundle,
-        'bundle_timestamp': bundle_timestamp,
-        'start': start,
-        'end': end,
-        'local_namespace': local_namespace,
-        'environ': None,
-        'analyze_live': analyze_live,
-        'stats_output': stats_output,
-        'algofile': algofile,
-        'output': output,
-        'print_algo': print_algo,
-        'live': live,
-        'exchange': exchange,
-        'algo_namespace': algo_namespace,
-        'base_currency': base_currency,
-        'live_graph': live_graph,
-        'simulate_orders': simulate_orders,
-        'auth_aliases': auth_aliases,
-    }}
-
-    response = requests.post(url,
-                             json=json.dumps(
-                                    json_file,
-                                    default=convert_date
-                                )
-                             )
-
-    if response.status_code == 500:
-        raise Exception("issues with cloud connections, "
-                        "unable to run catalyst on the cloud")
-    received_data = response.json()
-    cloud_log_tail = base64.b64decode(received_data["log"])
-    print(cloud_log_tail)

docs/source/install.rst

@@ -143,7 +143,8 @@ with the following steps:
     
    .. code-block:: bash
 
-      conda create --name catalyst python=2.7 scipy zlib
+      conda create --name catalyst python=3.6 scipy zlib
+
 
 3. Activate the environment:
 

etc/python2.7-environment.yml

@@ -23,9 +23,7 @@ dependencies:
   - bottleneck==1.2.1
   - chardet==3.0.4
   - ccxt==1.10.1094
-# The Enigma Data Marketplace requires Python3 because it depends on
+  - web3==4.0.0b7
-# web3, which requires Python3, as building its dependencies breaks in Python2
-#  - web3==4.0.0b7
   - requests-toolbelt==0.8.0
   - click==6.7
   - contextlib2==0.5.5

etc/requirements.txt

@@ -84,5 +84,5 @@ tables==3.3.0
 ccxt==1.10.1094
 boto3==1.4.8
 redo==1.6
-web3==4.0.0b11; python_version > '3.4'
+web3==4.0.0b7
 requests-toolbelt==0.8.0

etc/requirements_marketplace.txt

@@ -0,0 +1,2 @@
+web3==4.0.0b7
+requests-toolbelt==0.8.0

tests/exchange/test_exchange_utils.py

@@ -1,175 +0,0 @@
-from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \
-    forward_fill_df_if_needed, get_candles_df
-
-from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
-from datetime import timedelta
-from pandas import Timestamp, DataFrame, concat
-
-import numpy as np
-
-
-class TestExchangeUtils(WithLogger, ZiplineTestCase):
-    @classmethod
-    def get_specific_field_from_df(cls, df, field, asset):
-        new_df = DataFrame(df[field])
-        new_df.columns = [asset]
-        new_df.index.name = None
-        return new_df
-
-    @classmethod
-    def verify_forward_fill_df_if_needed(cls, candles, periods, expected_df):
-        observed_df = forward_fill_df_if_needed(
-            transform_candles_to_df(candles),
-            periods)
-        assert (expected_df.equals(observed_df))
-
-    @classmethod
-    def verify_get_candles_df(cls, assets, candles, end_fixed_dt,
-                              expected_df, check_next_candle=False):
-        # run on all the fields
-        for field in ['volume', 'open', 'close', 'high', 'low']:
-
-            field_dt = cls.get_specific_field_from_df(expected_df,
-                                                      field,
-                                                      assets[0])
-            # run on several timestamps
-            for delta in range(5):
-                end_dt = end_fixed_dt + timedelta(minutes=delta)
-                assert (field_dt.equals(get_candles_df({assets[0]: candles},
-                                                       field, '5T', 3,
-                                                       end_dt=end_dt)))
-
-                field_dt_a1 = cls.get_specific_field_from_df(expected_df,
-                                                             field,
-                                                             assets[0])
-                field_dt_a2 = cls.get_specific_field_from_df(expected_df,
-                                                             field,
-                                                             assets[1])
-                observed_df = get_candles_df({assets[0]: candles,
-                                              assets[1]: candles},
-                                             field, '5T', 3,
-                                             end_dt=end_dt)
-
-                assert (observed_df.equals(concat([field_dt_a1, field_dt_a2],
-                                                  axis=1)))
-
-            if check_next_candle:
-                # one candle forward
-                end_dt = end_fixed_dt + timedelta(minutes=6)
-                observed_df = get_candles_df({assets[0]: candles,
-                                              assets[1]: candles},
-                                             field, '5T', 3,
-                                             end_dt=end_dt)
-
-                assert (not observed_df.equals(concat([field_dt_a1,
-                                                       field_dt_a2],
-                                               axis=1)))
-                assert (concat([field_dt_a1, field_dt_a2],
-                               axis=1)[1:].equals(observed_df[:-1]))
-
-    def test_get_candles_df(self):
-        assets = ['btc_usdt', 'eth_usdt']
-
-        # test forward fill in the end
-        candles = [{'high': 595, 'volume': 10, 'low': 594,
-                    'close': 595, 'open': 594,
-                    'last_traded': Timestamp('2018-03-01 09:45:00+0000',
-                                             tz='UTC')
-                    },
-                   {'high': 594, 'volume': 108, 'low': 592,
-                    'close': 593, 'open': 592,
-                    'last_traded': Timestamp('2018-03-01 09:50:00+0000',
-                                             tz='UTC')
-                    }]
-
-        expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
-                     'close': 595.0, 'open': 594.0,
-                     'last_traded': Timestamp('2018-03-01 09:45:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 594.0, 'volume': 108.0, 'low': 592.0,
-                     'close': 593.0, 'open': 592.0,
-                     'last_traded': Timestamp('2018-03-01 09:50:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 593.0, 'volume': 0.0, 'low': 593.0,
-                     'close': 593.0, 'open': 593.0,
-                     'last_traded': Timestamp('2018-03-01 09:55:00+0000',
-                                              tz='UTC')
-                     }]
-
-        periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'),
-                   Timestamp('2018-03-01 09:50:00+0000', tz='UTC'),
-                   Timestamp('2018-03-01 09:55:00+0000', tz='UTC')]
-
-        expected_df = transform_candles_to_df(expected)
-
-        self.verify_forward_fill_df_if_needed(candles, periods,
-                                              expected_df)
-        self.verify_get_candles_df(assets, candles, periods[2],
-                                   expected_df, True)
-
-        # test forward fill in the middle
-        candles = [{'high': 595, 'volume': 10, 'low': 594,
-                    'close': 595, 'open': 594,
-                    'last_traded': Timestamp('2018-03-01 09:45:00+0000',
-                                             tz='UTC')
-                    },
-                   {'high': 594, 'volume': 108, 'low': 592,
-                    'close': 593, 'open': 592,
-                    'last_traded': Timestamp('2018-03-01 09:55:00+0000',
-                                             tz='UTC')
-                    }]
-
-        expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
-                     'close': 595.0, 'open': 594.0,
-                     'last_traded': Timestamp('2018-03-01 09:45:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 595.0, 'volume': 0.0, 'low': 595.0,
-                     'close': 595.0, 'open': 595.0,
-                     'last_traded': Timestamp('2018-03-01 09:50:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 594.0, 'volume': 108.0, 'low': 592.0,
-                     'close': 593.0, 'open': 592.0,
-                     'last_traded': Timestamp('2018-03-01 09:55:00+0000',
-                                              tz='UTC')
-                     }]
-
-        expected_df = transform_candles_to_df(expected)
-        self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
-        self.verify_get_candles_df(assets, candles, periods[2], expected_df)
-
-        # test "forward fill" at the beginning
-        candles = [{'high': 595, 'volume': 10, 'low': 594,
-                    'close': 595, 'open': 594,
-                    'last_traded': Timestamp('2018-03-01 09:50:00+0000',
-                                             tz='UTC')
-                    },
-                   {'high': 594, 'volume': 108, 'low': 592,
-                    'close': 593, 'open': 592,
-                    'last_traded': Timestamp('2018-03-01 09:55:00+0000',
-                                             tz='UTC')
-                    }]
-
-        expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN,
-                     'close': np.NaN, 'open': np.NaN,
-                     'last_traded': Timestamp('2018-03-01 09:45:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 595, 'volume': 10, 'low': 594,
-                     'close': 595, 'open': 594,
-                     'last_traded': Timestamp('2018-03-01 09:50:00+0000',
-                                              tz='UTC')
-                     },
-                    {'high': 594, 'volume': 108, 'low': 592,
-                     'close': 593, 'open': 592,
-                     'last_traded': Timestamp('2018-03-01 09:55:00+0000',
-                                              tz='UTC')
-                     }]
-
-        expected_df = transform_candles_to_df(expected)
-        self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
-        # Not the same due to dropna - commenting out for now
-        # self.verify_get_candles_df(assets, candles, periods[2], expected_df)

tests/exchange/test_suites/test_suite_bundle.py

@@ -107,14 +107,14 @@ class TestSuiteBundle:
             print('saved {} test results: {}'.format(end_dt, folder))
 
             assert_frame_equal(
-                right=data['bundle'][:-1],
+                right=data['bundle'],
-                left=data['exchange'][:-1],
+                left=data['exchange'],
                 check_less_precise=1,
             )
             try:
                 assert_frame_equal(
-                    right=data['bundle'][:-1],
+                    right=data['bundle'],
-                    left=data['exchange'][:-1],
+                    left=data['exchange'],
                     check_less_precise=min([a.decimals for a in assets]),
                 )
             except Exception as e: