fix typos

BLD: Dockerfile[-dev]: s/quantopian/enigmampc/g

Dockerfile

@@ -1,11 +1,11 @@
 #
 # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
 #
-#    docker build -t quantopian/catalyst .
+#    docker build -t enigmampc/catalyst .
 #
 # To run the container:
 #
-#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it quantopian/catalyst
+#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it enigmampc/catalyst
 #
 # To access Jupyter when running docker locally (you may need to add NAT rules):
 #

Dockerfile-dev

@@ -1,15 +1,15 @@
 #
 # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
 #
-#    docker build -t quantopian/catalystdev -f Dockerfile-dev .
+#    docker build -t enigmampc/catalystdev -f Dockerfile-dev .
 #
-# Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
+# Note: the dev build requires a enigmampc/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile .
+#    docker build -t enigmampc/catalyst -f Dockerfile .
 #
 # To run the container:
 #
-#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it quantopian/catalystdev
+#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it enigmampc/catalystdev
 #
 # To access Jupyter when running docker locally (you may need to add NAT rules):
 #
@@ -25,7 +25,7 @@
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #
-FROM quantopian/catalyst
+FROM enigmampc/catalyst
 
 WORKDIR /catalyst
 

README.rst

@@ -5,6 +5,7 @@
 
 |version tag|
 |version status|
+|forum|
 |discord|
 |twitter|
 
@@ -22,9 +23,11 @@ visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
 Catalyst builds on top of the well-established 
 `Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
 minimize structural changes to the general API to maximize compatibility with 
-existing trading algorithms, developer knowledge, and tutorials. Join us on 
-`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
-for questions around Catalyst, algorithmic trading and technical support.
+existing trading algorithms, developer knowledge, and tutorials. Join us on the 
+`Catalyst Forum <https://catalyst.enigma.co/>`_ for questions around Catalyst, 
+algorithmic trading and technical support. We also have a 
+`Discord <https://discord.gg/SJK32GY>`_ group with the *#catalyst_dev* and 
+*#catalyst_setup* dedicated channels.
 
 Overview
 ========
@@ -60,6 +63,9 @@ Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
 
 .. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
    :target: https://pypi.python.org/pypi/enigma-catalyst
+   
+.. |forum| image:: https://img.shields.io/badge/forum-join-green.svg
+   :target: https://catalyst.enigma.co/
 
 .. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
    :target: https://discordapp.com/invite/SJK32GY

catalyst/__main__.py

@@ -580,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
+    click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
                sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,

catalyst/_protocol.pyx

@@ -16,7 +16,6 @@ import warnings
 from contextlib import contextmanager
 from functools import wraps
 
-from pandas.tslib import normalize_date
 import pandas as pd
 import numpy as np
 
@@ -564,7 +563,7 @@ cdef class BarData:
             })
 
     cdef bool _is_stale_for_asset(self, asset, dt, adjusted_dt, data_portal):
-        session_label = normalize_date(dt) # FIXME
+        session_label = dt.normalize_date() # FIXME
 
         if not asset.is_alive_for_session(session_label):
             return False

catalyst/algorithm.py

@@ -21,7 +21,6 @@ import logbook
 import pytz
 import pandas as pd
 from contextlib2 import ExitStack
-from pandas.tseries.tools import normalize_date
 import numpy as np
 
 from itertools import chain, repeat
@@ -1345,7 +1344,7 @@ class TradingAlgorithm(object):
         # Make sure the asset exists, and that there is a last price for it.
         # FIXME: we should use BarData's can_trade logic here, but I haven't
         # yet found a good way to do that.
-        normalized_date = normalize_date(self.datetime)
+        normalized_date = self.datetime.normalize()
 
         if normalized_date < asset.start_date:
             raise CannotOrderDelistedAsset(
@@ -1392,7 +1391,7 @@ class TradingAlgorithm(object):
             )
 
         if asset.auto_close_date:
-            day = normalize_date(self.get_datetime())
+            day = self.get_datetime().normalize()
 
             if day > min(asset.end_date, asset.auto_close_date):
                 # If we are after the asset's end date or auto close date, warn
@@ -2475,7 +2474,7 @@ class TradingAlgorithm(object):
         """
         Internal implementation of `pipeline_output`.
         """
-        today = normalize_date(self.get_datetime())
+        today = self.get_datetime().normalize()
         data = NO_DATA = object()
         try:
             data = self._pipeline_cache.unwrap(today)

catalyst/assets/_assets.pyx

@@ -630,23 +630,28 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
-        #TODO: make sure that all fields set there
+        # added arguments for catalyst
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
                                  self.asset_name,
                                  self.sid,
                                  self.leverage,
+                                 self.end_daily,
+                                 self.end_minute,
                                  self.end_date,
+                                 self.exchange_symbol,
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
                                  self.min_trade_size,
                                  self.max_trade_size,
+                                 self.maker,
+                                 self.taker,
                                  self.lot,
                                  self.decimals,
-                                 self.taker,
-                                 self.maker))
+                                 self.trading_state,
+                                 self.data_source))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -24,8 +24,7 @@ AUTO_INGEST = False
 
 AUTH_SERVER = 'https://data.enigma.co'
 
-# TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
+ETH_REMOTE_NODE = 'https://mainnet.infura.io'
 
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                        'catalyst/master/catalyst/marketplace/' \
@@ -35,7 +34,6 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                            'catalyst/master/catalyst/marketplace/' \
                            'contract_marketplace_abi.json'
 
-# TODO: switch to mainnet
 ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                   'catalyst/master/catalyst/marketplace/' \
                   'contract_enigma_address.txt'
@@ -43,3 +41,6 @@ ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                       'catalyst/master/catalyst/marketplace/' \
                       'contract_enigma_abi.json'
+
+SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
+                     'key']

catalyst/data/data_portal.py

@@ -20,7 +20,6 @@ import numpy as np
 from numpy import float64, int64, nan
 import pandas as pd
 from pandas import isnull
-from pandas.tslib import normalize_date
 from six import iteritems
 from six.moves import reduce
 
@@ -439,7 +438,7 @@ class DataPortal(object):
                     (isinstance(asset, (Asset, ContinuousFuture))))
 
     def _get_fetcher_value(self, asset, field, dt):
-        day = normalize_date(dt)
+        day = dt.normalize()
 
         try:
             return \
@@ -1130,7 +1129,7 @@ class DataPortal(object):
         if self._asset_start_dates[sid] > dt:
             raise NoTradeDataAvailableTooEarly(
                 sid=sid,
-                dt=normalize_date(dt),
+                dt=dt.normalize(),
                 start_dt=start_date
             )
 
@@ -1138,7 +1137,7 @@ class DataPortal(object):
         if self._asset_end_dates[sid] < dt:
             raise NoTradeDataAvailableTooLate(
                 sid=sid,
-                dt=normalize_date(dt),
+                dt=dt.normalize(),
                 end_dt=end_date
             )
 
@@ -1262,7 +1261,7 @@ class DataPortal(object):
         if self._extra_source_df is None:
             return []
 
-        day = normalize_date(dt)
+        day = dt.normalize()
 
         if day in self._extra_source_df.index:
             assets = self._extra_source_df.loc[day]['sid']

catalyst/data/history_loader.py

@@ -21,7 +21,6 @@ from abc import (
 from numpy import concatenate
 from lru import LRU
 from pandas import isnull
-from pandas.tslib import normalize_date
 from toolz import sliding_window
 
 from six import with_metaclass
@@ -93,8 +92,8 @@ class HistoryCompatibleUSEquityAdjustmentReader(object):
             The adjustments as a dict of loc -> Float64Multiply
         """
         sid = int(asset)
-        start = normalize_date(dts[0])
-        end = normalize_date(dts[-1])
+        start = dts[0].normalize()
+        end = dts[-1].normalize()
         adjs = {}
         if field != 'volume':
             mergers = self._adjustments_reader.get_adjustments_for_sid(

catalyst/data/us_equity_pricing.py

@@ -49,7 +49,6 @@ from pandas import (
     to_datetime,
     Timestamp,
 )
-from pandas.tslib import iNaT
 from six import (
     iteritems,
     string_types,
@@ -422,7 +421,7 @@ class BcolzDailyBarWriter(object):
         )
 
         full_table.attrs['first_trading_day'] = (
-            earliest_date if earliest_date is not None else iNaT
+            earliest_date if earliest_date is not None else NaT
         )
 
         full_table.attrs['first_row'] = first_row

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -190,6 +190,9 @@ class CCXT(Exchange):
                 if data_frequency == 'minute' and not freq.endswith('T'):
                     continue
 
+                elif data_frequency == 'hourly' and not freq.endswith('D'):
+                    continue
+
                 elif data_frequency == 'daily' and not freq.endswith('D'):
                     continue
 

catalyst/exchange/exchange.py

@@ -13,7 +13,6 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     PricingDataNotLoadedError, \
     NoDataAvailableOnExchange, NoValueForField, \
     NoCandlesReceivedFromExchange, \
-    InvalidHistoryFrequencyAlias,  \
     TickerNotFoundError, NotEnoughCashError
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
@@ -200,12 +199,8 @@ class Exchange:
                 )
                 assets.append(asset)
 
-            except SymbolNotFoundOnExchange:
-                log.debug(
-                    'skipping non-existent market {} {}'.format(
-                        self.name, symbol
-                    )
-                )
+            except SymbolNotFoundOnExchange as e:
+                log.warn(e)
         return assets
 
     def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
@@ -509,10 +504,6 @@ class Exchange:
             frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
 
-        if unit == 'H':
-            raise InvalidHistoryFrequencyAlias(
-                freq=frequency)
-
         # we want to avoid receiving empty candles
         # so we request more than needed
         # TODO: consider defining a const per asset
@@ -616,7 +607,7 @@ class Exchange:
         # TODO: this function needs some work,
         # we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D']
         )
         adj_bar_count = candle_size * bar_count
         try:

catalyst/exchange/exchange_bundle.py

@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_df_from_arrays, get_assets
-from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
+from catalyst.exchange.utils.datetime_utils import get_start_dt, \
     get_period_label, get_month_start_end, get_year_start_end
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
@@ -232,12 +232,12 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                    name=asset.symbol,
-                    start_dt=asset.start_date.strftime(
-                        DATE_TIME_FORMAT),
-                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                    dates=[date.strftime(
-                        DATE_TIME_FORMAT) for date in dates])
+                                name=asset.symbol,
+                                start_dt=asset.start_date.strftime(
+                                    DATE_TIME_FORMAT),
+                                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                                dates=[date.strftime(
+                                    DATE_TIME_FORMAT) for date in dates])
 
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
@@ -286,12 +286,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                name=asset.symbol,
-                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                threshold=threshold,
-                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                       for date in dates])
+                        name=asset.symbol,
+                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                        threshold=threshold,
+                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                               for date in dates])
 
             problems.append(problem)
 
@@ -458,7 +458,7 @@ class ExchangeBundle:
                 last_entry = None
 
         if start is None or \
-            (earliest_trade is not None and earliest_trade > start):
+                (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
         if last_entry is not None and (end is None or end > last_entry):
@@ -598,16 +598,41 @@ class ExchangeBundle:
         # we want to give an end_date far in time
         writer = self.get_writer(start_dt, end_dt, data_frequency)
         if show_breakdown:
-            for asset in chunks:
+            if chunks:
+                for asset in chunks:
+                    with maybe_show_progress(
+                        chunks[asset],
+                        show_progress,
+                        label='Ingesting {frequency} price data for '
+                              '{symbol} on {exchange}'.format(
+                            exchange=self.exchange_name,
+                            frequency=data_frequency,
+                            symbol=asset.symbol
+                            )) as it:
+                        for chunk in it:
+                            problems += self.ingest_ctable(
+                                asset=chunk['asset'],
+                                data_frequency=data_frequency,
+                                period=chunk['period'],
+                                writer=writer,
+                                empty_rows_behavior='strip',
+                                cleanup=True
+                            )
+        else:
+            all_chunks = list(chain.from_iterable(itervalues(chunks)))
+            # We sort the chunks by end date to ingest most recent data first
+            if all_chunks:
+                all_chunks.sort(
+                    key=lambda chunk: pd.to_datetime(chunk['period'])
+                )
                 with maybe_show_progress(
-                    chunks[asset],
+                    all_chunks,
                     show_progress,
-                    label='Ingesting {frequency} price data for '
-                          '{symbol} on {exchange}'.format(
+                    label='Ingesting {frequency} price data on '
+                          '{exchange}'.format(
                         exchange=self.exchange_name,
                         frequency=data_frequency,
-                        symbol=asset.symbol
-                    )) as it:
+                        )) as it:
                     for chunk in it:
                         problems += self.ingest_ctable(
                             asset=chunk['asset'],
@@ -617,30 +642,6 @@ class ExchangeBundle:
                             empty_rows_behavior='strip',
                             cleanup=True
                         )
-        else:
-            all_chunks = list(chain.from_iterable(itervalues(chunks)))
-
-            # We sort the chunks by end date to ingest most recent data first
-            all_chunks.sort(
-                key=lambda chunk: pd.to_datetime(chunk['period'])
-            )
-            with maybe_show_progress(
-                all_chunks,
-                show_progress,
-                label='Ingesting {frequency} price data on '
-                      '{exchange}'.format(
-                    exchange=self.exchange_name,
-                    frequency=data_frequency,
-                )) as it:
-                for chunk in it:
-                    problems += self.ingest_ctable(
-                        asset=chunk['asset'],
-                        data_frequency=data_frequency,
-                        period=chunk['period'],
-                        writer=writer,
-                        empty_rows_behavior='strip',
-                        cleanup=True
-                    )
 
         if show_report and len(problems) > 0:
             log.info('problems during ingestion:{}\n'.format(

catalyst/exchange/exchange_data_portal.py

@@ -296,7 +296,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D']
         )
         adj_bar_count = candle_size * bar_count
 
@@ -312,7 +312,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             algo_end_dt=self._last_available_session,
         )
 
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
+        start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
         df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
         return df
 

catalyst/exchange/utils/datetime_utils.py

@@ -249,9 +249,12 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
+def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
     """
-    Get the frequency parameters.
+    Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
+    common denominator supported by the data bundles (e.g. 1T). The data
+    bundles only support 1T and 1D frequencies. If another frequency
+    is requested, Catalyst must request the underlying data and resample.
 
     Notes
     -----
@@ -306,14 +309,14 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
         data_frequency = 'minute'
 
     elif unit.lower() == 'h':
+        data_frequency = 'minute'
+
         if 'H' in supported_freqs:
             unit = 'H'
             alias = '{}H'.format(candle_size)
-
         else:
             candle_size = candle_size * 60
             alias = '{}T'.format(candle_size)
-            data_frequency = 'minute'
 
     else:
         raise InvalidHistoryFrequencyAlias(freq=freq)

catalyst/finance/performance/tracker.py

@@ -62,7 +62,6 @@ from __future__ import division
 import logbook
 
 import pandas as pd
-from pandas.tseries.tools import normalize_date
 
 from catalyst.finance.performance.period import PerformancePeriod
 from catalyst.errors import NoFurtherDataError
@@ -344,7 +343,7 @@ class PerformanceTracker(object):
         """
         self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
         self.update_performance()
-        todays_date = normalize_date(dt)
+        todays_date = dt.normalize()
         account = self.get_account(False)
 
         bench_returns = self.all_benchmark_returns.loc[todays_date:dt]

catalyst/finance/risk/cumulative.py

@@ -18,7 +18,6 @@ import logbook
 import numpy as np
 
 import pandas as pd
-from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
@@ -80,7 +79,7 @@ class RiskMetricsCumulative(object):
         # on the first day.
         self.day_before_start = self.start_session - self.sessions.freq
 
-        last_day = normalize_date(sim_params.end_session)
+        last_day = sim_params.end_session.normalize()
         if last_day not in self.sessions:
             last_day = pd.tseries.index.DatetimeIndex(
                 [last_day]

catalyst/finance/trading.py

@@ -16,7 +16,6 @@ from functools import partial
 
 import logbook
 import pandas as pd
-from pandas.tslib import normalize_date
 from six import string_types
 from sqlalchemy import create_engine
 
@@ -95,11 +94,24 @@ class TradingEnvironment(object):
         if not trading_calendar:
             trading_calendar = get_calendar("NYSE")
 
-        self.benchmark_returns, self.treasury_curves = load(
-            trading_calendar.day,
-            trading_calendar.schedule.index,
-            self.bm_symbol,
-        )
+        # todo: uncomment and add a well defined benchmark
+        # self.benchmark_returns, self.treasury_curves = load(
+        #     trading_calendar.day,
+        #     trading_calendar.schedule.index,
+        #     self.bm_symbol,
+        #     exchange=exchange,
+        # )
+
+        start_data = get_calendar('OPEN').first_trading_session
+        end_data = pd.Timestamp.utcnow()
+        treasure_cols = ['1month', '3month', '6month', '1year', '2year',
+                         '3year', '5year', '7year', '10year', '20year', '30year']
+        self.benchmark_returns = pd.DataFrame(data=0.001,
+                                              index=pd.date_range(start_data, end_data),
+                                              columns=['close'])
+        self.treasury_curves = pd.DataFrame(data=0.001,
+                                            index=pd.date_range(start_data, end_data),
+                                            columns=treasure_cols)
 
         self.exchange_tz = exchange_tz
 
@@ -151,8 +163,8 @@ class SimulationParameters(object):
         # chop off any minutes or hours on the given start and end dates,
         # as we only support session labels here (and we represent session
         # labels as midnight UTC).
-        self._start_session = normalize_date(start_session)
-        self._end_session = normalize_date(end_session)
+        self._start_session = start_session.normalize()
+        self._end_session = end_session.normalize()
         self._capital_base = capital_base
 
         self._emission_rate = emission_rate

catalyst/gens/tradesimulation.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 from contextlib2 import ExitStack
 from logbook import Logger, Processor
-from pandas.tslib import normalize_date
 from catalyst.protocol import BarData
 from catalyst.utils.api_support import ZiplineAPI
 from six import viewkeys
@@ -229,7 +228,7 @@ class AlgorithmSimulator(object):
                 elif action == SESSION_END:
                     # End of the session.
                     if emission_rate == 'daily':
-                        handle_benchmark(normalize_date(dt))
+                        handle_benchmark(dt).normalize()
                     execute_order_cancellation_policy()
 
                     yield self._get_daily_message(dt, algo, algo.perf_tracker)

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0x39a54f480d922a58c963de8091a6c9afc69db2cf
+0xf0ee6b27b759c9893ce4f094b49ad28fd15a23e4

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc
+0xa64927358a82254be92eb1f1cb01de68d1787004

catalyst/marketplace/marketplace.py

@@ -69,7 +69,10 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
-        abi = json.load(abi_url)
+        abi_url = abi_url.read().decode(
+                abi_url.info().get_content_charset())
+
+        abi = json.loads(abi_url)
 
         self.mkt_contract = self.web3.eth.contract(
             self.mkt_contract_address,
@@ -83,7 +86,10 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
-        abi = json.load(abi_url)
+        abi_url = abi_url.read().decode(
+                abi_url.info().get_content_charset())
+
+        abi = json.loads(abi_url)
 
         self.eng_contract = self.web3.eth.contract(
             self.eng_contract_address,
@@ -126,9 +132,10 @@ class Marketplace:
         else:
             while True:
                 for i in range(0, len(self.addresses)):
-                    print('{}\t{}\t{}'.format(
+                    print('{}\t{}\t{}\t{}'.format(
                         i,
                         self.addresses[i]['pubAddr'],
+                        self.addresses[i]['wallet'].ljust(10),
                         self.addresses[i]['desc'])
                     )
                 address_i = int(input('Choose your address associated with '
@@ -145,7 +152,7 @@ class Marketplace:
 
     def sign_transaction(self, tx):
 
-        url = 'https://www.myetherwallet.com/#offline-transaction'
+        url = 'https://www.mycrypto.com/#offline-transaction'
         print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
@@ -430,10 +437,9 @@ class Marketplace:
             merge_bundles(zsource, ztarget)
 
         else:
+            shutil.rmtree(bundle_folder, ignore_errors=True)
             os.rename(tmp_bundle, bundle_folder)
 
-        pass
-
     def ingest(self, ds_name=None, start=None, end=None, force_download=False):
 
         if ds_name is None:
@@ -498,20 +504,29 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         headers = get_signed_headers(ds_name, key, secret)
-        log.debug('Starting download of dataset for ingestion...')
+        log.info('Starting download of dataset for ingestion...')
         r = requests.post(
             '{}/marketplace/ingest'.format(AUTH_SERVER),
             headers=headers,
             stream=True,
         )
         if r.status_code == 200:
+            log.info('Dataset downloaded successfully. Processing dataset...')
             target_path = get_temp_bundles_folder()
             try:
                 decoder = MultipartDecoder.from_response(r)
+                # with maybe_show_progress(
+                #     iter(decoder.parts),
+                #     True,
+                #     label='Processing files') as part:
+                counter = 1
                 for part in decoder.parts:
+                    log.info("Processing file {} of {}".format(
+                        counter, len(decoder.parts)))
                     h = part.headers[b'Content-Disposition'].decode('utf-8')
                     # Extracting the filename from the header
                     name = re.search(r'filename="(.*)"', h).group(1)
@@ -525,6 +540,7 @@ class Marketplace:
                         f.write(part.content)
 
                     self.process_temp_bundle(ds_name, filename)
+                    counter += 1
 
             except NonMultipartContentTypeException:
                 response = r.json()
@@ -592,7 +608,6 @@ class Marketplace:
             folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
-        pass
 
     def create_metadata(self, key, secret, ds_name, data_frequency, desc,
                         has_history=True, has_live=True):
@@ -628,7 +643,7 @@ class Marketplace:
     def register(self):
         while True:
             desc = input('Enter the name of the dataset to register: ')
-            dataset = desc.lower()
+            dataset = desc.lower().strip()
             provider_info = self.mkt_contract.functions.getDataProviderInfo(
                 Web3.toHex(dataset)
             ).call()
@@ -684,7 +699,8 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         grains = to_grains(price)
 
@@ -765,28 +781,34 @@ class Marketplace:
             key = match['key']
             secret = match['secret']
         else:
-            key, secret = get_key_secret(provider_info[0])
+            key, secret = get_key_secret(provider_info[0], match['wallet'])
 
-        headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))
 
         if not filenames:
             raise MarketplaceNoCSVFiles(datadir=datadir)
 
         files = []
-        for file in filenames:
+        for idx, file in enumerate(filenames):
+            log.info('Uploading file {} of {}: {}'.format(
+                idx+1, len(filenames), file))
+            files = []
             files.append(('file', open(file, 'rb')))
 
-        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
-                          files=files,
-                          headers=headers)
+            headers = get_signed_headers(dataset, key, secret)
+            r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                              files=files,
+                              headers=headers)
 
-        if r.status_code != 200:
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.status_code)
+            if r.status_code != 200:
+                raise MarketplaceHTTPRequest(request='upload file',
+                                             error=r.status_code)
 
-        if 'error' in r.json():
-            raise MarketplaceHTTPRequest(request='upload file',
-                                         error=r.json()['error'])
+            if 'error' in r.json():
+                raise MarketplaceHTTPRequest(request='upload file',
+                                             error=r.json()['error'])
 
-        print('Dataset {} uploaded successfully.'.format(dataset))
+            log.info('File processed successfully.')
+
+        print('\nDataset {} uploaded and processed successfully.'.format(
+            dataset))

catalyst/marketplace/utils/auth_utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import hmac
+import webbrowser
 
 import requests
 import time
@@ -9,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import (
     MarketplaceEmptySignature)
 from catalyst.marketplace.utils.path_utils import (
     get_user_pubaddr, save_user_pubaddr)
-from catalyst.constants import AUTH_SERVER
+from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
 
 
-def get_key_secret(pubAddr, wallet='mew'):
+def get_key_secret(pubAddr, wallet):
     """
     Obtain a new key/secret pair from authentication server
 
@@ -42,14 +43,22 @@ def get_key_secret(pubAddr, wallet='mew'):
     auth_type, auth_info = header.split(None, 1)
     d = requests.utils.parse_dict_header(auth_info)
 
-    nonce = '0x{}'.format(d['nonce'])
+    nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
+
+    if wallet in SUPPORTED_WALLETS:
+        url = 'https://www.mycrypto.com/signmsg.html'
 
-    if wallet == 'mew':
         print('\nObtaining a key/secret pair to streamline all future '
               'requests with the authentication server.\n'
-              'Visit https://www.myetherwallet.com/signmsg.html and sign the '
-              'following message:\n{}'.format(nonce))
-        signature = input('Copy and Paste the "sig" field from '
+              'Visit {url} and sign the '
+              'following message (copy the entire line, without the '
+              'line break at the end):\n\n{nonce}'.format(
+                url=url,
+                nonce=nonce))
+
+        webbrowser.open_new(url)
+
+        signature = input('\nCopy and Paste the "sig" field from '
                           'the signature here (without the double quotes, '
                           'only the HEX value):\n')
     else:
@@ -83,7 +92,8 @@ def get_key_secret(pubAddr, wallet='mew'):
             addresses = get_user_pubaddr()
 
             match = next((l for l in addresses if
-                          l['pubAddr'] == pubAddr), None)
+                          l['pubAddr'].lower() == pubAddr.lower()), None)
+
             match['key'] = response.json()['key']
             match['secret'] = response.json()['secret']
 
@@ -113,7 +123,7 @@ def get_signed_headers(ds_name, key, secret):
     -------
 
     """
-    nonce = str(int(time.time()))
+    nonce = str(int(time.time() * 1000))
 
     signature = hmac.new(
         secret.encode('utf-8'),

catalyst/marketplace/utils/path_utils.py

@@ -2,6 +2,7 @@ import os
 import json
 import tarfile
 
+from catalyst.constants import SUPPORTED_WALLETS
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_root, ensure_directory
 from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
@@ -131,17 +132,63 @@ def get_user_pubaddr(environ=None):
             try:
                 d = data[0]['pubAddr']
             except Exception as e:
-                return [data, ]
+                data = [data, ]
+
+            changed = False
+
+            for idx, d in enumerate(data):
+                try:
+                    if d['wallet'] not in SUPPORTED_WALLETS:
+                        data[idx]['wallet'] = _choose_wallet(
+                            d['pubAddr'], False)
+                        changed = True
+                except KeyError:
+                    data[idx]['wallet'] = _choose_wallet(
+                        d['pubAddr'], True)
+                    changed = True
+
+            if changed:
+                save_user_pubaddr(data)
+
             return data
+
     else:
         data = []
-        data.append(dict(pubAddr='', desc=''))
+        data.append(dict(pubAddr='', desc='', wallet=''))
         with open(filename, 'w') as f:
             json.dump(data, f, sort_keys=False, indent=2,
                       separators=(',', ':'))
             return data
 
 
+def _choose_wallet(pubAddr, missing):
+    while True:
+        if missing:
+            print('\nYou need to specify a wallet for address '
+                  '{}.'.format(pubAddr))
+        else:
+            print('\nThe wallet specified for address {} is not '
+                  'supported.'.format(pubAddr))
+
+        print('Please choose among the following options:')
+        for idx, wallet in enumerate(SUPPORTED_WALLETS):
+            print('{}\t{}'.format(idx, wallet))
+
+        lw = len(SUPPORTED_WALLETS)-1
+        w = input('Choose a number between 0 and {}: '.format(
+                    lw))
+        try:
+            w = int(w)
+        except ValueError:
+            print('Enter a number between 0 and {}'.format(lw))
+        else:
+            if w not in range(0, lw+1):
+                print('Enter a number between 0 and '
+                      '{}'.format(lw))
+            else:
+                return SUPPORTED_WALLETS[w]
+
+
 def save_user_pubaddr(data, environ=None):
     """
     Saves the user's public addresses and their related metadata in

catalyst/support/issue_227.py

@@ -0,0 +1,49 @@
+import pytz
+from datetime import datetime
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+coin = 'btc'
+base_currency = 'usd'
+n_candles = 5
+
+
+def initialize(context):
+    context.symbol = symbol('%s_%s' % (coin, base_currency))
+
+
+def handle_data_polo_partial_candles(context, data):
+    history = data.history(symbol('btc_usdt'), ['volume'],
+                           bar_count=10,
+                           frequency='4H')
+    print('\nnow: %s\n%s' % (data.current_dt, history))
+    if not hasattr(context, 'i'):
+        context.i = 0
+    context.i += 1
+    if context.i > 5:
+        raise Exception('stop')
+
+
+live = False
+
+if live:
+    run_algorithm(initialize=lambda ctx: True,
+                  handle_data=handle_data_polo_partial_candles,
+                  exchange_name='poloniex',
+                  base_currency='usdt',
+                  algo_namespace='ns',
+                  live=True,
+                  data_frequency='minute',
+                  capital_base=3000)
+else:
+    run_algorithm(initialize=lambda ctx: True,
+                  handle_data=handle_data_polo_partial_candles,
+                  exchange_name='poloniex',
+                  base_currency='usdt',
+                  algo_namespace='ns',
+                  live=False,
+                  data_frequency='minute',
+                  capital_base=3000,
+                  start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
+                  end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc)
+                  )

catalyst/support/issue_274.py

@@ -0,0 +1,35 @@
+import pytz
+from datetime import datetime
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+coin = 'btc'
+base_currency = 'usd'
+
+
+def initialize(context):
+    context.symbol = symbol('%s_%s' % (coin, base_currency))
+
+
+def handle_data_polo_partial_candles(context, data):
+    history = data.history(symbol('btc_usdt'), ['volume'],
+                           bar_count=10,
+                           frequency='1D')
+    print('\nnow: %s\n%s' % (data.current_dt, history))
+    if not hasattr(context, 'i'):
+        context.i = 0
+    context.i += 1
+    if context.i > 5:
+        raise Exception('stop')
+
+
+run_algorithm(initialize=lambda ctx: True,
+              handle_data=handle_data_polo_partial_candles,
+              exchange_name='poloniex',
+              base_currency='usdt',
+              algo_namespace='ns',
+              live=False,
+              data_frequency='minute',
+              capital_base=3000,
+              start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
+              end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc))

catalyst/utils/calendars/exchange_calendar_ice.py

@@ -7,7 +7,7 @@ from pandas.tseries.holiday import (
     USLaborDay,
     USThanksgivingDay
 )
-from pandas.tslib import Timestamp
+from pandas import Timestamp
 from pytz import timezone
 
 from catalyst.utils.calendars import TradingCalendar

catalyst/utils/calendars/trading_calendar.py

@@ -640,12 +640,9 @@ class TradingCalendar(with_metaclass(ABCMeta)):
         """
         sched = self.schedule
 
-        # `market_open` and `market_close` should be timezone aware, but pandas
-        # 0.16.1 does not appear to support this:
-        # http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#datetime-with-tz  # noqa
         return (
-            sched.at[session_label, 'market_open'].tz_localize('UTC'),
-            sched.at[session_label, 'market_close'].tz_localize('UTC'),
+            sched.at[session_label, 'market_open'],
+            sched.at[session_label, 'market_close'],
         )
 
     def session_open(self, session_label):

catalyst/utils/factory.py

@@ -117,9 +117,9 @@ def create_dividend(sid, payment, declared_date, ex_date, pay_date):
         'net_amount': payment,
         'payment_sid': None,
         'ratio': None,
-        'declared_date': pd.tslib.normalize_date(declared_date),
-        'ex_date': pd.tslib.normalize_date(ex_date),
-        'pay_date': pd.tslib.normalize_date(pay_date),
+        'declared_date': declared_date.normalize(),
+        'ex_date': ex_date.normalize(),
+        'pay_date': pay_date.normalize(),
         'type': DATASOURCE_TYPE.DIVIDEND,
         'source_id': 'MockDividendSource'
     })
@@ -134,9 +134,9 @@ def create_stock_dividend(sid, payment_sid, ratio, declared_date,
         'ratio': ratio,
         'net_amount': None,
         'gross_amount': None,
-        'dt': pd.tslib.normalize_date(declared_date),
-        'ex_date': pd.tslib.normalize_date(ex_date),
-        'pay_date': pd.tslib.normalize_date(pay_date),
+        'dt': declared_date.normalize(),
+        'ex_date': ex_date.normalize(),
+        'pay_date': pay_date.normalize(),
         'type': DATASOURCE_TYPE.DIVIDEND,
         'source_id': 'MockDividendSource'
     })

catalyst/utils/run_algo.py

@@ -263,9 +263,9 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
-        if start != pd.tslib.normalize_date(start) or \
-                        end != pd.tslib.normalize_date(end):
-            # todo: add to Sim_Params the option to start & end at specific times 
+        if start != pd.Timestamp(start).normalize() or \
+                        end != pd.Timestamp(end).normalize():
+            # todo: add to Sim_Params the option to start & end at specific times
             log.warn(
                 "Catalyst currently starts and ends on the start and "
                 "end of the dates specified, respectively. We hope to "

docs/source/install.rst

@@ -143,7 +143,7 @@ with the following steps:
 
    .. code-block:: bash
 
-      conda create --name catalyst python=2.7 scipy zlib
+      conda create --name catalyst python=3.6 scipy zlib
 
 3. Activate the environment:
 
@@ -562,6 +562,10 @@ If after following the instructions above, and going through the
 *Troubleshooting* sections, you still experience problems installing Catalyst,
 you can seek additional help through the following channels:
 
+- Join our `Catalyst Forum <https://catalyst.enigma.co/>`_, and browse a variety
+  of topics and conversations around common issues that others face when using
+  Catalyst, and how to resolve them. And join the conversation!
+
 - Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
   the #catalyst_dev channel where many other users (as well as the project 
   developers) hang out, and can assist you with your particular issue. The 

docs/source/releases.rst

@@ -2,6 +2,57 @@
 Release Notes
 =============
 
+Version 0.5.8
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-29
+
+Bug Fixes
+~~~~~~~~~
+- Fix proper release of Data Marketplace on mainnet.
+
+
+Version 0.5.7
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-29
+
+Build
+~~~~~
+- Data Marketplace deployed on mainnet.
+- Added progress indicators for publishing data, and made the data publishing
+  synchronous to provide feedback to the publisher.
+
+Bug Fixes
+~~~~~~~~~
+- Added arguments to the ``reduce`` function in tha Asset class :issue:`214`, 
+  :issue:`287`
+
+Version 0.5.6
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-22
+
+Build
+~~~~~
+- Data Marketplace: ensures compatibility across wallets, now fully supporting 
+  ``ledger``, ``trezor``, ``keystore``, ``private key``. Partial support for 
+  ``metamask`` (includes sign_msg, but not sign_tx). Current support for 
+  ``Digital Bitbox`` is unknown, but believed to be supported.
+- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
+- Data Marketplace: Added progress indicator for data ingestion.
+
+Bug Fixes
+~~~~~~~~~
+- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary
+  fix for :issue:`271`, :issue:`285`
+
+Version 0.5.5
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-19
+
+Bug Fixes
+~~~~~~~~~
+- Fixed an issue with the data history in daily frequency :issue:`274`
+- Fix hourly frequency issues :issue:`227` and :issue:`114`
+
 Version 0.5.4
 ^^^^^^^^^^^^^
 **Release Date**: 2018-03-14
@@ -27,7 +78,7 @@ Bug Fixes
   :issue:`253`
 - Upgraded Data Marketplace to Web3==4.0.0b11, which was breaking some 
   functionality from prior version 4.0.0b7 :issue:`257`
-- Always request more data to avoid empty bars and always give the exact bar  
+- Always request more data to avoid empty bars and always give the exact bar
   number :issue:`260`
 
 Documentation

etc/python2.7-environment.yml

@@ -5,7 +5,6 @@ channels:
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
-- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -40,11 +39,11 @@ dependencies:
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.1.0
+  - matplotlib==2.1.2
   - multipledispatch==0.4.9
   - networkx==2.0
   - numexpr==2.6.4
-  - pandas==0.19.2
+  - pandas==0.22.0
   - pandas-datareader==0.5.0
   - patsy==0.4.1
   - pyparsing==2.2.0

etc/python3.6-environment.yml

@@ -62,7 +62,7 @@ dependencies:
   - multipledispatch==0.4.9
   - networkx==2.1
   - numexpr==2.6.4
-  - pandas==0.19.2
+  - pandas==0.22.0
   - pandas-datareader==0.6.0
   - patsy==0.5.0
   - pycares==2.3.0

etc/requirements.txt

@@ -19,7 +19,7 @@ requests-file==1.4.1
 # scipy and pandas are required for statsmodels,
 # statsmodels in turn is required for some pandas packages
 scipy==0.17.1
-pandas==0.19.2
+pandas==0.22.0
 pandas-datareader==0.2.1
 # Needed for parts of pandas.stats
 patsy==0.4.0

setup.py

@@ -165,7 +165,7 @@ def _filter_requirements(lines_iter, filter_names=None,
 
 REQ_UPPER_BOUNDS = {
     'bcolz': '<1',
-    'pandas': '<0.20',
+    'pandas': '>=0.22',
     'empyrical': '<0.2.2',
 }
 

tests/calendars/test_trading_calendar.py

@@ -24,7 +24,7 @@ import numpy as np
 import pandas as pd
 from nose_parameterized import parameterized
 from pandas import read_csv
-from pandas.tslib import Timedelta
+from pandas import Timedelta
 from pandas.util.testing import assert_index_equal
 from pytz import timezone
 from toolz import concat

tests/finance/test_slippage.py

@@ -21,7 +21,6 @@ import datetime
 from math import sqrt
 
 from nose_parameterized import parameterized
-from pandas.tslib import normalize_date
 import numpy as np
 import pandas as pd
 import pytz
@@ -1108,8 +1107,8 @@ class OrdersStopTestCase(WithSimParams,
             )),
         )
         days = pd.date_range(
-            start=normalize_date(self.minutes[0]),
-            end=normalize_date(self.minutes[-1])
+            start=self.minutes[0].normalize(),
+            end=self.minutes[-1].normalize()
         )
         with tmp_bcolz_equity_minute_bar_reader(
                 self.trading_calendar, days, assets) as reader:

tests/pipeline/test_pipeline_algo.py

@@ -27,7 +27,6 @@ from pandas import (
     Series,
     Timestamp,
 )
-from pandas.tseries.tools import normalize_date
 from six import iteritems, itervalues
 
 from catalyst.algorithm import TradingAlgorithm
@@ -530,7 +529,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs,
             attach_pipeline(pipeline, 'test')
 
         def handle_data(context, data):
-            today = normalize_date(get_datetime())
+            today = get_datetime().normalize()
             results = pipeline_output('test')
             expect_over_300 = {
                 AAPL: today < self.AAPL_split_date,

tests/test_data_portal.py

@@ -17,7 +17,7 @@ from collections import OrderedDict
 from numpy import array, append, nan, full
 from numpy.testing import assert_almost_equal
 import pandas as pd
-from pandas.tslib import Timedelta
+from pandas import Timedelta
 
 from catalyst.assets import Equity, Future
 from catalyst.data.data_portal import HISTORY_FREQUENCIES, OHLCV_FIELDS

tests/test_fetcher.py

@@ -395,7 +395,6 @@ def handle_data(context, data):
 
             algocode = """
 from pandas import Timestamp
-from pandas.tseries.tools import normalize_date
 from catalyst.api import fetch_csv, record, sid, get_datetime
 
 def initialize(context):
@@ -411,7 +410,7 @@ def initialize(context):
     context.bar_count = 0
 
 def handle_data(context, data):
-    expected = context.expected_sids[normalize_date(get_datetime())]
+    expected = context.expected_sids[get_datetime().normalize()]
     actual = data.fetcher_assets
     for stk in expected:
         if stk not in actual: