From 54dcc58ee83d7e783e64fc611072d0be2c7b00c7 Mon Sep 17 00:00:00 2001
From: fredfortier <fredfortier@gmail.com>
Date: Thu, 7 Dec 2017 20:26:37 -0500
Subject: [PATCH 1/4] BLD: improved stats display to better support multiple
 assets per algo

---
 catalyst/examples/mean_reversion_simple.py |   9 +-
 catalyst/exchange/exchange_algorithm.py    |   6 +-
 catalyst/exchange/exchange_blotter.py      |   4 +-
 catalyst/exchange/stats_utils.py           | 130 ++++++++++++++++-----
 4 files changed, 109 insertions(+), 40 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index da49e542..2027cdc8 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -97,10 +97,9 @@ def handle_data(context, data):
     # the record() method to save it. This data will be available as
     # a parameter of the analyze() function for further analysis.
     record(
-        price=price,
-        volume=current['volume'],
-        price_change=price_change,
-        rsi=rsi[-1],
+        volume=(context.market, current['volume']),
+        price_change=(context.market, price_change),
+        rsi=(context.market, rsi[-1]),
         cash=cash
     )
 
@@ -278,6 +277,6 @@ if __name__ == '__main__':
             algo_namespace=NAMESPACE,
             base_currency='eth',
             live_graph=False,
-            simulate_orders=False,
+            simulate_orders=True,
             stats_output=None
         )
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 9745a876..7042bde4 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -614,12 +614,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
             self.add_exposure_stats(frame_stats)
 
-            print_df = pd.DataFrame(list(self.frame_stats))
+            # print_df = pd.DataFrame(list(self.frame_stats))
             log.info(
                 'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
                     stats_minutes=self.stats_minutes,
                     stats=get_pretty_stats(
-                        df=print_df,
+                        stats=self.frame_stats,
                         recorded_cols=recorded_cols,
                         num_rows=self.stats_minutes
                     )
@@ -644,7 +644,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 if 's3://' in self.stats_output:
                     stats_to_s3(
                         uri=self.stats_output,
-                        df=print_df,
+                        stats=self.frame_stats,
                         algo_namespace=self.algo_namespace,
                         recorded_cols=recorded_cols,
                     )
diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py
index df8b352d..4ce45fce 100644
--- a/catalyst/exchange/exchange_blotter.py
+++ b/catalyst/exchange/exchange_blotter.py
@@ -6,7 +6,7 @@ from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import ExchangeRequestError, \
-    ExchangePortfolioDataError, OrphanOrderError, ExchangeTransactionError
+    ExchangePortfolioDataError, ExchangeTransactionError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.order import ORDER_STATUS
@@ -175,6 +175,8 @@ class ExchangeBlotter(Blotter):
 
     @expect_types(asset=TradingPair)
     def order(self, asset, amount, style, order_id=None):
+        log.debug('ordering {} {}'.format(amount, asset.symbol))
+
         if self.simulate_orders:
             return super(ExchangeBlotter, self).order(
                 asset, amount, style, order_id
diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py
index 3b486b77..8a34d533 100644
--- a/catalyst/exchange/stats_utils.py
+++ b/catalyst/exchange/stats_utils.py
@@ -1,10 +1,13 @@
 import numbers
 
+import copy
 import numpy as np
 import pandas as pd
 import boto3
 import time
 
+from catalyst.assets._assets import TradingPair
+
 s3 = boto3.resource('s3')
 
 
@@ -123,50 +126,115 @@ def vwap(df):
     return ret
 
 
-def format_positions(positions):
-    parts = []
-    for position in positions:
-        msg = '{amount:.2f}{base} cost basis {cost_basis:.8f}{quote}'.format(
-            amount=position['amount'],
-            base=position['sid'].base_currency,
-            cost_basis=position['cost_basis'],
-            quote=position['sid'].quote_currency
-        )
-        parts.append(msg)
-    return ', '.join(parts)
+def set_position_row(row, position_index, recorded_cols=None):
+    """
+    Apply the position data as individual columns.
 
+    Parameters
+    ----------
+    row: dict[str, Object]
+    position_index: int
+    recorded_cols: list[str]
+        If a recorded_col contains a tuple which first value is an asset
+        matching a position, its value will be displayed with the
+        position and not in the index.
 
-def prepare_stats(df, recorded_cols=None):
-    columns = ['starting_cash', 'ending_cash', 'portfolio_value',
-               'pnl', 'long_exposure', 'short_exposure', 'orders',
-               'transactions', 'positions']
+    Returns
+    -------
+
+    """
+    position = row['positions'][position_index]
+
+    asset = position['sid']
+    row['symbol'] = asset.symbol
+
+    columns = ['amount', 'cost_basis', 'last_sale_price']
+    for column in columns:
+        row[column] = position[column]
+
+    columns.insert(0, 'symbol')
 
+    if recorded_cols is not None:
+        for column in recorded_cols[:]:
+            value = row[column]
+            if type(value) in [list, tuple] and \
+                    isinstance(value[0], TradingPair) and asset == value[0]:
+                row[column] = value[1]
+
+                columns.append(column)
+                # Removing the asset specific entries
+                recorded_cols.remove(column)
+
+    return columns
+
+
+def prepare_stats(stats, recorded_cols=None):
+    """
+    Prepare the stats DataFrame for user-friendly output.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    position_cols = None
+
+    # Using a copy since we are adding rows inside the loop.
+    for row_index, row_data in enumerate(list(stats)):
+        if len(row_data['positions']) == 1:
+            row = stats[row_index]
+            columns = set_position_row(row, 0, recorded_cols)
+
+        elif len(row_data['positions']) > 1:
+            for pos_index, position in enumerate(row_data['positions']):
+                if pos_index > 0:
+                    row = row_data
+                    stats.append(row)
+
+                else:
+                    row = stats[row_index]
+
+                columns = set_position_row(row, pos_index, recorded_cols)
+
+        else:
+            break
+
+        if position_cols is None:
+            position_cols = columns
+
+    df = pd.DataFrame(list(stats))
+
+    index_cols = [
+        'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
+        'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
+    ]
     if recorded_cols is not None:
         for column in recorded_cols:
-            columns.append(column)
-
-    df = df.copy(True)
-
-    df.set_index('period_close', drop=True, inplace=True)
-    df.dropna(axis=1, how='all', inplace=True)
+            index_cols.append(column)
 
     df['orders'] = df['orders'].apply(lambda orders: len(orders))
     df['transactions'] = df['transactions'].apply(
         lambda transactions: len(transactions)
     )
-    df['positions'] = df['positions'].apply(format_positions)
 
-    return df, columns
+    df.set_index(index_cols, drop=True, inplace=True)
+    df.dropna(axis=1, how='all', inplace=True)
+
+    return df, position_cols
 
 
-def get_pretty_stats(df, recorded_cols=None, num_rows=10):
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
 
     Parameters
     ----------
-    df: pd.DataFrame
+    stats: list[Object]
     num_rows: int
 
     Returns
@@ -174,7 +242,7 @@ def get_pretty_stats(df, recorded_cols=None, num_rows=10):
     str
 
     """
-    df, columns = prepare_stats(df, recorded_cols=recorded_cols)
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
 
     pd.set_option('display.expand_frame_repr', False)
     pd.set_option('precision', 3)
@@ -191,21 +259,21 @@ def get_pretty_stats(df, recorded_cols=None, num_rows=10):
     )
 
 
-def get_csv_stats(df, recorded_cols=None):
+def get_csv_stats(stats, recorded_cols=None):
     """
     Create a CSV buffer from the stats DataFrame.
 
     Parameters
     ----------
     path: str
-    df: pd.DataFrame
+    stats: list[Object]
     recorded_cols: list[str]
 
     Returns
     -------
 
     """
-    df, columns = prepare_stats(df, recorded_cols=recorded_cols)
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
 
     return df.to_csv(
         None,
@@ -214,8 +282,8 @@ def get_csv_stats(df, recorded_cols=None):
     ).encode()
 
 
-def stats_to_s3(uri, df, algo_namespace, recorded_cols=None):
-    bytes_to_write = get_csv_stats(df, recorded_cols=recorded_cols)
+def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None):
+    bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
 
     timestr = time.strftime('%Y%m%d')
 

From 2e7aabd97372ecc334e194cf1bbbeeeef94f371b Mon Sep 17 00:00:00 2001
From: fredfortier <fredfortier@gmail.com>
Date: Thu, 7 Dec 2017 22:14:49 -0500
Subject: [PATCH 2/4] BLD: tested stats with multiple assets

---
 catalyst/examples/mean_reversion_simple.py |  2 +-
 catalyst/exchange/exchange_algorithm.py    |  2 +-
 catalyst/exchange/exchange_blotter.py      |  3 +
 catalyst/exchange/stats_utils.py           | 88 +++++++++++++---------
 4 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 2027cdc8..450a73f7 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -96,13 +96,13 @@ def handle_data(context, data):
     # Now that we've collected all current data for this frame, we use
     # the record() method to save it. This data will be available as
     # a parameter of the analyze() function for further analysis.
+
     record(
         volume=(context.market, current['volume']),
         price_change=(context.market, price_change),
         rsi=(context.market, rsi[-1]),
         cash=cash
     )
-
     # We are trying to avoid over-trading by limiting our trades to
     # one per day.
     if context.traded_today:
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 7042bde4..147d87e4 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -326,7 +326,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.retry_order = 2
         self.retry_delay = 5
 
-        self.stats_minutes = 5
+        self.stats_minutes = 20
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py
index 4ce45fce..7b89f9b9 100644
--- a/catalyst/exchange/exchange_blotter.py
+++ b/catalyst/exchange/exchange_blotter.py
@@ -176,6 +176,9 @@ class ExchangeBlotter(Blotter):
     @expect_types(asset=TradingPair)
     def order(self, asset, amount, style, order_id=None):
         log.debug('ordering {} {}'.format(amount, asset.symbol))
+        if amount == 0:
+            log.warn('skipping 0 amount orders')
+            return None
 
         if self.simulate_orders:
             return super(ExchangeBlotter, self).order(
diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py
index 8a34d533..03df1740 100644
--- a/catalyst/exchange/stats_utils.py
+++ b/catalyst/exchange/stats_utils.py
@@ -126,15 +126,15 @@ def vwap(df):
     return ret
 
 
-def set_position_row(row, position_index, recorded_cols=None):
+def set_position_row(row, asset, asset_values=list()):
     """
     Apply the position data as individual columns.
 
     Parameters
     ----------
     row: dict[str, Object]
-    position_index: int
-    recorded_cols: list[str]
+    asset: TradingPair
+    asset_values: list[str]
         If a recorded_col contains a tuple which first value is an asset
         matching a position, its value will be displayed with the
         position and not in the index.
@@ -143,32 +143,31 @@ def set_position_row(row, position_index, recorded_cols=None):
     -------
 
     """
-    position = row['positions'][position_index]
-
-    asset = position['sid']
+    asset_cols = ['symbol']
     row['symbol'] = asset.symbol
 
+    position = next((p for p in row['positions'] if p['sid'] == asset), None)
+
     columns = ['amount', 'cost_basis', 'last_sale_price']
     for column in columns:
-        row[column] = position[column]
+        if position is not None:
+            row[column] = position[column]
 
-    columns.insert(0, 'symbol')
+        else:
+            row[column] = 0
 
-    if recorded_cols is not None:
-        for column in recorded_cols[:]:
-            value = row[column]
-            if type(value) in [list, tuple] and \
-                    isinstance(value[0], TradingPair) and asset == value[0]:
-                row[column] = value[1]
+        asset_cols.append(column)
 
-                columns.append(column)
-                # Removing the asset specific entries
-                recorded_cols.remove(column)
+    values = asset_values[asset] if asset in asset_values else list()
+    for column in values:
+        row[column] = values[column]
 
-    return columns
+        asset_cols.append(column)
+
+    return asset_cols
 
 
-def prepare_stats(stats, recorded_cols=None):
+def prepare_stats(stats, recorded_cols=list()):
     """
     Prepare the stats DataFrame for user-friendly output.
 
@@ -181,30 +180,43 @@ def prepare_stats(stats, recorded_cols=None):
     -------
 
     """
-    position_cols = None
+    asset_cols = list()
 
     # Using a copy since we are adding rows inside the loop.
     for row_index, row_data in enumerate(list(stats)):
-        if len(row_data['positions']) == 1:
-            row = stats[row_index]
-            columns = set_position_row(row, 0, recorded_cols)
+        assets = [p['sid'] for p in row_data['positions']]
 
-        elif len(row_data['positions']) > 1:
-            for pos_index, position in enumerate(row_data['positions']):
-                if pos_index > 0:
-                    row = row_data
+        asset_values = dict()
+        for column in recorded_cols[:]:
+            value = row_data[column]
+            if type(value) is dict:
+                for asset in value:
+                    if not isinstance(asset, TradingPair):
+                        break
+
+                    if asset not in assets:
+                        assets.append(asset)
+
+                    if asset not in asset_values:
+                        asset_values[asset] = dict()
+
+                    asset_values[asset][column] = value[asset]
+
+        if len(assets) == 1:
+            row = stats[row_index]
+            asset_cols = set_position_row(row, assets[0], asset_values)
+
+        elif len(assets) > 1:
+            for asset_index, asset in enumerate(assets):
+                if asset_index > 0:
+                    row = copy.deepcopy(row_data)
                     stats.append(row)
 
                 else:
                     row = stats[row_index]
 
-                columns = set_position_row(row, pos_index, recorded_cols)
-
-        else:
-            break
-
-        if position_cols is None:
-            position_cols = columns
+                asset_cols = set_position_row(row, assets[asset_index],
+                                              asset_values)
 
     df = pd.DataFrame(list(stats))
 
@@ -212,6 +224,9 @@ def prepare_stats(stats, recorded_cols=None):
         'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
         'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
     ]
+
+    # Removing the asset specific entries
+    recorded_cols = [x for x in recorded_cols if x not in asset_cols]
     if recorded_cols is not None:
         for column in recorded_cols:
             index_cols.append(column)
@@ -223,8 +238,9 @@ def prepare_stats(stats, recorded_cols=None):
 
     df.set_index(index_cols, drop=True, inplace=True)
     df.dropna(axis=1, how='all', inplace=True)
+    df.sort_index(inplace=True)
 
-    return df, position_cols
+    return df, asset_cols
 
 
 def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
@@ -245,7 +261,7 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
 
     pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 3)
+    pd.set_option('precision', 8)
     pd.set_option('display.width', 1000)
     pd.set_option('display.max_colwidth', 1000)
 

From 1c03d837cc935b53515df1dbeb6b5b0e448f335d Mon Sep 17 00:00:00 2001
From: fredfortier <fredfortier@gmail.com>
Date: Thu, 7 Dec 2017 22:26:01 -0500
Subject: [PATCH 3/4] BUG: fixed issue with stats output

---
 catalyst/exchange/stats_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py
index 03df1740..43e9e5ec 100644
--- a/catalyst/exchange/stats_utils.py
+++ b/catalyst/exchange/stats_utils.py
@@ -1,3 +1,4 @@
+import csv
 import numbers
 
 import copy
@@ -294,7 +295,8 @@ def get_csv_stats(stats, recorded_cols=None):
     return df.to_csv(
         None,
         columns=columns,
-        encoding='utf-8',
+        # encoding='utf-8',
+        quoting=csv.QUOTE_NONNUMERIC
     ).encode()
 
 

From b644c947e365486c65eb3bac962ce75fd6514d22 Mon Sep 17 00:00:00 2001
From: fredfortier <fredfortier@gmail.com>
Date: Thu, 7 Dec 2017 23:01:12 -0500
Subject: [PATCH 4/4] BUG: fixed issue with stats output

---
 catalyst/exchange/stats_utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py
index 43e9e5ec..ec89a340 100644
--- a/catalyst/exchange/stats_utils.py
+++ b/catalyst/exchange/stats_utils.py
@@ -183,6 +183,7 @@ def prepare_stats(stats, recorded_cols=list()):
     """
     asset_cols = list()
 
+    stats = copy.deepcopy(list(stats))
     # Using a copy since we are adding rows inside the loop.
     for row_index, row_data in enumerate(list(stats)):
         assets = [p['sid'] for p in row_data['positions']]
@@ -219,7 +220,7 @@ def prepare_stats(stats, recorded_cols=list()):
                 asset_cols = set_position_row(row, assets[asset_index],
                                               asset_values)
 
-    df = pd.DataFrame(list(stats))
+    df = pd.DataFrame(stats)
 
     index_cols = [
         'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
@@ -239,7 +240,7 @@ def prepare_stats(stats, recorded_cols=list()):
 
     df.set_index(index_cols, drop=True, inplace=True)
     df.dropna(axis=1, how='all', inplace=True)
-    df.sort_index(inplace=True)
+    df.sort_index(axis=0, level=0, inplace=True)
 
     return df, asset_cols