ENH: Allows history to be dynamic and grow the container at runtime.

Previously, all specs had to be pre-allocated by using the 'add_history'
function. This is now no longer required and instead serves as a hint to
the HistoryContainer to pre-allocate the space for the given spec.

History can grow by increasing the length for a frequency, adding a
frequency, or adding a field. It can grow with any combination of
these.

HistoryContainer now is aware of the data_frequency of the algorithm,
and no longer uses the daily_at_midnight flag; instead, this is the
default behavior.

zipline/algorithm.py

@@ -181,6 +181,9 @@ class TradingAlgorithm(object):
         self._portfolio = None
         self._account = None
 
+        self.history_container_class = kwargs.pop(
+            'history_container_class', HistoryContainer,
+        )
         self.history_container = None
         self.history_specs = {}
 
@@ -435,11 +438,13 @@ class TradingAlgorithm(object):
             self.sim_params._update_internal()
 
         # Create history containers
-        if len(self.history_specs) != 0:
-            self.history_container = HistoryContainer(
+        if self.history_specs:
+            self.history_container = self.history_container_class(
                 self.history_specs,
                 self.sim_params.sids,
-                self.sim_params.first_open)
+                self.sim_params.first_open,
+                self.sim_params.data_frequency,
+            )
 
         # Create transforms by wrapping them into StatefulTransforms
         self.transforms = []
@@ -912,21 +917,54 @@ class TradingAlgorithm(object):
         self.blotter.cancel(order_id)
 
     @api_method
-    def add_history(self, bar_count, frequency, field,
-                    ffill=True):
+    def add_history(self, bar_count, frequency, field, ffill=True):
         data_frequency = self.sim_params.data_frequency
-        daily_at_midnight = (data_frequency == 'daily')
-
         history_spec = HistorySpec(bar_count, frequency, field, ffill,
-                                   daily_at_midnight=daily_at_midnight,
                                    data_frequency=data_frequency)
         self.history_specs[history_spec.key_str] = history_spec
+        if self.initialized:
+            if self.history_container:
+                self.history_container.ensure_spec(history_spec, self.datetime)
+            else:
+                self.history_container = self.history_container_class(
+                    self.trade_sources.history_backfill,
+                    self.history_specs,
+                    self.multiverse.current_sids,
+                    self.sim_params.first_open,
+                    self.sim_params.data_frequency,
+                )
+
+    def get_history_spec(self, bar_count, frequency, field, ffill):
+        spec_key = HistorySpec.spec_key(bar_count, frequency, field, ffill)
+        if spec_key not in self.history_specs:
+            data_freq = self.sim_params.data_frequency
+            spec = HistorySpec(
+                bar_count,
+                frequency,
+                field,
+                ffill,
+                data_frequency=data_freq,
+            )
+            self.history_specs[spec_key] = spec
+            if not self.history_container:
+                self.history_container = self.history_container_class(
+                    self.history_specs,
+                    self.current_universe(),
+                    self.datetime,
+                    self.sim_params.data_frequency,
+                    shift_digest=True,
+                )
+            self.history_container.ensure_spec(spec, self.datetime)
+        return self.history_specs[spec_key]
 
     @api_method
     def history(self, bar_count, frequency, field, ffill=True):
-        spec_key_str = HistorySpec.spec_key(
-            bar_count, frequency, field, ffill)
-        history_spec = self.history_specs[spec_key_str]
+        history_spec = self.get_history_spec(
+            bar_count,
+            frequency,
+            field,
+            ffill,
+        )
         return self.history_container.get_history(history_spec, self.datetime)
 
     ####################
@@ -994,6 +1032,9 @@ class TradingAlgorithm(object):
         """
         self.register_trading_control(LongOnly())
 
+    def current_universe(self):
+        return self.sim_params.sids
+
     @classmethod
     def all_api_methods(cls):
         """