Merge branch 'master' into cheetah

.gitignore

@@ -43,3 +43,6 @@ docs/_build/*
 
 # credentials and other uncheckinables
 host_settings.py
+
+# database of vbench
+benchmarks.db

etc/requirements_dev.txt

@@ -23,13 +23,6 @@ Sphinx==1.1.2
 Paver==1.0.5
 Paved==0.3
 
-# Testing
-nose==1.1.2
-coverage==3.5.1
-mock==0.7.2
-nosexcover==1.0.7
-pylint==0.25.1
-
 # pycco deps
 Markdown==2.1.1
 Pycco==0.3.0
@@ -39,3 +32,4 @@ wsgiref==0.1.2
 
 # misc
 pycleaner==1.1.1
+-e git://github.com/pydata/vbench.git#egg=vbench

tests/test_logger.py

@@ -1,7 +1,7 @@
 import logging
 import uuid
 
-from zipline.utils.logger import configure_logging
+from zipline.utils.logger import configure_logging, tail
 from unittest2 import TestCase
 
 
@@ -17,35 +17,6 @@ class LoggerTestCase(TestCase):
         self.LOG.info(test_msg)
         logfile = open('/var/log/zipline/zipline.log','r')
         with logfile:
-            last_line = tail(logfile)
+            last_line = tail(logfile, window=1)
             logged_msg = last_line.split(" - ")[1]
             self.assertEqual(test_msg, logged_msg)
-
-def tail( f, window=20 ):
-    """
-    from
-    http://stackoverflow.com/questions/136168/get-last-n-lines-of-a-file- \
-            with-python-similar-to-tail
-    """
-    BUFSIZ = 1024
-    f.seek(0, 2)
-    bytes = f.tell()
-    size = window
-    block = -1
-    data = []
-    while size > 0 and bytes > 0:
-        if (bytes - BUFSIZ > 0):
-            # Seek back one whole BUFSIZ
-            f.seek(block*BUFSIZ, 2)
-            # read BUFFER
-            data.append(f.read(BUFSIZ))
-        else:
-            # file too small, start from begining
-            f.seek(0,0)
-            # only read what was not read
-            data.append(f.read(bytes))
-        linesFound = data[-1].count('\n')
-        size -= linesFound
-        bytes -= BUFSIZ
-        block -= 1
-    return '\n'.join(''.join(data).splitlines()[-window:])

vb_suite/run_suite.py

@@ -0,0 +1,12 @@
+from vbench.api import BenchmarkRunner
+from suite import *
+
+def run_process():
+    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
+                             BUILD, DB_PATH, TMP_DIR, PREPARE,
+                             run_option='all', start_date=START_DATE,
+                             module_dependencies=dependencies)
+    runner.run()
+
+if __name__ == '__main__':
+    run_process()

vb_suite/suite.py

@@ -0,0 +1,111 @@
+from vbench.api import Benchmark, GitRepo
+from datetime import datetime
+
+import os
+
+modules = ['ziplines']
+
+by_module = {}
+benchmarks = []
+
+for modname in modules:
+    ref = __import__(modname)
+    by_module[modname] = [v for v in ref.__dict__.values()
+                          if isinstance(v, Benchmark)]
+    benchmarks.extend(by_module[modname])
+
+for bm in benchmarks:
+    assert(bm.name is not None)
+
+import getpass
+import sys
+
+USERNAME = getpass.getuser()
+
+if sys.platform == 'darwin':
+    HOME = '/Users/%s' % USERNAME
+else:
+    HOME = '/home/%s' % USERNAME
+
+REPO_PATH = os.path.join(HOME, 'projects/qexec/zipline_repo')
+REPO_URL = 'git@github.com:quantopian/zipline.git'
+DB_PATH = os.path.join(REPO_PATH, 'vb_suite/benchmarks.db')
+TMP_DIR = os.path.join(HOME, 'tmp/vb_zipline')
+
+PREPARE = """
+"""
+BUILD = """
+"""
+dependencies = ['zipline_bench_functions.py']
+
+START_DATE = datetime(2011, 6, 1)
+
+repo = GitRepo(REPO_PATH)
+
+RST_BASE = 'source'
+
+# HACK!
+
+#timespan = [datetime(2011, 1, 1), datetime(2012, 1, 1)]
+
+def generate_rst_files(benchmarks):
+    import matplotlib as mpl
+    mpl.use('Agg')
+    import matplotlib.pyplot as plt
+
+    vb_path = os.path.join(RST_BASE, 'vbench')
+    fig_base_path = os.path.join(vb_path, 'figures')
+
+    if not os.path.exists(vb_path):
+        print 'creating %s' % vb_path
+        os.makedirs(vb_path)
+
+    if not os.path.exists(fig_base_path):
+        print 'creating %s' % fig_base_path
+        os.makedirs(fig_base_path)
+
+    for bmk in benchmarks:
+        print 'Generating rst file for %s' % bmk.name
+        rst_path = os.path.join(RST_BASE, 'vbench/%s.txt' % bmk.name)
+
+        fig_full_path = os.path.join(fig_base_path, '%s.png' % bmk.name)
+
+        # make the figure
+        plt.figure(figsize=(10, 6))
+        ax = plt.gca()
+        bmk.plot(DB_PATH, ax=ax)
+
+        start, end = ax.get_xlim()
+
+        plt.xlim([start - 30, end + 30])
+        plt.savefig(fig_full_path, bbox_inches='tight')
+        plt.close('all')
+
+        fig_rel_path = 'vbench/figures/%s.png' % bmk.name
+        rst_text = bmk.to_rst(image_path=fig_rel_path)
+        with open(rst_path, 'w') as f:
+            f.write(rst_text)
+
+    with open(os.path.join(RST_BASE, 'index.rst'), 'w') as f:
+        print >> f, """
+Performance Benchmarks
+======================
+
+These historical benchmark graphs were produced with `vbench
+<http://github.com/pydata/vbench>`__.
+
+.. toctree::
+    :hidden:
+    :maxdepth: 3
+"""
+        for modname, mod_bmks in sorted(by_module.items()):
+            print >> f, '    vb_%s' % modname
+            modpath = os.path.join(RST_BASE, 'vb_%s.rst' % modname)
+            with open(modpath, 'w') as mh:
+                header = '%s\n%s\n\n' % (modname, '=' * len(modname))
+                print >> mh, header
+
+                for bmk in mod_bmks:
+                    print >> mh, bmk.name
+                    print >> mh, '-' * len(bmk.name)
+                    print >> mh, '.. include:: vbench/%s.txt\n' % bmk.name

vb_suite/zipline_bench_functions.py

@@ -0,0 +1,43 @@
+try:
+    from zipline.simulator import AddressAllocator
+    pass
+except Exception, e:
+    from zipline.core.devsimulator import AddressAllocator
+
+from zipline.lines import SimulatedTrading
+
+allocator = AddressAllocator(1001)
+
+
+def get_zipline():
+    zipline_test_config = {
+        'allocator':allocator,
+        'sid':133
+    }
+
+    zipline = SimulatedTrading.create_test_zipline(
+        **zipline_test_config
+    )
+
+    return zipline
+
+def run_basic_zipline():
+    zipline = get_zipline()
+    zipline.simulate(blocking=True)
+
+def load_ndict():
+    from zipline import ndict
+    nd = ndict({})
+    keyname = 'a %i'
+    for i in xrange(1000000):
+        nd[keyname % i] = i
+
+    for i in xrange(1000000):
+        nd[keyname % i]
+
+def mass_create_ndict():
+    from zipline import ndict
+    data = dict(('a %d' % a,a) for a in xrange(1000))
+
+    for i in xrange(10000):
+        ndict(data)

vb_suite/ziplines.py

@@ -0,0 +1,27 @@
+from vbench.api import Benchmark
+from datetime import datetime
+
+setup = """
+from zipline_bench_functions import *
+"""
+
+basic_zipline = Benchmark(
+        'run_basic_zipline()',
+        setup=setup,
+        start_date=datetime(2012,5,15),
+        name='basic_zipline_test'
+)
+
+load_ndict = Benchmark(
+        'load_ndict()',
+        setup=setup,
+        start_date=datetime(2012,5,15),
+        name='load_ndict'
+)
+
+mass_create_ndict = Benchmark(
+        'mass_create_ndict()',
+        setup=setup,
+        start_date=datetime(2012,5,1),
+        name='create_ndict'
+)

zipline/finance/performance.py

@@ -114,8 +114,6 @@ Performance Period
 import logging
 import datetime
 import pytz
-import msgpack
-import pandas
 import math
 
 import zmq

zipline/logging.cfg

@@ -20,7 +20,7 @@ qualname=ZiplineLogger
 class=handlers.RotatingFileHandler
 level=DEBUG
 formatter=ziplineformat
-args=("/var/log/zipline/zipline.log",10*1024*1024,5)
+args=("/var/log/zipline/zipline.log",'a',10*1024*1024,5)
 propagate=1
 
 [handler_consoleHandler]

zipline/utils/__init__.py

@@ -1,5 +1,4 @@
 from protocol_utils import ndict
-
 __all__ = [
     ndict,
 ]

zipline/utils/factory.py

@@ -1,7 +1,7 @@
 """
 Factory functions to prepare useful data for tests.
 """
-
+import os
 import pytz
 import msgpack
 import random
@@ -21,6 +21,11 @@ def data_path():
     data_path = dirname(abspath(data.__file__))
     return data_path
 
+def logger_path():
+    import zipline
+    log_path = dirname(abspath(zipline.__file__))
+    return os.join(log_path, 'logging.cfg')
+
 def load_market_data():
     fp_bm = open(join(data_path(), "benchmark.msgpack"), "rb")
     bm_list = msgpack.loads(fp_bm.read())

zipline/utils/logger.py

@@ -3,11 +3,49 @@ Small classes to assist with timezone calculations, LOGGER configuration,
 and other common operations.
 """
 
+
 import logging
 import logging.config
+from os.path import join, abspath, dirname
 
 def configure_logging():
     logging.config.fileConfig(
-        'logging.cfg',
+        logger_path(),
         disable_existing_loggers = False
     )
+
+def logger_path():
+    import zipline
+    log_path = dirname(abspath(zipline.__file__))
+    return join(log_path, 'logging.cfg')
+
+
+# utility for tailing a log file.
+def tail( f, window=20 ):
+    """
+    from
+    http://stackoverflow.com/questions/136168/get-last-n-lines-of-a-file- \
+            with-python-similar-to-tail
+    """
+    BUFSIZ = 1024
+    f.seek(0, 2)
+    bytes = f.tell()
+    size = window
+    block = -1
+    data = []
+    while size > 0 and bytes > 0:
+        if (bytes - BUFSIZ > 0):
+            # Seek back one whole BUFSIZ
+            f.seek(block*BUFSIZ, 2)
+            # read BUFFER
+            data.append(f.read(BUFSIZ))
+        else:
+            # file too small, start from begining
+            f.seek(0,0)
+            # only read what was not read
+            data.append(f.read(bytes))
+        linesFound = data[-1].count('\n')
+        size -= linesFound
+        bytes -= BUFSIZ
+        block -= 1
+    return '\n'.join(''.join(data).splitlines()[-window:])

zipline/utils/protocol_utils.py

@@ -40,9 +40,14 @@ class ndict(MutableMapping):
     this time.
     """
 
+    cls = None
+    __slots__ = ['cls', '__internal']
+
     def __init__(self, dct=None):
         self.__internal = dict()
-        self.cls = frozenset(dir(self))
+
+        if not ndict.cls:
+            ndict.cls = frozenset(dir(self))
 
         if dct:
             self.__internal.update(dct)
@@ -51,8 +56,8 @@ class ndict(MutableMapping):
     # -----------------
 
     def __setattr__(self, key, value):
-        if '_ndict' in key or key == 'cls':
-            self.__dict__[key] = value
+        if key == 'cls' or key == '__internal' or '_ndict' in key:
+            super(ndict, self).__setattr__(key, value)
         else:
             self.__internal[key] = value
         return value
@@ -68,7 +73,7 @@ class ndict(MutableMapping):
 
     def __getattr__(self, key):
         if key in self.cls:
-            return self.__dict__[key]
+            super(ndict, self).__getattr__(key)
         else:
             return self.__internal[key]