import argparse import os import sys sys.path.append('qexec') import pandas as pd import cProfile from line_profiler import LineProfiler import datetime from functools import partial from mem_util import get_memusage_mb def profile_qexec(algo_text, results_file_name, start_date, end_date, capital_base, granularity, profiler_type=None, names_to_profile=None, live_algo=False, session_start_date=None, inception_date=None, data_delay=None): # Import inside profile function, so that modules that take a while # to import, e.g. tradingcalendar, don't trigger when there are # invalid parameters, which should be a quick fail. # TODO: Fix load time of tradingcalendar. from algo_profile import run_algo from qexec.algo.validation import unittest results, ok = unittest(algo_text, granularity) if not ok: for res in results: if not res['passed']: print res._data sys.exit('Did not pass validation.') algo_runner = partial( run_algo, algo_text, start_date, end_date, capital_base, granularity, session_start=session_start_date, inception_date=inception_date, data_delay=data_delay, live_algo=live_algo ) if profiler_type == 'cProfile': results_dir = os.path.join('results', 'cprofile') if not os.path.exists(results_dir): os.makedirs(results_dir) results_file = os.path.join(results_dir, results_file_name) cProfile.runctx('algo_runner()', globals(), locals(), results_file) print ("Wrote results to: {0}".format(results_file)) elif profiler_type == 'line_profiler': results_dir = os.path.join('results', 'line_profiler') if not os.path.exists(results_dir): os.makedirs(results_dir) results_file = os.path.join(results_dir, results_file_name) profiler = LineProfiler() for name_to_profile in names_to_profile: name_parts = name_to_profile.split('.') obj = __import__(name_parts[0]) for name in name_parts[1:]: obj = getattr(obj, name) profiler.add_function(obj) profiler.runctx('algo_runner()', globals(), locals()) with open(results_file, 'w') as f: profiler.print_stats(stream=f) print ("Wrote results to: {0}".format(results_file)) elif not profiler_type: algo_runner() def create_parser(): parser = argparse.ArgumentParser() parser.add_argument('--algofile', '-f', type=argparse.FileType('r'), required=True) parser.add_argument('--data-frequency', default='minute', choices=('minute', 'daily')) parser.add_argument('--start-date', default='2012-01-01') parser.add_argument('--end-date', default='2012-12-31') parser.add_argument('--start-epoch', type=int) parser.add_argument('--end-epoch', type=int) parser.add_argument('--capital-base', default='10e6') parser.add_argument('--live-algo', action='store_true', default=False) parser.add_argument('--session-start-date', default='2014-01-03') parser.add_argument('--inception-date', default='2013-12-03') parser.add_argument('--data-delay', type=int, default=15 * 60) parser.add_argument('--is-inception', action='store_true', default=False) parser.add_argument('--profiler-type', choices=('cProfile', 'line_profiler')), parser.add_argument( '--name-to-profile', action='append', default=[ # Good proxy for overall performance/main gen 'zipline.gens.tradesimulation.AlgorithmSimulator.transform', # Proxy for network time, will eventually have to change if # we change data access style. 'pymongo.cursor.Cursor.next' ]) return parser if __name__ == "__main__": parser = create_parser() args = parser.parse_args() start_date = pd.Timestamp(args.start_date, tz='UTC') end_date = pd.Timestamp(args.end_date, tz='UTC') if args.start_epoch: start_date = pd.Timestamp(args.start_epoch, tz='UTC') if args.end_epoch: end_date = pd.Timestamp(args.end_epoch, tz='UTC') algo_text = args.algofile.read() # Remove the extension algo_name_base = os.path.splitext(args.algofile.name)[0] algo_name = os.path.basename(algo_name_base) results_file_name = \ 'qexec-prof-{algo_name}-{commitish}-{granularity}-{time}'.format( algo_name=algo_name.replace('.', '_'), commitish='local', granularity=args.data_frequency, time=str(datetime.datetime.now()).replace(' ', '-'). replace(':', '-')) if args.live_algo: session_start_date = pd.Timestamp(args.session_start_date, tz='UTC') inception_date = pd.Timestamp(args.inception_date, tz='UTC') else: session_start_date = None inception_date = None profile_qexec( algo_text, results_file_name, start_date, end_date, float(args.capital_base), args.data_frequency, args.profiler_type, args.name_to_profile, live_algo=args.live_algo, session_start_date=session_start_date, inception_date=inception_date, data_delay=args.data_delay ) import objgraph print 'finished running algo, now doing memory inspection' maxrss_in_mb = get_memusage_mb() print "Max memory consumption={maxrss_in_mb}MB".format( maxrss_in_mb=maxrss_in_mb) print objgraph.show_growth(limit=20) print objgraph.show_most_common_types(limit=35)