mirror of
https://github.com/wassname/catalyst.git
synced 2026-06-29 15:42:03 +08:00
Frederic Fortier
652 lines
41 KiB
HTML
652 lines
41 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
<title>Data Bundles — Catalyst 0.4 documentation</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="top" title="Catalyst 0.4 documentation" href="index.html"/>
|
|
|
|
|
|
<script src="_static/js/modernizr.min.js"></script>
|
|
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav" role="document">
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-nav-search">
|
|
|
|
|
|
|
|
<a href="index.html" class="icon icon-home"> Catalyst
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
|
|
|
|
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="install.html">Install</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#installing-with-conda">Installing with <code class="docutils literal"><span class="pre">conda</span></code></a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="install.html#troubleshooting-conda-install">Troubleshooting <code class="docutils literal"><span class="pre">conda</span></code> Install</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#installing-with-pip">Installing with <code class="docutils literal"><span class="pre">pip</span></code></a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="install.html#troubleshooting-pip-install">Troubleshooting <code class="docutils literal"><span class="pre">pip</span></code> Install</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#installing-with-pipenv">Installing with <code class="docutils literal"><span class="pre">pipenv</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#gnu-linux-requirements">GNU/Linux Requirements</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="install.html#amazon-linux-ami-notes">Amazon Linux AMI Notes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#macos-requirements">MacOS Requirements</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="install.html#macos-virtualenv-conda-matplotlib">MacOS + virtualenv/conda + matplotlib</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#windows-requirements">Windows Requirements</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#updating-catalyst">Updating Catalyst</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="install.html#getting-help">Getting Help</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="beginner-tutorial.html">Catalyst Beginner Tutorial</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#basics">Basics</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#my-first-algorithm">My first algorithm</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#ingesting-data">Ingesting data</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#running-the-algorithm">Running the algorithm</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="beginner-tutorial.html#command-line-interface">Command line interface</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#access-to-previous-prices-using-history">Access to previous prices using <code class="docutils literal"><span class="pre">history</span></code></a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="beginner-tutorial.html#working-example-dual-moving-average-cross-over">Working example: Dual Moving Average Cross-Over</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#jupyter-notebook">Jupyter Notebook</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="beginner-tutorial.html#install">Install</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="beginner-tutorial.html#running-algorithms">Running Algorithms</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="beginner-tutorial.html#next-steps">Next steps</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="live-trading.html">Live Trading</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="live-trading.html#supported-exchanges">Supported Exchanges</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="live-trading.html#paper-trading-vs-live-trading-modes">Paper Trading vs Live Trading modes</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="live-trading.html#authentication">Authentication</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="live-trading.html#currency-symbols">Currency Symbols</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="live-trading.html#trading-an-algorithm">Trading an Algorithm</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="features.html">Features</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="features.html#current-functionality">Current Functionality</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="features.html#upcoming-features">Upcoming features</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="features.html#naming-convention">Naming Convention</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="example-algos.html">Example Algorithms</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#overview">Overview</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#buy-btc-simple-algorithm">Buy BTC Simple Algorithm</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#buy-and-hodl-algorithm">Buy and Hodl Algorithm</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#dual-moving-average-crossover">Dual Moving Average Crossover</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#mean-reversion-algorithm">Mean Reversion Algorithm</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#simple-universe">Simple Universe</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="example-algos.html#portfolio-optimization">Portfolio Optimization</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="utilities.html">Utilities</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="utilities.html#output-to-csv-file">Output to CSV file</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="utilities.html#extracting-market-data">Extracting market data</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="videos.html">Videos</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="videos.html#installation-macos">Installation: MacOS</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="videos.html#installation-windows">Installation: Windows</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="videos.html#backtesting-a-strategy">Backtesting a Strategy</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="videos.html#live-trading-a-strategy">Live Trading a Strategy</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="resources.html">Resources</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="resources.html#related-3rd-party-apis">Related 3rd Party APIs</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="development-guidelines.html">Development Guidelines</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="development-guidelines.html#creating-a-development-environment">Creating a Development Environment</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="development-guidelines.html#git-branching-structure">Git Branching Structure</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="development-guidelines.html#contributing-to-the-docs">Contributing to the Docs</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="development-guidelines.html#commit-messages">Commit messages</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="development-guidelines.html#formatting-docstrings">Formatting Docstrings</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="releases.html">Release Notes</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-5-1">Version 0.5.1</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#bug-fixes">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#build">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-7">Version 0.4.7</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id1">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id2">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-6">Version 0.4.6</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id3">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id4">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-5">Version 0.4.5</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id5">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-4">Version 0.4.4</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id6">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-3">Version 0.4.3</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id7">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-2">Version 0.4.2</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id8">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id9">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-4-0">Version 0.4.0</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id10">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id11">Build</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#version-0-3-10">Version 0.3.10</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id12">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-9">Version 0.3.9</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id13">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id14">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-8">Version 0.3.8</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id15">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-7">Version 0.3.7</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id16">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id17">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-6">Version 0.3.6</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id18">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-5">Version 0.3.5</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id19">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-4">Version 0.3.4</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id20">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id21">Build</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#documentation">Documentation</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-3">Version 0.3.3</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id23">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id24">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-2">Version 0.3.2</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id25">Bug Fixes</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id26">Build</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3-1">Version 0.3.1</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="releases.html#id27">Bug Fixes</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-3">Version 0.3</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-2-dev5">Version 0.2.dev5</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-2-dev4">Version 0.2.dev4</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-2-dev3">Version 0.2.dev3</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-2-dev2">Version 0.2.dev2</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-2-dev1">Version 0.2.dev1</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-1-dev9">Version 0.1.dev9</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-1-dev8">Version 0.1.dev8</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-1-dev7">Version 0.1.dev7</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="releases.html#version-0-1-dev6">Version 0.1.dev6</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="index.html">Catalyst</a>
|
|
</nav>
|
|
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="index.html">Docs</a> »</li>
|
|
|
|
<li>Data Bundles</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
|
|
<a href="_sources/bundles.txt" rel="nofollow"> View page source</a>
|
|
|
|
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document">
|
|
|
|
<div class="section" id="data-bundles">
|
|
<span id="id1"></span><h1>Data Bundles<a class="headerlink" href="#data-bundles" title="Permalink to this headline">¶</a></h1>
|
|
<p>A data bundle is a collection of pricing data, adjustment data, and an asset
|
|
database. Bundles allow us to preload all of the data we will need to run
|
|
backtests and store the data for future runs.</p>
|
|
<div class="section" id="discovering-available-bundles">
|
|
<span id="bundles-command"></span><h2>Discovering Available Bundles<a class="headerlink" href="#discovering-available-bundles" title="Permalink to this headline">¶</a></h2>
|
|
<p>Zipline comes with a few bundles by default as well as the ability to register
|
|
new bundles. To see which bundles we have have available, we may run the
|
|
<code class="docutils literal"><span class="pre">bundles</span></code> command, for example:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ </span>zipline bundles
|
|
my-custom-bundle 2016-05-05 20:35:19.809398
|
|
my-custom-bundle 2016-05-05 20:34:53.654082
|
|
my-custom-bundle 2016-05-05 20:34:48.401767
|
|
quandl <no ingestions>
|
|
quantopian-quandl 2016-05-05 20:06:40.894956
|
|
</pre></div>
|
|
</div>
|
|
<p>The output here shows that there are 3 bundles available:</p>
|
|
<ul class="simple">
|
|
<li><code class="docutils literal"><span class="pre">my-custom-bundle</span></code> (added by the user)</li>
|
|
<li><code class="docutils literal"><span class="pre">quandl</span></code> (provided by zipline)</li>
|
|
<li><code class="docutils literal"><span class="pre">quantopian-quandl</span></code> (provided by zipline)</li>
|
|
</ul>
|
|
<p>The dates and times next to the name show the times when the data for this
|
|
bundle was ingested. We have run three different ingestions for
|
|
<code class="docutils literal"><span class="pre">my-custom-bundle</span></code>. We have never ingested any data for the <code class="docutils literal"><span class="pre">quandl</span></code> bundle
|
|
so it just shows <code class="docutils literal"><span class="pre"><no</span> <span class="pre">ingestions></span></code> instead. Finally, there is only one
|
|
ingestion for <code class="docutils literal"><span class="pre">quantopian-quandl</span></code>.</p>
|
|
</div>
|
|
<div class="section" id="ingesting-data">
|
|
<span id="id2"></span><h2>Ingesting Data<a class="headerlink" href="#ingesting-data" title="Permalink to this headline">¶</a></h2>
|
|
<p>The first step to using a data bundle is to ingest the data. The ingestion
|
|
process will invoke some custom bundle command and then write the data to a
|
|
standard location that zipline can find. By default the location where ingested
|
|
data will be written is <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT/data/<bundle></span></code> where by default
|
|
<code class="docutils literal"><span class="pre">ZIPLINE_ROOT=~/.zipline</span></code>. The ingestion step may take some time as it could
|
|
involve downloading and processing a lot of data. This can be run with:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ </span>zipline ingest <span class="o">[</span>-b <bundle><span class="o">]</span>
|
|
</pre></div>
|
|
</div>
|
|
<p>where <code class="docutils literal"><span class="pre"><bundle></span></code> is the name of the bundle to ingest, defaulting to
|
|
<a class="reference internal" href="#quantopian-quandl-mirror"><span>quantopian-quandl</span></a>.</p>
|
|
</div>
|
|
<div class="section" id="old-data">
|
|
<h2>Old Data<a class="headerlink" href="#old-data" title="Permalink to this headline">¶</a></h2>
|
|
<p>When the <code class="docutils literal"><span class="pre">ingest</span></code> command is used it will write the new data to a subdirectory
|
|
of <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT/data/<bundle></span></code> which is named with the current date. This
|
|
makes it possible to look at older data or even run backtests with the older
|
|
copies. Running a backtest with an old ingestion makes it easier to reproduce
|
|
backtest results later.</p>
|
|
<p>One drawback of saving all of the data by default is that the data directory
|
|
may grow quite large even if you do not want to use the data. As shown earlier,
|
|
we can list all of the ingestions with the <a class="reference internal" href="#bundles-command"><span>bundles command</span></a>. To solve the problem of leaking old data there is another
|
|
command: <code class="docutils literal"><span class="pre">clean</span></code>, which will clear data bundles based on some time
|
|
constraints.</p>
|
|
<p>For example:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="c"># clean everything older than <date></span>
|
|
<span class="nv">$ </span>zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --before <date>
|
|
|
|
<span class="c"># clean everything newer than <date></span>
|
|
<span class="nv">$ </span>zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --after <date>
|
|
|
|
<span class="c"># keep everything in the range of [before, after] and delete the rest</span>
|
|
<span class="nv">$ </span>zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --before <date> --after <after>
|
|
|
|
<span class="c"># clean all but the last <int> runs</span>
|
|
<span class="nv">$ </span>zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --keep-last <int>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="running-backtests-with-data-bundles">
|
|
<h2>Running Backtests with Data Bundles<a class="headerlink" href="#running-backtests-with-data-bundles" title="Permalink to this headline">¶</a></h2>
|
|
<p>Now that the data has been ingested we can use it to run backtests with the
|
|
<code class="docutils literal"><span class="pre">run</span></code> command. The bundle to use can be specified with the <code class="docutils literal"><span class="pre">--bundle</span></code> option
|
|
like:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ </span>zipline run --bundle <bundle> --algofile algo.py ...
|
|
</pre></div>
|
|
</div>
|
|
<p>We may also specify the date to use to look up the bundle data with the
|
|
<code class="docutils literal"><span class="pre">--bundle-date</span></code> option. Setting the <code class="docutils literal"><span class="pre">--bundle-date</span></code> will cause run to use
|
|
the most recent bundle ingestion that is less than or equal to the
|
|
<code class="docutils literal"><span class="pre">bundle-date</span></code>. This is how we can run backtests with older data. The reason
|
|
that <code class="docutils literal"><span class="pre">-bundle-date</span></code> uses a less than or equal to relationship is that we can
|
|
specify the date that we ran an old backtest and get the same data that would
|
|
have been available to us on that date. The <code class="docutils literal"><span class="pre">bundle-date</span></code> defaults to the
|
|
current day to use the most recent data.</p>
|
|
</div>
|
|
<div class="section" id="default-data-bundles">
|
|
<h2>Default Data Bundles<a class="headerlink" href="#default-data-bundles" title="Permalink to this headline">¶</a></h2>
|
|
<div class="section" id="quandl-wiki-bundle">
|
|
<span id="quandl-data-bundle"></span><h3>Quandl WIKI Bundle<a class="headerlink" href="#quandl-wiki-bundle" title="Permalink to this headline">¶</a></h3>
|
|
<p>By default zipline comes with the <code class="docutils literal"><span class="pre">quandl</span></code> data bundle which uses quandl’s
|
|
<a class="reference external" href="https://www.quandl.com/data/WIKI">WIKI dataset</a>. The quandl data bundle
|
|
includes daily pricing data, splits, cash dividends, and asset metadata. To
|
|
ingest the <code class="docutils literal"><span class="pre">quandl</span></code> data bundle we recommend creating an account on quandl.com
|
|
to get an API key to be able to make more API requests per day. Once we have an
|
|
API key we may run:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ QUANDL_API_KEY</span><span class="o">=</span><api-key> zipline ingest -b quandl
|
|
</pre></div>
|
|
</div>
|
|
<p>though we may still run <code class="docutils literal"><span class="pre">ingest</span></code> as an anonymous quandl user (with no API
|
|
key). We may also set the <code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> environment variable to
|
|
an integer which is the number of attempts that should be made to download data
|
|
from quandls servers. By default <code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> will be 5, meaning
|
|
that we will retry each attempt 5 times.</p>
|
|
<div class="admonition note">
|
|
<p class="first admonition-title">Note</p>
|
|
<p class="last"><code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> is not the total number of allowed failures,
|
|
just the number of allowed failures per request. The quandl loader will make
|
|
one request per 100 equities for the metadata followed by one request per
|
|
equity.</p>
|
|
</div>
|
|
<div class="section" id="quantopian-quandl-wiki-mirror">
|
|
<span id="quantopian-quandl-mirror"></span><h4>Quantopian Quandl WIKI Mirror<a class="headerlink" href="#quantopian-quandl-wiki-mirror" title="Permalink to this headline">¶</a></h4>
|
|
<p>Quantopian provides a mirror of the quandl WIKI dataset with the data in the
|
|
formats that zipline expects. This is available under the name:
|
|
<code class="docutils literal"><span class="pre">quantopian-quandl</span></code> and is the default bundle for zipline.</p>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="yahoo-bundle-factories">
|
|
<h3>Yahoo Bundle Factories<a class="headerlink" href="#yahoo-bundle-factories" title="Permalink to this headline">¶</a></h3>
|
|
<p>Zipline also ships with a factory function for creating a data bundle out of a
|
|
set of tickers from yahoo: <code class="xref py py-func docutils literal"><span class="pre">yahoo_equities()</span></code>.
|
|
<code class="xref py py-func docutils literal"><span class="pre">yahoo_equities()</span></code> makes it easy to pre-download and
|
|
cache the data for a set of equities from yahoo. The yahoo bundles include daily
|
|
pricing data along with splits, cash dividends, and inferred asset metadata. To
|
|
create a bundle from a set of equities, add the following to your
|
|
<code class="docutils literal"><span class="pre">~/.zipline/extensions.py</span></code> file:</p>
|
|
<div class="highlight-python"><div class="highlight"><pre><span class="kn">from</span> <span class="nn">zipline.data.bundles</span> <span class="kn">import</span> <span class="n">register</span><span class="p">,</span> <span class="n">yahoo_equities</span>
|
|
|
|
<span class="c"># these are the tickers you would like data for</span>
|
|
<span class="n">equities</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s">'AAPL'</span><span class="p">,</span>
|
|
<span class="s">'MSFT'</span><span class="p">,</span>
|
|
<span class="s">'GOOG'</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="n">register</span><span class="p">(</span>
|
|
<span class="s">'my-yahoo-equities-bundle'</span><span class="p">,</span> <span class="c"># name this whatever you like</span>
|
|
<span class="n">yahoo_equities</span><span class="p">(</span><span class="n">equities</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
<p>This may now be used like:</p>
|
|
<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ </span>zipline ingest -b my-yahoo-equities-bundle
|
|
<span class="nv">$ </span>zipline run -f algo.py --bundle my-yahoo-equities-bundle
|
|
</pre></div>
|
|
</div>
|
|
<p>More than one yahoo equities bundle may be registered as long as they use
|
|
different names.</p>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="writing-a-new-bundle">
|
|
<h2>Writing a New Bundle<a class="headerlink" href="#writing-a-new-bundle" title="Permalink to this headline">¶</a></h2>
|
|
<p>Data bundles exist to make it easy to use different data sources with
|
|
zipline. To add a new bundle, one must implement an <code class="docutils literal"><span class="pre">ingest</span></code> function.</p>
|
|
<p>The <code class="docutils literal"><span class="pre">ingest</span></code> function is responsible for loading the data into memory and
|
|
passing it to a set of writer objects provided by zipline to convert the data to
|
|
zipline’s internal format. The ingest function may work by downloading data from
|
|
a remote location like the <code class="docutils literal"><span class="pre">quandl</span></code> bundle or yahoo bundles or it may just
|
|
load files that are already on the machine. The function is provided with
|
|
writers that will write the data to the correct location transactionally. If an
|
|
ingestion fails part way through the bundle will not be written in an incomplete
|
|
state.</p>
|
|
<p>The signature of the ingest function should be:</p>
|
|
<div class="highlight-python"><div class="highlight"><pre><span class="n">ingest</span><span class="p">(</span><span class="n">environ</span><span class="p">,</span>
|
|
<span class="n">asset_db_writer</span><span class="p">,</span>
|
|
<span class="n">minute_bar_writer</span><span class="p">,</span>
|
|
<span class="n">daily_bar_writer</span><span class="p">,</span>
|
|
<span class="n">adjustment_writer</span><span class="p">,</span>
|
|
<span class="n">calendar</span><span class="p">,</span>
|
|
<span class="n">start_session</span><span class="p">,</span>
|
|
<span class="n">end_session</span><span class="p">,</span>
|
|
<span class="n">cache</span><span class="p">,</span>
|
|
<span class="n">show_progress</span><span class="p">,</span>
|
|
<span class="n">output_dir</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
<div class="section" id="environ">
|
|
<h3><code class="docutils literal"><span class="pre">environ</span></code><a class="headerlink" href="#environ" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">environ</span></code> is a mapping representing the environment variables to use. This is
|
|
where any custom arguments needed for the ingestion should be passed, for
|
|
example: the <code class="docutils literal"><span class="pre">quandl</span></code> bundle uses the enviornment to pass the API key and the
|
|
download retry attempt count.</p>
|
|
</div>
|
|
<div class="section" id="asset-db-writer">
|
|
<h3><code class="docutils literal"><span class="pre">asset_db_writer</span></code><a class="headerlink" href="#asset-db-writer" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">asset_db_writer</span></code> is an instance of <code class="xref py py-class docutils literal"><span class="pre">AssetDBWriter</span></code>.
|
|
This is the writer for the asset metadata which provides the asset lifetimes and
|
|
the symbol to asset id (sid) mapping. This may also contain the asset name,
|
|
exchange and a few other columns. To write data, invoke
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with dataframes for the various
|
|
pieces of metadata. More information about the format of the data exists in the
|
|
docs for write.</p>
|
|
</div>
|
|
<div class="section" id="minute-bar-writer">
|
|
<h3><code class="docutils literal"><span class="pre">minute_bar_writer</span></code><a class="headerlink" href="#minute-bar-writer" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">minute_bar_writer</span></code> is an instance of
|
|
<code class="xref py py-class docutils literal"><span class="pre">BcolzMinuteBarWriter</span></code>. This writer is used to
|
|
convert data to zipline’s internal bcolz format to later be read by a
|
|
<code class="xref py py-class docutils literal"><span class="pre">BcolzMinuteBarReader</span></code>. If minute data is
|
|
provided, users should call
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with an iterable of
|
|
(sid, dataframe) tuples. The <code class="docutils literal"><span class="pre">show_progress</span></code> argument should also be forwarded
|
|
to this method. If the data source does not provide minute level data, then
|
|
there is no need to call the write method. It is also acceptable to pass an
|
|
empty iterator to <code class="xref py py-meth docutils literal"><span class="pre">write()</span></code>
|
|
to signal that there is no minutely data.</p>
|
|
<div class="admonition note">
|
|
<p class="first admonition-title">Note</p>
|
|
<p class="last">The data passed to
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> may be a lazy
|
|
iterator or generator to avoid loading all of the minute data into memory at
|
|
a single time. A given sid may also appear multiple times in the data as long
|
|
as the dates are strictly increasing.</p>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="daily-bar-writer">
|
|
<h3><code class="docutils literal"><span class="pre">daily_bar_writer</span></code><a class="headerlink" href="#daily-bar-writer" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">daily_bar_writer</span></code> is an instance of
|
|
<code class="xref py py-class docutils literal"><span class="pre">BcolzDailyBarWriter</span></code>. This writer is
|
|
used to convert data into zipline’s internal bcolz format to later be read by a
|
|
<code class="xref py py-class docutils literal"><span class="pre">BcolzDailyBarReader</span></code>. If daily data is
|
|
provided, users should call
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with an iterable of
|
|
(sid dataframe) tuples. The <code class="docutils literal"><span class="pre">show_progress</span></code> argument should also be forwarded
|
|
to this method. If the data shource does not provide daily data, then there is
|
|
no need to call the write method. It is also acceptable to pass an empty
|
|
iterable to <code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> to
|
|
signal that there is no daily data. If no daily data is provided but minute data
|
|
is provided, a daily rollup will happen to service daily history requests.</p>
|
|
<div class="admonition note">
|
|
<p class="first admonition-title">Note</p>
|
|
<p class="last">Like the <code class="docutils literal"><span class="pre">minute_bar_writer</span></code>, the data passed to
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> may be a lazy
|
|
iterable or generator to avoid loading all of the data into memory at once.
|
|
Unlike the <code class="docutils literal"><span class="pre">minute_bar_writer</span></code>, a sid may only appear once in the data
|
|
iterable.</p>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="adjustment-writer">
|
|
<h3><code class="docutils literal"><span class="pre">adjustment_writer</span></code><a class="headerlink" href="#adjustment-writer" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">adjustment_writer</span></code> is an instance of
|
|
<code class="xref py py-class docutils literal"><span class="pre">SQLiteAdjustmentWriter</span></code>. This writer is
|
|
used to store splits, mergers, dividends, and stock dividends. The data should
|
|
be provided as dataframes and passed to
|
|
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code>. Each of
|
|
these fields are optional, but the writer can accept as much of the data as you
|
|
have.</p>
|
|
</div>
|
|
<div class="section" id="calendar">
|
|
<h3><code class="docutils literal"><span class="pre">calendar</span></code><a class="headerlink" href="#calendar" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">calendar</span></code> is an instance of
|
|
<code class="xref py py-class docutils literal"><span class="pre">zipline.utils.calendars.TradingCalendar</span></code>. The calendar is provided to
|
|
help some bundles generate queries for the days needed.</p>
|
|
</div>
|
|
<div class="section" id="start-session">
|
|
<h3><code class="docutils literal"><span class="pre">start_session</span></code><a class="headerlink" href="#start-session" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">start_session</span></code> is a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/generated/pandas.Timestamp.html#pandas.Timestamp" title="(in pandas v0.22.0)"><code class="xref py py-class docutils literal"><span class="pre">pandas.Timestamp</span></code></a> object indicating the first
|
|
day that the bundle should load data for.</p>
|
|
</div>
|
|
<div class="section" id="end-session">
|
|
<h3><code class="docutils literal"><span class="pre">end_session</span></code><a class="headerlink" href="#end-session" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">end_session</span></code> is a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/generated/pandas.Timestamp.html#pandas.Timestamp" title="(in pandas v0.22.0)"><code class="xref py py-class docutils literal"><span class="pre">pandas.Timestamp</span></code></a> object indicating the last day
|
|
that the bundle should load data for.</p>
|
|
</div>
|
|
<div class="section" id="cache">
|
|
<h3><code class="docutils literal"><span class="pre">cache</span></code><a class="headerlink" href="#cache" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">cache</span></code> is an instance of <code class="xref py py-class docutils literal"><span class="pre">dataframe_cache</span></code>. This
|
|
object is a mapping from strings to dataframes. This object is provided in case
|
|
an ingestion crashes part way through. The idea is that the ingest function
|
|
should check the cache for raw data, if it doesn’t exist in the cache, it should
|
|
acquire it and then store it in the cache. Then it can parse and write the
|
|
data. The cache will be cleared only after a successful load, this prevents the
|
|
ingest function from needing to redownload all the data if there is some bug in
|
|
the parsing. If it is very fast to get the data, for example if it is coming
|
|
from another local file, then there is no need to use this cache.</p>
|
|
</div>
|
|
<div class="section" id="show-progress">
|
|
<h3><code class="docutils literal"><span class="pre">show_progress</span></code><a class="headerlink" href="#show-progress" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">show_progress</span></code> is a boolean indicating that the user would like to receive
|
|
feedback about the ingest function’s progress fetching and writing the
|
|
data. Some examples for where to show how many files you have downloaded out of
|
|
the total needed, or how far into some data conversion the ingest function
|
|
is. One tool that may help with implementing <code class="docutils literal"><span class="pre">show_progress</span></code> for a loop is
|
|
<code class="xref py py-class docutils literal"><span class="pre">maybe_show_progress</span></code>. This argument should always be
|
|
forwarded to <code class="docutils literal"><span class="pre">minute_bar_writer.write</span></code> and <code class="docutils literal"><span class="pre">daily_bar_writer.write</span></code>.</p>
|
|
</div>
|
|
<div class="section" id="output-dir">
|
|
<h3><code class="docutils literal"><span class="pre">output_dir</span></code><a class="headerlink" href="#output-dir" title="Permalink to this headline">¶</a></h3>
|
|
<p><code class="docutils literal"><span class="pre">output_dir</span></code> is a string representing the file path where all the data will be
|
|
written. <code class="docutils literal"><span class="pre">output_dir</span></code> will be some subdirectory of <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT</span></code> and will
|
|
contain the time of the start of the current ingestion. This can be used to
|
|
directly move resources here if for some reason your ingest function can produce
|
|
it’s own outputs without the writers. For example, the <code class="docutils literal"><span class="pre">quantopian:quandl</span></code>
|
|
bundle uses this to directly untar the bundle into the <code class="docutils literal"><span class="pre">output_dir</span></code>.</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
<footer>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>
|
|
© Copyright 2018, Enigma MPC, Inc..
|
|
</p>
|
|
</div>
|
|
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
var DOCUMENTATION_OPTIONS = {
|
|
URL_ROOT:'./',
|
|
VERSION:'0.4',
|
|
COLLAPSE_INDEX:false,
|
|
FILE_SUFFIX:'.html',
|
|
HAS_SOURCE: true
|
|
};
|
|
</script>
|
|
<script type="text/javascript" src="_static/jquery.js"></script>
|
|
<script type="text/javascript" src="_static/underscore.js"></script>
|
|
<script type="text/javascript" src="_static/doctools.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" src="_static/js/theme.js"></script>
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
jQuery(function () {
|
|
SphinxRtdTheme.StickyNav.enable();
|
|
});
|
|
</script>
|
|
|
|
|
|
</body>
|
|
</html> |