mirror of
https://github.com/wassname/catalyst.git
synced 2026-06-27 19:30:28 +08:00
Victor Grau Serrat
524 lines
29 KiB
HTML
524 lines
29 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||
<head>
|
||
<meta charset="utf-8">
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
|
||
<title>Data Bundles — Catalyst 0.4 documentation</title>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="index" title="Index"
|
||
href="genindex.html"/>
|
||
<link rel="search" title="Search" href="search.html"/>
|
||
<link rel="top" title="Catalyst 0.4 documentation" href="index.html"/>
|
||
|
||
|
||
<script src="_static/js/modernizr.min.js"></script>
|
||
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav" role="document">
|
||
|
||
|
||
<div class="wy-grid-for-nav">
|
||
|
||
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search">
|
||
|
||
|
||
|
||
<a href="index.html" class="icon icon-home"> Catalyst
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="install.html">Install</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="beginner-tutorial.html">Catalyst Beginner Tutorial</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="live-trading.html">Live Trading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="features.html">Features</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="example-algos.html">Example Algorithms</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="utilities.html">Utilities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="videos.html">Videos</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="resources.html">Resources</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="development-guidelines.html">Development Guidelines</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="releases.html">Release Notes</a></li>
|
||
</ul>
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||
|
||
|
||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="index.html">Catalyst</a>
|
||
|
||
</nav>
|
||
|
||
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||
|
||
<ul class="wy-breadcrumbs">
|
||
|
||
<li><a href="index.html">Docs</a> »</li>
|
||
|
||
<li>Data Bundles</li>
|
||
|
||
|
||
<li class="wy-breadcrumbs-aside">
|
||
|
||
|
||
<a href="_sources/bundles.rst.txt" rel="nofollow"> View page source</a>
|
||
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<div class="section" id="data-bundles">
|
||
<span id="id1"></span><h1>Data Bundles<a class="headerlink" href="#data-bundles" title="Permalink to this headline">¶</a></h1>
|
||
<p>A data bundle is a collection of pricing data, adjustment data, and an asset
|
||
database. Bundles allow us to preload all of the data we will need to run
|
||
backtests and store the data for future runs.</p>
|
||
<div class="section" id="discovering-available-bundles">
|
||
<span id="bundles-command"></span><h2>Discovering Available Bundles<a class="headerlink" href="#discovering-available-bundles" title="Permalink to this headline">¶</a></h2>
|
||
<p>Zipline comes with a few bundles by default as well as the ability to register
|
||
new bundles. To see which bundles we have have available, we may run the
|
||
<code class="docutils literal"><span class="pre">bundles</span></code> command, for example:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ zipline bundles
|
||
my-custom-bundle <span class="m">2016</span>-05-05 <span class="m">20</span>:35:19.809398
|
||
my-custom-bundle <span class="m">2016</span>-05-05 <span class="m">20</span>:34:53.654082
|
||
my-custom-bundle <span class="m">2016</span>-05-05 <span class="m">20</span>:34:48.401767
|
||
quandl <no ingestions>
|
||
quantopian-quandl <span class="m">2016</span>-05-05 <span class="m">20</span>:06:40.894956
|
||
</pre></div>
|
||
</div>
|
||
<p>The output here shows that there are 3 bundles available:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal"><span class="pre">my-custom-bundle</span></code> (added by the user)</li>
|
||
<li><code class="docutils literal"><span class="pre">quandl</span></code> (provided by zipline)</li>
|
||
<li><code class="docutils literal"><span class="pre">quantopian-quandl</span></code> (provided by zipline)</li>
|
||
</ul>
|
||
<p>The dates and times next to the name show the times when the data for this
|
||
bundle was ingested. We have run three different ingestions for
|
||
<code class="docutils literal"><span class="pre">my-custom-bundle</span></code>. We have never ingested any data for the <code class="docutils literal"><span class="pre">quandl</span></code> bundle
|
||
so it just shows <code class="docutils literal"><span class="pre"><no</span> <span class="pre">ingestions></span></code> instead. Finally, there is only one
|
||
ingestion for <code class="docutils literal"><span class="pre">quantopian-quandl</span></code>.</p>
|
||
</div>
|
||
<div class="section" id="ingesting-data">
|
||
<span id="id2"></span><h2>Ingesting Data<a class="headerlink" href="#ingesting-data" title="Permalink to this headline">¶</a></h2>
|
||
<p>The first step to using a data bundle is to ingest the data. The ingestion
|
||
process will invoke some custom bundle command and then write the data to a
|
||
standard location that zipline can find. By default the location where ingested
|
||
data will be written is <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT/data/<bundle></span></code> where by default
|
||
<code class="docutils literal"><span class="pre">ZIPLINE_ROOT=~/.zipline</span></code>. The ingestion step may take some time as it could
|
||
involve downloading and processing a lot of data. This can be run with:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ zipline ingest <span class="o">[</span>-b <bundle><span class="o">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>where <code class="docutils literal"><span class="pre"><bundle></span></code> is the name of the bundle to ingest, defaulting to
|
||
<a class="reference internal" href="#quantopian-quandl-mirror"><span class="std std-ref">quantopian-quandl</span></a>.</p>
|
||
</div>
|
||
<div class="section" id="old-data">
|
||
<h2>Old Data<a class="headerlink" href="#old-data" title="Permalink to this headline">¶</a></h2>
|
||
<p>When the <code class="docutils literal"><span class="pre">ingest</span></code> command is used it will write the new data to a subdirectory
|
||
of <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT/data/<bundle></span></code> which is named with the current date. This
|
||
makes it possible to look at older data or even run backtests with the older
|
||
copies. Running a backtest with an old ingestion makes it easier to reproduce
|
||
backtest results later.</p>
|
||
<p>One drawback of saving all of the data by default is that the data directory
|
||
may grow quite large even if you do not want to use the data. As shown earlier,
|
||
we can list all of the ingestions with the <a class="reference internal" href="#bundles-command"><span class="std std-ref">bundles command</span></a>. To solve the problem of leaking old data there is another
|
||
command: <code class="docutils literal"><span class="pre">clean</span></code>, which will clear data bundles based on some time
|
||
constraints.</p>
|
||
<p>For example:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># clean everything older than <date></span>
|
||
$ zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --before <date>
|
||
|
||
<span class="c1"># clean everything newer than <date></span>
|
||
$ zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --after <date>
|
||
|
||
<span class="c1"># keep everything in the range of [before, after] and delete the rest</span>
|
||
$ zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --before <date> --after <after>
|
||
|
||
<span class="c1"># clean all but the last <int> runs</span>
|
||
$ zipline clean <span class="o">[</span>-b <bundle><span class="o">]</span> --keep-last <int>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="running-backtests-with-data-bundles">
|
||
<h2>Running Backtests with Data Bundles<a class="headerlink" href="#running-backtests-with-data-bundles" title="Permalink to this headline">¶</a></h2>
|
||
<p>Now that the data has been ingested we can use it to run backtests with the
|
||
<code class="docutils literal"><span class="pre">run</span></code> command. The bundle to use can be specified with the <code class="docutils literal"><span class="pre">--bundle</span></code> option
|
||
like:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ zipline run --bundle <bundle> --algofile algo.py ...
|
||
</pre></div>
|
||
</div>
|
||
<p>We may also specify the date to use to look up the bundle data with the
|
||
<code class="docutils literal"><span class="pre">--bundle-date</span></code> option. Setting the <code class="docutils literal"><span class="pre">--bundle-date</span></code> will cause run to use
|
||
the most recent bundle ingestion that is less than or equal to the
|
||
<code class="docutils literal"><span class="pre">bundle-date</span></code>. This is how we can run backtests with older data. The reason
|
||
that <code class="docutils literal"><span class="pre">-bundle-date</span></code> uses a less than or equal to relationship is that we can
|
||
specify the date that we ran an old backtest and get the same data that would
|
||
have been available to us on that date. The <code class="docutils literal"><span class="pre">bundle-date</span></code> defaults to the
|
||
current day to use the most recent data.</p>
|
||
</div>
|
||
<div class="section" id="default-data-bundles">
|
||
<h2>Default Data Bundles<a class="headerlink" href="#default-data-bundles" title="Permalink to this headline">¶</a></h2>
|
||
<div class="section" id="quandl-wiki-bundle">
|
||
<span id="quandl-data-bundle"></span><h3>Quandl WIKI Bundle<a class="headerlink" href="#quandl-wiki-bundle" title="Permalink to this headline">¶</a></h3>
|
||
<p>By default zipline comes with the <code class="docutils literal"><span class="pre">quandl</span></code> data bundle which uses quandl’s
|
||
<a class="reference external" href="https://www.quandl.com/data/WIKI">WIKI dataset</a>. The quandl data bundle
|
||
includes daily pricing data, splits, cash dividends, and asset metadata. To
|
||
ingest the <code class="docutils literal"><span class="pre">quandl</span></code> data bundle we recommend creating an account on quandl.com
|
||
to get an API key to be able to make more API requests per day. Once we have an
|
||
API key we may run:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ <span class="nv">QUANDL_API_KEY</span><span class="o">=</span><api-key> zipline ingest -b quandl
|
||
</pre></div>
|
||
</div>
|
||
<p>though we may still run <code class="docutils literal"><span class="pre">ingest</span></code> as an anonymous quandl user (with no API
|
||
key). We may also set the <code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> environment variable to
|
||
an integer which is the number of attempts that should be made to download data
|
||
from quandls servers. By default <code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> will be 5, meaning
|
||
that we will retry each attempt 5 times.</p>
|
||
<div class="admonition note">
|
||
<p class="first admonition-title">Note</p>
|
||
<p class="last"><code class="docutils literal"><span class="pre">QUANDL_DOWNLOAD_ATTEMPTS</span></code> is not the total number of allowed failures,
|
||
just the number of allowed failures per request. The quandl loader will make
|
||
one request per 100 equities for the metadata followed by one request per
|
||
equity.</p>
|
||
</div>
|
||
<div class="section" id="quantopian-quandl-wiki-mirror">
|
||
<span id="quantopian-quandl-mirror"></span><h4>Quantopian Quandl WIKI Mirror<a class="headerlink" href="#quantopian-quandl-wiki-mirror" title="Permalink to this headline">¶</a></h4>
|
||
<p>Quantopian provides a mirror of the quandl WIKI dataset with the data in the
|
||
formats that zipline expects. This is available under the name:
|
||
<code class="docutils literal"><span class="pre">quantopian-quandl</span></code> and is the default bundle for zipline.</p>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="yahoo-bundle-factories">
|
||
<h3>Yahoo Bundle Factories<a class="headerlink" href="#yahoo-bundle-factories" title="Permalink to this headline">¶</a></h3>
|
||
<p>Zipline also ships with a factory function for creating a data bundle out of a
|
||
set of tickers from yahoo: <code class="xref py py-func docutils literal"><span class="pre">yahoo_equities()</span></code>.
|
||
<code class="xref py py-func docutils literal"><span class="pre">yahoo_equities()</span></code> makes it easy to pre-download and
|
||
cache the data for a set of equities from yahoo. The yahoo bundles include daily
|
||
pricing data along with splits, cash dividends, and inferred asset metadata. To
|
||
create a bundle from a set of equities, add the following to your
|
||
<code class="docutils literal"><span class="pre">~/.zipline/extensions.py</span></code> file:</p>
|
||
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">zipline.data.bundles</span> <span class="kn">import</span> <span class="n">register</span><span class="p">,</span> <span class="n">yahoo_equities</span>
|
||
|
||
<span class="c1"># these are the tickers you would like data for</span>
|
||
<span class="n">equities</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'AAPL'</span><span class="p">,</span>
|
||
<span class="s1">'MSFT'</span><span class="p">,</span>
|
||
<span class="s1">'GOOG'</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
<span class="n">register</span><span class="p">(</span>
|
||
<span class="s1">'my-yahoo-equities-bundle'</span><span class="p">,</span> <span class="c1"># name this whatever you like</span>
|
||
<span class="n">yahoo_equities</span><span class="p">(</span><span class="n">equities</span><span class="p">),</span>
|
||
<span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This may now be used like:</p>
|
||
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ zipline ingest -b my-yahoo-equities-bundle
|
||
$ zipline run -f algo.py --bundle my-yahoo-equities-bundle
|
||
</pre></div>
|
||
</div>
|
||
<p>More than one yahoo equities bundle may be registered as long as they use
|
||
different names.</p>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="writing-a-new-bundle">
|
||
<h2>Writing a New Bundle<a class="headerlink" href="#writing-a-new-bundle" title="Permalink to this headline">¶</a></h2>
|
||
<p>Data bundles exist to make it easy to use different data sources with
|
||
zipline. To add a new bundle, one must implement an <code class="docutils literal"><span class="pre">ingest</span></code> function.</p>
|
||
<p>The <code class="docutils literal"><span class="pre">ingest</span></code> function is responsible for loading the data into memory and
|
||
passing it to a set of writer objects provided by zipline to convert the data to
|
||
zipline’s internal format. The ingest function may work by downloading data from
|
||
a remote location like the <code class="docutils literal"><span class="pre">quandl</span></code> bundle or yahoo bundles or it may just
|
||
load files that are already on the machine. The function is provided with
|
||
writers that will write the data to the correct location transactionally. If an
|
||
ingestion fails part way through the bundle will not be written in an incomplete
|
||
state.</p>
|
||
<p>The signature of the ingest function should be:</p>
|
||
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">ingest</span><span class="p">(</span><span class="n">environ</span><span class="p">,</span>
|
||
<span class="n">asset_db_writer</span><span class="p">,</span>
|
||
<span class="n">minute_bar_writer</span><span class="p">,</span>
|
||
<span class="n">daily_bar_writer</span><span class="p">,</span>
|
||
<span class="n">adjustment_writer</span><span class="p">,</span>
|
||
<span class="n">calendar</span><span class="p">,</span>
|
||
<span class="n">start_session</span><span class="p">,</span>
|
||
<span class="n">end_session</span><span class="p">,</span>
|
||
<span class="n">cache</span><span class="p">,</span>
|
||
<span class="n">show_progress</span><span class="p">,</span>
|
||
<span class="n">output_dir</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<div class="section" id="environ">
|
||
<h3><code class="docutils literal"><span class="pre">environ</span></code><a class="headerlink" href="#environ" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">environ</span></code> is a mapping representing the environment variables to use. This is
|
||
where any custom arguments needed for the ingestion should be passed, for
|
||
example: the <code class="docutils literal"><span class="pre">quandl</span></code> bundle uses the enviornment to pass the API key and the
|
||
download retry attempt count.</p>
|
||
</div>
|
||
<div class="section" id="asset-db-writer">
|
||
<h3><code class="docutils literal"><span class="pre">asset_db_writer</span></code><a class="headerlink" href="#asset-db-writer" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">asset_db_writer</span></code> is an instance of <code class="xref py py-class docutils literal"><span class="pre">AssetDBWriter</span></code>.
|
||
This is the writer for the asset metadata which provides the asset lifetimes and
|
||
the symbol to asset id (sid) mapping. This may also contain the asset name,
|
||
exchange and a few other columns. To write data, invoke
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with dataframes for the various
|
||
pieces of metadata. More information about the format of the data exists in the
|
||
docs for write.</p>
|
||
</div>
|
||
<div class="section" id="minute-bar-writer">
|
||
<h3><code class="docutils literal"><span class="pre">minute_bar_writer</span></code><a class="headerlink" href="#minute-bar-writer" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">minute_bar_writer</span></code> is an instance of
|
||
<code class="xref py py-class docutils literal"><span class="pre">BcolzMinuteBarWriter</span></code>. This writer is used to
|
||
convert data to zipline’s internal bcolz format to later be read by a
|
||
<code class="xref py py-class docutils literal"><span class="pre">BcolzMinuteBarReader</span></code>. If minute data is
|
||
provided, users should call
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with an iterable of
|
||
(sid, dataframe) tuples. The <code class="docutils literal"><span class="pre">show_progress</span></code> argument should also be forwarded
|
||
to this method. If the data source does not provide minute level data, then
|
||
there is no need to call the write method. It is also acceptable to pass an
|
||
empty iterator to <code class="xref py py-meth docutils literal"><span class="pre">write()</span></code>
|
||
to signal that there is no minutely data.</p>
|
||
<div class="admonition note">
|
||
<p class="first admonition-title">Note</p>
|
||
<p class="last">The data passed to
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> may be a lazy
|
||
iterator or generator to avoid loading all of the minute data into memory at
|
||
a single time. A given sid may also appear multiple times in the data as long
|
||
as the dates are strictly increasing.</p>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="daily-bar-writer">
|
||
<h3><code class="docutils literal"><span class="pre">daily_bar_writer</span></code><a class="headerlink" href="#daily-bar-writer" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">daily_bar_writer</span></code> is an instance of
|
||
<code class="xref py py-class docutils literal"><span class="pre">BcolzDailyBarWriter</span></code>. This writer is
|
||
used to convert data into zipline’s internal bcolz format to later be read by a
|
||
<code class="xref py py-class docutils literal"><span class="pre">BcolzDailyBarReader</span></code>. If daily data is
|
||
provided, users should call
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> with an iterable of
|
||
(sid dataframe) tuples. The <code class="docutils literal"><span class="pre">show_progress</span></code> argument should also be forwarded
|
||
to this method. If the data shource does not provide daily data, then there is
|
||
no need to call the write method. It is also acceptable to pass an empty
|
||
iterable to <code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> to
|
||
signal that there is no daily data. If no daily data is provided but minute data
|
||
is provided, a daily rollup will happen to service daily history requests.</p>
|
||
<div class="admonition note">
|
||
<p class="first admonition-title">Note</p>
|
||
<p class="last">Like the <code class="docutils literal"><span class="pre">minute_bar_writer</span></code>, the data passed to
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code> may be a lazy
|
||
iterable or generator to avoid loading all of the data into memory at once.
|
||
Unlike the <code class="docutils literal"><span class="pre">minute_bar_writer</span></code>, a sid may only appear once in the data
|
||
iterable.</p>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="adjustment-writer">
|
||
<h3><code class="docutils literal"><span class="pre">adjustment_writer</span></code><a class="headerlink" href="#adjustment-writer" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">adjustment_writer</span></code> is an instance of
|
||
<code class="xref py py-class docutils literal"><span class="pre">SQLiteAdjustmentWriter</span></code>. This writer is
|
||
used to store splits, mergers, dividends, and stock dividends. The data should
|
||
be provided as dataframes and passed to
|
||
<code class="xref py py-meth docutils literal"><span class="pre">write()</span></code>. Each of
|
||
these fields are optional, but the writer can accept as much of the data as you
|
||
have.</p>
|
||
</div>
|
||
<div class="section" id="calendar">
|
||
<h3><code class="docutils literal"><span class="pre">calendar</span></code><a class="headerlink" href="#calendar" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">calendar</span></code> is an instance of
|
||
<code class="xref py py-class docutils literal"><span class="pre">zipline.utils.calendars.TradingCalendar</span></code>. The calendar is provided to
|
||
help some bundles generate queries for the days needed.</p>
|
||
</div>
|
||
<div class="section" id="start-session">
|
||
<h3><code class="docutils literal"><span class="pre">start_session</span></code><a class="headerlink" href="#start-session" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">start_session</span></code> is a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/generated/pandas.Timestamp.html#pandas.Timestamp" title="(in pandas v0.22.0)"><code class="xref py py-class docutils literal"><span class="pre">pandas.Timestamp</span></code></a> object indicating the first
|
||
day that the bundle should load data for.</p>
|
||
</div>
|
||
<div class="section" id="end-session">
|
||
<h3><code class="docutils literal"><span class="pre">end_session</span></code><a class="headerlink" href="#end-session" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">end_session</span></code> is a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/generated/pandas.Timestamp.html#pandas.Timestamp" title="(in pandas v0.22.0)"><code class="xref py py-class docutils literal"><span class="pre">pandas.Timestamp</span></code></a> object indicating the last day
|
||
that the bundle should load data for.</p>
|
||
</div>
|
||
<div class="section" id="cache">
|
||
<h3><code class="docutils literal"><span class="pre">cache</span></code><a class="headerlink" href="#cache" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">cache</span></code> is an instance of <code class="xref py py-class docutils literal"><span class="pre">dataframe_cache</span></code>. This
|
||
object is a mapping from strings to dataframes. This object is provided in case
|
||
an ingestion crashes part way through. The idea is that the ingest function
|
||
should check the cache for raw data, if it doesn’t exist in the cache, it should
|
||
acquire it and then store it in the cache. Then it can parse and write the
|
||
data. The cache will be cleared only after a successful load, this prevents the
|
||
ingest function from needing to redownload all the data if there is some bug in
|
||
the parsing. If it is very fast to get the data, for example if it is coming
|
||
from another local file, then there is no need to use this cache.</p>
|
||
</div>
|
||
<div class="section" id="show-progress">
|
||
<h3><code class="docutils literal"><span class="pre">show_progress</span></code><a class="headerlink" href="#show-progress" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">show_progress</span></code> is a boolean indicating that the user would like to receive
|
||
feedback about the ingest function’s progress fetching and writing the
|
||
data. Some examples for where to show how many files you have downloaded out of
|
||
the total needed, or how far into some data conversion the ingest function
|
||
is. One tool that may help with implementing <code class="docutils literal"><span class="pre">show_progress</span></code> for a loop is
|
||
<code class="xref py py-class docutils literal"><span class="pre">maybe_show_progress</span></code>. This argument should always be
|
||
forwarded to <code class="docutils literal"><span class="pre">minute_bar_writer.write</span></code> and <code class="docutils literal"><span class="pre">daily_bar_writer.write</span></code>.</p>
|
||
</div>
|
||
<div class="section" id="output-dir">
|
||
<h3><code class="docutils literal"><span class="pre">output_dir</span></code><a class="headerlink" href="#output-dir" title="Permalink to this headline">¶</a></h3>
|
||
<p><code class="docutils literal"><span class="pre">output_dir</span></code> is a string representing the file path where all the data will be
|
||
written. <code class="docutils literal"><span class="pre">output_dir</span></code> will be some subdirectory of <code class="docutils literal"><span class="pre">$ZIPLINE_ROOT</span></code> and will
|
||
contain the time of the start of the current ingestion. This can be used to
|
||
directly move resources here if for some reason your ingest function can produce
|
||
it’s own outputs without the writers. For example, the <code class="docutils literal"><span class="pre">quantopian:quandl</span></code>
|
||
bundle uses this to directly untar the bundle into the <code class="docutils literal"><span class="pre">output_dir</span></code>.</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div class="articleComments">
|
||
|
||
</div>
|
||
</div>
|
||
<footer>
|
||
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>
|
||
© Copyright 2018, Enigma MPC, Inc..
|
||
|
||
</p>
|
||
</div>
|
||
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</section>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript">
|
||
var DOCUMENTATION_OPTIONS = {
|
||
URL_ROOT:'./',
|
||
VERSION:'0.4',
|
||
COLLAPSE_INDEX:false,
|
||
FILE_SUFFIX:'.html',
|
||
HAS_SOURCE: true,
|
||
SOURCELINK_SUFFIX: '.txt'
|
||
};
|
||
</script>
|
||
<script type="text/javascript" src="_static/jquery.js"></script>
|
||
<script type="text/javascript" src="_static/underscore.js"></script>
|
||
<script type="text/javascript" src="_static/doctools.js"></script>
|
||
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript" src="_static/js/theme.js"></script>
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript">
|
||
jQuery(function () {
|
||
SphinxRtdTheme.StickyNav.enable();
|
||
});
|
||
</script>
|
||
|
||
|
||
</body>
|
||
</html> |