diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index 9d357e23..4510bce4 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -7,22 +7,42 @@ import numpy as np import pandas as pd import pytz -from catalyst.data.bundles import from_bundle_ingest_dirname from catalyst.data.bundles.core import download_without_progress -from catalyst.exchange.exchange_errors import NoDataAvailableOnExchange from catalyst.exchange.exchange_utils import get_exchange_bundles_folder -from catalyst.utils.deprecate import deprecated -from catalyst.utils.paths import data_path EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] API_URL = 'http://data.enigma.co/api/v1' def get_date_from_ms(ms): + """ + The date from the number of miliseconds from the epoch. + + Parameters + ---------- + ms: int + + Returns + ------- + datetime + + """ return datetime.fromtimestamp(ms / 1000.0) def get_seconds_from_date(date): + """ + The number of seconds from the epoch. + + Parameters + ---------- + date: datetime + + Returns + ------- + int + + """ epoch = datetime.utcfromtimestamp(0) epoch = epoch.replace(tzinfo=pytz.UTC) @@ -33,16 +53,19 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period): """ Download and extract a bcolz bundle. - :param exchange_name: - :param symbol: - :param data_frequency: - :param period: - :return: + Parameters + ---------- + exchange_name: str + symbol: str + data_frequency: str + period: str - Note: + Returns + ------- + str Filename: bitfinex-daily-neo_eth-2017-10.tar.gz - """ + """ root = get_exchange_bundles_folder(exchange_name) name = '{exchange}-{frequency}-{symbol}-{period}'.format( exchange=exchange_name, @@ -67,11 +90,38 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period): def get_delta(periods, data_frequency): + """ + Get a time delta based on the specified data frequency. + + Parameters + ---------- + periods: int + data_frequency: str + + Returns + ------- + timedelta + + """ return timedelta(minutes=periods) \ if data_frequency == 'minute' else timedelta(days=periods) def get_periods_range(start_dt, end_dt, freq): + """ + Get a date range for the specified parameters. + + Parameters + ---------- + start_dt: datetime + end_dt: datetime + freq: str + + Returns + ------- + DateTimeIndex + + """ if freq == 'minute': freq = 'T' @@ -82,10 +132,38 @@ def get_periods_range(start_dt, end_dt, freq): def get_periods(start_dt, end_dt, freq): + """ + The number of periods in the specified range. + + Parameters + ---------- + start_dt: datetime + end_dt: datetime + freq: str + + Returns + ------- + int + + """ return len(get_periods_range(start_dt, end_dt, freq)) def get_start_dt(end_dt, bar_count, data_frequency): + """ + The start date based on specified end date and data frequency. + + Parameters + ---------- + end_dt: datetime + bar_count: int + data_frequency: str + + Returns + ------- + datetime + + """ periods = bar_count if periods > 1: delta = get_delta(periods, data_frequency) @@ -100,9 +178,15 @@ def get_period_label(dt, data_frequency): """ The period label for the specified date and frequency. - :param dt: - :param data_frequency: - :return: + Parameters + ---------- + dt: datetime + data_frequency: str + + Returns + ------- + str + """ return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \ else '{}'.format(dt.year) @@ -112,10 +196,16 @@ def get_month_start_end(dt, first_day=None, last_day=None): """ The first and last day of the month for the specified date. - :param dt: - :param first_day - :param last_day - :return: + Parameters + ---------- + dt: datetime + first_day: datetime + last_day: datetime + + Returns + ------- + datetime, datetime + """ month_range = calendar.monthrange(dt.year, dt.month) @@ -140,10 +230,17 @@ def get_year_start_end(dt, first_day=None, last_day=None): """ The first and last day of the year for the specified date. - :param dt: - :param first_day - :param last_day - :return: + Parameters + ---------- + + dt: datetime + first_day: datetime + last_day: datetime + + Returns + ------- + datetime, datetime + """ year_start = first_day if first_day \ else pd.to_datetime(date(dt.year, 1, 1), utc=True) @@ -154,6 +251,19 @@ def get_year_start_end(dt, first_day=None, last_day=None): def get_df_from_arrays(arrays, periods): + """ + A DataFrame from the specified OHCLV arrays. + + Parameters + ---------- + arrays: Object + periods: DateTimeIndex + + Returns + ------- + DataFrame + + """ ohlcv = dict() for index, field in enumerate( ['open', 'high', 'low', 'close', 'volume']): @@ -171,11 +281,17 @@ def range_in_bundle(asset, start_dt, end_dt, reader): Evaluate whether price data of an asset is included has been ingested in the exchange bundle for the given date range. - :param asset: - :param start_dt: - :param end_dt: - :param reader: - :return: + Parameters + ---------- + asset: TradingPair + start_dt: datetime + end_dt: datetime + reader: BcolzBarMinuteReader + + Returns + ------- + bool + """ has_data = True if has_data and reader is not None: @@ -199,35 +315,3 @@ def range_in_bundle(asset, start_dt, end_dt, reader): has_data = False return has_data - - -@deprecated -def find_most_recent_time(bundle_name): - """ - Find most recent "time folder" for a given bundle. - - :param bundle_name: - The name of the targeted bundle. - - :return folder: - The name of the time folder. - """ - try: - bundle_folders = os.listdir( - data_path([bundle_name]), - ) - except OSError: - return None - - most_recent_bundle = dict() - for folder in bundle_folders: - date = from_bundle_ingest_dirname(folder) - if not most_recent_bundle or date > \ - most_recent_bundle[list(most_recent_bundle.keys())[0]]: - most_recent_bundle = dict() - most_recent_bundle[folder] = date - - if most_recent_bundle: - return list(most_recent_bundle.keys())[0] - else: - return None diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index b1ad3fb9..f10bbaf0 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -397,17 +397,24 @@ class Exchange: """ Similar to 'get_spot_value' but for a single asset - Note - ---- + Notes + ----- We're writing each minute bar to disk using zipline's machinery. This is especially useful when running multiple algorithms concurrently. By using local data when possible, we try to reaching request limits on exchanges. - :param asset: - :param field: - :param data_frequency: - :return value: The spot value of the given asset / field + Parameters + ---------- + asset: TradingPair + field: str + data_frequency: str + + Returns + ------- + float + The spot value of the given asset / field + """ log.debug( 'fetching spot value {field} for symbol {symbol}'.format( @@ -503,7 +510,9 @@ class Exchange: Returns ------- - A dataframe containing the requested data. + DataFrame + A dataframe containing the requested data. + """ start_dt = get_start_dt(end_dt, bar_count, data_frequency) diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index ee242ab2..4db41d9e 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -63,9 +63,15 @@ def download_exchange_symbols(exchange_name, environ=None): """ Downloads the exchange's symbols.json from the repository. - :param exchange_name: - :param environ: - :return: response + Parameters + ---------- + exchange_name: str + environ: + + Returns + ------- + str + """ filename = get_exchange_symbols_filename(exchange_name) url = SYMBOLS_URL.format(exchange=exchange_name) @@ -77,9 +83,15 @@ def get_exchange_symbols(exchange_name, environ=None): """ The de-serialized content of the exchange's symbols.json. - :param exchange_name: - :param environ: - :return: + Parameters + ---------- + exchange_name: str + environ: + + Returns + ------- + Object + """ filename = get_exchange_symbols_filename(exchange_name) @@ -104,8 +116,14 @@ def get_symbols_string(assets): """ A concatenated string of symbols from a list of assets. - :param assets: - :return: + Parameters + ---------- + assets: list[TradingPair] + + Returns + ------- + str + """ array = [assets] if isinstance(assets, TradingPair) else assets return ', '.join([asset.symbol for asset in array]) @@ -115,9 +133,15 @@ def get_exchange_auth(exchange_name, environ=None): """ The de-serialized contend of the exchange's auth.json file. - :param exchange_name: - :param environ: - :return: + Parameters + ---------- + exchange_name: str + environ: + + Returns + ------- + Object + """ exchange_folder = get_exchange_folder(exchange_name, environ) filename = os.path.join(exchange_folder, 'auth.json') @@ -138,9 +162,15 @@ def get_algo_folder(algo_name, environ=None): """ The algorithm root folder of the algorithm. - :param algo_name: - :param environ: - :return: + Parameters + ---------- + algo_name: str + environ: + + Returns + ------- + str + """ if not environ: environ = os.environ @@ -156,11 +186,17 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None): """ The de-serialized object of the algo name and key. - :param algo_name: - :param key: - :param environ: - :param rel_path: - :return: + Parameters + ---------- + algo_name: str + key: str + environ: + rel_path: str + + Returns + ------- + Object + """ if algo_name is None: return None @@ -186,12 +222,14 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None): """ Serialize and save an object by algo name and key. - :param algo_name: - :param key: - :param obj: - :param environ: - :param rel_path: - :return: + Parameters + ---------- + algo_name: str + key: str + obj: Object + environ: + rel_path: str + """ folder = get_algo_folder(algo_name, environ) @@ -209,11 +247,17 @@ def get_algo_df(algo_name, key, environ=None, rel_path=None): """ The de-serialized DataFrame of an algo name and key. - :param algo_name: - :param key: - :param environ: - :param rel_path: - :return: + Parameters + ---------- + algo_name: str + key: str + environ: + rel_path: str + + Returns + ------- + DataFrame + """ folder = get_algo_folder(algo_name, environ) @@ -236,12 +280,14 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None): """ Serialize to csv and save a DataFrame by algo name and key. - :param algo_name: - :param key: - :param df: - :param environ: - :param rel_path: - :return: + Parameters + ---------- + algo_name: str + key: str + df: DataFrame + environ: + rel_path: str + """ folder = get_algo_folder(algo_name, environ) @@ -259,9 +305,15 @@ def get_exchange_minute_writer_root(exchange_name, environ=None): """ The minute writer folder for the exchange. - :param exchange_name: - :param environ: - :return: + Parameters + ---------- + exchange_name: str + environ: + + Returns + ------- + BcolzExchangeBarWriter + """ exchange_folder = get_exchange_folder(exchange_name, environ) @@ -275,9 +327,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None): """ The temp folder for bundle downloads by algo name. - :param exchange_name: - :param environ: - :return: + Parameters + ---------- + exchange_name: str + environ: + + Returns + ------- + str + """ exchange_folder = get_exchange_folder(exchange_name, environ) @@ -291,8 +349,14 @@ def perf_serial(obj): """ JSON serializer for objects not serializable by default json code - :param obj: - :return: + Parameters + ---------- + obj: Object + + Returns + ------- + str + """ if isinstance(obj, (datetime, date)): return obj.isoformat() @@ -304,8 +368,14 @@ def get_common_assets(exchanges): """ The assets available in all specified exchanges. - :param exchanges: - :return: + Parameters + ---------- + exchanges: list[Exchange] + + Returns + ------- + list[TradingPair] + """ symbols = [] for exchange_name in exchanges: @@ -324,6 +394,23 @@ def get_common_assets(exchanges): def get_frequency(freq, data_frequency): + """ + Get the frequency parameters. + + Notes + ----- + We're trying to use Pandas convention for frequency aliases. + + Parameters + ---------- + freq: str + data_frequency: str + + Returns + ------- + str, int, str, str + + """ if freq == 'minute': unit = 'T' candle_size = 1 @@ -368,6 +455,20 @@ def get_frequency(freq, data_frequency): def resample_history_df(df, freq, field): + """ + Resample the OHCLV DataFrame using the specified frequency. + + Parameters + ---------- + df: DataFrame + freq: str + field: str + + Returns + ------- + DataFrame + + """ if field == 'open': agg = 'first' elif field == 'high': diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index ecc83677..6f674455 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -1,16 +1,3 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import pandas as pd from catalyst.gens.sim_engine import ( BAR, @@ -33,8 +20,8 @@ class LiveGraphClock(object): This mixes the clock with a live graph. - Note - ---- + Notes + ----- This seemingly awkward approach allows us to run the program using a single thread. This is important because Matplotlib does not play nice with multi-threaded environments. Zipline probably does not either. @@ -53,7 +40,7 @@ class LiveGraphClock(object): def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')): - global mdates, plt #TODO: Could be cleaner + global mdates, plt # TODO: Could be cleaner import matplotlib.dates as mdates from matplotlib import pyplot as plt from matplotlib import style @@ -95,11 +82,12 @@ class LiveGraphClock(object): """ Trying to assign reasonable parameters to the time axis. - TODO: room for improvement + Parameters + ---------- + ax: - :param ax: - :return: """ + # TODO: room for improvement ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) ax.xaxis.set_major_formatter(self.fmt) @@ -113,9 +101,21 @@ class LiveGraphClock(object): ax.grid(True) def set_legend(self, ax): + """ + Set legend on the chart. + + Parameters + ---------- + ax + + """ ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1) def draw_pnl(self): + """ + Draw p&l line on the chart. + + """ ax = self.ax_pnl df = self.context.pnl_stats @@ -136,6 +136,10 @@ class LiveGraphClock(object): self.format_ax(ax) def draw_custom_signals(self): + """ + Draw custom signals on the chart. + + """ ax = self.ax_custom_signals df = self.context.custom_signals_stats @@ -154,6 +158,10 @@ class LiveGraphClock(object): self.format_ax(ax) def draw_exposure(self): + """ + Draw exposure line on the chart. + + """ ax = self.ax_exposure context = self.context df = context.exposure_stats diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py index bd968dfe..b7bfda98 100644 --- a/catalyst/exchange/stats_utils.py +++ b/catalyst/exchange/stats_utils.py @@ -1,5 +1,5 @@ -import pandas as pd import numpy as np +import pandas as pd def crossover(source, target): @@ -8,9 +8,15 @@ def crossover(source, target): of `x` is greater than the value of `y` and the value of `x` was less than the value of `y` on the bar immediately preceding the current bar. - :param source: - :param target: - :return: + Parameters + ---------- + source: Series + target: Series + + Returns + ------- + bool + """ if source[-1] is np.nan or source[-2] is np.nan \ or target[-1] is np.nan or target[-2] is np.nan: @@ -27,9 +33,16 @@ def crossunder(source, target): The `x`-series is defined as having crossed under `y`-series if the value of `x` is less than the value of `y` and the value of `x` was greater than the value of `y` on the bar immediately preceding the current bar. - :param source: - :param target: - :return: + + Parameters + ---------- + source: Series + target: Series + + Returns + ------- + bool + """ if source[-1] is np.nan or source[-2] is np.nan \ or target[-1] is np.nan or target[-2] is np.nan: @@ -46,9 +59,15 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10): Format and print the last few rows of a statistics DataFrame. See the pyfolio project for the data structure. - :param stats_df: - :param num_rows: - :return: + Parameters + ---------- + stats_df: DataFrame + num_rows: int + + Returns + ------- + str + """ stats_df.set_index('period_close', drop=True, inplace=True) stats_df.dropna(axis=1, how='all', inplace=True) @@ -92,6 +111,18 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10): def df_to_string(df): + """ + Create a formatted str representation of the DataFrame. + + Parameters + ---------- + df: DataFrame + + Returns + ------- + str + + """ pd.set_option('display.expand_frame_repr', False) pd.set_option('precision', 8) pd.set_option('display.width', 1000)