BLD: housekeeping, reorganizing files into smaller packages

This commit is contained in:
Frederic Fortier
2017-12-28 17:49:15 -05:00
parent 44614a75c2
commit cd1f5ca97b
37 changed files with 117 additions and 95 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ from six import text_type
from catalyst.data import bundles as bundles_module
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_utils import delete_algo_folder
from catalyst.exchange.utils.exchange_utils import delete_algo_folder
from catalyst.utils.cli import Date, Timestamp
from catalyst.utils.run_algo import _run, load_extensions
+2 -1
View File
@@ -17,6 +17,7 @@
"""
Cythonized Asset object.
"""
import hashlib
cimport cython
@@ -38,7 +39,7 @@ from numpy cimport int64_t
import warnings
cimport numpy as np
from catalyst.exchange.exchange_utils import get_sid
from catalyst.exchange.utils.exchange_utils import get_sid
from catalyst.utils.calendars import get_calendar
from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
+7 -7
View File
@@ -1,16 +1,16 @@
import os
import time
import shutil
import json
import csv
import json
import os
import shutil
import time
from datetime import datetime
import logbook
import pandas as pd
import requests
import logbook
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
from catalyst.exchange.utils.exchange_utils import \
get_exchange_symbols_filename
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
DT_END = pd.to_datetime('today').value // 10 ** 9
+2 -3
View File
@@ -22,6 +22,7 @@ from pandas_datareader.data import DataReader
from six import iteritems
from six.moves.urllib_error import HTTPError
from catalyst.constants import LOG_LEVEL
from catalyst.utils.calendars import get_calendar
from . import treasuries, treasuries_can
from .benchmarks import get_benchmark_returns
@@ -31,8 +32,6 @@ from ..utils.paths import (
data_root,
)
from catalyst.constants import LOG_LEVEL
logger = logbook.Logger('Loader', level=LOG_LEVEL)
# Mapping from index symbol to appropriate bond data
@@ -143,7 +142,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
if exchange is None:
# This is exceptional, since placing the import at the module scope
# breaks things and it's only needed here
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.utils.factory import get_exchange
exchange = get_exchange(
exchange_name='poloniex', base_currency='usdt'
)
@@ -6,7 +6,7 @@ from catalyst.api import (
symbol,
get_open_orders
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.exchange.utils.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'arbitrage_eth_btc'
+2 -2
View File
@@ -10,6 +10,7 @@ needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
instructions on how to install the required dependencies.
'''
import pandas as pd
import talib
from logbook import Logger
@@ -21,8 +22,7 @@ from catalyst.api import (
record,
get_open_orders,
)
from catalyst.exchange.stats_utils import get_pretty_stats
import pandas as pd
from catalyst.exchange.utils.stats_utils import get_pretty_stats
algo_namespace = 'buy_low_sell_high_xrp'
log = Logger(algo_namespace)
+1 -2
View File
@@ -1,7 +1,6 @@
import talib
from logbook import Logger
import pandas as pd
from catalyst.api import (
order,
order_target_percent,
@@ -9,7 +8,7 @@ from catalyst.api import (
record,
get_open_orders,
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.exchange.utils.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'buy_the_dip_live'
+2 -2
View File
@@ -1,12 +1,12 @@
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
+1 -2
View File
@@ -12,8 +12,7 @@ from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
from catalyst.exchange.utils.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
+2 -2
View File
@@ -1,10 +1,10 @@
import talib
import pandas as pd
import talib
from logbook import Logger, INFO
from catalyst import run_algorithm
from catalyst.api import symbol, record
from catalyst.exchange.stats_utils import get_pretty_stats, \
from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
extract_transactions
log = Logger('simple_loop', level=INFO)
+1 -1
View File
@@ -35,8 +35,8 @@ import numpy as np
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
def initialize(context):
+1 -1
View File
@@ -23,7 +23,7 @@ from catalyst.api import (
order_target_percent,
symbol,
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.exchange.utils.stats_utils import get_pretty_stats
algo_namespace = 'talib_sample'
log = Logger(algo_namespace)
+2 -2
View File
@@ -23,11 +23,11 @@ from catalyst.exchange.exchange_errors import (
InvalidOrderStyle, OrderCancelError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
from catalyst.exchange.utils.exchange_utils import \
get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.protocol import Account
# Trying to account for REST api instability
# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
from catalyst.utils.deprecate import deprecated
+2 -2
View File
@@ -13,11 +13,11 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
CreateOrderError
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
from catalyst.exchange.utils.exchange_utils import \
get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.finance.execution import LimitOrder, StopLimitOrder
from catalyst.finance.order import Order, ORDER_STATUS
# TODO: consider using this: https://github.com/mondeja/bittrex_v2
from catalyst.utils.deprecate import deprecated
+3 -3
View File
@@ -1,17 +1,17 @@
import json
import os
import re
from collections import defaultdict
import ccxt
import os
import pandas as pd
import six
from catalyst.assets._assets import TradingPair
from ccxt import ExchangeNotAvailable, InvalidOrder
from logbook import Logger
from six import string_types
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -19,7 +19,7 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import mixin_market_params, \
from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
get_exchange_auth
from catalyst.finance.order import Order, ORDER_STATUS
+4 -4
View File
@@ -9,15 +9,15 @@ from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
TickerNotFoundError, BalanceNotFoundError, NotEnoughCashError
from catalyst.exchange.exchange_utils import get_exchange_symbols, \
TickerNotFoundError, NotEnoughCashError
from catalyst.exchange.utils.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
get_frequency, resample_history_df, has_bundle
log = Logger('Exchange', level=LOG_LEVEL)
+5 -8
View File
@@ -31,28 +31,25 @@ from catalyst.exchange.exchange_errors import (
ExchangePortfolioDataError,
OrderTypeNotSupported)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import (
from catalyst.exchange.live_graph_clock import LiveGraphClock
from catalyst.exchange.simple_clock import SimpleClock
from catalyst.exchange.utils.exchange_utils import (
save_algo_object,
get_algo_object,
get_algo_folder,
get_algo_df,
save_algo_df,
group_assets_by_exchange, )
from catalyst.exchange.live_graph_clock import LiveGraphClock
from catalyst.exchange.simple_clock import SimpleClock
from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
stats_to_algo_folder
from catalyst.exchange.utils.serialization import portfolio_to_dict
from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance import PerformanceTracker, Position
from catalyst.finance.performance import PerformanceTracker
from catalyst.finance.performance.period import calc_period_stats
from catalyst.finance.performance.position import positiondict
from catalyst.gens.tradesimulation import AlgorithmSimulator
from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
from catalyst.utils.preprocess import preprocess
from catalyst.protocol import Portfolio
log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
+2 -4
View File
@@ -1,10 +1,8 @@
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.errors import SidsNotFound
from catalyst.exchange.factory import find_exchanges
import pandas as pd
from catalyst.exchange.utils.factory import find_exchanges
log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
+7 -7
View File
@@ -18,17 +18,17 @@ from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
from catalyst.constants import LOG_LEVEL
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
BcolzMinuteBarMetadata
from catalyst.exchange.bundle_utils import range_in_bundle, \
get_bcolz_chunk, get_month_start_end, \
get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
get_delta, get_assets
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
BcolzExchangeBarWriter
from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
TempBundleNotFoundError, \
NoDataAvailableOnExchange, \
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.exchange_utils import get_exchange_folder, \
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
get_bcolz_chunk, get_month_start_end, \
get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
get_delta, get_assets
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
@@ -668,7 +668,7 @@ class ExchangeBundle:
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.utils.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
problems = []
@@ -805,7 +805,7 @@ class ExchangeBundle:
else:
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.utils.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
assets = get_assets(
+1 -1
View File
@@ -13,7 +13,7 @@ from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
ExchangeBarDataError,
PricingDataNotLoadedError)
from catalyst.exchange.exchange_utils import get_frequency, \
from catalyst.exchange.utils.exchange_utils import get_frequency, \
resample_history_df, group_assets_by_exchange
log = Logger('DataPortalExchange', level=LOG_LEVEL)
+2 -2
View File
@@ -19,9 +19,9 @@ from numpy import (
from catalyst.constants import LOG_LEVEL
from catalyst.data.us_equity_pricing import BcolzDailyBarReader
from catalyst.exchange.factory import get_exchange
from catalyst.lib.adjusted_array import AdjustedArray
from catalyst.errors import NoFurtherDataError
from catalyst.exchange.utils.factory import get_exchange
from catalyst.lib.adjusted_array import AdjustedArray
from catalyst.pipeline.data import DataSet, Column
from catalyst.pipeline.loaders.base import PipelineLoader
from catalyst.utils.calendars import get_calendar
+3 -2
View File
@@ -22,9 +22,10 @@ from catalyst.exchange.exchange_errors import (
OrphanOrderReverseError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
from catalyst.exchange.utils.exchange_utils import \
get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from catalyst.protocol import Account
@@ -8,7 +8,7 @@ import pandas as pd
import pytz
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'
@@ -14,7 +14,7 @@ from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
from catalyst.exchange.utils.serialization import ExchangeJSONEncoder, \
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
ExchangeJSONDecoder
from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time
@@ -6,7 +6,7 @@ from catalyst.constants import LOG_LEVEL
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.exchange_utils import get_exchange_auth, \
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_exchange_folder, is_blacklist
log = Logger('factory', level=LOG_LEVEL)
@@ -9,7 +9,7 @@ import numpy as np
import pandas as pd
from catalyst.assets._assets import TradingPair
from catalyst.exchange.exchange_utils import get_algo_folder
from catalyst.exchange.utils.exchange_utils import get_algo_folder
from catalyst.utils.paths import data_root, ensure_directory
s3_conn = []
@@ -1,11 +1,11 @@
import os
import random
import tempfile
from catalyst.assets._assets import TradingPair
from catalyst.exchange.exchange_utils import get_exchange_folder
from catalyst.exchange.factory import find_exchanges
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import find_exchanges
from catalyst.utils.paths import ensure_directory
+28
View File
@@ -0,0 +1,28 @@
from catalyst.api import symbol
from catalyst.utils.run_algo import run_algorithm
def initialize(context):
context.asset = symbol('bcc_usdt')
def handle_data(context, data):
data.history(context.asset, ['close'], bar_count=100, frequency='5T')
def analyze(context=None, results=None):
pass
if __name__ == '__main__':
run_algorithm(
capital_base=100,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
algo_namespace="bittrex_is_broken",
base_currency='usdt',
data_frequency='minute',
simulate_orders=True,
live=True)
+1 -1
View File
@@ -14,7 +14,7 @@ from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
TradingPairPricing
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.utils.factory import get_exchange
try:
from pygments import highlight
+6 -7
View File
@@ -1,15 +1,14 @@
import shutil
import random
import shutil
import tempfile
import pandas as pd
from catalyst.exchange.exchange_bundle import ExchangeBundle
import pandas as pd
from nose.tools import assert_equals
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
BcolzExchangeBarReader
from catalyst.exchange.bundle_utils import get_df_from_arrays
from nose.tools import assert_equals
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.utils.bundle_utils import get_df_from_arrays
class TestBcolzWriter(object):
+5 -5
View File
@@ -5,15 +5,15 @@ from logging import getLogger
import pandas as pd
from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
get_start_dt, get_df_from_arrays
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
BcolzExchangeBarWriter
from catalyst.exchange.exchange_bundle import ExchangeBundle, \
BUNDLE_NAME_TEMPLATE
from catalyst.exchange.exchange_utils import get_exchange_folder
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.stats_utils import df_to_string
from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
get_start_dt, get_df_from_arrays
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import get_exchange
from catalyst.exchange.utils.stats_utils import df_to_string
from catalyst.utils.paths import ensure_directory
log = getLogger('test_exchange_bundle')
+2 -2
View File
@@ -1,10 +1,10 @@
import pandas as pd
from logbook import Logger
from base import BaseExchangeTestCase
from base import BaseExchangeTestCase
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.utils.exchange_utils import get_exchange_auth
from catalyst.finance.order import Order
from catalyst.exchange.exchange_utils import get_exchange_auth
log = Logger('test_ccxt')
+2 -2
View File
@@ -7,8 +7,8 @@ from catalyst.exchange.exchange_data_portal import (
DataPortalExchangeBacktest,
DataPortalExchangeLive
)
from catalyst.exchange.exchange_utils import get_common_assets
from catalyst.exchange.factory import get_exchanges
from catalyst.exchange.utils.exchange_utils import get_common_assets
from catalyst.exchange.utils.factory import get_exchanges
from test_utils import rnd_history_date_days, rnd_bar_count
log = Logger('test_bitfinex')
+7 -6
View File
@@ -1,17 +1,18 @@
import os
import importlib
import os
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.finance import candlestick2_ohlc
# from matplotlib.finance import volume_overlay
import matplotlib.ticker as ticker
import pandas as pd
from matplotlib.finance import candlestick2_ohlc
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.utils.bundle_utils import get_df_from_arrays, \
get_bcolz_chunk
from catalyst.exchange.utils.factory import get_exchange
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
exchanges = dict((e, getattr(importlib.import_module(
+3 -3
View File
@@ -7,9 +7,9 @@ from pandas.util.testing import assert_frame_equal
from catalyst import get_calendar
from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
from catalyst.exchange.exchange_utils import get_candles_df
from catalyst.exchange.factory import get_exchange
from catalyst.exchange.test_utils import output_df, \
from catalyst.exchange.utils.exchange_utils import get_candles_df
from catalyst.exchange.utils.factory import get_exchange
from catalyst.exchange.utils.test_utils import output_df, \
select_random_assets
log = Logger('TestSuiteExchange')
+2 -2
View File
@@ -8,8 +8,8 @@ import pandas as pd
from catalyst.exchange.exchange_errors import ExchangeRequestError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import get_exchange_folder
from catalyst.exchange.test_utils import select_random_exchanges, \
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.test_utils import select_random_exchanges, \
handle_exchange_error, select_random_assets
log = Logger('TestSuiteExchange')