mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-24 13:00:57 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1bd65397b6 | ||
|
|
c768b207bc | ||
|
|
e18686d5c5 | ||
|
|
b7779cf363 | ||
|
|
28819b8a32 | ||
|
|
a56d7f34c7 | ||
|
|
9f0b3303f1 | ||
|
|
9d7a35658b | ||
|
|
c58cebd1eb | ||
|
|
027cdba474 | ||
|
|
d223529100 | ||
|
|
1e02506ab4 | ||
|
|
2a97ade68e | ||
|
|
9648767e9a | ||
|
|
98449b2088 | ||
|
|
91d16aba3b | ||
|
|
9eb649371b | ||
|
|
7f2ded65bc | ||
|
|
b76b4458cb | ||
|
|
decbdbf6ea | ||
|
|
685ce25b85 | ||
|
|
1cafcc1417 | ||
|
|
0d77854782 | ||
|
|
4cb8d54d97 | ||
|
|
7b796a4276 | ||
|
|
41a4c7072f |
@@ -580,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
|
|||||||
|
|
||||||
exchange_bundle = ExchangeBundle(exchange_name)
|
exchange_bundle = ExchangeBundle(exchange_name)
|
||||||
|
|
||||||
click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
|
click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
|
||||||
sys.stdout)
|
sys.stdout)
|
||||||
exchange_bundle.ingest(
|
exchange_bundle.ingest(
|
||||||
data_frequency=data_frequency,
|
data_frequency=data_frequency,
|
||||||
|
|||||||
@@ -43,3 +43,6 @@ ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
|
|||||||
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
|
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
|
||||||
'catalyst/master/catalyst/marketplace/' \
|
'catalyst/master/catalyst/marketplace/' \
|
||||||
'contract_enigma_abi.json'
|
'contract_enigma_abi.json'
|
||||||
|
|
||||||
|
SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
|
||||||
|
'key']
|
||||||
|
|||||||
@@ -190,6 +190,9 @@ class CCXT(Exchange):
|
|||||||
if data_frequency == 'minute' and not freq.endswith('T'):
|
if data_frequency == 'minute' and not freq.endswith('T'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
elif data_frequency == 'hourly' and not freq.endswith('D'):
|
||||||
|
continue
|
||||||
|
|
||||||
elif data_frequency == 'daily' and not freq.endswith('D'):
|
elif data_frequency == 'daily' and not freq.endswith('D'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
|
|||||||
PricingDataNotLoadedError, \
|
PricingDataNotLoadedError, \
|
||||||
NoDataAvailableOnExchange, NoValueForField, \
|
NoDataAvailableOnExchange, NoValueForField, \
|
||||||
NoCandlesReceivedFromExchange, \
|
NoCandlesReceivedFromExchange, \
|
||||||
InvalidHistoryFrequencyAlias, \
|
|
||||||
TickerNotFoundError, NotEnoughCashError
|
TickerNotFoundError, NotEnoughCashError
|
||||||
from catalyst.exchange.utils.datetime_utils import get_delta, \
|
from catalyst.exchange.utils.datetime_utils import get_delta, \
|
||||||
get_periods_range, \
|
get_periods_range, \
|
||||||
@@ -200,12 +199,8 @@ class Exchange:
|
|||||||
)
|
)
|
||||||
assets.append(asset)
|
assets.append(asset)
|
||||||
|
|
||||||
except SymbolNotFoundOnExchange:
|
except SymbolNotFoundOnExchange as e:
|
||||||
log.debug(
|
log.warn(e)
|
||||||
'skipping non-existent market {} {}'.format(
|
|
||||||
self.name, symbol
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return assets
|
return assets
|
||||||
|
|
||||||
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
|
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
|
||||||
@@ -509,10 +504,6 @@ class Exchange:
|
|||||||
frequency, data_frequency, supported_freqs=['T', 'D', 'H']
|
frequency, data_frequency, supported_freqs=['T', 'D', 'H']
|
||||||
)
|
)
|
||||||
|
|
||||||
if unit == 'H':
|
|
||||||
raise InvalidHistoryFrequencyAlias(
|
|
||||||
freq=frequency)
|
|
||||||
|
|
||||||
# we want to avoid receiving empty candles
|
# we want to avoid receiving empty candles
|
||||||
# so we request more than needed
|
# so we request more than needed
|
||||||
# TODO: consider defining a const per asset
|
# TODO: consider defining a const per asset
|
||||||
@@ -616,7 +607,7 @@ class Exchange:
|
|||||||
# TODO: this function needs some work,
|
# TODO: this function needs some work,
|
||||||
# we're currently using it just for benchmark data
|
# we're currently using it just for benchmark data
|
||||||
freq, candle_size, unit, data_frequency = get_frequency(
|
freq, candle_size, unit, data_frequency = get_frequency(
|
||||||
frequency, data_frequency
|
frequency, data_frequency, supported_freqs=['T', 'D']
|
||||||
)
|
)
|
||||||
adj_bar_count = candle_size * bar_count
|
adj_bar_count = candle_size * bar_count
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
|
|||||||
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
|
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
|
||||||
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
|
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
|
||||||
get_bcolz_chunk, get_df_from_arrays, get_assets
|
get_bcolz_chunk, get_df_from_arrays, get_assets
|
||||||
from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
|
from catalyst.exchange.utils.datetime_utils import get_start_dt, \
|
||||||
get_period_label, get_month_start_end, get_year_start_end
|
get_period_label, get_month_start_end, get_year_start_end
|
||||||
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
|
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
|
||||||
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
|
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
|
||||||
@@ -232,12 +232,12 @@ class ExchangeBundle:
|
|||||||
|
|
||||||
problem = '{name} ({start_dt} to {end_dt}) has empty ' \
|
problem = '{name} ({start_dt} to {end_dt}) has empty ' \
|
||||||
'periods: {dates}'.format(
|
'periods: {dates}'.format(
|
||||||
name=asset.symbol,
|
name=asset.symbol,
|
||||||
start_dt=asset.start_date.strftime(
|
start_dt=asset.start_date.strftime(
|
||||||
DATE_TIME_FORMAT),
|
DATE_TIME_FORMAT),
|
||||||
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
||||||
dates=[date.strftime(
|
dates=[date.strftime(
|
||||||
DATE_TIME_FORMAT) for date in dates])
|
DATE_TIME_FORMAT) for date in dates])
|
||||||
|
|
||||||
if empty_rows_behavior == 'warn':
|
if empty_rows_behavior == 'warn':
|
||||||
log.warn(problem)
|
log.warn(problem)
|
||||||
@@ -286,12 +286,12 @@ class ExchangeBundle:
|
|||||||
|
|
||||||
problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
|
problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
|
||||||
'identical close values on: {dates}'.format(
|
'identical close values on: {dates}'.format(
|
||||||
name=asset.symbol,
|
name=asset.symbol,
|
||||||
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
|
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
|
||||||
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
||||||
threshold=threshold,
|
threshold=threshold,
|
||||||
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
|
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
|
||||||
for date in dates])
|
for date in dates])
|
||||||
|
|
||||||
problems.append(problem)
|
problems.append(problem)
|
||||||
|
|
||||||
@@ -458,7 +458,7 @@ class ExchangeBundle:
|
|||||||
last_entry = None
|
last_entry = None
|
||||||
|
|
||||||
if start is None or \
|
if start is None or \
|
||||||
(earliest_trade is not None and earliest_trade > start):
|
(earliest_trade is not None and earliest_trade > start):
|
||||||
start = earliest_trade
|
start = earliest_trade
|
||||||
|
|
||||||
if last_entry is not None and (end is None or end > last_entry):
|
if last_entry is not None and (end is None or end > last_entry):
|
||||||
@@ -598,16 +598,41 @@ class ExchangeBundle:
|
|||||||
# we want to give an end_date far in time
|
# we want to give an end_date far in time
|
||||||
writer = self.get_writer(start_dt, end_dt, data_frequency)
|
writer = self.get_writer(start_dt, end_dt, data_frequency)
|
||||||
if show_breakdown:
|
if show_breakdown:
|
||||||
for asset in chunks:
|
if chunks:
|
||||||
|
for asset in chunks:
|
||||||
|
with maybe_show_progress(
|
||||||
|
chunks[asset],
|
||||||
|
show_progress,
|
||||||
|
label='Ingesting {frequency} price data for '
|
||||||
|
'{symbol} on {exchange}'.format(
|
||||||
|
exchange=self.exchange_name,
|
||||||
|
frequency=data_frequency,
|
||||||
|
symbol=asset.symbol
|
||||||
|
)) as it:
|
||||||
|
for chunk in it:
|
||||||
|
problems += self.ingest_ctable(
|
||||||
|
asset=chunk['asset'],
|
||||||
|
data_frequency=data_frequency,
|
||||||
|
period=chunk['period'],
|
||||||
|
writer=writer,
|
||||||
|
empty_rows_behavior='strip',
|
||||||
|
cleanup=True
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
all_chunks = list(chain.from_iterable(itervalues(chunks)))
|
||||||
|
# We sort the chunks by end date to ingest most recent data first
|
||||||
|
if all_chunks:
|
||||||
|
all_chunks.sort(
|
||||||
|
key=lambda chunk: pd.to_datetime(chunk['period'])
|
||||||
|
)
|
||||||
with maybe_show_progress(
|
with maybe_show_progress(
|
||||||
chunks[asset],
|
all_chunks,
|
||||||
show_progress,
|
show_progress,
|
||||||
label='Ingesting {frequency} price data for '
|
label='Ingesting {frequency} price data on '
|
||||||
'{symbol} on {exchange}'.format(
|
'{exchange}'.format(
|
||||||
exchange=self.exchange_name,
|
exchange=self.exchange_name,
|
||||||
frequency=data_frequency,
|
frequency=data_frequency,
|
||||||
symbol=asset.symbol
|
)) as it:
|
||||||
)) as it:
|
|
||||||
for chunk in it:
|
for chunk in it:
|
||||||
problems += self.ingest_ctable(
|
problems += self.ingest_ctable(
|
||||||
asset=chunk['asset'],
|
asset=chunk['asset'],
|
||||||
@@ -617,30 +642,6 @@ class ExchangeBundle:
|
|||||||
empty_rows_behavior='strip',
|
empty_rows_behavior='strip',
|
||||||
cleanup=True
|
cleanup=True
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
all_chunks = list(chain.from_iterable(itervalues(chunks)))
|
|
||||||
|
|
||||||
# We sort the chunks by end date to ingest most recent data first
|
|
||||||
all_chunks.sort(
|
|
||||||
key=lambda chunk: pd.to_datetime(chunk['period'])
|
|
||||||
)
|
|
||||||
with maybe_show_progress(
|
|
||||||
all_chunks,
|
|
||||||
show_progress,
|
|
||||||
label='Ingesting {frequency} price data on '
|
|
||||||
'{exchange}'.format(
|
|
||||||
exchange=self.exchange_name,
|
|
||||||
frequency=data_frequency,
|
|
||||||
)) as it:
|
|
||||||
for chunk in it:
|
|
||||||
problems += self.ingest_ctable(
|
|
||||||
asset=chunk['asset'],
|
|
||||||
data_frequency=data_frequency,
|
|
||||||
period=chunk['period'],
|
|
||||||
writer=writer,
|
|
||||||
empty_rows_behavior='strip',
|
|
||||||
cleanup=True
|
|
||||||
)
|
|
||||||
|
|
||||||
if show_report and len(problems) > 0:
|
if show_report and len(problems) > 0:
|
||||||
log.info('problems during ingestion:{}\n'.format(
|
log.info('problems during ingestion:{}\n'.format(
|
||||||
|
|||||||
@@ -296,7 +296,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
|||||||
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
|
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
|
||||||
|
|
||||||
freq, candle_size, unit, adj_data_frequency = get_frequency(
|
freq, candle_size, unit, adj_data_frequency = get_frequency(
|
||||||
frequency, data_frequency
|
frequency, data_frequency, supported_freqs=['T', 'D']
|
||||||
)
|
)
|
||||||
adj_bar_count = candle_size * bar_count
|
adj_bar_count = candle_size * bar_count
|
||||||
|
|
||||||
@@ -312,7 +312,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
|||||||
algo_end_dt=self._last_available_session,
|
algo_end_dt=self._last_available_session,
|
||||||
)
|
)
|
||||||
|
|
||||||
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
|
start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
|
||||||
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
|
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
|
||||||
return df
|
return df
|
||||||
|
|
||||||
|
|||||||
@@ -249,9 +249,12 @@ def get_year_start_end(dt, first_day=None, last_day=None):
|
|||||||
return year_start, year_end
|
return year_start, year_end
|
||||||
|
|
||||||
|
|
||||||
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
|
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
|
||||||
"""
|
"""
|
||||||
Get the frequency parameters.
|
Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
|
||||||
|
common denominator supported by the data bundles (e.g. 1T). The data
|
||||||
|
bundles only support 1T and 1D frequencies. If another frequency
|
||||||
|
is requested, Catalyst must request the underlying data and resample.
|
||||||
|
|
||||||
Notes
|
Notes
|
||||||
-----
|
-----
|
||||||
@@ -306,14 +309,14 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
|
|||||||
data_frequency = 'minute'
|
data_frequency = 'minute'
|
||||||
|
|
||||||
elif unit.lower() == 'h':
|
elif unit.lower() == 'h':
|
||||||
|
data_frequency = 'minute'
|
||||||
|
|
||||||
if 'H' in supported_freqs:
|
if 'H' in supported_freqs:
|
||||||
unit = 'H'
|
unit = 'H'
|
||||||
alias = '{}H'.format(candle_size)
|
alias = '{}H'.format(candle_size)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
candle_size = candle_size * 60
|
candle_size = candle_size * 60
|
||||||
alias = '{}T'.format(candle_size)
|
alias = '{}T'.format(candle_size)
|
||||||
data_frequency = 'minute'
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise InvalidHistoryFrequencyAlias(freq=freq)
|
raise InvalidHistoryFrequencyAlias(freq=freq)
|
||||||
|
|||||||
@@ -95,11 +95,24 @@ class TradingEnvironment(object):
|
|||||||
if not trading_calendar:
|
if not trading_calendar:
|
||||||
trading_calendar = get_calendar("NYSE")
|
trading_calendar = get_calendar("NYSE")
|
||||||
|
|
||||||
self.benchmark_returns, self.treasury_curves = load(
|
# todo: uncomment and add a well defined benchmark
|
||||||
trading_calendar.day,
|
# self.benchmark_returns, self.treasury_curves = load(
|
||||||
trading_calendar.schedule.index,
|
# trading_calendar.day,
|
||||||
self.bm_symbol,
|
# trading_calendar.schedule.index,
|
||||||
)
|
# self.bm_symbol,
|
||||||
|
# exchange=exchange,
|
||||||
|
# )
|
||||||
|
|
||||||
|
start_data = get_calendar('OPEN').first_trading_session
|
||||||
|
end_data = pd.Timestamp.utcnow()
|
||||||
|
treasure_cols = ['1month', '3month', '6month', '1year', '2year',
|
||||||
|
'3year', '5year', '7year', '10year', '20year', '30year']
|
||||||
|
self.benchmark_returns = pd.DataFrame(data=0.001,
|
||||||
|
index=pd.date_range(start_data, end_data),
|
||||||
|
columns=['close'])
|
||||||
|
self.treasury_curves = pd.DataFrame(data=0.001,
|
||||||
|
index=pd.date_range(start_data, end_data),
|
||||||
|
columns=treasure_cols)
|
||||||
|
|
||||||
self.exchange_tz = exchange_tz
|
self.exchange_tz = exchange_tz
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from requests_toolbelt.multipart.decoder import \
|
|||||||
from catalyst.constants import (
|
from catalyst.constants import (
|
||||||
LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
|
LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
|
||||||
MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
|
MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
|
||||||
|
from catalyst.utils.cli import maybe_show_progress
|
||||||
from catalyst.exchange.utils.stats_utils import set_print_settings
|
from catalyst.exchange.utils.stats_utils import set_print_settings
|
||||||
from catalyst.marketplace.marketplace_errors import (
|
from catalyst.marketplace.marketplace_errors import (
|
||||||
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
|
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
|
||||||
@@ -126,9 +127,10 @@ class Marketplace:
|
|||||||
else:
|
else:
|
||||||
while True:
|
while True:
|
||||||
for i in range(0, len(self.addresses)):
|
for i in range(0, len(self.addresses)):
|
||||||
print('{}\t{}\t{}'.format(
|
print('{}\t{}\t{}\t{}'.format(
|
||||||
i,
|
i,
|
||||||
self.addresses[i]['pubAddr'],
|
self.addresses[i]['pubAddr'],
|
||||||
|
self.addresses[i]['wallet'].ljust(10),
|
||||||
self.addresses[i]['desc'])
|
self.addresses[i]['desc'])
|
||||||
)
|
)
|
||||||
address_i = int(input('Choose your address associated with '
|
address_i = int(input('Choose your address associated with '
|
||||||
@@ -145,7 +147,7 @@ class Marketplace:
|
|||||||
|
|
||||||
def sign_transaction(self, tx):
|
def sign_transaction(self, tx):
|
||||||
|
|
||||||
url = 'https://www.myetherwallet.com/#offline-transaction'
|
url = 'https://www.mycrypto.com/#offline-transaction'
|
||||||
print('\nVisit {url} and enter the following parameters:\n\n'
|
print('\nVisit {url} and enter the following parameters:\n\n'
|
||||||
'From Address:\t\t{_from}\n'
|
'From Address:\t\t{_from}\n'
|
||||||
'\n\tClick the "Generate Information" button\n\n'
|
'\n\tClick the "Generate Information" button\n\n'
|
||||||
@@ -430,10 +432,9 @@ class Marketplace:
|
|||||||
merge_bundles(zsource, ztarget)
|
merge_bundles(zsource, ztarget)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
shutil.rmtree(bundle_folder, ignore_errors=True)
|
||||||
os.rename(tmp_bundle, bundle_folder)
|
os.rename(tmp_bundle, bundle_folder)
|
||||||
|
|
||||||
pass
|
|
||||||
|
|
||||||
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
|
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
|
||||||
|
|
||||||
if ds_name is None:
|
if ds_name is None:
|
||||||
@@ -498,20 +499,29 @@ class Marketplace:
|
|||||||
key = self.addresses[address_i]['key']
|
key = self.addresses[address_i]['key']
|
||||||
secret = self.addresses[address_i]['secret']
|
secret = self.addresses[address_i]['secret']
|
||||||
else:
|
else:
|
||||||
key, secret = get_key_secret(address)
|
key, secret = get_key_secret(address,
|
||||||
|
self.addresses[address_i]['wallet'])
|
||||||
|
|
||||||
headers = get_signed_headers(ds_name, key, secret)
|
headers = get_signed_headers(ds_name, key, secret)
|
||||||
log.debug('Starting download of dataset for ingestion...')
|
log.info('Starting download of dataset for ingestion...')
|
||||||
r = requests.post(
|
r = requests.post(
|
||||||
'{}/marketplace/ingest'.format(AUTH_SERVER),
|
'{}/marketplace/ingest'.format(AUTH_SERVER),
|
||||||
headers=headers,
|
headers=headers,
|
||||||
stream=True,
|
stream=True,
|
||||||
)
|
)
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
|
log.info('Dataset downloaded successfully. Processing dataset...')
|
||||||
target_path = get_temp_bundles_folder()
|
target_path = get_temp_bundles_folder()
|
||||||
try:
|
try:
|
||||||
decoder = MultipartDecoder.from_response(r)
|
decoder = MultipartDecoder.from_response(r)
|
||||||
|
# with maybe_show_progress(
|
||||||
|
# iter(decoder.parts),
|
||||||
|
# True,
|
||||||
|
# label='Processing files') as part:
|
||||||
|
counter = 0
|
||||||
for part in decoder.parts:
|
for part in decoder.parts:
|
||||||
|
log.info("Processing file {} of {}".format(
|
||||||
|
counter, len(decoder.parts)))
|
||||||
h = part.headers[b'Content-Disposition'].decode('utf-8')
|
h = part.headers[b'Content-Disposition'].decode('utf-8')
|
||||||
# Extracting the filename from the header
|
# Extracting the filename from the header
|
||||||
name = re.search(r'filename="(.*)"', h).group(1)
|
name = re.search(r'filename="(.*)"', h).group(1)
|
||||||
@@ -525,6 +535,7 @@ class Marketplace:
|
|||||||
f.write(part.content)
|
f.write(part.content)
|
||||||
|
|
||||||
self.process_temp_bundle(ds_name, filename)
|
self.process_temp_bundle(ds_name, filename)
|
||||||
|
counter += 1
|
||||||
|
|
||||||
except NonMultipartContentTypeException:
|
except NonMultipartContentTypeException:
|
||||||
response = r.json()
|
response = r.json()
|
||||||
@@ -592,7 +603,6 @@ class Marketplace:
|
|||||||
folder = get_bundle_folder(ds_name, data_frequency)
|
folder = get_bundle_folder(ds_name, data_frequency)
|
||||||
|
|
||||||
shutil.rmtree(folder)
|
shutil.rmtree(folder)
|
||||||
pass
|
|
||||||
|
|
||||||
def create_metadata(self, key, secret, ds_name, data_frequency, desc,
|
def create_metadata(self, key, secret, ds_name, data_frequency, desc,
|
||||||
has_history=True, has_live=True):
|
has_history=True, has_live=True):
|
||||||
@@ -684,7 +694,8 @@ class Marketplace:
|
|||||||
key = self.addresses[address_i]['key']
|
key = self.addresses[address_i]['key']
|
||||||
secret = self.addresses[address_i]['secret']
|
secret = self.addresses[address_i]['secret']
|
||||||
else:
|
else:
|
||||||
key, secret = get_key_secret(address)
|
key, secret = get_key_secret(address,
|
||||||
|
self.addresses[address_i]['wallet'])
|
||||||
|
|
||||||
grains = to_grains(price)
|
grains = to_grains(price)
|
||||||
|
|
||||||
@@ -765,7 +776,7 @@ class Marketplace:
|
|||||||
key = match['key']
|
key = match['key']
|
||||||
secret = match['secret']
|
secret = match['secret']
|
||||||
else:
|
else:
|
||||||
key, secret = get_key_secret(provider_info[0])
|
key, secret = get_key_secret(provider_info[0], match['wallet'])
|
||||||
|
|
||||||
headers = get_signed_headers(dataset, key, secret)
|
headers = get_signed_headers(dataset, key, secret)
|
||||||
filenames = glob.glob(os.path.join(datadir, '*.csv'))
|
filenames = glob.glob(os.path.join(datadir, '*.csv'))
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
|
import webbrowser
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
import time
|
import time
|
||||||
@@ -9,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import (
|
|||||||
MarketplaceEmptySignature)
|
MarketplaceEmptySignature)
|
||||||
from catalyst.marketplace.utils.path_utils import (
|
from catalyst.marketplace.utils.path_utils import (
|
||||||
get_user_pubaddr, save_user_pubaddr)
|
get_user_pubaddr, save_user_pubaddr)
|
||||||
from catalyst.constants import AUTH_SERVER
|
from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
|
||||||
|
|
||||||
|
|
||||||
def get_key_secret(pubAddr, wallet='mew'):
|
def get_key_secret(pubAddr, wallet):
|
||||||
"""
|
"""
|
||||||
Obtain a new key/secret pair from authentication server
|
Obtain a new key/secret pair from authentication server
|
||||||
|
|
||||||
@@ -42,14 +43,22 @@ def get_key_secret(pubAddr, wallet='mew'):
|
|||||||
auth_type, auth_info = header.split(None, 1)
|
auth_type, auth_info = header.split(None, 1)
|
||||||
d = requests.utils.parse_dict_header(auth_info)
|
d = requests.utils.parse_dict_header(auth_info)
|
||||||
|
|
||||||
nonce = '0x{}'.format(d['nonce'])
|
nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
|
||||||
|
|
||||||
|
if wallet in SUPPORTED_WALLETS:
|
||||||
|
url = 'https://www.mycrypto.com/signmsg.html'
|
||||||
|
|
||||||
if wallet == 'mew':
|
|
||||||
print('\nObtaining a key/secret pair to streamline all future '
|
print('\nObtaining a key/secret pair to streamline all future '
|
||||||
'requests with the authentication server.\n'
|
'requests with the authentication server.\n'
|
||||||
'Visit https://www.myetherwallet.com/signmsg.html and sign the '
|
'Visit {url} and sign the '
|
||||||
'following message:\n{}'.format(nonce))
|
'following message (copy the entire line, without the '
|
||||||
signature = input('Copy and Paste the "sig" field from '
|
'line break at the end):\n\n{nonce}'.format(
|
||||||
|
url=url,
|
||||||
|
nonce=nonce))
|
||||||
|
|
||||||
|
webbrowser.open_new(url)
|
||||||
|
|
||||||
|
signature = input('\nCopy and Paste the "sig" field from '
|
||||||
'the signature here (without the double quotes, '
|
'the signature here (without the double quotes, '
|
||||||
'only the HEX value):\n')
|
'only the HEX value):\n')
|
||||||
else:
|
else:
|
||||||
@@ -83,7 +92,8 @@ def get_key_secret(pubAddr, wallet='mew'):
|
|||||||
addresses = get_user_pubaddr()
|
addresses = get_user_pubaddr()
|
||||||
|
|
||||||
match = next((l for l in addresses if
|
match = next((l for l in addresses if
|
||||||
l['pubAddr'] == pubAddr), None)
|
l['pubAddr'].lower() == pubAddr.lower()), None)
|
||||||
|
|
||||||
match['key'] = response.json()['key']
|
match['key'] = response.json()['key']
|
||||||
match['secret'] = response.json()['secret']
|
match['secret'] = response.json()['secret']
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import os
|
|||||||
import json
|
import json
|
||||||
import tarfile
|
import tarfile
|
||||||
|
|
||||||
|
from catalyst.constants import SUPPORTED_WALLETS
|
||||||
from catalyst.utils.deprecate import deprecated
|
from catalyst.utils.deprecate import deprecated
|
||||||
from catalyst.utils.paths import data_root, ensure_directory
|
from catalyst.utils.paths import data_root, ensure_directory
|
||||||
from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
|
from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
|
||||||
@@ -131,17 +132,63 @@ def get_user_pubaddr(environ=None):
|
|||||||
try:
|
try:
|
||||||
d = data[0]['pubAddr']
|
d = data[0]['pubAddr']
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return [data, ]
|
data = [data, ]
|
||||||
|
|
||||||
|
changed = False
|
||||||
|
|
||||||
|
for idx, d in enumerate(data):
|
||||||
|
try:
|
||||||
|
if d['wallet'] not in SUPPORTED_WALLETS:
|
||||||
|
data[idx]['wallet'] = _choose_wallet(
|
||||||
|
d['pubAddr'], False)
|
||||||
|
changed = True
|
||||||
|
except KeyError:
|
||||||
|
data[idx]['wallet'] = _choose_wallet(
|
||||||
|
d['pubAddr'], True)
|
||||||
|
changed = True
|
||||||
|
|
||||||
|
if changed:
|
||||||
|
save_user_pubaddr(data)
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
else:
|
else:
|
||||||
data = []
|
data = []
|
||||||
data.append(dict(pubAddr='', desc=''))
|
data.append(dict(pubAddr='', desc='', wallet=''))
|
||||||
with open(filename, 'w') as f:
|
with open(filename, 'w') as f:
|
||||||
json.dump(data, f, sort_keys=False, indent=2,
|
json.dump(data, f, sort_keys=False, indent=2,
|
||||||
separators=(',', ':'))
|
separators=(',', ':'))
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _choose_wallet(pubAddr, missing):
|
||||||
|
while True:
|
||||||
|
if missing:
|
||||||
|
print('\nYou need to specify a wallet for address '
|
||||||
|
'{}.'.format(pubAddr))
|
||||||
|
else:
|
||||||
|
print('\nThe wallet specified for address {} is not '
|
||||||
|
'supported.'.format(pubAddr))
|
||||||
|
|
||||||
|
print('Please choose among the following options:')
|
||||||
|
for idx, wallet in enumerate(SUPPORTED_WALLETS):
|
||||||
|
print('{}\t{}'.format(idx, wallet))
|
||||||
|
|
||||||
|
lw = len(SUPPORTED_WALLETS)-1
|
||||||
|
w = input('Choose a number between 0 and {}: '.format(
|
||||||
|
lw))
|
||||||
|
try:
|
||||||
|
w = int(w)
|
||||||
|
except ValueError:
|
||||||
|
print('Enter a number between 0 and {}'.format(lw))
|
||||||
|
else:
|
||||||
|
if w not in range(0, lw+1):
|
||||||
|
print('Enter a number between 0 and '
|
||||||
|
'{}'.format(lw))
|
||||||
|
else:
|
||||||
|
return SUPPORTED_WALLETS[w]
|
||||||
|
|
||||||
|
|
||||||
def save_user_pubaddr(data, environ=None):
|
def save_user_pubaddr(data, environ=None):
|
||||||
"""
|
"""
|
||||||
Saves the user's public addresses and their related metadata in
|
Saves the user's public addresses and their related metadata in
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import pytz
|
||||||
|
from datetime import datetime
|
||||||
|
from catalyst.api import symbol
|
||||||
|
from catalyst.utils.run_algo import run_algorithm
|
||||||
|
|
||||||
|
coin = 'btc'
|
||||||
|
base_currency = 'usd'
|
||||||
|
n_candles = 5
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(context):
|
||||||
|
context.symbol = symbol('%s_%s' % (coin, base_currency))
|
||||||
|
|
||||||
|
|
||||||
|
def handle_data_polo_partial_candles(context, data):
|
||||||
|
history = data.history(symbol('btc_usdt'), ['volume'],
|
||||||
|
bar_count=10,
|
||||||
|
frequency='4H')
|
||||||
|
print('\nnow: %s\n%s' % (data.current_dt, history))
|
||||||
|
if not hasattr(context, 'i'):
|
||||||
|
context.i = 0
|
||||||
|
context.i += 1
|
||||||
|
if context.i > 5:
|
||||||
|
raise Exception('stop')
|
||||||
|
|
||||||
|
|
||||||
|
live = False
|
||||||
|
|
||||||
|
if live:
|
||||||
|
run_algorithm(initialize=lambda ctx: True,
|
||||||
|
handle_data=handle_data_polo_partial_candles,
|
||||||
|
exchange_name='poloniex',
|
||||||
|
base_currency='usdt',
|
||||||
|
algo_namespace='ns',
|
||||||
|
live=True,
|
||||||
|
data_frequency='minute',
|
||||||
|
capital_base=3000)
|
||||||
|
else:
|
||||||
|
run_algorithm(initialize=lambda ctx: True,
|
||||||
|
handle_data=handle_data_polo_partial_candles,
|
||||||
|
exchange_name='poloniex',
|
||||||
|
base_currency='usdt',
|
||||||
|
algo_namespace='ns',
|
||||||
|
live=False,
|
||||||
|
data_frequency='minute',
|
||||||
|
capital_base=3000,
|
||||||
|
start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
|
||||||
|
end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc)
|
||||||
|
)
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import pytz
|
||||||
|
from datetime import datetime
|
||||||
|
from catalyst.api import symbol
|
||||||
|
from catalyst.utils.run_algo import run_algorithm
|
||||||
|
|
||||||
|
coin = 'btc'
|
||||||
|
base_currency = 'usd'
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(context):
|
||||||
|
context.symbol = symbol('%s_%s' % (coin, base_currency))
|
||||||
|
|
||||||
|
|
||||||
|
def handle_data_polo_partial_candles(context, data):
|
||||||
|
history = data.history(symbol('btc_usdt'), ['volume'],
|
||||||
|
bar_count=10,
|
||||||
|
frequency='1D')
|
||||||
|
print('\nnow: %s\n%s' % (data.current_dt, history))
|
||||||
|
if not hasattr(context, 'i'):
|
||||||
|
context.i = 0
|
||||||
|
context.i += 1
|
||||||
|
if context.i > 5:
|
||||||
|
raise Exception('stop')
|
||||||
|
|
||||||
|
|
||||||
|
run_algorithm(initialize=lambda ctx: True,
|
||||||
|
handle_data=handle_data_polo_partial_candles,
|
||||||
|
exchange_name='poloniex',
|
||||||
|
base_currency='usdt',
|
||||||
|
algo_namespace='ns',
|
||||||
|
live=False,
|
||||||
|
data_frequency='minute',
|
||||||
|
capital_base=3000,
|
||||||
|
start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
|
||||||
|
end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc))
|
||||||
@@ -143,7 +143,7 @@ with the following steps:
|
|||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
conda create --name catalyst python=2.7 scipy zlib
|
conda create --name catalyst python=3.6 scipy zlib
|
||||||
|
|
||||||
3. Activate the environment:
|
3. Activate the environment:
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,33 @@
|
|||||||
Release Notes
|
Release Notes
|
||||||
=============
|
=============
|
||||||
|
|
||||||
|
Version 0.5.6
|
||||||
|
^^^^^^^^^^^^^
|
||||||
|
**Release Date**: 2018-03-22
|
||||||
|
|
||||||
|
Build
|
||||||
|
~~~~~
|
||||||
|
- Data Marketplace: ensures compatibility across wallets, now fully supporting
|
||||||
|
`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask`
|
||||||
|
(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is
|
||||||
|
unknown.
|
||||||
|
- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
|
||||||
|
- Data Marketplace: Added progress indicator for data ingestion.
|
||||||
|
|
||||||
|
Bug Fixes
|
||||||
|
~~~~~~~~~
|
||||||
|
- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary
|
||||||
|
fix for :issue:`271`, :issue:`285`
|
||||||
|
|
||||||
|
Version 0.5.5
|
||||||
|
^^^^^^^^^^^^^
|
||||||
|
**Release Date**: 2018-03-19
|
||||||
|
|
||||||
|
Bug Fixes
|
||||||
|
~~~~~~~~~
|
||||||
|
- Fixed an issue with the data history in daily frequency :issue:`274`
|
||||||
|
- Fix hourly frequency issues :issue:`227` and :issue:`114`
|
||||||
|
|
||||||
Version 0.5.4
|
Version 0.5.4
|
||||||
^^^^^^^^^^^^^
|
^^^^^^^^^^^^^
|
||||||
**Release Date**: 2018-03-14
|
**Release Date**: 2018-03-14
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ channels:
|
|||||||
dependencies:
|
dependencies:
|
||||||
- certifi=2016.2.28=py27_0
|
- certifi=2016.2.28=py27_0
|
||||||
- mkl=2017.0.3
|
- mkl=2017.0.3
|
||||||
- matplotlib=2.1.2=py36_0
|
|
||||||
- numpy=1.13.1=py27_0
|
- numpy=1.13.1=py27_0
|
||||||
- openssl=1.0.2l
|
- openssl=1.0.2l
|
||||||
- pip=9.0.1=py27_1
|
- pip=9.0.1=py27_1
|
||||||
@@ -40,7 +39,7 @@ dependencies:
|
|||||||
- lru-dict==1.1.6
|
- lru-dict==1.1.6
|
||||||
- mako==1.0.7
|
- mako==1.0.7
|
||||||
- markupsafe==1.0
|
- markupsafe==1.0
|
||||||
- matplotlib==2.1.0
|
- matplotlib==2.1.2
|
||||||
- multipledispatch==0.4.9
|
- multipledispatch==0.4.9
|
||||||
- networkx==2.0
|
- networkx==2.0
|
||||||
- numexpr==2.6.4
|
- numexpr==2.6.4
|
||||||
|
|||||||
Reference in New Issue
Block a user