mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-24 13:00:57 +08:00
BLD: register dataset in marketplace
This commit is contained in:
Victor Grau Serrat
+7
-10
@@ -1,9 +1,9 @@
|
||||
import errno
|
||||
import os
|
||||
import sys
|
||||
from functools import wraps
|
||||
|
||||
import click
|
||||
import sys
|
||||
import logbook
|
||||
import pandas as pd
|
||||
from catalyst.marketplace.marketplace import Marketplace
|
||||
@@ -752,10 +752,11 @@ def marketplace(ctx):
|
||||
@marketplace.command()
|
||||
@click.pass_context
|
||||
def ls(ctx):
|
||||
click.echo('Listing of available data sources on the marketplace:')
|
||||
click.echo('Listing of available data sources on the marketplace:',
|
||||
sys.stdout)
|
||||
marketplace = Marketplace()
|
||||
marketplace.list()
|
||||
click.echo('Functionality not yet implemented.')
|
||||
click.echo('Functionality not yet implemented.', sys.stdout)
|
||||
|
||||
|
||||
@marketplace.command()
|
||||
@@ -808,9 +809,7 @@ def ingest(ctx, dataset, data_frequency, start, end):
|
||||
ctx.fail("must specify a dataset to clean with '--dataset'\n"
|
||||
"List available dataset on the marketplace with "
|
||||
"'catalyst marketplace ls'")
|
||||
click.echo(
|
||||
'Ingesting data: {}'.format(dataset)
|
||||
)
|
||||
click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
|
||||
marketplace = Marketplace()
|
||||
marketplace.ingest(dataset, data_frequency, start, end)
|
||||
|
||||
@@ -827,12 +826,10 @@ def clean(ctx, dataset):
|
||||
ctx.fail("must specify a dataset to ingest with '--dataset'\n"
|
||||
"List available dataset on the marketplace with "
|
||||
"'catalyst marketplace ls'")
|
||||
click.echo(
|
||||
'Cleaning data source: {}'.format(dataset)
|
||||
)
|
||||
click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
|
||||
marketplace = Marketplace()
|
||||
marketplace.clean(dataset)
|
||||
click.echo('Done')
|
||||
click.echo('Done', sys.stdout)
|
||||
|
||||
|
||||
@marketplace.command()
|
||||
|
||||
@@ -1,23 +1,31 @@
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import urllib
|
||||
import sys
|
||||
|
||||
import binascii
|
||||
import bcolz
|
||||
import logbook
|
||||
import pandas as pd
|
||||
import six
|
||||
from web3 import Web3, HTTPProvider
|
||||
|
||||
from catalyst.constants import ROOT_DIR, LOG_LEVEL
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.utils.stats_utils import set_print_settings
|
||||
from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty
|
||||
from catalyst.marketplace.utils.bundle_utils import merge_bundles
|
||||
from catalyst.marketplace.utils.path_utils import get_data_source, \
|
||||
get_bundle_folder, get_data_source_folder
|
||||
get_bundle_folder, get_data_source_folder, get_marketplace_folder, \
|
||||
get_user_pubaddr
|
||||
|
||||
if sys.version_info.major < 3:
|
||||
import urllib
|
||||
else:
|
||||
import urllib.request as urllib
|
||||
|
||||
# TODO: host our own node on aws?
|
||||
# TODO: switch to mainnet
|
||||
REMOTE_NODE = 'https://ropsten.infura.io/'
|
||||
REMOTE_NODE = 'https://ropsten.infura.io/'
|
||||
|
||||
# TODO: move to MASTER branch on github
|
||||
CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
|
||||
@@ -32,28 +40,28 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL)
|
||||
class Marketplace:
|
||||
def __init__(self):
|
||||
|
||||
contract_url = urllib.urlopen(CONTRACT_PATH)
|
||||
CONTRACT_ADDRESS = Web3.toChecksumAddress(
|
||||
self.addresses = get_user_pubaddr()
|
||||
|
||||
if self.addresses[0]['pubAddr'] == '':
|
||||
raise MarketplacePubAddressEmpty(
|
||||
filename=os.path.join(
|
||||
get_marketplace_folder(), 'addresses.json')
|
||||
)
|
||||
self.default_account = self.addresses[0]['pubAddr']
|
||||
|
||||
contract_url = urllib.urlopen(CONTRACT_PATH)
|
||||
self.contract_address = Web3.toChecksumAddress(
|
||||
contract_url.readline().strip())
|
||||
|
||||
abi_url = urllib.urlopen(CONTRACT_ABI)
|
||||
abi = json.load(abi_url)
|
||||
|
||||
w3 = Web3(HTTPProvider(REMOTE_NODE))
|
||||
self.web3 = Web3(HTTPProvider(REMOTE_NODE))
|
||||
|
||||
self.contract = w3.eth.contract(
|
||||
CONTRACT_ADDRESS,
|
||||
self.contract = self.web3.eth.contract(
|
||||
self.contract_address,
|
||||
abi=abi,
|
||||
) # Type: Contract
|
||||
|
||||
# TODO: Set default address correctly from user-provided config
|
||||
DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None)
|
||||
if DEFAULT_ETH_ADDRESS is None:
|
||||
raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an '
|
||||
'environment variable. Quitting...')
|
||||
|
||||
self.default_account = DEFAULT_ETH_ADDRESS
|
||||
pass
|
||||
)
|
||||
|
||||
def get_data_sources_map(self):
|
||||
return [
|
||||
@@ -122,11 +130,11 @@ class Marketplace:
|
||||
|
||||
pass
|
||||
|
||||
def register(self, data_source_name):
|
||||
def subscribe(self, dataset):
|
||||
data_sources = self.get_data_sources_map()
|
||||
index = next(
|
||||
(index for (index, d) in enumerate(data_sources) if
|
||||
d['name'].lower() == data_source_name.lower()),
|
||||
d['name'].lower() == dataset.lower()),
|
||||
None
|
||||
)
|
||||
if index is None:
|
||||
@@ -140,7 +148,7 @@ class Marketplace:
|
||||
).subscribe(index)
|
||||
print(
|
||||
'Subscribed to data source {} successfully'.format(
|
||||
data_source_name
|
||||
dataset
|
||||
)
|
||||
)
|
||||
|
||||
@@ -207,3 +215,91 @@ class Marketplace:
|
||||
|
||||
shutil.rmtree(folder)
|
||||
pass
|
||||
|
||||
def register(self):
|
||||
dataset = input('Enter the name of the dataset to register: ')
|
||||
price = int(input('Enter the price for a monthly subscription to '
|
||||
'this dataset in ENG: '))
|
||||
|
||||
# while True:
|
||||
# freq = input('Enter the data frequency [daily, hourly, minute]: ')
|
||||
# if freq.lower() not in ('daily', 'houlry', 'minute'):
|
||||
# print("Not a valid frequency.")
|
||||
# else:
|
||||
# break
|
||||
|
||||
# while True:
|
||||
# reg_pub = input('Can data be published every hour at a regular '
|
||||
# 'time? [default: Y]: ') or 'y'
|
||||
# if reg_pub.lower() not in ('y', 'n'):
|
||||
# print("Please answer Y or N.")
|
||||
# else:
|
||||
# if reg_pub.lower() == 'y':
|
||||
# reg_pub = True
|
||||
# else:
|
||||
# reg_pub = False
|
||||
# break
|
||||
|
||||
# while True:
|
||||
# hist = input('Does it include historical data? '
|
||||
# '[default: Y]') or 'y'
|
||||
# if hist.lower() not in ('y', 'n'):
|
||||
# print("Please answer Y or N.")
|
||||
# else:
|
||||
# if hist.lower() == 'y':
|
||||
# hist = True
|
||||
# else:
|
||||
# hist = False
|
||||
# break
|
||||
|
||||
while True:
|
||||
for i in range(0, len(self.addresses)):
|
||||
print('{}\t{}\t{}'.format(
|
||||
i,
|
||||
self.addresses[i]['pubAddr'],
|
||||
self.addresses[i]['desc'])
|
||||
)
|
||||
address_i = int(input('Choose the your address associated with '
|
||||
'this transaction: [default: 0] ') or 0)
|
||||
if not (0 <= address_i < len(self.addresses)):
|
||||
print('Please choose a number between 0 and {}\n'.format(
|
||||
len(self.addresses)-1))
|
||||
else:
|
||||
address = Web3.toChecksumAddress(
|
||||
self.addresses[address_i]['pubAddr'])
|
||||
break
|
||||
|
||||
tx = self.contract.functions.register(
|
||||
binascii.hexlify(dataset.encode('utf-8')),
|
||||
price,
|
||||
address,
|
||||
).buildTransaction(
|
||||
{'nonce': self.web3.eth.getTransactionCount(address)})
|
||||
|
||||
tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas
|
||||
|
||||
print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
|
||||
'enter the following parameters:\n\n'
|
||||
'From Address:\t\t{_from}\n'
|
||||
'To Address:\t\t{to}\n'
|
||||
'Value / Amount to Send:\t{value}\n'
|
||||
'Gas Limit:\t\t{gas}\n'
|
||||
'Nonce:\t\t\t{nonce}\n'
|
||||
'Data:\t\t\t{data}\n'.format(
|
||||
_from=address,
|
||||
to=tx['to'],
|
||||
value=tx['value'],
|
||||
gas=tx['gas'],
|
||||
nonce=tx['nonce'],
|
||||
data=tx['data'],
|
||||
)
|
||||
)
|
||||
|
||||
signed_tx = input('Copy and Paste the "Signed Transaction" '
|
||||
'field here:\n')
|
||||
|
||||
tx_hash = '0x{}'.format(binascii.hexlify(
|
||||
self.web3.eth.sendRawTransaction(signed_tx)
|
||||
).decode('utf-8'))
|
||||
|
||||
print('\nThis is the TxHash for this transaction: {}'.format(tx_hash))
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from catalyst.errors import ZiplineError
|
||||
|
||||
|
||||
def silent_except_hook(exctype, excvalue, exctraceback):
|
||||
if exctype in [MarketplacePubAddressEmpty]:
|
||||
fn = traceback.extract_tb(exctraceback)[-1][0]
|
||||
ln = traceback.extract_tb(exctraceback)[-1][1]
|
||||
print("Error traceback: {1} (line {2})\n"
|
||||
"{0.__name__}: {3}".format(exctype, fn, ln, excvalue))
|
||||
else:
|
||||
sys.__excepthook__(exctype, excvalue, exctraceback)
|
||||
|
||||
|
||||
sys.excepthook = silent_except_hook
|
||||
|
||||
|
||||
class MarketplacePubAddressEmpty(ZiplineError):
|
||||
msg = (
|
||||
'Please enter your public address to use in the Data Marketplace '
|
||||
'in the following file: {filename}'
|
||||
).strip()
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import json
|
||||
import tarfile
|
||||
|
||||
import shutil
|
||||
@@ -7,6 +8,29 @@ from catalyst.data.bundles.core import download_without_progress
|
||||
from catalyst.utils.paths import data_root, ensure_directory
|
||||
|
||||
|
||||
def get_marketplace_folder(environ=None):
|
||||
"""
|
||||
The root path of the marketplace folder.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
environ:
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
if not environ:
|
||||
environ = os.environ
|
||||
|
||||
root = data_root(environ)
|
||||
marketplace_folder = os.path.join(root, 'marketplace')
|
||||
ensure_directory(marketplace_folder)
|
||||
|
||||
return marketplace_folder
|
||||
|
||||
|
||||
def get_data_source_folder(data_source_name, environ=None):
|
||||
"""
|
||||
The root path of an data_source folder.
|
||||
@@ -104,3 +128,35 @@ def get_data_source(data_source_name, period, force_download=False):
|
||||
tar.extractall(path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def get_user_pubaddr(environ=None):
|
||||
"""
|
||||
The de-serialized contend of the user's addresses.json file.
|
||||
Parameters
|
||||
----------
|
||||
environ:
|
||||
|
||||
Returns
|
||||
-------
|
||||
Object
|
||||
|
||||
"""
|
||||
marketplace_folder = get_marketplace_folder(environ)
|
||||
|
||||
filename = os.path.join(marketplace_folder, 'addresses.json')
|
||||
|
||||
if os.path.isfile(filename):
|
||||
with open(filename) as data_file:
|
||||
data = json.load(data_file)
|
||||
try:
|
||||
d = data[0]['pubAddr']
|
||||
except Exception as e:
|
||||
return [data, ]
|
||||
return data
|
||||
else:
|
||||
data = dict(pubAddr='', desc='')
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(data, f, sort_keys=False, indent=2,
|
||||
separators=(',', ':'))
|
||||
return data
|
||||
|
||||
Reference in New Issue
Block a user