diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4b1cfe8b..af4c4aef 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,22 +1,31 @@ -import json import os -import shutil import sys +import json +import hmac +import glob +import time +import shutil +import hashlib -import binascii import bcolz import logbook import pandas as pd import six +import requests from web3 import Web3, HTTPProvider from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty +from catalyst.marketplace.marketplace_errors import ( + MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: import urllib @@ -34,6 +43,8 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' +AUTH_SERVER = 'http://localhost:5000' + log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -259,7 +270,7 @@ class Marketplace: self.addresses[i]['pubAddr'], self.addresses[i]['desc']) ) - address_i = int(input('Choose the your address associated with ' + address_i = int(input('Choose your address associated with ' 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( @@ -270,7 +281,7 @@ class Marketplace: break tx = self.contract.functions.register( - binascii.hexlify(dataset.encode('utf-8')), + bytes32(dataset), price, address, ).buildTransaction( @@ -298,8 +309,63 @@ class Marketplace: signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') - tx_hash = '0x{}'.format(binascii.hexlify( - self.web3.eth.sendRawTransaction(signed_tx) - ).decode('utf-8')) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + def publish(self, dataset, datadir, watch): + + datasource = self.contract.functions.getDataSource( + bytes32(dataset)).call() + + if not datasource[4]: + raise MarketplaceDatasetNotFound(dataset=dataset) + + match = next((l for l in self.addresses if + l['pubAddr'] == datasource[0]), None) + + if not match: + raise MarketplaceNoAddressMatch( + dataset=dataset, + address=datasource[0]) + + print('Using address: {} to publish this dataset.'.format( + datasource[0])) + + if 'key' in match: + key = match['key'] + secret = match['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + nonce.encode('utf-8'), + hashlib.sha512).hexdigest() + headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + filenames = glob.glob(os.path.join(datadir, '*.csv')) + + if not filenames: + raise MarketplaceNoCSVFiles(datadir=datadir) + + files = [] + for file in filenames: + files.append(('file', open(file, 'rb'))) + + r = requests.post('{}/publish'.format(AUTH_SERVER), + files=files, + headers=headers) + + if r.status_code != 200: + raise MarketplaceHTTPRequest(request='upload file', + error=r.status_code) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest(request='upload file', + error=r.json()['error']) + + print('Dataset {} published successfully.'.format(dataset)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 04fca091..3992158d 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -5,7 +5,9 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): - if exctype in [MarketplacePubAddressEmpty]: + if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -22,3 +24,28 @@ class MarketplacePubAddressEmpty(ZiplineError): 'Please enter your public address to use in the Data Marketplace ' 'in the following file: {filename}' ).strip() + + +class MarketplaceDatasetNotFound(ZiplineError): + msg = ( + 'The dataset "{dataset}" is not registered in the Data Marketplace.' + ).strip() + + +class MarketplaceNoAddressMatch(ZiplineError): + msg = ( + 'The address registered with the dataset {dataset}: {address} ' + 'does not match any of your addresses.' + ).strip() + + +class MarketplaceHTTPRequest(ZiplineError): + msg = ( + 'Request to remote server to {request} failed: {error}' + ).strip() + + +class MarketplaceNoCSVFiles(ZiplineError): + msg = ( + 'No CSV files found on {datadir} to upload.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py new file mode 100644 index 00000000..f1193758 --- /dev/null +++ b/catalyst/marketplace/utils/auth_utils.py @@ -0,0 +1,47 @@ +import requests + +from catalyst.marketplace.marketplace_errors import ( + MarketplaceHTTPRequest) +from catalyst.marketplace.utils.path_utils import ( + get_user_pubaddr, save_user_pubaddr) + +AUTH_SERVER = 'http://localhost:5000' + + +def get_key_secret(pubAddr, dataset): + """ + Obtain a new key/secret pair from authentication server + + Parameters + ---------- + pubAddr: str + dataset: str + + Returns + ------- + key: str + secret: str + + """ + session = requests.Session() + response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + headers={ + 'pubAddr': pubAddr, + 'dataset': dataset}) + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + + return match['key'], match['secret'] diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 532f46ce..b58595ac 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -1,8 +1,8 @@ -import bcolz import os -import pandas as pd import shutil +import bcolz + def merge_bundles(zsource, ztarget): """ diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py new file mode 100644 index 00000000..c2be4bd6 --- /dev/null +++ b/catalyst/marketplace/utils/eth_utils.py @@ -0,0 +1,33 @@ +import binascii + + +def bytes32(string): + """ + Convert string to bytes32 data type for smart contract + + Parameters + ---------- + string: str + + Returns + ------- + list + + """ + return binascii.hexlify(string.encode('utf-8')) + + +def b32_str(bytes32): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(bytes32.decode('utf-8')) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 2cee9303..62f8beac 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,6 +133,7 @@ def get_data_source(data_source_name, period, force_download=False): def get_user_pubaddr(environ=None): """ The de-serialized contend of the user's addresses.json file. + Parameters ---------- environ: @@ -143,7 +144,6 @@ def get_user_pubaddr(environ=None): """ marketplace_folder = get_marketplace_folder(environ) - filename = os.path.join(marketplace_folder, 'addresses.json') if os.path.isfile(filename): @@ -160,3 +160,27 @@ def get_user_pubaddr(environ=None): json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) return data + + +def save_user_pubaddr(data, environ=None): + """ + Saves the user's public addresses and their related metadata in + the corresponding addresses.json file. + + Parameters + ---------- + data: dict + + Returns + ------- + True + + """ + marketplace_folder = get_marketplace_folder(environ) + filename = os.path.join(marketplace_folder, 'addresses.json') + + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + + return True