From 94d5b4a4d564697e4aa25b51ff2f9b8ac05ed7f2 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 17:16:54 -0500 Subject: [PATCH 01/41] BLD: defined first version of commands and marketplace class --- catalyst/__main__.py | 70 ++++++++++++++++++++++++++++++++ catalyst/alt_data/__init__.py | 0 catalyst/alt_data/marketplace.py | 15 +++++++ 3 files changed, 85 insertions(+) create mode 100644 catalyst/alt_data/__init__.py create mode 100644 catalyst/alt_data/marketplace.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f8f9136..d86276ba 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,6 +5,7 @@ from functools import wraps import click import logbook import pandas as pd +from catalyst.alt_data.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -573,6 +574,75 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) +@main.command(name='ls-data') +@click.pass_context +def ls_data(ctx): + click.echo( + 'Listing available alternative data sources' + ) + marketplace = Marketplace() + marketplace.list() + click.echo('Done') + + +@main.command(name='register-data') +@click.argument('data_source_name') +@click.pass_context +def register_data(ctx, data_source_name): + click.echo( + 'Registering to data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.register(data_source_name) + click.echo('Done') + + +@main.command(name='ingest-data') +@click.argument('data_source_name') +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest_data(ctx, data_source_name, data_frequency, start, end): + click.echo( + 'Ingesting data: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.ingest(data_source_name, data_frequency, start, end) + click.echo('Done') + + +@main.command(name='clean-data') +@click.argument('data_source_name') +@click.pass_context +def clean_data(ctx, data_source_name): + click.echo( + 'Cleaning data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.clean(data_source_name) + click.echo('Done') + + @main.command(name='clean-algo') @click.option( '-n', diff --git a/catalyst/alt_data/__init__.py b/catalyst/alt_data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/alt_data/marketplace.py b/catalyst/alt_data/marketplace.py new file mode 100644 index 00000000..8f4610cc --- /dev/null +++ b/catalyst/alt_data/marketplace.py @@ -0,0 +1,15 @@ +class Marketplace: + def __init__(self): + pass + + def list(self): + pass + + def register(self, data_source_name): + pass + + def ingest(self, data_source_name, data_frequency, start, end): + pass + + def clean(self, data_source_name): + pass \ No newline at end of file From aab501aa1791c8b14f398ebd0cb1c75808b50f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 19:08:53 -0500 Subject: [PATCH 02/41] BLD: first test of the marketplace smart contract --- .../{alt_data => marketplace}/__init__.py | 0 .../{alt_data => marketplace}/marketplace.py | 0 etc/requirements_marketplace.txt | 1 + marketplace/contracts/Marketplace.sol | 19 +++++++++ marketplace/contracts/Migrations.sol | 23 +++++++++++ marketplace/marketplace.iml | 14 +++++++ marketplace/migrations/1_initial_migration.js | 5 +++ marketplace/migrations/2_deploy_contracts.js | 5 +++ marketplace/test/TestMarketplace.sol | 39 +++++++++++++++++++ marketplace/truffle-config.js | 4 ++ marketplace/truffle.js | 11 ++++++ 11 files changed, 121 insertions(+) rename catalyst/{alt_data => marketplace}/__init__.py (100%) rename catalyst/{alt_data => marketplace}/marketplace.py (100%) create mode 100644 etc/requirements_marketplace.txt create mode 100644 marketplace/contracts/Marketplace.sol create mode 100644 marketplace/contracts/Migrations.sol create mode 100644 marketplace/marketplace.iml create mode 100644 marketplace/migrations/1_initial_migration.js create mode 100644 marketplace/migrations/2_deploy_contracts.js create mode 100644 marketplace/test/TestMarketplace.sol create mode 100644 marketplace/truffle-config.js create mode 100644 marketplace/truffle.js diff --git a/catalyst/alt_data/__init__.py b/catalyst/marketplace/__init__.py similarity index 100% rename from catalyst/alt_data/__init__.py rename to catalyst/marketplace/__init__.py diff --git a/catalyst/alt_data/marketplace.py b/catalyst/marketplace/marketplace.py similarity index 100% rename from catalyst/alt_data/marketplace.py rename to catalyst/marketplace/marketplace.py diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt new file mode 100644 index 00000000..cbb3a5a3 --- /dev/null +++ b/etc/requirements_marketplace.txt @@ -0,0 +1 @@ +web3==4.0.0b5 \ No newline at end of file diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol new file mode 100644 index 00000000..54a14d1a --- /dev/null +++ b/marketplace/contracts/Marketplace.sol @@ -0,0 +1,19 @@ +pragma solidity ^0.4.17; + +contract Marketplace { + address[16] public subscribers; + + // Adopting a pet + function subscribe(uint dataSourceId) public returns (uint) { + require(dataSourceId >= 0 && dataSourceId <= 1); + + subscribers[dataSourceId] = msg.sender; + + return dataSourceId; + } + + // Retrieving the subscribers + function getSubscribers() public view returns (address[16]) { + return subscribers; + } +} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol new file mode 100644 index 00000000..f170cb4f --- /dev/null +++ b/marketplace/contracts/Migrations.sol @@ -0,0 +1,23 @@ +pragma solidity ^0.4.17; + +contract Migrations { + address public owner; + uint public last_completed_migration; + + modifier restricted() { + if (msg.sender == owner) _; + } + + function Migrations() public { + owner = msg.sender; + } + + function setCompleted(uint completed) public restricted { + last_completed_migration = completed; + } + + function upgrade(address new_address) public restricted { + Migrations upgraded = Migrations(new_address); + upgraded.setCompleted(last_completed_migration); + } +} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml new file mode 100644 index 00000000..cf4ef5ed --- /dev/null +++ b/marketplace/marketplace.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js new file mode 100644 index 00000000..4d5f3f9b --- /dev/null +++ b/marketplace/migrations/1_initial_migration.js @@ -0,0 +1,5 @@ +var Migrations = artifacts.require("./Migrations.sol"); + +module.exports = function(deployer) { + deployer.deploy(Migrations); +}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js new file mode 100644 index 00000000..e3935400 --- /dev/null +++ b/marketplace/migrations/2_deploy_contracts.js @@ -0,0 +1,5 @@ +var Marketplace = artifacts.require ("Marketplace"); + +module.exports = function (deployer) { + deployer.deploy (Marketplace); +}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol new file mode 100644 index 00000000..ea1c7fb3 --- /dev/null +++ b/marketplace/test/TestMarketplace.sol @@ -0,0 +1,39 @@ +pragma solidity ^0.4.17; + +import "truffle/Assert.sol"; +import "truffle/DeployedAddresses.sol"; +import "../contracts/Marketplace.sol"; + +contract TestMarketplace { + Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); + + // Testing the adopt() function + function testUserCanSubscribe() public { + uint returnedId = marketplace.subscribe(0); + + uint expected = 0; + + Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + } + + // Testing retrieval of a single pet's owner + function testGetSubscriberAddressByDataSourceId() public { + // Expected owner is this contract + address expected = this; + + address adopter = marketplace.subscribers(0); + + Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + } + + // Testing retrieval of all pet owners + function testGetSubscriberAddressByDataSourceIdInArray() public { + // Expected owner is this contract + address expected = this; + + // Store adopters in memory rather than contract's storage + address[16] memory subscribers = marketplace.getSubscribers(); + + Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + } +} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js new file mode 100644 index 00000000..a6330d6d --- /dev/null +++ b/marketplace/truffle-config.js @@ -0,0 +1,4 @@ +module.exports = { + // See + // to customize your Truffle configuration! +}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js new file mode 100644 index 00000000..67f2dd44 --- /dev/null +++ b/marketplace/truffle.js @@ -0,0 +1,11 @@ +module.exports = { + // See + // to customize your Truffle configuration! + networks: { + development: { + host: "localhost", + port: 7545, + network_id: "*" // Match any network id + } + } +}; From 22a850ab141556a194c5aa8648a5bdefe6aae6ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 20:41:03 -0500 Subject: [PATCH 03/41] BLD: testing basic smart contract --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++++++-- marketplace/contracts/Marketplace.sol | 2 +- tests/marketplace/__init__.py | 0 tests/marketplace/test_marketplace.py | 4 +++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/marketplace/__init__.py create mode 100644 tests/marketplace/test_marketplace.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 8f4610cc..a6c0ff0b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,15 +1,52 @@ +import json +import os + +from web3 import Web3, HTTPProvider + +CONTRACT_PATH = os.path.join( + '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' +) +CONTRACT_ADDRESS = Web3.toChecksumAddress( + '0x345ca3e014aaf5dca488057592ee47305d9b3e10' +) + + +# we'll use one of our default accounts to deploy from. every write to the chain requires a +# payment of ethereum called "gas". if we were running an actual test ethereum node locally, +# then we'd have to go on the test net and get some free ethereum to play with. that is beyond +# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and +# the only chain we're using is our own local one + + class Marketplace: def __init__(self): - pass + with open(CONTRACT_PATH) as handle: + json_interface = json.load(handle) + + w3 = Web3(HTTPProvider('http://localhost:7545')) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=json_interface['abi'], + ) # Type: Contract + self.default_account = w3.eth.accounts[1] + + pass def list(self): + subscribers = self.contract.call( + {'from': self.default_account} + ).getSubscribers() pass def register(self, data_source_name): + test = self.contract.transact( + {'from': self.default_account} + ).subscribe(0) pass def ingest(self, data_source_name, data_frequency, start, end): pass def clean(self, data_source_name): - pass \ No newline at end of file + pass diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 54a14d1a..30401e5b 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 1); + require(dataSourceId >= 0 && dataSourceId <= 5); subscribers[dataSourceId] = msg.sender; diff --git a/tests/marketplace/__init__.py b/tests/marketplace/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py new file mode 100644 index 00000000..e03efc43 --- /dev/null +++ b/tests/marketplace/test_marketplace.py @@ -0,0 +1,4 @@ +from catalyst.marketplace.marketplace import Marketplace + +marketplace = Marketplace() +pass \ No newline at end of file From e0c8178932a17632ca8ef7fcf0363b7f9436341a Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 22:42:09 -0500 Subject: [PATCH 04/41] BLD: improved smart contract and catalyst commands --- catalyst/__main__.py | 4 +- catalyst/constants.py | 2 + catalyst/marketplace/marketplace.py | 95 +++++++++++++++++++++++---- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 8 +-- tests/marketplace/test_marketplace.py | 14 +++- 6 files changed, 102 insertions(+), 23 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index d86276ba..5f70b985 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,7 +5,7 @@ from functools import wraps import click import logbook import pandas as pd -from catalyst.alt_data.marketplace import Marketplace +from catalyst.marketplace.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -578,7 +578,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def ls_data(ctx): click.echo( - 'Listing available alternative data sources' + 'The alternative data sources:' ) marketplace = Marketplace() marketplace.list() diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..6b3fc235 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,4 +15,6 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + AUTO_INGEST = False diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a6c0ff0b..9de33328 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,29 +1,26 @@ import json import os +import pandas as pd from web3 import Web3, HTTPProvider +from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.constants import ROOT_DIR + +REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( - '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' + ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0x345ca3e014aaf5dca488057592ee47305d9b3e10' + '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' ) -# we'll use one of our default accounts to deploy from. every write to the chain requires a -# payment of ethereum called "gas". if we were running an actual test ethereum node locally, -# then we'd have to go on the test net and get some free ethereum to play with. that is beyond -# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and -# the only chain we're using is our own local one - - class Marketplace: def __init__(self): with open(CONTRACT_PATH) as handle: json_interface = json.load(handle) - - w3 = Web3(HTTPProvider('http://localhost:7545')) + w3 = Web3(HTTPProvider(REMOTE_NODE)) self.contract = w3.eth.contract( CONTRACT_ADDRESS, @@ -33,16 +30,86 @@ class Marketplace: pass + def get_data_sources_map(self): + return [ + dict( + name='Marketcap', + desc='The marketcap value in USD.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='GitHub', + desc='The rate of development activity on GitHub.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='Influencers', + desc='Tweets and related sentiments by selected influencers.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + ] + def list(self): subscribers = self.contract.call( {'from': self.default_account} ).getSubscribers() + + subscribed = [] + for index, address in enumerate(subscribers): + if address == self.default_account: + subscribed.append(index) + + data_sources = self.get_data_sources_map() + + data = [] + for index, data_source in enumerate(data_sources): + data.append( + dict( + id=index, + subscribed=index in subscribed, + **data_source, + ) + ) + + df = pd.DataFrame(data) + df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) + set_print_settings() + + formatters = dict( + subscribed=lambda s: u'\u2713' if s else '', + ) + print(df.to_string(formatters=formatters)) + pass def register(self, data_source_name): - test = self.contract.transact( - {'from': self.default_account} - ).subscribe(0) + data_sources = self.get_data_sources_map() + index = next( + (index for (index, d) in enumerate(data_sources) if + d['name'].lower() == data_source_name.lower()), + None + ) + if index is None: + raise ValueError( + 'Data source not found.' + ) + + try: + self.contract.transact( + {'from': self.default_account} + ).subscribe(index) + print( + 'Subscribed to data source {} successfully'.format( + data_source_name + ) + ) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + pass def ingest(self, data_source_name, data_frequency, start, end): diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 30401e5b..1cc1cf75 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 5); + require(dataSourceId >= 0 && dataSourceId <= 15); subscribers[dataSourceId] = msg.sender; diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index ea1c7fb3..5c4f6411 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -9,9 +9,9 @@ contract TestMarketplace { // Testing the adopt() function function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(0); + uint returnedId = marketplace.subscribe(2); - uint expected = 0; + uint expected = 2; Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } @@ -21,7 +21,7 @@ contract TestMarketplace { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(0); + address adopter = marketplace.subscribers(2); Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); } @@ -34,6 +34,6 @@ contract TestMarketplace { // Store adopters in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); } } \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index e03efc43..1faf0648 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,4 +1,14 @@ from catalyst.marketplace.marketplace import Marketplace +from catalyst.testing.fixtures import WithLogger, ZiplineTestCase -marketplace = Marketplace() -pass \ No newline at end of file + +class TestMarketplace(WithLogger, ZiplineTestCase): + def test_list(self): + marketplace = Marketplace() + marketplace.list() + pass + + def test_register(self): + marketplace = Marketplace() + marketplace.register('GitHub') + pass From 1e4af12e3daa4222a93338f0feed67f136c8aae6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 13:31:08 -0500 Subject: [PATCH 05/41] BLD: improved smart contract and catalyst commands --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 9de33328..e8288037 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -12,7 +12,7 @@ CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' + '0xaa862ddac09f6736a61e1124040fd883a6533c19' ) From 354e422be84d50bef799453fed9020c9dd0a8416 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 21:54:09 -0500 Subject: [PATCH 06/41] BLD: working on ingesting alt data sources --- catalyst/marketplace/marketplace.py | 7 ++- catalyst/marketplace/utils/__init__.py | 0 catalyst/marketplace/utils/paths.py | 87 ++++++++++++++++++++++++++ tests/marketplace/test_marketplace.py | 10 +++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 catalyst/marketplace/utils/__init__.py create mode 100644 catalyst/marketplace/utils/paths.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e8288037..e22d3f49 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,13 +6,14 @@ from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR +from catalyst.marketplace.utils.paths import get_temp_bundles_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xaa862ddac09f6736a61e1124040fd883a6533c19' + '0xe2b6cf3863240892d59664d209a28289a73ef644' ) @@ -112,7 +113,9 @@ class Marketplace: pass - def ingest(self, data_source_name, data_frequency, start, end): + def ingest(self, data_source_name, data_frequency=None, start=None, + end=None): + temp_folder = get_temp_bundles_folder(data_source_name) pass def clean(self, data_source_name): diff --git a/catalyst/marketplace/utils/__init__.py b/catalyst/marketplace/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/paths.py new file mode 100644 index 00000000..008ed2b7 --- /dev/null +++ b/catalyst/marketplace/utils/paths.py @@ -0,0 +1,87 @@ +import os +import tarfile + +from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.paths import data_root, ensure_directory + + +def get_data_source_folder(data_source_name, environ=None): + """ + The root path of an data_source folder. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + data_source_folder = os.path.join(root, 'marketplace', data_source_name) + ensure_directory(data_source_folder) + + return data_source_folder + + +def get_temp_bundles_folder(data_source_name, environ=None): + """ + The temp folder for bundle downloads by algo name. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + data_source_folder = get_data_source_folder(data_source_name, environ) + + temp_bundles = os.path.join(data_source_folder, 'temp_bundles') + ensure_directory(temp_bundles) + + return temp_bundles + + +def get_data_source(data_source_name, period): + """ + Download and extract a bcolz bundle. + + Parameters + ---------- + exchange_name: str + symbol: str + data_frequency: str + period: str + + Returns + ------- + str + Filename: bitfinex-daily-neo_eth-2017-10.tar.gz + + """ + root = get_temp_bundles_folder(data_source_name) + name = '{data_source}-{period}'.format( + data_source=data_source_name, + period=period, + ) + path = os.path.join(root, name) + + if not os.path.isdir(path): + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) + + return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 1faf0648..3feeb9a6 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,5 +1,6 @@ from catalyst.marketplace.marketplace import Marketplace from catalyst.testing.fixtures import WithLogger, ZiplineTestCase +import pandas as pd class TestMarketplace(WithLogger, ZiplineTestCase): @@ -12,3 +13,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.register('GitHub') pass + + def test_ingest(self): + marketplace = Marketplace() + marketplace.ingest( + data_source_name='GitHub', + data_frequency='finest', + start=pd.Timestamp.utcnow(), + ) + pass From 50ded59c3ce1a201080a5c8281f18ac36291d4a3 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:01:00 -0500 Subject: [PATCH 07/41] DOC: documented the marketplace structure --- marketplace/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 marketplace/README.md diff --git a/marketplace/README.md b/marketplace/README.md new file mode 100644 index 00000000..599d8112 --- /dev/null +++ b/marketplace/README.md @@ -0,0 +1,18 @@ +This module contains smart contracts for the data marketplace. +It was generated with Truffle. Truffle can be used to compile and test +the smart contracts. It's being tested on a test blockhain using Ganache. + +Steps to test: +* Download an run Ganache +* Ensure that your Ganache endpoint matches `truffle.js` +* In this `marketplace` folder: + * `truffle compile` + * `truffle test` + * `truffle migrate --reset` + * `truffle console` + * From the console: `Marketplace.deployed()` +* The deployed method displays info about the deployed smart contract +including its address. Copy/paste the address into +`catalyst/marketplace/marketplace.py`. +* Run the catalyst marketplace unit tests: +`tests/marketplace/test_marketplace.py` \ No newline at end of file From 9361570895868f9b71b39da56af733073625da07 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:02 -0500 Subject: [PATCH 08/41] DOC: improved code comments --- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 1cc1cf75..115b6110 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -3,7 +3,7 @@ pragma solidity ^0.4.17; contract Marketplace { address[16] public subscribers; - // Adopting a pet + // Subscribing to a data source function subscribe(uint dataSourceId) public returns (uint) { require(dataSourceId >= 0 && dataSourceId <= 15); diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 5c4f6411..77a03aea 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -7,7 +7,7 @@ import "../contracts/Marketplace.sol"; contract TestMarketplace { Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - // Testing the adopt() function + // Testing the subscribe() function function testUserCanSubscribe() public { uint returnedId = marketplace.subscribe(2); @@ -16,24 +16,24 @@ contract TestMarketplace { Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } - // Testing retrieval of a single pet's owner + // Testing retrieval of a single subscriber function testGetSubscriberAddressByDataSourceId() public { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(2); + address subscriber = marketplace.subscribers(2); - Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); } - // Testing retrieval of all pet owners + // Testing retrieval of all subscribers function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected owner is this contract + // Expected subscriber is this contract address expected = this; - // Store adopters in memory rather than contract's storage + // Store subscribers in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); + Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); } } \ No newline at end of file From 3ea00e783c6d931d1a906cf729ceb7a9f0582689 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:52 -0500 Subject: [PATCH 09/41] DOC: improved code comments --- marketplace/test/TestMarketplace.sol | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 77a03aea..f169d728 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -13,7 +13,7 @@ contract TestMarketplace { uint expected = 2; - Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + Assert.equal(returnedId, expected, "Data source 2 should be recorded."); } // Testing retrieval of a single subscriber @@ -23,7 +23,7 @@ contract TestMarketplace { address subscriber = marketplace.subscribers(2); - Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); } // Testing retrieval of all subscribers From b1d63c9a5b7f34cee8f24774701aaa9aebd88f10 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 11 Jan 2018 12:15:49 -0700 Subject: [PATCH 10/41] Update README.md --- marketplace/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marketplace/README.md b/marketplace/README.md index 599d8112..61f63842 100644 --- a/marketplace/README.md +++ b/marketplace/README.md @@ -3,9 +3,9 @@ It was generated with Truffle. Truffle can be used to compile and test the smart contracts. It's being tested on a test blockhain using Ganache. Steps to test: -* Download an run Ganache +* Download and run Ganache * Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder: +* In this `marketplace` folder, run the following commands: * `truffle compile` * `truffle test` * `truffle migrate --reset` @@ -15,4 +15,4 @@ Steps to test: including its address. Copy/paste the address into `catalyst/marketplace/marketplace.py`. * Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` \ No newline at end of file +`tests/marketplace/test_marketplace.py` From 88aa7154db5b57cef9b6fe8d27571bd68e5761b4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 20:04:53 -0500 Subject: [PATCH 11/41] BLD: ingesting and cleaning marketplace data sources --- catalyst/exchange/utils/bundle_utils.py | 3 ++ catalyst/marketplace/marketplace.py | 35 ++++++++++++++-- catalyst/marketplace/utils/bundle_utils.py | 36 ++++++++++++++++ .../utils/{paths.py => path_utils.py} | 42 ++++++++++++++----- tests/marketplace/test_marketplace.py | 3 +- 5 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 catalyst/marketplace/utils/bundle_utils.py rename catalyst/marketplace/utils/{paths.py => path_utils.py} (61%) diff --git a/catalyst/exchange/utils/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py index e9207511..992cabbb 100644 --- a/catalyst/exchange/utils/bundle_utils.py +++ b/catalyst/exchange/utils/bundle_utils.py @@ -1,6 +1,8 @@ import calendar import os import tarfile + +import bcolz from datetime import timedelta, datetime, date import numpy as np @@ -355,3 +357,4 @@ def get_assets(exchange, include_symbols, exclude_symbols): else: return all_assets + diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e22d3f49..c71cef1c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,12 +1,17 @@ import json import os + +import bcolz import pandas as pd +import shutil from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR -from catalyst.marketplace.utils.paths import get_temp_bundles_folder +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ + get_data_source, get_bundle_folder, get_data_source_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( @@ -114,9 +119,31 @@ class Marketplace: pass def ingest(self, data_source_name, data_frequency=None, start=None, - end=None): - temp_folder = get_temp_bundles_folder(data_source_name) + end=None, force_download=False): + data_source_name = data_source_name.lower() + + period = start.strftime('%Y-%m-%d') + tmp_folder = get_data_source(data_source_name, period, force_download) + + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_folder, bundle_folder) + pass - def clean(self, data_source_name): + def clean(self, data_source_name, data_frequency=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + folder = get_data_source_folder(data_source_name) + + else: + forlder = get_bundle_folder(data_source_name, data_frequency) + + shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py new file mode 100644 index 00000000..916722da --- /dev/null +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -0,0 +1,36 @@ +import bcolz +import os +import pandas as pd +import shutil + + +def merge_bundles(zsource, ztarget): + """ + Merge + Parameters + ---------- + zsource + ztarget + + Returns + ------- + + """ + # TODO: find a way to do this iteratively instead of in-memory + df_source = zsource.todataframe() + df_source.set_index('last_updated', drop=False, inplace=True) + df_target = ztarget.todataframe() + df_target.set_index('last_updated', drop=False, inplace=True) + + df = df_target.merge( + right=df_source, + how='right', + ) # type: pd.DataFrame + + dirname = os.path.basename(ztarget.rootdir) + bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname)) + os.rename(ztarget.rootdir, bak_dir) + + z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir) + shutil.rmtree(bak_dir) + return z diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/path_utils.py similarity index 61% rename from catalyst/marketplace/utils/paths.py rename to catalyst/marketplace/utils/path_utils.py index 008ed2b7..6d301880 100644 --- a/catalyst/marketplace/utils/paths.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,6 +1,8 @@ import os import tarfile +import shutil + from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory @@ -29,6 +31,17 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +def get_bundle_folder(data_source_name, data_frequency, environ=None): + data_source_folder = get_data_source_folder(data_source_name, environ) + + subfolder = data_frequency if data_frequency is not None else 'data' + bundle_folder = os.path.join(data_source_folder, subfolder) + + ensure_directory(bundle_folder) + + return bundle_folder + + def get_temp_bundles_folder(data_source_name, environ=None): """ The temp folder for bundle downloads by algo name. @@ -51,7 +64,7 @@ def get_temp_bundles_folder(data_source_name, environ=None): return temp_bundles -def get_data_source(data_source_name, period): +def get_data_source(data_source_name, period, force_download=False): """ Download and extract a bcolz bundle. @@ -65,23 +78,30 @@ def get_data_source(data_source_name, period): Returns ------- str - Filename: bitfinex-daily-neo_eth-2017-10.tar.gz """ root = get_temp_bundles_folder(data_source_name) - name = '{data_source}-{period}'.format( + name = '{data_source}_{period}'.format( data_source=data_source_name, period=period, ) path = os.path.join(root, name) - if not os.path.isdir(path): - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) + if os.path.isdir(path): + if force_download: + shutil.rmtree(path) + + else: + return path + + ensure_directory(path) + + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 3feeb9a6..68990791 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -17,8 +17,9 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() marketplace.ingest( - data_source_name='GitHub', + data_source_name='Marketcap', data_frequency='finest', start=pd.Timestamp.utcnow(), + force_download=True, ) pass From 70b9fd8e03903a576a3422ad6a8946e1a49cab96 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 12 Jan 2018 00:39:14 -0500 Subject: [PATCH 12/41] BLD: completed data market place first integration and created an algo to test it --- catalyst/api.pyi | 76 +++++- catalyst/constants.py | 5 +- .../examples/mean_reversion_by_marketcap.py | 233 ++++++++++++++++++ catalyst/exchange/exchange_algorithm.py | 13 + catalyst/marketplace/marketplace.py | 60 ++++- catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/path_utils.py | 3 +- tests/marketplace/test_marketplace.py | 11 +- 8 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 catalyst/examples/mean_reversion_by_marketcap.py diff --git a/catalyst/api.pyi b/catalyst/api.pyi index bf104664..6e46aba9 100644 --- a/catalyst/api.pyi +++ b/catalyst/api.pyi @@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None): :func:`catalyst.api.pipeline_output` """ + def batch_market_order(share_counts): """Place a batch market order for multiple assets. @@ -48,6 +49,7 @@ def batch_market_order(share_counts): Index of ids for newly-created orders. """ + def cancel_order(order_param): """Cancel an open order. @@ -57,7 +59,9 @@ def cancel_order(order_param): The order_id or order object to cancel. """ -def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'): + +def continuous_future(root_symbol_str, offset=0, roll='volume', + adjustment='mul'): """Create a specifier for a continuous contract. Parameters @@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul' The continuous future specifier. """ -def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs): + +def fetch_csv(url, pre_func=None, post_func=None, date_column='date', + date_format=None, timezone='UTC', symbol=None, mask=True, + symbol_column=None, special_params_checker=None, **kwargs): """Fetch a csv from a remote url and register the data so that it is queryable from the ``data`` object. @@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma A requests source that will pull data from the url specified. """ + def future_symbol(symbol): """Lookup a futures contract with a given symbol. @@ -144,6 +152,7 @@ def future_symbol(symbol): Raised when no contract named 'symbol' is found. """ + def get_datetime(tz=None): """ Returns the current simulation datetime. @@ -159,6 +168,7 @@ dt : datetime The current simulation datetime converted to ``tz``. """ + def get_environment(field='platform'): """Query the execution environment. @@ -198,6 +208,7 @@ def get_environment(field='platform'): Raised when ``field`` is not a valid option. """ + def get_order(order_id): """Lookup an order based on the order id returned from one of the order functions. @@ -213,10 +224,12 @@ def get_order(order_id): The order object. """ + def history(bar_count, frequency, field, ffill=True): """DEPRECATED: use ``data.history`` instead. """ + def order(asset, amount, limit_price=None, stop_price=None, style=None): """Place an order. @@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ -def order_percent(asset, percent, limit_price=None, stop_price=None, style=None): + +def order_percent(asset, percent, limit_price=None, stop_price=None, + style=None): """Place an order in the specified asset corresponding to the given percent of the current portfolio value. @@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None) :func:`catalyst.api.order_value` """ + def order_target(asset, target, limit_price=None, stop_price=None, style=None): """Place an order to adjust a position to a target number of shares. If the position doesn't already exist, this is equivalent to placing a new @@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_target_value` """ -def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_percent(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target percent of the current portfolio value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is @@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style :func:`catalyst.api.order_target_value` """ -def order_target_value(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_value(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is equivalent to placing an @@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N :func:`catalyst.api.order_target_percent` """ + def order_value(asset, value, limit_price=None, stop_price=None, style=None): """Place an order by desired value rather than desired number of shares. @@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ + def pipeline_output(name): """Get the results of the pipeline that was attached with the name: ``name``. @@ -514,6 +536,7 @@ def pipeline_output(name): :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline` """ + def record(*args, **kwargs): """Track and record values each day. @@ -529,7 +552,9 @@ def record(*args, **kwargs): :func:`~catalyst.run_algorithm`. """ -def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None): + +def schedule_function(func, date_rule=None, time_rule=None, half_days=True, + calendar=None): """Schedules a function to be called according to some timed rules. Parameters @@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale :class:`catalyst.api.time_rules` """ + def set_asset_restrictions(restrictions, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'): catalyst.finance.asset_restrictions.Restrictions """ + def set_benchmark(benchmark): """Set the benchmark asset. @@ -576,6 +603,7 @@ def set_benchmark(benchmark): automatically reinvested. """ + def set_cancel_policy(cancel_policy): """Sets the order cancellation policy for the simulation. @@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy): :class:`catalyst.api.NeverCancel` """ + def set_commission(commission): """Sets the commission model for the simulation. @@ -605,6 +634,7 @@ def set_commission(commission): :class:`catalyst.finance.commission.PerDollar` """ + def set_do_not_order_list(restricted_list, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'): The assets that cannot be ordered. """ + def set_long_only(on_error='fail'): """Set a rule specifying that this algorithm cannot take short positions. """ + def set_max_leverage(max_leverage): """Set a limit on the maximum leverage of the algorithm. @@ -629,6 +661,7 @@ def set_max_leverage(max_leverage): be no maximum. """ + def set_max_order_count(max_count, on_error='fail'): """Set a limit on the number of orders that can be placed in a single day. @@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'): The maximum number of orders that can be placed on any single day. """ -def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_order_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value of any single order placed for sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. @@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error= The maximum value that can be ordered at one time. """ -def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_position_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value held for the given sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. This means @@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err The maximum value to hold for an asset. """ + def set_slippage(slippage): """Set the slippage model for the simulation. @@ -694,6 +732,7 @@ def set_slippage(slippage): :class:`catalyst.finance.slippage.SlippageModel` """ + def set_symbol_lookup_date(dt): """Set the date for which symbols will be resolved to their assets (symbols may map to different firms or underlying assets at @@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt): The new symbol lookup date. """ + def sid(sid): """Lookup an Asset by its unique asset identifier. @@ -724,6 +764,7 @@ def sid(sid): When a requested ``sid`` does not map to any asset. """ + def symbol(symbol_str): """Lookup an Equity by its ticker symbol. @@ -748,6 +789,7 @@ def symbol(symbol_str): :func:`catalyst.api.set_symbol_lookup_date` """ + def symbols(*args): """Lookup multuple Equities as a list. @@ -773,3 +815,21 @@ def symbols(*args): :func:`catalyst.api.set_symbol_lookup_date` """ + +def get_data_source(data_source_name, data_frequency=None, + start=None, end=None): + """ + Lookup a data source from the marketplace + + Parameters + ---------- + self + data_source_name + data_frequency + start + end + + Returns + ------- + + """ diff --git a/catalyst/constants.py b/catalyst/constants.py index 6b3fc235..5668bc83 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,6 +15,9 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +try: + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +except Exception as e: + print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py new file mode 100644 index 00000000..6d31dab7 --- /dev/null +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -0,0 +1,233 @@ +# For this example, we're going to write a simple momentum script. When the +# stock goes up quickly, we're going to buy; when it goes down quickly, we're +# going to sell. Hopefully we'll ride the waves. +import os +import tempfile +import time + +import pandas as pd +import talib +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import symbol, record, order_target_percent, get_data_source +from catalyst.exchange.utils.stats_utils import set_print_settings, \ + get_pretty_stats +# We give a name to the algorithm which Catalyst will use to persist its state. +# In this example, Catalyst will create the `.catalyst/data/live_algos` +# directory. If we stop and start the algorithm, Catalyst will resume its +# state using the files included in the folder. +from catalyst.utils.paths import ensure_directory + +NAMESPACE = 'mean_reversion_simple' +log = Logger(NAMESPACE) + + +# To run an algorithm in Catalyst, you need two functions: initialize and +# handle_data. + +def initialize(context): + # This initialize function sets any data or variables that you'll use in + # your algorithm. For instance, you'll want to define the trading pair (or + # trading pairs) you want to backtest. You'll also want to define any + # parameters or values you're going to use. + + # In our example, we're looking at Neo in Ether. + df = get_data_source( + 'marketcap', start=context.datetime + ) # type: pd.DataFrame + + # Keep only the top coins by market cap + df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))] + + set_print_settings() + + df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True) + print('the marketplace data:\n{}'.format(df)) + + # Pick the 5 assets with the lowest market cap for trading + quote_currency = 'eth' + exchange = context.exchanges[next(iter(context.exchanges))] + symbols = [a.symbol for a in exchange.assets + if a.start_date < context.datetime] + context.assets = [] + for currency, price in df['market_cap_usd'].iteritems(): + if len(context.assets) >= 5: + break + + s = '{}_{}'.format(currency.decode('utf-8'), quote_currency) + if s in symbols: + context.assets.append(symbol(s)) + + context.base_price = None + context.current_day = None + + context.RSI_OVERSOLD = 55 + context.RSI_OVERBOUGHT = 60 + context.CANDLE_SIZE = '5T' + + context.start_time = time.time() + + +def handle_data(context, data): + # This handle_data function is where the real work is done. Our data is + # minute-level tick data, and each minute is called a frame. This function + # runs on each frame of the data. + + # We flag the first period of each day. + # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle + # would only execute once. This method works with minute and daily + # frequencies. + today = data.current_dt.floor('1D') + if today != context.current_day: + context.traded_today = dict() + context.current_day = today + + # Preparing dictionaries for asset-level data points + volumes = dict() + rsis = dict() + price_values = dict() + cash = context.portfolio.cash + + for asset in context.assets: + # We're computing the volume-weighted-average-price of the security + # defined above, in the context.assets variable. For this example, + # we're using three bars on the 15 min bars. + + # The frequency attribute determine the bar size. We use this + # convention for the frequency alias: + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + prices = data.history( + asset, + fields='close', + bar_count=50, + frequency=context.CANDLE_SIZE + ) + + # Ta-lib calculates various technical indicator based on price and + # volume arrays. + + # In this example, we are comp + rsi = talib.RSI(prices.values, timeperiod=14) + + # We need a variable for the current price of the security to compare + # to the average. Since we are requesting two fields, data.current() + # returns a DataFrame with + current = data.current(asset, fields=['close', 'volume']) + price = current['close'] + + # If base_price is not set, we use the current value. This is the + # price at the first bar which we reference to calculate price_change. + # if asset not in context.base_price: + # context.base_price[asset] = price + # + # base_price = context.base_price[asset] + # price_change = (price - base_price) / base_price + + # Tracking the relevant data + volumes[asset] = current['volume'] + rsis[asset] = rsi[-1] + price_values[asset] = price + # price_changes[asset] = price_change + + # We are trying to avoid over-trading by limiting our trades to + # one per day. + if asset in context.traded_today: + continue + + # Exit if we cannot trade + if not data.can_trade(asset): + continue + + # Another powerful built-in feature of the Catalyst backtester is the + # portfolio object. The portfolio object tracks your positions, cash, + # cost basis of specific holdings, and more. In this line, we + # calculate how long or short our position is at this minute. + pos_amount = context.portfolio.positions[asset].amount + + if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: + log.info( + '{}: buying - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + # Set a style for limit orders, + limit_price = price * 1.005 + target = 1.0 / len(context.assets) + order_target_percent( + asset, target, limit_price=limit_price + ) + context.traded_today[asset] = True + + elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: + log.info( + '{}: selling - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + limit_price = price * 0.995 + order_target_percent( + asset, 0, limit_price=limit_price + ) + context.traded_today[asset] = True + + # Now that we've collected all current data for this frame, we use + # the record() method to save it. This data will be available as + # a parameter of the analyze() function for further analysis. + record( + current_price=price_values, + volume=volumes, + rsi=rsis, + cash=cash, + ) + + +def analyze(context=None, perf=None): + stats = get_pretty_stats(perf) + print('the algo stats:\n{}'.format(stats)) + pass + + +if __name__ == '__main__': + # The execution mode: backtest or live + live = False + + if live: + run_algorithm( + capital_base=0.1, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + live=True, + algo_namespace=NAMESPACE, + base_currency='btc', + live_graph=False, + simulate_orders=False, + stats_output=None, + ) + + else: + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', NAMESPACE + ) + ensure_directory(folder) + + timestr = time.strftime('%Y%m%d-%H%M%S') + out = os.path.join(folder, '{}.p'.format(timestr)) + # catalyst run -f catalyst/examples/mean_reversion_simple.py \ + # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ + # --data-frequency minute --capital-base 10000 + run_algorithm( + capital_base=100, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace=NAMESPACE, + base_currency='eth', + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-10-15', utc=True), + ) + log.info('saved perf stats: {}'.format(out)) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 782b7ec8..3ba1b5ea 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -43,6 +43,7 @@ from catalyst.finance.execution import MarketOrder from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator +from catalyst.marketplace.marketplace import Marketplace from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest @@ -92,6 +93,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): attempts=self.attempts, ) + self._marketplace = None + @staticmethod def __convert_order_params_for_blotter(limit_price, stop_price, style): """ @@ -167,6 +170,16 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): """ return round_nearest(amount, asset.min_trade_size) + @api_method + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + if self._marketplace is None: + self._marketplace = Marketplace() + + return self._marketplace.get_data_source( + data_source_name, data_frequency, start, end, + ) + @api_method @preprocess(symbol_str=ensure_upper_case) def symbol(self, symbol_str, exchange_name=None): diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index c71cef1c..041fb004 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,19 +1,22 @@ import json import os - -import bcolz -import pandas as pd import shutil +import bcolz +import logbook +import pandas as pd +import six from web3 import Web3, HTTPProvider +from catalyst.constants import ROOT_DIR, LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.constants import ROOT_DIR from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ - get_data_source, get_bundle_folder, get_data_source_folder +from catalyst.marketplace.utils.path_utils import get_data_source, \ + get_bundle_folder, get_data_source_folder +# TODO: host our own node on aws? REMOTE_NODE = 'http://localhost:7545' +# TODO: read from GitHub CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) @@ -21,6 +24,8 @@ CONTRACT_ADDRESS = Web3.toChecksumAddress( '0xe2b6cf3863240892d59664d209a28289a73ef644' ) +log = logbook.Logger('Marketplace', level=LOG_LEVEL) + class Marketplace: def __init__(self): @@ -43,21 +48,33 @@ class Marketplace: desc='The marketcap value in USD.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily'], ), dict( name='GitHub', desc='The rate of development activity on GitHub.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour'], ), dict( name='Influencers', desc='Tweets and related sentiments by selected influencers.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour', 'minute'], ), ] + def get_data_source_def(self, data_source_name): + data_source_name = data_source_name.lower() + dsm = self.get_data_sources_map() + + ds = six.next( + (d for d in dsm if d['name'].lower() == data_source_name), None + ) + return ds + def list(self): subscribers = self.contract.call( {'from': self.default_account} @@ -136,6 +153,35 @@ class Marketplace: pass + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + ds_def = self.get_data_source_def(data_source_name) + freqs = ds_def['data_frequencies'] + data_frequency = freqs[0] + + if len(freqs) > 1: + log.warn( + 'no data frequencies specified for data source {}, ' + 'selected the first one by default: {}'.format( + data_source_name, data_frequency + ) + ) + + # TODO: filter ctable by start and end date + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + z = bcolz.ctable(rootdir=bundle_folder, mode='r') + + df = z.todataframe() # type: pd.DataFrame + df.set_index(['date', 'symbol'], drop=False, inplace=True) + + if start and end is None: + df = df.xs(start, level=0) + + return df + def clean(self, data_source_name, data_frequency=None): data_source_name = data_source_name.lower() @@ -143,7 +189,7 @@ class Marketplace: folder = get_data_source_folder(data_source_name) else: - forlder = get_bundle_folder(data_source_name, data_frequency) + folder = get_bundle_folder(data_source_name, data_frequency) shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 916722da..532f46ce 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -18,9 +18,9 @@ def merge_bundles(zsource, ztarget): """ # TODO: find a way to do this iteratively instead of in-memory df_source = zsource.todataframe() - df_source.set_index('last_updated', drop=False, inplace=True) + df_source.set_index('date', drop=False, inplace=True) df_target = ztarget.todataframe() - df_target.set_index('last_updated', drop=False, inplace=True) + df_target.set_index('date', drop=False, inplace=True) df = df_target.merge( right=df_source, diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 6d301880..890eaa20 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -34,8 +34,7 @@ def get_data_source_folder(data_source_name, environ=None): def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) - subfolder = data_frequency if data_frequency is not None else 'data' - bundle_folder = os.path.join(data_source_folder, subfolder) + bundle_folder = os.path.join(data_source_folder, data_frequency) ensure_directory(bundle_folder) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 68990791..9292b3a0 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,10 +16,17 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() + ds_def = marketplace.get_data_source_def('Marketcap') + marketplace.ingest( data_source_name='Marketcap', - data_frequency='finest', - start=pd.Timestamp.utcnow(), + data_frequency=ds_def['data_frequencies'][0], + start=pd.to_datetime('2017-10-01'), force_download=True, ) pass + + def test_clean(self): + marketplace = Marketplace() + marketplace.clean('marketcap') + pass From 76e183b5f7fc6fb6548ff19da4245a701564a90b Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 12:48:42 -0700 Subject: [PATCH 13/41] BLD: contract address+abi on testnet --- catalyst/marketplace/contract_abi.json | 1 + catalyst/marketplace/contract_address.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 catalyst/marketplace/contract_abi.json create mode 100644 catalyst/marketplace/contract_address.txt diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_abi.json new file mode 100644 index 00000000..a3dda75f --- /dev/null +++ b/catalyst/marketplace/contract_abi.json @@ -0,0 +1 @@ +[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_address.txt new file mode 100644 index 00000000..ec7bf029 --- /dev/null +++ b/catalyst/marketplace/contract_address.txt @@ -0,0 +1 @@ +0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file From 04fed4140c9e3e37ca1c80b38c0a92f0665bfe12 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 14:12:59 -0700 Subject: [PATCH 14/41] BLD: sourcing contract address+abi from github --- catalyst/marketplace/marketplace.py | 50 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 041fb004..a28d1c9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,7 @@ import json import os import shutil +import urllib import bcolz import logbook @@ -15,31 +16,44 @@ from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder # TODO: host our own node on aws? -REMOTE_NODE = 'http://localhost:7545' -# TODO: read from GitHub -CONTRACT_PATH = os.path.join( - ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' -) -CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xe2b6cf3863240892d59664d209a28289a73ef644' -) +# TODO: switch to mainnet +REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_address.txt' + +CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - with open(CONTRACT_PATH) as handle: - json_interface = json.load(handle) - w3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, - abi=json_interface['abi'], - ) # Type: Contract - self.default_account = w3.eth.accounts[1] + contract_url = urllib.urlopen(CONTRACT_PATH) + CONTRACT_ADDRESS = Web3.toChecksumAddress( + contract_url.readline().strip()) - pass + abi_url = urllib.urlopen(CONTRACT_ABI) + abi = json.load(abi_url) + + w3 = Web3(HTTPProvider(REMOTE_NODE)) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=abi, + ) # Type: Contract + + # TODO: Set default address correctly from user-provided config + DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) + if DEFAULT_ETH_ADDRESS is None: + raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' + 'environment variable. Quitting...') + + self.default_account = DEFAULT_ETH_ADDRESS + pass def get_data_sources_map(self): return [ @@ -93,7 +107,7 @@ class Marketplace: dict( id=index, subscribed=index in subscribed, - **data_source, + **data_source ) ) From 2ec9aa2ca96c8121b36d53868c5c68529ee7180e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 11:55:25 -0700 Subject: [PATCH 15/41] BLD: CLI implementation for the marketplace --- catalyst/__main__.py | 197 ++++++++++++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 69 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f70b985..10c6c6ac 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -574,75 +574,6 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) -@main.command(name='ls-data') -@click.pass_context -def ls_data(ctx): - click.echo( - 'The alternative data sources:' - ) - marketplace = Marketplace() - marketplace.list() - click.echo('Done') - - -@main.command(name='register-data') -@click.argument('data_source_name') -@click.pass_context -def register_data(ctx, data_source_name): - click.echo( - 'Registering to data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.register(data_source_name) - click.echo('Done') - - -@main.command(name='ingest-data') -@click.argument('data_source_name') -@click.option( - '-f', - '--data-frequency', - type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), - default='daily', - show_default=True, - help='The data frequency of the desired OHLCV bars.', -) -@click.option( - '-s', - '--start', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The start date of the data range. (default: one year from end date)', -) -@click.option( - '-e', - '--end', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The end date of the data range. (default: today)', -) -@click.pass_context -def ingest_data(ctx, data_source_name, data_frequency, start, end): - click.echo( - 'Ingesting data: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.ingest(data_source_name, data_frequency, start, end) - click.echo('Done') - - -@main.command(name='clean-data') -@click.argument('data_source_name') -@click.pass_context -def clean_data(ctx, data_source_name): - click.echo( - 'Cleaning data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.clean(data_source_name) - click.echo('Done') - - @main.command(name='clean-algo') @click.option( '-n', @@ -810,5 +741,133 @@ def bundles(): click.echo("%s %s" % (bundle, timestamp)) +@main.group() +@click.pass_context +def marketplace(ctx): + pass + + +@marketplace.command() +@click.pass_context +def ls(ctx): + click.echo('Listing of available data sources on the marketplace:') + marketplace = Marketplace() + marketplace.list() + click.echo('Functionality not yet implemented.') + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def subscribe(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + marketplace = Marketplace() + marketplace.subscribe(dataset) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest(ctx, dataset, data_frequency, start, end): + if dataset is None: + ctx.fail("must specify a dataset to clean with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Ingesting data: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.ingest(dataset, data_frequency, start, end) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def clean(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to ingest with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Cleaning data source: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.clean(dataset) + click.echo('Done') + + +@marketplace.command() +@click.pass_context +def register(ctx): + marketplace = Marketplace() + marketplace.register() + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the Marketplace dataset to publish data for.', +) +@click.option( + '--datadir', + default=None, + help='The folder that contains the CSV data files to publish.', +) +@click.option( + '--watch/--no-watch', + is_flag=True, + default=False, + help='Whether to watch the datadir for live data.', +) +@click.pass_context +def publish(ctx, dataset, datadir, watch): + marketplace = Marketplace() + if dataset is None: + ctx.fail("must specify a dataset to publish data for " + " with '--dataset'\n") + if datadir is None: + ctx.fail("must specify a datadir where to find the files to publish " + " with '--datadir'\n") + marketplace.publish(dataset, datadir, watch) + + if __name__ == '__main__': main() From 940f625ec1fe37995193e30a783844cb16f3db11 Mon Sep 17 00:00:00 2001 From: Avishai Weingarten <33716232+AvishaiW@users.noreply.github.com> Date: Thu, 25 Jan 2018 16:02:44 +0200 Subject: [PATCH 16/41] fix for click.echo added sys.stdout to click.echo to prevent errors on jupyter --- catalyst/__main__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 10c6c6ac..411f712f 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -3,6 +3,7 @@ import os from functools import wraps import click +import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -258,7 +259,7 @@ def run(ctx, if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") - click.echo('Running in backtesting mode.') + click.echo('Running in backtesting mode.', sys.stdout) perf = _run( initialize=None, @@ -290,7 +291,7 @@ def run(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -443,10 +444,10 @@ def live(ctx, ctx.fail("must specify a capital base with '--capital-base'") if simulate_orders: - click.echo('Running in paper trading mode.') + click.echo('Running in paper trading mode.', sys.stdout) else: - click.echo('Running in live trading mode.') + click.echo('Running in live trading mode.', sys.stdout) perf = _run( initialize=None, @@ -478,7 +479,7 @@ def live(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -560,7 +561,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -583,10 +584,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def clean_algo(ctx, algo_namespace): click.echo( - 'Cleaning algo state: {}'.format(algo_namespace) + 'Cleaning algo state: {}'.format(algo_namespace), + sys.stdout ) delete_algo_folder(algo_namespace) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command(name='clean-exchange') @@ -613,11 +615,11 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command() @@ -738,7 +740,7 @@ def bundles(): # because there were no entries, print a single message indicating that # no ingestions have yet been made. for timestamp in ingestions or [""]: - click.echo("%s %s" % (bundle, timestamp)) + click.echo("%s %s" % (bundle, timestamp), sys.stdout) @main.group() From ff989ba52405e344c5365d9b6430afeb4490f425 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 22:39:54 -0700 Subject: [PATCH 17/41] BLD: register dataset in marketplace --- catalyst/__main__.py | 17 ++- catalyst/marketplace/marketplace.py | 140 +++++++++++++++++---- catalyst/marketplace/marketplace_errors.py | 24 ++++ catalyst/marketplace/utils/path_utils.py | 56 +++++++++ 4 files changed, 205 insertions(+), 32 deletions(-) create mode 100644 catalyst/marketplace/marketplace_errors.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 411f712f..3833f3e4 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,9 +1,9 @@ import errno import os +import sys from functools import wraps import click -import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -752,10 +752,11 @@ def marketplace(ctx): @marketplace.command() @click.pass_context def ls(ctx): - click.echo('Listing of available data sources on the marketplace:') + click.echo('Listing of available data sources on the marketplace:', + sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.') + click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() @@ -808,9 +809,7 @@ def ingest(ctx, dataset, data_frequency, start, end): ctx.fail("must specify a dataset to clean with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Ingesting data: {}'.format(dataset) - ) + click.echo('Ingesting data: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.ingest(dataset, data_frequency, start, end) @@ -827,12 +826,10 @@ def clean(ctx, dataset): ctx.fail("must specify a dataset to ingest with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Cleaning data source: {}'.format(dataset) - ) + click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.clean(dataset) - click.echo('Done') + click.echo('Done', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a28d1c9f..4b1cfe8b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,23 +1,31 @@ import json import os import shutil -import urllib +import sys +import binascii import bcolz import logbook import pandas as pd import six from web3 import Web3, HTTPProvider -from catalyst.constants import ROOT_DIR, LOG_LEVEL +from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder + get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr + +if sys.version_info.major < 3: + import urllib +else: + import urllib.request as urllib # TODO: host our own node on aws? # TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' +REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ @@ -32,28 +40,28 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - contract_url = urllib.urlopen(CONTRACT_PATH) - CONTRACT_ADDRESS = Web3.toChecksumAddress( + self.addresses = get_user_pubaddr() + + if self.addresses[0]['pubAddr'] == '': + raise MarketplacePubAddressEmpty( + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) + self.default_account = self.addresses[0]['pubAddr'] + + contract_url = urllib.urlopen(CONTRACT_PATH) + self.contract_address = Web3.toChecksumAddress( contract_url.readline().strip()) abi_url = urllib.urlopen(CONTRACT_ABI) abi = json.load(abi_url) - w3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, + self.contract = self.web3.eth.contract( + self.contract_address, abi=abi, - ) # Type: Contract - - # TODO: Set default address correctly from user-provided config - DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) - if DEFAULT_ETH_ADDRESS is None: - raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' - 'environment variable. Quitting...') - - self.default_account = DEFAULT_ETH_ADDRESS - pass + ) def get_data_sources_map(self): return [ @@ -122,11 +130,11 @@ class Marketplace: pass - def register(self, data_source_name): + def subscribe(self, dataset): data_sources = self.get_data_sources_map() index = next( (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == data_source_name.lower()), + d['name'].lower() == dataset.lower()), None ) if index is None: @@ -140,7 +148,7 @@ class Marketplace: ).subscribe(index) print( 'Subscribed to data source {} successfully'.format( - data_source_name + dataset ) ) @@ -207,3 +215,91 @@ class Marketplace: shutil.rmtree(folder) pass + + def register(self): + dataset = input('Enter the name of the dataset to register: ') + price = int(input('Enter the price for a monthly subscription to ' + 'this dataset in ENG: ')) + + # while True: + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): + # print("Not a valid frequency.") + # else: + # break + + # while True: + # reg_pub = input('Can data be published every hour at a regular ' + # 'time? [default: Y]: ') or 'y' + # if reg_pub.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if reg_pub.lower() == 'y': + # reg_pub = True + # else: + # reg_pub = False + # break + + # while True: + # hist = input('Does it include historical data? ' + # '[default: Y]') or 'y' + # if hist.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if hist.lower() == 'y': + # hist = True + # else: + # hist = False + # break + + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose the your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + tx = self.contract.functions.register( + binascii.hexlify(dataset.encode('utf-8')), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') + + tx_hash = '0x{}'.format(binascii.hexlify( + self.web3.eth.sendRawTransaction(signed_tx) + ).decode('utf-8')) + + print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py new file mode 100644 index 00000000..04fca091 --- /dev/null +++ b/catalyst/marketplace/marketplace_errors.py @@ -0,0 +1,24 @@ +import sys +import traceback + +from catalyst.errors import ZiplineError + + +def silent_except_hook(exctype, excvalue, exctraceback): + if exctype in [MarketplacePubAddressEmpty]: + fn = traceback.extract_tb(exctraceback)[-1][0] + ln = traceback.extract_tb(exctraceback)[-1][1] + print("Error traceback: {1} (line {2})\n" + "{0.__name__}: {3}".format(exctype, fn, ln, excvalue)) + else: + sys.__excepthook__(exctype, excvalue, exctraceback) + + +sys.excepthook = silent_except_hook + + +class MarketplacePubAddressEmpty(ZiplineError): + msg = ( + 'Please enter your public address to use in the Data Marketplace ' + 'in the following file: {filename}' + ).strip() diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 890eaa20..2cee9303 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,4 +1,5 @@ import os +import json import tarfile import shutil @@ -7,6 +8,29 @@ from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory +def get_marketplace_folder(environ=None): + """ + The root path of the marketplace folder. + + Parameters + ---------- + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + marketplace_folder = os.path.join(root, 'marketplace') + ensure_directory(marketplace_folder) + + return marketplace_folder + + def get_data_source_folder(data_source_name, environ=None): """ The root path of an data_source folder. @@ -104,3 +128,35 @@ def get_data_source(data_source_name, period, force_download=False): tar.extractall(path) return path + + +def get_user_pubaddr(environ=None): + """ + The de-serialized contend of the user's addresses.json file. + Parameters + ---------- + environ: + + Returns + ------- + Object + + """ + marketplace_folder = get_marketplace_folder(environ) + + filename = os.path.join(marketplace_folder, 'addresses.json') + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + try: + d = data[0]['pubAddr'] + except Exception as e: + return [data, ] + return data + else: + data = dict(pubAddr='', desc='') + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + return data From 68faad40989bd5b37e56eb9d45bca9083feb72fb Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 26 Jan 2018 15:32:53 -0700 Subject: [PATCH 18/41] BLD: publish dataset in marketplace --- catalyst/marketplace/marketplace.py | 84 +++++++++++++++++++--- catalyst/marketplace/marketplace_errors.py | 29 +++++++- catalyst/marketplace/utils/auth_utils.py | 47 ++++++++++++ catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/eth_utils.py | 33 +++++++++ catalyst/marketplace/utils/path_utils.py | 26 ++++++- 6 files changed, 210 insertions(+), 13 deletions(-) create mode 100644 catalyst/marketplace/utils/auth_utils.py create mode 100644 catalyst/marketplace/utils/eth_utils.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4b1cfe8b..af4c4aef 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,22 +1,31 @@ -import json import os -import shutil import sys +import json +import hmac +import glob +import time +import shutil +import hashlib -import binascii import bcolz import logbook import pandas as pd import six +import requests from web3 import Web3, HTTPProvider from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty +from catalyst.marketplace.marketplace_errors import ( + MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: import urllib @@ -34,6 +43,8 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' +AUTH_SERVER = 'http://localhost:5000' + log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -259,7 +270,7 @@ class Marketplace: self.addresses[i]['pubAddr'], self.addresses[i]['desc']) ) - address_i = int(input('Choose the your address associated with ' + address_i = int(input('Choose your address associated with ' 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( @@ -270,7 +281,7 @@ class Marketplace: break tx = self.contract.functions.register( - binascii.hexlify(dataset.encode('utf-8')), + bytes32(dataset), price, address, ).buildTransaction( @@ -298,8 +309,63 @@ class Marketplace: signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') - tx_hash = '0x{}'.format(binascii.hexlify( - self.web3.eth.sendRawTransaction(signed_tx) - ).decode('utf-8')) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + def publish(self, dataset, datadir, watch): + + datasource = self.contract.functions.getDataSource( + bytes32(dataset)).call() + + if not datasource[4]: + raise MarketplaceDatasetNotFound(dataset=dataset) + + match = next((l for l in self.addresses if + l['pubAddr'] == datasource[0]), None) + + if not match: + raise MarketplaceNoAddressMatch( + dataset=dataset, + address=datasource[0]) + + print('Using address: {} to publish this dataset.'.format( + datasource[0])) + + if 'key' in match: + key = match['key'] + secret = match['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + nonce.encode('utf-8'), + hashlib.sha512).hexdigest() + headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + filenames = glob.glob(os.path.join(datadir, '*.csv')) + + if not filenames: + raise MarketplaceNoCSVFiles(datadir=datadir) + + files = [] + for file in filenames: + files.append(('file', open(file, 'rb'))) + + r = requests.post('{}/publish'.format(AUTH_SERVER), + files=files, + headers=headers) + + if r.status_code != 200: + raise MarketplaceHTTPRequest(request='upload file', + error=r.status_code) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest(request='upload file', + error=r.json()['error']) + + print('Dataset {} published successfully.'.format(dataset)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 04fca091..3992158d 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -5,7 +5,9 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): - if exctype in [MarketplacePubAddressEmpty]: + if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -22,3 +24,28 @@ class MarketplacePubAddressEmpty(ZiplineError): 'Please enter your public address to use in the Data Marketplace ' 'in the following file: {filename}' ).strip() + + +class MarketplaceDatasetNotFound(ZiplineError): + msg = ( + 'The dataset "{dataset}" is not registered in the Data Marketplace.' + ).strip() + + +class MarketplaceNoAddressMatch(ZiplineError): + msg = ( + 'The address registered with the dataset {dataset}: {address} ' + 'does not match any of your addresses.' + ).strip() + + +class MarketplaceHTTPRequest(ZiplineError): + msg = ( + 'Request to remote server to {request} failed: {error}' + ).strip() + + +class MarketplaceNoCSVFiles(ZiplineError): + msg = ( + 'No CSV files found on {datadir} to upload.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py new file mode 100644 index 00000000..f1193758 --- /dev/null +++ b/catalyst/marketplace/utils/auth_utils.py @@ -0,0 +1,47 @@ +import requests + +from catalyst.marketplace.marketplace_errors import ( + MarketplaceHTTPRequest) +from catalyst.marketplace.utils.path_utils import ( + get_user_pubaddr, save_user_pubaddr) + +AUTH_SERVER = 'http://localhost:5000' + + +def get_key_secret(pubAddr, dataset): + """ + Obtain a new key/secret pair from authentication server + + Parameters + ---------- + pubAddr: str + dataset: str + + Returns + ------- + key: str + secret: str + + """ + session = requests.Session() + response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + headers={ + 'pubAddr': pubAddr, + 'dataset': dataset}) + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + + return match['key'], match['secret'] diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 532f46ce..b58595ac 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -1,8 +1,8 @@ -import bcolz import os -import pandas as pd import shutil +import bcolz + def merge_bundles(zsource, ztarget): """ diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py new file mode 100644 index 00000000..c2be4bd6 --- /dev/null +++ b/catalyst/marketplace/utils/eth_utils.py @@ -0,0 +1,33 @@ +import binascii + + +def bytes32(string): + """ + Convert string to bytes32 data type for smart contract + + Parameters + ---------- + string: str + + Returns + ------- + list + + """ + return binascii.hexlify(string.encode('utf-8')) + + +def b32_str(bytes32): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(bytes32.decode('utf-8')) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 2cee9303..62f8beac 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,6 +133,7 @@ def get_data_source(data_source_name, period, force_download=False): def get_user_pubaddr(environ=None): """ The de-serialized contend of the user's addresses.json file. + Parameters ---------- environ: @@ -143,7 +144,6 @@ def get_user_pubaddr(environ=None): """ marketplace_folder = get_marketplace_folder(environ) - filename = os.path.join(marketplace_folder, 'addresses.json') if os.path.isfile(filename): @@ -160,3 +160,27 @@ def get_user_pubaddr(environ=None): json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) return data + + +def save_user_pubaddr(data, environ=None): + """ + Saves the user's public addresses and their related metadata in + the corresponding addresses.json file. + + Parameters + ---------- + data: dict + + Returns + ------- + True + + """ + marketplace_folder = get_marketplace_folder(environ) + filename = os.path.join(marketplace_folder, 'addresses.json') + + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + + return True From c26d78cf0556a5807409deec1e55f20cd2ab001d Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 30 Jan 2018 11:20:06 -0700 Subject: [PATCH 19/41] BLD: set AUTH_SERVER as a Catalyst constant --- catalyst/constants.py | 2 ++ catalyst/marketplace/marketplace.py | 4 +--- catalyst/marketplace/utils/auth_utils.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 5668bc83..abfa1974 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -21,3 +21,5 @@ except Exception as e: print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False + +AUTH_SERVER = 'https://data.enigma.co' diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index af4c4aef..bbd0be9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,7 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL +from catalyst.constants import LOG_LEVEL, AUTH_SERVER from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -43,8 +43,6 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' -AUTH_SERVER = 'http://localhost:5000' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index f1193758..35fa3504 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -4,8 +4,7 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) - -AUTH_SERVER = 'http://localhost:5000' +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): From 0f55f0e9a6db6439fc6436f31f1bb3a215bbd6ad Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 10:50:10 -0700 Subject: [PATCH 20/41] BLD: passing dataset to marketplace publish request --- catalyst/marketplace/marketplace.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index bbd0be9f..f276d997 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -341,9 +341,13 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - nonce.encode('utf-8'), + '{}{}'.format(dataset,nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} filenames = glob.glob(os.path.join(datadir, '*.csv')) From d1cd95c492032877cb1db5e94a3fea73ac72bc06 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 13:39:59 -0700 Subject: [PATCH 21/41] BLD: marketplace subscribe + refactoring --- catalyst/marketplace/marketplace.py | 144 ++++++++++++++---------- catalyst/marketplace/utils/eth_utils.py | 2 +- 2 files changed, 88 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index f276d997..e0c6b6b0 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,6 +6,7 @@ import glob import time import shutil import hashlib +import binascii import bcolz import logbook @@ -97,6 +98,52 @@ class Marketplace: ), ] + def choose_pubaddr(self): + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + return address + + def sign_transaction(self, from_address, tx): + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + + if signed_tx.startswith('0x'): + signed_tx = signed_tx[2:] + + return signed_tx + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -137,35 +184,51 @@ class Marketplace: ) print(df.to_string(formatters=formatters)) - pass - def subscribe(self, dataset): - data_sources = self.get_data_sources_map() - index = next( - (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == dataset.lower()), - None - ) - if index is None: - raise ValueError( - 'Data source not found.' - ) + + # data_sources = self.get_data_sources_map() + # index = next( + # (index for (index, d) in enumerate(data_sources) if + # d['name'].lower() == dataset.lower()), + # None + # ) + + # Check if dataset exists, if not throw exception + # if index is None: + # raise ValueError( + # 'Data source not found.' + # ) + + address = self.choose_pubaddr() + + tx = self.contract.functions.subscribe( + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = 4700000 + + signed_tx = self.sign_transaction(address, tx) try: - self.contract.transact( - {'from': self.default_account} - ).subscribe(index) - print( - 'Subscribed to data source {} successfully'.format( - dataset - ) - ) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) - pass - def ingest(self, data_source_name, data_frequency=None, start=None, end=None, force_download=False): data_source_name = data_source_name.lower() @@ -261,22 +324,7 @@ class Marketplace: # hist = False # break - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + address = self.choose_pubaddr() tx = self.contract.functions.register( bytes32(dataset), @@ -287,25 +335,7 @@ class Marketplace: tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas - print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' - 'enter the following parameters:\n\n' - 'From Address:\t\t{_from}\n' - 'To Address:\t\t{to}\n' - 'Value / Amount to Send:\t{value}\n' - 'Gas Limit:\t\t{gas}\n' - 'Nonce:\t\t\t{nonce}\n' - 'Data:\t\t\t{data}\n'.format( - _from=address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) - - signed_tx = input('Copy and Paste the "Signed Transaction" ' - 'field here:\n') + signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(b32_str( self.web3.eth.sendRawTransaction(signed_tx))) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index c2be4bd6..fd6ba1e1 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,4 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32.decode('utf-8')) + return binascii.hexlify(bytes32).decode('utf-8') From e906315969529e9a49a4baf81a8abd1f2b669012 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 22:06:52 -0700 Subject: [PATCH 22/41] BLD: marketplace: enigma contract files --- catalyst/marketplace/contract_enigma_abi.json | 302 ++++++++++++++++++ .../marketplace/contract_enigma_address.txt | 1 + ...abi.json => contract_marketplace_abi.json} | 0 ...s.txt => contract_marketplace_address.txt} | 0 4 files changed, 303 insertions(+) create mode 100644 catalyst/marketplace/contract_enigma_abi.json create mode 100644 catalyst/marketplace/contract_enigma_address.txt rename catalyst/marketplace/{contract_abi.json => contract_marketplace_abi.json} (100%) rename catalyst/marketplace/{contract_address.txt => contract_marketplace_address.txt} (100%) diff --git a/catalyst/marketplace/contract_enigma_abi.json b/catalyst/marketplace/contract_enigma_abi.json new file mode 100644 index 00000000..4dd70538 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_abi.json @@ -0,0 +1,302 @@ +[ + { + "constant": true, + "inputs": [], + "name": "name", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_from", + "type": "address" + }, + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "INITIAL_SUPPLY", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "decimals", + "outputs": [ + { + "name": "", + "type": "uint8" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_subtractedValue", + "type": "uint256" + } + ], + "name": "decreaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [], + "name": "getAfterApproveTest", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "name": "balance", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "symbol", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_addedValue", + "type": "uint256" + } + ], + "name": "increaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + }, + { + "name": "_spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "name": "testValue", + "type": "address" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "from", + "type": "address" + }, + { + "indexed": true, + "name": "to", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + } + ] \ No newline at end of file diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt new file mode 100644 index 00000000..157e2c23 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -0,0 +1 @@ +0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_marketplace_abi.json similarity index 100% rename from catalyst/marketplace/contract_abi.json rename to catalyst/marketplace/contract_marketplace_abi.json diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_marketplace_address.txt similarity index 100% rename from catalyst/marketplace/contract_address.txt rename to catalyst/marketplace/contract_marketplace_address.txt From a04a99373d6d7276301f335f655314e5f90fe6fd Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 08:07:17 -0700 Subject: [PATCH 23/41] BLD: marketplace: subscription to dataset --- catalyst/marketplace/marketplace.py | 268 ++++++++++++++++----- catalyst/marketplace/marketplace_errors.py | 19 +- catalyst/marketplace/utils/eth_utils.py | 19 +- 3 files changed, 248 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e0c6b6b0..d6bf5293 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,7 +6,6 @@ import glob import time import shutil import hashlib -import binascii import bcolz import logbook @@ -20,12 +19,13 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles) + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: @@ -38,11 +38,23 @@ else: REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github -CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_address.txt' +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' -CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -59,17 +71,31 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - contract_url = urllib.urlopen(CONTRACT_PATH) - self.contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) - - abi_url = urllib.urlopen(CONTRACT_ABI) - abi = json.load(abi_url) - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = self.web3.eth.contract( - self.contract_address, + contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) + + self.mkt_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) + abi = json.load(abi_url) + + self.mkt_contract = self.web3.eth.contract( + self.mkt_contract_address, + abi=abi, + ) + + contract_url = urllib.urlopen(ENIGMA_CONTRACT) + + self.eng_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) + abi = json.load(abi_url) + + self.eng_contract = self.web3.eth.contract( + self.eng_contract_address, abi=abi, ) @@ -137,7 +163,8 @@ class Marketplace: ) ) - signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') if signed_tx.startswith('0x'): signed_tx = signed_tx[2:] @@ -186,57 +213,185 @@ class Marketplace: def subscribe(self, dataset): - # data_sources = self.get_data_sources_map() - # index = next( - # (index for (index, d) in enumerate(data_sources) if - # d['name'].lower() == dataset.lower()), - # None - # ) - - # Check if dataset exists, if not throw exception - # if index is None: - # raise ValueError( - # 'Data source not found.' - # ) - address = self.choose_pubaddr() - tx = self.contract.functions.subscribe( + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + price = dataset_info[1] + + print('\nThe price for a monthly subscription to this dataset is' + ' {} ENG'.format(price)) + + print('Checking that the ENG balance in {} is greater than ' + '{} ENG... '.format(address, price), end='') + + balance = self.web3.eth.call({ + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) + + balance = int(balance[2:], 16) // 10 ** 8 + + if balance > price: + print('OK.') + else: + print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower ' + 'than the price of the dataset that you are trying to\n' + 'buy: {} ENG. Get enough ENG to cover the costs of the ' + 'monthly\nsubscription for what you are trying to buy, ' + 'and try again.'.format(address, balance, price)) + return + + while True: + agree_pay = input('Please confirm that you agree to pay {} ENG ' + 'for a monthly subscription to the dataset "{}" ' + 'starting today. [default: Y] '.format( + price, dataset)) or 'y' + if agree_pay.lower() not in ('y', 'n'): + print("Please answer Y or N.") + else: + if agree_pay.lower() == 'y': + break + else: + return + + print('Ready to subscribe to dataset {}.\n'.format(dataset)) + print('In order to execute the subscription, you will need to sign ' + 'two different transactions:\n' + '1. First transaction is to authorize the Marketplace contract ' + 'to spend {} ENG on your behalf.\n' + '2. Second transaction is the actual subscription for the ' + 'desired dataset'.format(price)) + + tx = self.eng_contract.functions.approve( + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) + + signed_tx = self.sign_transaction(address, tx) + + try: + tx_hash = '0x{}'.format(bin_hex( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + return + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + + print('Waiting for the first transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nFirst transaction successful!\n' + 'Now processing second transaction.') + + tx = self.mkt_contract.functions.subscribe( bytes32(dataset), ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in REMOTE_NODE: - tx['gas'] = 4700000 + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) try: - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) - if 'ropsten' in REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}'.format(etherscan)) - except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) + return - def ingest(self, data_source_name, data_frequency=None, start=None, + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) + + print('Waiting for the second transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nSecond transaction successful!\n' + 'You have successfully subscribed to dataset {} with' + 'address {}.\n' + 'You can now ingest this dataset anytime during the ' + 'next month by running the following command:\n' + 'catalyst marketplace ingest --dataset={}'.format( + dataset, address, dataset)) + + def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - data_source_name = data_source_name.lower() + + address = self.choose_pubaddr() + + check_sub = self.mkt_contract.functions.checkAddressSubscription( + address, bytes32(dataset)).call() + + if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + raise MarketplaceContractDataNoMatch( + params='address: {}, dataset: {}'.format( + address, dataset)) + + if not check_sub[5]: + raise MarketplaceSubscriptionExpired( + dataset=dataset, + date=check_sub[4]) + + # WIP + + dataset = dataset.lower() period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(data_source_name, period, force_download) + tmp_folder = get_data_source(dataset, period, force_download) - bundle_folder = get_bundle_folder(data_source_name, data_frequency) + bundle_folder = get_bundle_folder(dataset, data_frequency) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') @@ -294,10 +449,10 @@ class Marketplace: 'this dataset in ENG: ')) # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): # print("Not a valid frequency.") - # else: + # else: # break # while True: @@ -326,25 +481,26 @@ class Marketplace: address = self.choose_pubaddr() - tx = self.contract.functions.register( + tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): - datasource = self.contract.functions.getDataSource( + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() if not datasource[4]: @@ -371,11 +527,11 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset,nonce).encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, - 'Key': key, + headers = {'Sign': signature, + 'Key': key, 'Nonce': nonce, 'Dataset': dataset} diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 3992158d..ed3d0fa4 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -7,7 +7,8 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles]: + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -49,3 +50,19 @@ class MarketplaceNoCSVFiles(ZiplineError): msg = ( 'No CSV files found on {datadir} to upload.' ) + + +class MarketplaceContractDataNoMatch(ZiplineError): + msg = ( + 'The information found on the contract does not match the ' + 'requested data:\n{params}.' + ) + + +class MarketplaceSubscriptionExpired(ZiplineError): + msg = ( + 'Your subscription to dataset "{dataset}" expired on {date} ' + 'and is no longer active. You have to subscribe again running the ' + 'following command:\n' + 'catalyst marketplace subscribe --dataset={dataset}' + ) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index fd6ba1e1..f5fe8b7b 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,21 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32).decode('utf-8') + return binascii.unhexlify( + bytes32.decode('utf-8').rstrip('\0')).decode('ascii') + + +def bin_hex(binary): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(binary).decode('utf-8') From 18123d7f7e75514dd0194f0e406bc99513bfde20 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 11:47:25 -0700 Subject: [PATCH 24/41] BLD: marketplace constants in constants.py --- catalyst/constants.py | 21 +++++ catalyst/marketplace/marketplace.py | 120 ++++++++++++++-------------- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index abfa1974..df0ea82b 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -23,3 +23,24 @@ except Exception as e: AUTO_INGEST = False AUTH_SERVER = 'https://data.enigma.co' + +# TODO: switch to mainnet +ETH_REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' \ No newline at end of file diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index d6bf5293..08249775 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,9 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL, AUTH_SERVER +from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, + MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, + ENIGMA_CONTRACT_ABI ) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -33,28 +35,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib -# TODO: host our own node on aws? -# TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' - -# TODO: move to MASTER branch on github -MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_address.txt' - -MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_abi.json' - -# TODO: switch to mainnet -ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_address.txt' - -ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_abi.json' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -71,7 +51,7 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) @@ -99,30 +79,30 @@ class Marketplace: abi=abi, ) - def get_data_sources_map(self): - return [ - dict( - name='Marketcap', - desc='The marketcap value in USD.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily'], - ), - dict( - name='GitHub', - desc='The rate of development activity on GitHub.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour'], - ), - dict( - name='Influencers', - desc='Tweets and related sentiments by selected influencers.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour', 'minute'], - ), - ] + # def get_data_sources_map(self): + # return [ + # dict( + # name='Marketcap', + # desc='The marketcap value in USD.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily'], + # ), + # dict( + # name='GitHub', + # desc='The rate of development activity on GitHub.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour'], + # ), + # dict( + # name='Influencers', + # desc='Tweets and related sentiments by selected influencers.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour', 'minute'], + # ), + # ] def choose_pubaddr(self): while True: @@ -142,7 +122,7 @@ class Marketplace: self.addresses[address_i]['pubAddr']) break - return address + return address, address_i def sign_transaction(self, from_address, tx): @@ -213,7 +193,7 @@ class Marketplace: def subscribe(self, dataset): - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -272,7 +252,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -287,7 +267,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -319,7 +299,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -334,7 +314,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -369,7 +349,7 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - address = self.choose_pubaddr() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( address, bytes32(dataset)).call() @@ -384,7 +364,31 @@ class Marketplace: dataset=dataset, date=check_sub[4]) - # WIP + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), + hashlib.sha512).hexdigest() + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} + + r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + + + print(r) + exit(0) + + dataset = dataset.lower() @@ -479,7 +483,7 @@ class Marketplace: # hist = False # break - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( bytes32(dataset), @@ -488,7 +492,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) From 56bd3db7a2f8ee2ec0c765f9e81215b223251a9f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 16:54:24 -0700 Subject: [PATCH 25/41] BLD: marketplace - ingestion downloads multiple files --- catalyst/marketplace/marketplace.py | 40 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 08249775..30860551 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,4 +1,5 @@ import os +import re import sys import json import hmac @@ -11,12 +12,13 @@ import bcolz import logbook import pandas as pd import six +from requests_toolbelt import MultipartDecoder import requests from web3 import Web3, HTTPProvider -from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, - MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, - ENIGMA_CONTRACT_ABI ) +from catalyst.constants import ( + LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, + MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -97,7 +99,7 @@ class Marketplace: # ), # dict( # name='Influencers', - # desc='Tweets and related sentiments by selected influencers.', + # desc='Tweets & related sentiments by selected influencers.', # start_date=pd.to_datetime('2017-01-01'), # end_date=pd.to_datetime('2018-01-15'), # data_frequencies=['daily', 'hour', 'minute'], @@ -349,6 +351,8 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): + dataset = dataset.lower() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -369,7 +373,7 @@ class Marketplace: secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) + key, secret = get_key_secret(address, dataset) nonce = str(int(time.time())) @@ -382,15 +386,27 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} - r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, stream=True) + if r.status_code == 200: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + else: + raise MarketplaceHTTPRequest(request='ingest dataset', + error=r.status_code) - print(r) + print('Download successful, now we need to ingest...') exit(0) - - - - dataset = dataset.lower() + # TODO: sync with Fred on the below and + # exiting for now period = start.strftime('%Y-%m-%d') tmp_folder = get_data_source(dataset, period, force_download) @@ -404,8 +420,6 @@ class Marketplace: else: os.rename(tmp_folder, bundle_folder) - pass - def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): data_source_name = data_source_name.lower() From 37980f35806c63fa6c87eedb1aaef12ade913e4e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 20:43:48 -0500 Subject: [PATCH 26/41] BLD: working on marketplace integration --- catalyst/marketplace/marketplace.py | 99 +++++++++++++-------------- etc/requirements_marketplace.txt | 2 +- tests/marketplace/test_marketplace.py | 3 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 30860551..4c55b2da 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -37,7 +37,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -48,9 +47,9 @@ class Marketplace: if self.addresses[0]['pubAddr'] == '': raise MarketplacePubAddressEmpty( - filename=os.path.join( - get_marketplace_folder(), 'addresses.json') - ) + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) self.default_account = self.addresses[0]['pubAddr'] self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) @@ -58,7 +57,7 @@ class Marketplace: contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) self.mkt_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -71,7 +70,7 @@ class Marketplace: contract_url = urllib.urlopen(ENIGMA_CONTRACT) self.eng_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) @@ -118,10 +117,10 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) + self.addresses[address_i]['pubAddr']) break return address, address_i @@ -136,14 +135,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -198,7 +197,7 @@ class Marketplace: address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() price = dataset_info[1] @@ -209,11 +208,11 @@ class Marketplace: '{} ENG... '.format(address, price), end='') balance = self.web3.eth.call({ - 'from': address, - 'to': self.eng_contract_address, - 'data': '0x70a08231000000000000000000000000{}'.format( - address[2:]) - }) + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) balance = int(balance[2:], 16) // 10 ** 8 @@ -231,7 +230,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -249,10 +248,10 @@ class Marketplace: 'desired dataset'.format(price)) tx = self.eng_contract.functions.approve( - self.mkt_contract_address, - price, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -261,7 +260,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -271,7 +270,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -297,9 +296,9 @@ class Marketplace: 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - bytes32(dataset), - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -308,7 +307,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -318,7 +317,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -346,7 +345,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): @@ -356,17 +355,17 @@ class Marketplace: address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(dataset)).call() if check_sub[0] != address or b32_str(check_sub[1]) != dataset: raise MarketplaceContractDataNoMatch( - params='address: {}, dataset: {}'.format( - address, dataset)) + params='address: {}, dataset: {}'.format( + address, dataset)) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=dataset, + date=check_sub[4]) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -500,11 +499,11 @@ class Marketplace: address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( - bytes32(dataset), - price, - address, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 @@ -512,14 +511,14 @@ class Marketplace: signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() if not datasource[4]: raise MarketplaceDatasetNotFound(dataset=dataset) @@ -529,11 +528,11 @@ class Marketplace: if not match: raise MarketplaceNoAddressMatch( - dataset=dataset, - address=datasource[0]) + dataset=dataset, + address=datasource[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + datasource[0])) if 'key' in match: key = match['key'] diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index cbb3a5a3..8062d192 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1 @@ -web3==4.0.0b5 \ No newline at end of file +web3==4.0.0b7 \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 9292b3a0..815ff885 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -11,7 +11,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_register(self): marketplace = Marketplace() - marketplace.register('GitHub') + marketplace.register() pass def test_ingest(self): @@ -30,3 +30,4 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass + From b9130fd968507d22b6e507e48255911e674403e4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 22:03:16 -0500 Subject: [PATCH 27/41] BLD: added missing requirement --- etc/requirements_marketplace.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index 8062d192..2a56c200 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1,2 @@ -web3==4.0.0b7 \ No newline at end of file +web3==4.0.0b7 +requests-toolbelt==0.8.0 From e4602900565d2998dc95d7f5bc3a4e935e28c5b0 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 21:57:13 -0700 Subject: [PATCH 28/41] BLD: better exception handling for failed multipart download --- catalyst/marketplace/marketplace.py | 65 ++++++++++++++++++----------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4c55b2da..43c15f19 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -13,6 +13,8 @@ import logbook import pandas as pd import six from requests_toolbelt import MultipartDecoder +from requests_toolbelt.multipart.decoder import \ + NonMultipartContentTypeException import requests from web3 import Web3, HTTPProvider @@ -106,22 +108,28 @@ class Marketplace: # ] def choose_pubaddr(self): - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses) - 1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + + if len(self.addresses) == 1: + address = self.addresses[0]['pubAddr'] + address_i = 0 + print('Using {} for this transaction.'.format(address)) + else: + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break return address, address_i @@ -385,19 +393,26 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} + print('Starting download of dataset for ingestion...') + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), headers=headers, stream=True) if r.status_code == 200: - decoder = MultipartDecoder.from_response(r) - for part in decoder.parts: - h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) - with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) - f.write(part.content) + try: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + except NonMultipartContentTypeException: + response = r.json() + raise MarketplaceHTTPRequest(request='ingest dataset', + error=response) else: raise MarketplaceHTTPRequest(request='ingest dataset', error=r.status_code) From 0decdf5aab874ab56d5889ffe77766e94c4b2e9e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 22:30:24 -0700 Subject: [PATCH 29/41] BLD: check for name duplicates when registering dataset --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 43c15f19..ced5bfde 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -160,6 +160,17 @@ class Marketplace: return signed_tx + def check_transaction(self, tx_hash): + + if 'ropsten' in ETH_REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('\nYou can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -276,14 +287,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in ETH_REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}\n\n'.format(etherscan)) + self.check_transaction(tx_hash) print('Waiting for the first transaction to succeed...') @@ -476,7 +480,20 @@ class Marketplace: pass def register(self): - dataset = input('Enter the name of the dataset to register: ') + while True: + dataset = input('Enter the name of the dataset to register: ') + dataset = dataset.lower() + + datasource = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if datasource[4]: + print('There is already a dataset registered under ' + 'the name "{}". Please choose a different ' + 'name.'.format(dataset)) + else: + break + price = int(input('Enter the price for a monthly subscription to ' 'this dataset in ENG: ')) @@ -520,8 +537,8 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: - tx['gas'] = 4700000 + if 'ropsten' in ETH_REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -530,6 +547,8 @@ class Marketplace: print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + self.check_transaction(tx_hash) + def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( From db4ee4c01ec3337da1c7695747ef4f6cb0293251 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 23:42:40 -0700 Subject: [PATCH 30/41] BLD: check for existence of dataset in ingestion & subscription --- catalyst/marketplace/marketplace.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ced5bfde..a59d5ed7 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -213,11 +213,18 @@ class Marketplace: def subscribe(self, dataset): + dataset = dataset.lower() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + price = dataset_info[1] print('\nThe price for a monthly subscription to this dataset is' @@ -364,6 +371,14 @@ class Marketplace: dataset = dataset.lower() + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -551,6 +566,8 @@ class Marketplace: def publish(self, dataset, datadir, watch): + dataset = dataset.lower() + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -607,4 +624,4 @@ class Marketplace: raise MarketplaceHTTPRequest(request='upload file', error=r.json()['error']) - print('Dataset {} published successfully.'.format(dataset)) + print('Dataset {} uploaded successfully.'.format(dataset)) From 9ef000e529c0dc6af15f9199eb17c0992d8db1ef Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Feb 2018 12:06:57 -0700 Subject: [PATCH 31/41] BUG: balance returns different types --- catalyst/marketplace/marketplace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a59d5ed7..828224ab 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -240,7 +240,10 @@ class Marketplace: address[2:]) }) - balance = int(balance[2:], 16) // 10 ** 8 + try: + balance = int(balance[2:], 16) // 10 ** 8 + except ValueError: + balance = int(bin_hex(balance), 16) // 10 ** 8 if balance > price: print('OK.') From 99dece953f157570f78e96404ce92a58bad046ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 16:20:33 -0500 Subject: [PATCH 32/41] BLD: ingesting marketplace bundles --- catalyst/marketplace/marketplace.py | 147 ++++++++++++++--------- catalyst/marketplace/utils/path_utils.py | 52 +++----- tests/marketplace/test_marketplace.py | 9 +- 3 files changed, 109 insertions(+), 99 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 828224ab..14721a82 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,21 +1,21 @@ +import glob +import hashlib +import hmac +import json import os import re -import sys -import json -import hmac -import glob -import time import shutil -import hashlib +import sys +import time import bcolz import logbook import pandas as pd +import requests import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException -import requests from web3 import Web3, HTTPProvider from catalyst.constants import ( @@ -27,12 +27,12 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ - get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex +from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ + get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr, get_temp_bundles_folder, extract_bundle if sys.version_info.major < 3: import urllib @@ -125,7 +125,7 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( self.addresses[address_i]['pubAddr']) @@ -369,92 +369,119 @@ class Marketplace: 'catalyst marketplace ingest --dataset={}'.format( dataset, address, dataset)) - def ingest(self, dataset, data_frequency=None, start=None, - end=None, force_download=False): + def process_temp_bundle(self, ds_name, path): + """ + Merge the temp bundle into the main bundle for the specified + data source. - dataset = dataset.lower() + Parameters + ---------- + ds_name + path + Returns + ------- + + """ + tmp_bundle = extract_bundle(path) + bundle_folder = get_data_source_folder(ds_name) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_bundle, bundle_folder) + + pass + + def ingest(self, ds_name, start=None, end=None, force_download=False): + + ds_name = ds_name.lower() dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(ds_name) + ).call() if not dataset_info[4]: print('The requested "{}" dataset is not registered in ' - 'the Data Marketplace.'.format(dataset)) + 'the Data Marketplace.'.format(ds_name)) return address, address_i = self.choose_pubaddr() - check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(ds_name) + ).call() - if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + if check_sub[0] != address or b32_str(check_sub[1]) != ds_name: raise MarketplaceContractDataNoMatch( params='address: {}, dataset: {}'.format( - address, dataset)) + address, ds_name + ) + ) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=ds_name, + date=check_sub[4], + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, dataset) + key, secret = get_key_secret(address, ds_name) nonce = str(int(time.time())) + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} - - print('Starting download of dataset for ingestion...') - - r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), - headers=headers, stream=True) - + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + log.debug('Starting download of dataset for ingestion...') + r = requests.post( + '{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, + stream=True, + ) if r.status_code == 200: + target_path = get_temp_bundles_folder() try: decoder = MultipartDecoder.from_response(r) for part in decoder.parts: h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) + # Extracting the filename from the header + name = re.search(r'filename="(.*)"', h).group(1) + + filename = os.path.join(target_path, name) with open(filename, 'wb') as f: # for chunk in part.content.iter_content(chunk_size=1024): # if chunk: # filter out keep-alive new chunks # f.write(chunk) f.write(part.content) + + self.process_temp_bundle(ds_name, filename) + except NonMultipartContentTypeException: response = r.json() - raise MarketplaceHTTPRequest(request='ingest dataset', - error=response) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=response, + ) else: - raise MarketplaceHTTPRequest(request='ingest dataset', - error=r.status_code) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=r.status_code, + ) - print('Download successful, now we need to ingest...') - exit(0) - # TODO: sync with Fred on the below and - # exiting for now - - period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(dataset, period, force_download) - - bundle_folder = get_bundle_folder(dataset, data_frequency) - if os.listdir(bundle_folder): - zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') - ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') - merge_bundles(zsource, ztarget) - - else: - os.rename(tmp_folder, bundle_folder) + log.info('{} ingested successfully'.format(ds_name)) def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 62f8beac..b19e1921 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -5,6 +5,7 @@ import tarfile import shutil from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory @@ -55,6 +56,7 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +@deprecated def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) @@ -65,13 +67,13 @@ def get_bundle_folder(data_source_name, data_frequency, environ=None): return bundle_folder -def get_temp_bundles_folder(data_source_name, environ=None): +def get_temp_bundles_folder(environ=None): """ The temp folder for bundle downloads by algo name. Parameters ---------- - data_source_name: str + ds_name: str environ: Returns @@ -79,55 +81,31 @@ def get_temp_bundles_folder(data_source_name, environ=None): str """ - data_source_folder = get_data_source_folder(data_source_name, environ) + root = data_root(environ) + folder = os.path.join(root, 'marketplace', 'temp_bundles') + ensure_directory(folder) - temp_bundles = os.path.join(data_source_folder, 'temp_bundles') - ensure_directory(temp_bundles) - - return temp_bundles + return folder -def get_data_source(data_source_name, period, force_download=False): +def extract_bundle(tar_filename): """ - Download and extract a bcolz bundle. + Extract a bcolz bundle. Parameters ---------- - exchange_name: str - symbol: str - data_frequency: str - period: str + ds_name Returns ------- str """ - root = get_temp_bundles_folder(data_source_name) - name = '{data_source}_{period}'.format( - data_source=data_source_name, - period=period, - ) - path = os.path.join(root, name) + target_path = tar_filename.replace('.tar.gz', '') + with tarfile.open(tar_filename, 'r') as tar: + tar.extractall(target_path) - if os.path.isdir(path): - if force_download: - shutil.rmtree(path) - - else: - return path - - ensure_directory(path) - - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) - - return path + return target_path def get_user_pubaddr(environ=None): diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 815ff885..7315629a 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -14,9 +14,15 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace.register() pass + def test_subscribe(self): + marketplace = Marketplace() + marketplace.subscribe('Victor2') + pass + def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.get_data_source_def('Marketcap') + ds_def = marketplace.ingest('Victor2') + pass marketplace.ingest( data_source_name='Marketcap', @@ -30,4 +36,3 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass - From d1529afeb318f9dcd0e5d4aa5b0cb20d9f4cb411 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:10:10 -0500 Subject: [PATCH 33/41] BLD: enhancing registration features --- catalyst/marketplace/marketplace.py | 78 ++++++++++++++------------- tests/marketplace/test_marketplace.py | 5 ++ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 14721a82..18294072 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -539,57 +539,61 @@ class Marketplace: else: break - price = int(input('Enter the price for a monthly subscription to ' - 'this dataset in ENG: ')) + price = int( + input( + 'Enter the price for a monthly subscription to ' + 'this dataset in ENG: ' + ) + ) + while True: + freq = input('Enter the data frequency [daily, hourly, minute]: ') + if freq.lower() not in ('daily', 'hourly', 'minute'): + print('Not a valid frequency.') + else: + break - # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): - # print("Not a valid frequency.") - # else: - # break + while True: + reg_pub = input( + 'Doest it include historical data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_history = True + else: + has_history = False + break - # while True: - # reg_pub = input('Can data be published every hour at a regular ' - # 'time? [default: Y]: ') or 'y' - # if reg_pub.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if reg_pub.lower() == 'y': - # reg_pub = True - # else: - # reg_pub = False - # break - - # while True: - # hist = input('Does it include historical data? ' - # '[default: Y]') or 'y' - # if hist.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if hist.lower() == 'y': - # hist = True - # else: - # hist = False - # break + while True: + reg_pub = input( + 'Doest it include live data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_live = True + else: + has_live = False + break address = self.choose_pubaddr()[0] - tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + {'nonce': self.web3.eth.getTransactionCount(address)} + ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - - tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) - + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) self.check_transaction(tx_hash) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 7315629a..c7c7c467 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -32,6 +32,11 @@ class TestMarketplace(WithLogger, ZiplineTestCase): ) pass + def test_publish(self): + marketplace = Marketplace() + datadir = '/Users/fredfortier/Downloads/marketcap_test_single' + marketplace.publish('marketcap1', datadir, False) + def test_clean(self): marketplace = Marketplace() marketplace.clean('marketcap') From b9bedfda21045d074abdba3cf8b1a2cfea599aba Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:23:13 -0500 Subject: [PATCH 34/41] BLD: enhancing registration features --- tests/marketplace/test_marketplace.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index c7c7c467..d2511ce8 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,20 +16,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('Victor2') + marketplace.subscribe('marketcap1') pass def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('Victor2') - pass - - marketplace.ingest( - data_source_name='Marketcap', - data_frequency=ds_def['data_frequencies'][0], - start=pd.to_datetime('2017-10-01'), - force_download=True, - ) + ds_def = marketplace.ingest('marketcap1') pass def test_publish(self): From e94c9db2ec41c45583c0a126e0ada83cebcfb756 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Feb 2018 18:29:11 +0200 Subject: [PATCH 35/41] Updated contract --- catalyst/marketplace/contract_enigma_address.txt | 2 +- catalyst/marketplace/contract_marketplace_abi.json | 2 +- catalyst/marketplace/contract_marketplace_address.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt index 157e2c23..90fcd06a 100644 --- a/catalyst/marketplace/contract_enigma_address.txt +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -1 +1 @@ -0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file +0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10 \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_abi.json b/catalyst/marketplace/contract_marketplace_abi.json index a3dda75f..4f0e2460 100644 --- a/catalyst/marketplace/contract_marketplace_abi.json +++ b/catalyst/marketplace/contract_marketplace_abi.json @@ -1 +1 @@ -[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file +[{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"isActive","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mBegin","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isPunished","type":"bool"}],"name":"setPunishProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataProviderInfo","outputs":[{"name":"owner","type":"address"},{"name":"price","type":"uint256"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"isPunished","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mProvidersSize","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"withdrawProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getAllProviders","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getMarketplaceTotalBalance","outputs":[{"name":"totalBalance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"refundSubscriber","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mCurrent","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getWithdrawAmount","outputs":[{"name":"withdrawAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"subscriber","type":"address"},{"name":"dataSourceName","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isUnExpired","type":"bool"},{"name":"isPaid","type":"bool"},{"name":"isPunishedProvider","type":"bool"},{"name":"isOrder","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isActive","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"mProviders","outputs":[{"name":"owner","type":"address"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"name","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"isPunished","type":"bool"},{"name":"punishTimeStamp","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"nextProvider","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"},{"name":"","type":"uint256"}],"name":"mOrders","outputs":[{"name":"dataSourceName","type":"bytes32"},{"name":"subscriber","type":"address"},{"name":"provider","type":"address"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isPaid","type":"bool"},{"name":"isOrder","type":"bool"},{"name":"isRefundPaid","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"owner","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"getRefundAmount","outputs":[{"name":"refundAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"MARKETPLACE_VERSION","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"isExpired","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"previousOwner","type":"address"},{"indexed":true,"name":"newOwner","type":"address"}],"name":"OwnershipTransferred","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionDeposited","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"TransferToProvider","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"ProviderWithdraw","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"isPunished","type":"bool"}],"name":"ProviderPunishStatus","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"refundAmount","type":"uint256"}],"name":"SubscriberRefund","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_address.txt b/catalyst/marketplace/contract_marketplace_address.txt index ec7bf029..86810a5f 100644 --- a/catalyst/marketplace/contract_marketplace_address.txt +++ b/catalyst/marketplace/contract_marketplace_address.txt @@ -1 +1 @@ -0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file +0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482 \ No newline at end of file From 444fcbb2b33f6eecc336c568b8fd49973b7ffa5f Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:04:23 -0500 Subject: [PATCH 36/41] BLD: adjustments for the new contract and developing register --- catalyst/marketplace/marketplace.py | 130 ++++++++++++++--------- catalyst/marketplace/utils/auth_utils.py | 46 +++++++- 2 files changed, 118 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 18294072..5de8f538 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -27,7 +27,8 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.auth_utils import get_key_secret, \ + get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ @@ -212,20 +213,19 @@ class Marketplace: print(df.to_string(formatters=formatters)) def subscribe(self, dataset): - dataset = dataset.lower() address = self.choose_pubaddr()[0] + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(dataset)) return - price = dataset_info[1] + price = provider_info[1] print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -239,7 +239,6 @@ class Marketplace: 'data': '0x70a08231000000000000000000000000{}'.format( address[2:]) }) - try: balance = int(balance[2:], 16) // 10 ** 8 except ValueError: @@ -398,11 +397,11 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): ds_name = ds_name.lower() - dataset_info = self.mkt_contract.functions.getDataSource( + provider_info = self.mkt_contract.functions.getDataProviderInfo( bytes32(ds_name) ).call() - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(ds_name)) return @@ -432,19 +431,7 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, ds_name) - nonce = str(int(time.time())) - signature = hmac.new( - secret.encode('utf-8'), - '{}{}'.format(ds_name, nonce).encode('utf-8'), - hashlib.sha512 - ).hexdigest() - - headers = { - 'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': ds_name, - } + headers = get_signed_headers(ds_name, key, secret) log.debug('Starting download of dataset for ingestion...') r = requests.post( '{}/marketplace/ingest'.format(AUTH_SERVER), @@ -524,15 +511,46 @@ class Marketplace: shutil.rmtree(folder) pass + def create_metadata(self, key, secret, ds_name, data_frequency, desc, + has_history=True, has_live=True): + """ + + Returns + ------- + + """ + headers = get_signed_headers(ds_name, key, secret) + r = requests.post( + '{}/marketplace/register'.format(AUTH_SERVER), + data=dict( + name=ds_name, + desc=desc, + data_frequency=data_frequency, + has_history=has_history, + has_live=has_live, + ), + headers=headers, + ) + + if r.status_code != 200: + raise MarketplaceHTTPRequest( + request='register', error=r.status_code + ) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest( + request='upload file', error=r.json()['error'] + ) + def register(self): while True: - dataset = input('Enter the name of the dataset to register: ') - dataset = dataset.lower() + desc = input('Enter the name of the dataset to register: ') + dataset = desc.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if datasource[4]: + if provider_info[4]: print('There is already a dataset registered under ' 'the name "{}". Please choose a different ' 'name.'.format(dataset)) @@ -554,7 +572,7 @@ class Marketplace: while True: reg_pub = input( - 'Doest it include historical data? [default: Y]: ' + 'Does it include historical data? [default: Y]: ' ) or 'y' if reg_pub.lower() not in ('y', 'n'): print('Please answer Y or N.') @@ -578,7 +596,23 @@ class Marketplace: has_live = False break - address = self.choose_pubaddr()[0] + address, address_i = self.choose_pubaddr() + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(address, dataset) + + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -599,44 +633,34 @@ class Marketplace: self.check_transaction(tx_hash) def publish(self, dataset, datadir, watch): - dataset = dataset.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not datasource[4]: + if not provider_info[4]: raise MarketplaceDatasetNotFound(dataset=dataset) - match = next((l for l in self.addresses if - l['pubAddr'] == datasource[0]), None) - + match = next( + (l for l in self.addresses if l['pubAddr'] == provider_info[0]), + None + ) if not match: raise MarketplaceNoAddressMatch( dataset=dataset, - address=datasource[0]) + address=provider_info[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + provider_info[0])) if 'key' in match: key = match['key'] secret = match['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) - - nonce = str(int(time.time())) - - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} + key, secret = get_key_secret(provider_info[0], dataset) + headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) if not filenames: diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 35fa3504..452dadb7 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -1,10 +1,14 @@ +import hashlib +import hmac + import requests +import time from catalyst.marketplace.marketplace_errors import ( - MarketplaceHTTPRequest) + MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( - get_user_pubaddr, save_user_pubaddr) -from catalyst.constants import AUTH_SERVER + get_user_pubaddr, save_user_pubaddr) +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): @@ -25,8 +29,8 @@ def get_key_secret(pubAddr, dataset): session = requests.Session() response = session.get('{}/getkeysecret'.format(AUTH_SERVER), headers={ - 'pubAddr': pubAddr, - 'dataset': dataset}) + 'pubAddr': pubAddr, + 'dataset': dataset}) if 'error' in response.json(): raise MarketplaceHTTPRequest(request=str('obtain key/secret'), @@ -44,3 +48,35 @@ def get_key_secret(pubAddr, dataset): save_user_pubaddr(addresses) return match['key'], match['secret'] + + +def get_signed_headers(ds_name, key, secret): + """ + Return a new request header including the key / secret signature + + Parameters + ---------- + ds_name + key + secret + + Returns + ------- + + """ + nonce = str(int(time.time())) + + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() + + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + + return headers From 3bc54a6c2ebebeb8c09458a6f9a522de1107f6b5 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:46:09 -0500 Subject: [PATCH 37/41] BLD: finalized the register implementation --- catalyst/marketplace/marketplace.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 5de8f538..363171c1 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -522,8 +522,8 @@ class Marketplace: headers = get_signed_headers(ds_name, key, secret) r = requests.post( '{}/marketplace/register'.format(AUTH_SERVER), - data=dict( - name=ds_name, + json=dict( + ds_name=ds_name, desc=desc, data_frequency=data_frequency, has_history=has_history, @@ -604,15 +604,6 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, dataset) - self.create_metadata( - key=key, - secret=secret, - ds_name=dataset, - data_frequency=freq, - desc=desc, - has_history=has_history, - has_live=has_live, - ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -632,6 +623,18 @@ class Marketplace: self.check_transaction(tx_hash) + print('\nWarming up the {} dataset'.format(dataset)) + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) + print('\n{} registered successfully'.format(dataset)) + def publish(self, dataset, datadir, watch): dataset = dataset.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( From 2168fb0d5b9534c5eb901305deca3905b9d0a6db Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:17:21 -0700 Subject: [PATCH 38/41] BUG: address json initialized with array of 1 dict, instead of dict --- catalyst/marketplace/utils/path_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index b19e1921..4bbf2854 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,7 +133,8 @@ def get_user_pubaddr(environ=None): return [data, ] return data else: - data = dict(pubAddr='', desc='') + data = [] + data.append(dict(pubAddr='', desc='')) with open(filename, 'w') as f: json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) From 4c1a9b1dd7f0f8ebb08b9d719d4d1494a3f91362 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:58:40 -0700 Subject: [PATCH 39/41] BLD: marketplace added listing of datasets --- catalyst/__main__.py | 1 - catalyst/marketplace/marketplace.py | 57 +++++++++--------------- catalyst/marketplace/utils/path_utils.py | 3 -- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 3833f3e4..ecbe7c23 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -756,7 +756,6 @@ def ls(ctx): sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 363171c1..a5e7d4f5 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,4 @@ import glob -import hashlib -import hmac import json import os import re @@ -144,14 +142,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -182,35 +180,21 @@ class Marketplace: return ds def list(self): - subscribers = self.contract.call( - {'from': self.default_account} - ).getSubscribers() - subscribed = [] - for index, address in enumerate(subscribers): - if address == self.default_account: - subscribed.append(index) - - data_sources = self.get_data_sources_map() + data_sources = self.mkt_contract.functions.getAllProviders().call() data = [] for index, data_source in enumerate(data_sources): - data.append( - dict( - id=index, - subscribed=index in subscribed, - **data_source + if index > 0: + data.append( + dict( + dataset=data_source.decode('utf-8').rstrip('\0') + ) ) - ) df = pd.DataFrame(data) - df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) set_print_settings() - - formatters = dict( - subscribed=lambda s: u'\u2713' if s else '', - ) - print(df.to_string(formatters=formatters)) + print(df) def subscribe(self, dataset): dataset = dataset.lower() @@ -258,7 +242,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -366,7 +350,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -449,9 +433,10 @@ class Marketplace: filename = os.path.join(target_path, name) with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) + # for chunk in part.content.iter_content( + # chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) f.write(part.content) self.process_temp_bundle(ds_name, filename) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 4bbf2854..6565b49d 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -2,9 +2,6 @@ import os import json import tarfile -import shutil - -from catalyst.data.bundles.core import download_without_progress from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory From d97890fc4e779aa683f91da68df45225ea6dff2f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 11:43:41 -0700 Subject: [PATCH 40/41] BLD: marketplace: moving AUTH_SERVER paths to /marketplace/* --- catalyst/marketplace/marketplace.py | 2 +- catalyst/marketplace/utils/auth_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a5e7d4f5..61ed7fec 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -658,7 +658,7 @@ class Marketplace: for file in filenames: files.append(('file', open(file, 'rb'))) - r = requests.post('{}/publish'.format(AUTH_SERVER), + r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER), files=files, headers=headers) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 452dadb7..d761cf8d 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -27,7 +27,7 @@ def get_key_secret(pubAddr, dataset): """ session = requests.Session() - response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), headers={ 'pubAddr': pubAddr, 'dataset': dataset}) From 5ccddf6d2178bd59a6115d870616beb8a975ea10 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 14:17:48 -0500 Subject: [PATCH 41/41] BLD: removed dummy smart contract --- marketplace/README.md | 18 --------- marketplace/contracts/Marketplace.sol | 19 --------- marketplace/contracts/Migrations.sol | 23 ----------- marketplace/marketplace.iml | 14 ------- marketplace/migrations/1_initial_migration.js | 5 --- marketplace/migrations/2_deploy_contracts.js | 5 --- marketplace/test/TestMarketplace.sol | 39 ------------------- marketplace/truffle-config.js | 4 -- marketplace/truffle.js | 11 ------ 9 files changed, 138 deletions(-) delete mode 100644 marketplace/README.md delete mode 100644 marketplace/contracts/Marketplace.sol delete mode 100644 marketplace/contracts/Migrations.sol delete mode 100644 marketplace/marketplace.iml delete mode 100644 marketplace/migrations/1_initial_migration.js delete mode 100644 marketplace/migrations/2_deploy_contracts.js delete mode 100644 marketplace/test/TestMarketplace.sol delete mode 100644 marketplace/truffle-config.js delete mode 100644 marketplace/truffle.js diff --git a/marketplace/README.md b/marketplace/README.md deleted file mode 100644 index 61f63842..00000000 --- a/marketplace/README.md +++ /dev/null @@ -1,18 +0,0 @@ -This module contains smart contracts for the data marketplace. -It was generated with Truffle. Truffle can be used to compile and test -the smart contracts. It's being tested on a test blockhain using Ganache. - -Steps to test: -* Download and run Ganache -* Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder, run the following commands: - * `truffle compile` - * `truffle test` - * `truffle migrate --reset` - * `truffle console` - * From the console: `Marketplace.deployed()` -* The deployed method displays info about the deployed smart contract -including its address. Copy/paste the address into -`catalyst/marketplace/marketplace.py`. -* Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol deleted file mode 100644 index 115b6110..00000000 --- a/marketplace/contracts/Marketplace.sol +++ /dev/null @@ -1,19 +0,0 @@ -pragma solidity ^0.4.17; - -contract Marketplace { - address[16] public subscribers; - - // Subscribing to a data source - function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 15); - - subscribers[dataSourceId] = msg.sender; - - return dataSourceId; - } - - // Retrieving the subscribers - function getSubscribers() public view returns (address[16]) { - return subscribers; - } -} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol deleted file mode 100644 index f170cb4f..00000000 --- a/marketplace/contracts/Migrations.sol +++ /dev/null @@ -1,23 +0,0 @@ -pragma solidity ^0.4.17; - -contract Migrations { - address public owner; - uint public last_completed_migration; - - modifier restricted() { - if (msg.sender == owner) _; - } - - function Migrations() public { - owner = msg.sender; - } - - function setCompleted(uint completed) public restricted { - last_completed_migration = completed; - } - - function upgrade(address new_address) public restricted { - Migrations upgraded = Migrations(new_address); - upgraded.setCompleted(last_completed_migration); - } -} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml deleted file mode 100644 index cf4ef5ed..00000000 --- a/marketplace/marketplace.iml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js deleted file mode 100644 index 4d5f3f9b..00000000 --- a/marketplace/migrations/1_initial_migration.js +++ /dev/null @@ -1,5 +0,0 @@ -var Migrations = artifacts.require("./Migrations.sol"); - -module.exports = function(deployer) { - deployer.deploy(Migrations); -}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js deleted file mode 100644 index e3935400..00000000 --- a/marketplace/migrations/2_deploy_contracts.js +++ /dev/null @@ -1,5 +0,0 @@ -var Marketplace = artifacts.require ("Marketplace"); - -module.exports = function (deployer) { - deployer.deploy (Marketplace); -}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol deleted file mode 100644 index f169d728..00000000 --- a/marketplace/test/TestMarketplace.sol +++ /dev/null @@ -1,39 +0,0 @@ -pragma solidity ^0.4.17; - -import "truffle/Assert.sol"; -import "truffle/DeployedAddresses.sol"; -import "../contracts/Marketplace.sol"; - -contract TestMarketplace { - Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - - // Testing the subscribe() function - function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(2); - - uint expected = 2; - - Assert.equal(returnedId, expected, "Data source 2 should be recorded."); - } - - // Testing retrieval of a single subscriber - function testGetSubscriberAddressByDataSourceId() public { - // Expected owner is this contract - address expected = this; - - address subscriber = marketplace.subscribers(2); - - Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); - } - - // Testing retrieval of all subscribers - function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected subscriber is this contract - address expected = this; - - // Store subscribers in memory rather than contract's storage - address[16] memory subscribers = marketplace.getSubscribers(); - - Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); - } -} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js deleted file mode 100644 index a6330d6d..00000000 --- a/marketplace/truffle-config.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! -}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js deleted file mode 100644 index 67f2dd44..00000000 --- a/marketplace/truffle.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! - networks: { - development: { - host: "localhost", - port: 7545, - network_id: "*" // Match any network id - } - } -};