From 94d5b4a4d564697e4aa25b51ff2f9b8ac05ed7f2 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 17:16:54 -0500 Subject: [PATCH 01/85] BLD: defined first version of commands and marketplace class --- catalyst/__main__.py | 70 ++++++++++++++++++++++++++++++++ catalyst/alt_data/__init__.py | 0 catalyst/alt_data/marketplace.py | 15 +++++++ 3 files changed, 85 insertions(+) create mode 100644 catalyst/alt_data/__init__.py create mode 100644 catalyst/alt_data/marketplace.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f8f9136..d86276ba 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,6 +5,7 @@ from functools import wraps import click import logbook import pandas as pd +from catalyst.alt_data.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -573,6 +574,75 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) +@main.command(name='ls-data') +@click.pass_context +def ls_data(ctx): + click.echo( + 'Listing available alternative data sources' + ) + marketplace = Marketplace() + marketplace.list() + click.echo('Done') + + +@main.command(name='register-data') +@click.argument('data_source_name') +@click.pass_context +def register_data(ctx, data_source_name): + click.echo( + 'Registering to data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.register(data_source_name) + click.echo('Done') + + +@main.command(name='ingest-data') +@click.argument('data_source_name') +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest_data(ctx, data_source_name, data_frequency, start, end): + click.echo( + 'Ingesting data: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.ingest(data_source_name, data_frequency, start, end) + click.echo('Done') + + +@main.command(name='clean-data') +@click.argument('data_source_name') +@click.pass_context +def clean_data(ctx, data_source_name): + click.echo( + 'Cleaning data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.clean(data_source_name) + click.echo('Done') + + @main.command(name='clean-algo') @click.option( '-n', diff --git a/catalyst/alt_data/__init__.py b/catalyst/alt_data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/alt_data/marketplace.py b/catalyst/alt_data/marketplace.py new file mode 100644 index 00000000..8f4610cc --- /dev/null +++ b/catalyst/alt_data/marketplace.py @@ -0,0 +1,15 @@ +class Marketplace: + def __init__(self): + pass + + def list(self): + pass + + def register(self, data_source_name): + pass + + def ingest(self, data_source_name, data_frequency, start, end): + pass + + def clean(self, data_source_name): + pass \ No newline at end of file From aab501aa1791c8b14f398ebd0cb1c75808b50f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 19:08:53 -0500 Subject: [PATCH 02/85] BLD: first test of the marketplace smart contract --- .../{alt_data => marketplace}/__init__.py | 0 .../{alt_data => marketplace}/marketplace.py | 0 etc/requirements_marketplace.txt | 1 + marketplace/contracts/Marketplace.sol | 19 +++++++++ marketplace/contracts/Migrations.sol | 23 +++++++++++ marketplace/marketplace.iml | 14 +++++++ marketplace/migrations/1_initial_migration.js | 5 +++ marketplace/migrations/2_deploy_contracts.js | 5 +++ marketplace/test/TestMarketplace.sol | 39 +++++++++++++++++++ marketplace/truffle-config.js | 4 ++ marketplace/truffle.js | 11 ++++++ 11 files changed, 121 insertions(+) rename catalyst/{alt_data => marketplace}/__init__.py (100%) rename catalyst/{alt_data => marketplace}/marketplace.py (100%) create mode 100644 etc/requirements_marketplace.txt create mode 100644 marketplace/contracts/Marketplace.sol create mode 100644 marketplace/contracts/Migrations.sol create mode 100644 marketplace/marketplace.iml create mode 100644 marketplace/migrations/1_initial_migration.js create mode 100644 marketplace/migrations/2_deploy_contracts.js create mode 100644 marketplace/test/TestMarketplace.sol create mode 100644 marketplace/truffle-config.js create mode 100644 marketplace/truffle.js diff --git a/catalyst/alt_data/__init__.py b/catalyst/marketplace/__init__.py similarity index 100% rename from catalyst/alt_data/__init__.py rename to catalyst/marketplace/__init__.py diff --git a/catalyst/alt_data/marketplace.py b/catalyst/marketplace/marketplace.py similarity index 100% rename from catalyst/alt_data/marketplace.py rename to catalyst/marketplace/marketplace.py diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt new file mode 100644 index 00000000..cbb3a5a3 --- /dev/null +++ b/etc/requirements_marketplace.txt @@ -0,0 +1 @@ +web3==4.0.0b5 \ No newline at end of file diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol new file mode 100644 index 00000000..54a14d1a --- /dev/null +++ b/marketplace/contracts/Marketplace.sol @@ -0,0 +1,19 @@ +pragma solidity ^0.4.17; + +contract Marketplace { + address[16] public subscribers; + + // Adopting a pet + function subscribe(uint dataSourceId) public returns (uint) { + require(dataSourceId >= 0 && dataSourceId <= 1); + + subscribers[dataSourceId] = msg.sender; + + return dataSourceId; + } + + // Retrieving the subscribers + function getSubscribers() public view returns (address[16]) { + return subscribers; + } +} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol new file mode 100644 index 00000000..f170cb4f --- /dev/null +++ b/marketplace/contracts/Migrations.sol @@ -0,0 +1,23 @@ +pragma solidity ^0.4.17; + +contract Migrations { + address public owner; + uint public last_completed_migration; + + modifier restricted() { + if (msg.sender == owner) _; + } + + function Migrations() public { + owner = msg.sender; + } + + function setCompleted(uint completed) public restricted { + last_completed_migration = completed; + } + + function upgrade(address new_address) public restricted { + Migrations upgraded = Migrations(new_address); + upgraded.setCompleted(last_completed_migration); + } +} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml new file mode 100644 index 00000000..cf4ef5ed --- /dev/null +++ b/marketplace/marketplace.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js new file mode 100644 index 00000000..4d5f3f9b --- /dev/null +++ b/marketplace/migrations/1_initial_migration.js @@ -0,0 +1,5 @@ +var Migrations = artifacts.require("./Migrations.sol"); + +module.exports = function(deployer) { + deployer.deploy(Migrations); +}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js new file mode 100644 index 00000000..e3935400 --- /dev/null +++ b/marketplace/migrations/2_deploy_contracts.js @@ -0,0 +1,5 @@ +var Marketplace = artifacts.require ("Marketplace"); + +module.exports = function (deployer) { + deployer.deploy (Marketplace); +}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol new file mode 100644 index 00000000..ea1c7fb3 --- /dev/null +++ b/marketplace/test/TestMarketplace.sol @@ -0,0 +1,39 @@ +pragma solidity ^0.4.17; + +import "truffle/Assert.sol"; +import "truffle/DeployedAddresses.sol"; +import "../contracts/Marketplace.sol"; + +contract TestMarketplace { + Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); + + // Testing the adopt() function + function testUserCanSubscribe() public { + uint returnedId = marketplace.subscribe(0); + + uint expected = 0; + + Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + } + + // Testing retrieval of a single pet's owner + function testGetSubscriberAddressByDataSourceId() public { + // Expected owner is this contract + address expected = this; + + address adopter = marketplace.subscribers(0); + + Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + } + + // Testing retrieval of all pet owners + function testGetSubscriberAddressByDataSourceIdInArray() public { + // Expected owner is this contract + address expected = this; + + // Store adopters in memory rather than contract's storage + address[16] memory subscribers = marketplace.getSubscribers(); + + Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + } +} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js new file mode 100644 index 00000000..a6330d6d --- /dev/null +++ b/marketplace/truffle-config.js @@ -0,0 +1,4 @@ +module.exports = { + // See + // to customize your Truffle configuration! +}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js new file mode 100644 index 00000000..67f2dd44 --- /dev/null +++ b/marketplace/truffle.js @@ -0,0 +1,11 @@ +module.exports = { + // See + // to customize your Truffle configuration! + networks: { + development: { + host: "localhost", + port: 7545, + network_id: "*" // Match any network id + } + } +}; From 22a850ab141556a194c5aa8648a5bdefe6aae6ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 20:41:03 -0500 Subject: [PATCH 03/85] BLD: testing basic smart contract --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++++++-- marketplace/contracts/Marketplace.sol | 2 +- tests/marketplace/__init__.py | 0 tests/marketplace/test_marketplace.py | 4 +++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/marketplace/__init__.py create mode 100644 tests/marketplace/test_marketplace.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 8f4610cc..a6c0ff0b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,15 +1,52 @@ +import json +import os + +from web3 import Web3, HTTPProvider + +CONTRACT_PATH = os.path.join( + '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' +) +CONTRACT_ADDRESS = Web3.toChecksumAddress( + '0x345ca3e014aaf5dca488057592ee47305d9b3e10' +) + + +# we'll use one of our default accounts to deploy from. every write to the chain requires a +# payment of ethereum called "gas". if we were running an actual test ethereum node locally, +# then we'd have to go on the test net and get some free ethereum to play with. that is beyond +# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and +# the only chain we're using is our own local one + + class Marketplace: def __init__(self): - pass + with open(CONTRACT_PATH) as handle: + json_interface = json.load(handle) + + w3 = Web3(HTTPProvider('http://localhost:7545')) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=json_interface['abi'], + ) # Type: Contract + self.default_account = w3.eth.accounts[1] + + pass def list(self): + subscribers = self.contract.call( + {'from': self.default_account} + ).getSubscribers() pass def register(self, data_source_name): + test = self.contract.transact( + {'from': self.default_account} + ).subscribe(0) pass def ingest(self, data_source_name, data_frequency, start, end): pass def clean(self, data_source_name): - pass \ No newline at end of file + pass diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 54a14d1a..30401e5b 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 1); + require(dataSourceId >= 0 && dataSourceId <= 5); subscribers[dataSourceId] = msg.sender; diff --git a/tests/marketplace/__init__.py b/tests/marketplace/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py new file mode 100644 index 00000000..e03efc43 --- /dev/null +++ b/tests/marketplace/test_marketplace.py @@ -0,0 +1,4 @@ +from catalyst.marketplace.marketplace import Marketplace + +marketplace = Marketplace() +pass \ No newline at end of file From e0c8178932a17632ca8ef7fcf0363b7f9436341a Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 22:42:09 -0500 Subject: [PATCH 04/85] BLD: improved smart contract and catalyst commands --- catalyst/__main__.py | 4 +- catalyst/constants.py | 2 + catalyst/marketplace/marketplace.py | 95 +++++++++++++++++++++++---- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 8 +-- tests/marketplace/test_marketplace.py | 14 +++- 6 files changed, 102 insertions(+), 23 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index d86276ba..5f70b985 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,7 +5,7 @@ from functools import wraps import click import logbook import pandas as pd -from catalyst.alt_data.marketplace import Marketplace +from catalyst.marketplace.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -578,7 +578,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def ls_data(ctx): click.echo( - 'Listing available alternative data sources' + 'The alternative data sources:' ) marketplace = Marketplace() marketplace.list() diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..6b3fc235 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,4 +15,6 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + AUTO_INGEST = False diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a6c0ff0b..9de33328 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,29 +1,26 @@ import json import os +import pandas as pd from web3 import Web3, HTTPProvider +from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.constants import ROOT_DIR + +REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( - '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' + ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0x345ca3e014aaf5dca488057592ee47305d9b3e10' + '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' ) -# we'll use one of our default accounts to deploy from. every write to the chain requires a -# payment of ethereum called "gas". if we were running an actual test ethereum node locally, -# then we'd have to go on the test net and get some free ethereum to play with. that is beyond -# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and -# the only chain we're using is our own local one - - class Marketplace: def __init__(self): with open(CONTRACT_PATH) as handle: json_interface = json.load(handle) - - w3 = Web3(HTTPProvider('http://localhost:7545')) + w3 = Web3(HTTPProvider(REMOTE_NODE)) self.contract = w3.eth.contract( CONTRACT_ADDRESS, @@ -33,16 +30,86 @@ class Marketplace: pass + def get_data_sources_map(self): + return [ + dict( + name='Marketcap', + desc='The marketcap value in USD.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='GitHub', + desc='The rate of development activity on GitHub.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='Influencers', + desc='Tweets and related sentiments by selected influencers.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + ] + def list(self): subscribers = self.contract.call( {'from': self.default_account} ).getSubscribers() + + subscribed = [] + for index, address in enumerate(subscribers): + if address == self.default_account: + subscribed.append(index) + + data_sources = self.get_data_sources_map() + + data = [] + for index, data_source in enumerate(data_sources): + data.append( + dict( + id=index, + subscribed=index in subscribed, + **data_source, + ) + ) + + df = pd.DataFrame(data) + df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) + set_print_settings() + + formatters = dict( + subscribed=lambda s: u'\u2713' if s else '', + ) + print(df.to_string(formatters=formatters)) + pass def register(self, data_source_name): - test = self.contract.transact( - {'from': self.default_account} - ).subscribe(0) + data_sources = self.get_data_sources_map() + index = next( + (index for (index, d) in enumerate(data_sources) if + d['name'].lower() == data_source_name.lower()), + None + ) + if index is None: + raise ValueError( + 'Data source not found.' + ) + + try: + self.contract.transact( + {'from': self.default_account} + ).subscribe(index) + print( + 'Subscribed to data source {} successfully'.format( + data_source_name + ) + ) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + pass def ingest(self, data_source_name, data_frequency, start, end): diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 30401e5b..1cc1cf75 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 5); + require(dataSourceId >= 0 && dataSourceId <= 15); subscribers[dataSourceId] = msg.sender; diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index ea1c7fb3..5c4f6411 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -9,9 +9,9 @@ contract TestMarketplace { // Testing the adopt() function function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(0); + uint returnedId = marketplace.subscribe(2); - uint expected = 0; + uint expected = 2; Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } @@ -21,7 +21,7 @@ contract TestMarketplace { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(0); + address adopter = marketplace.subscribers(2); Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); } @@ -34,6 +34,6 @@ contract TestMarketplace { // Store adopters in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); } } \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index e03efc43..1faf0648 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,4 +1,14 @@ from catalyst.marketplace.marketplace import Marketplace +from catalyst.testing.fixtures import WithLogger, ZiplineTestCase -marketplace = Marketplace() -pass \ No newline at end of file + +class TestMarketplace(WithLogger, ZiplineTestCase): + def test_list(self): + marketplace = Marketplace() + marketplace.list() + pass + + def test_register(self): + marketplace = Marketplace() + marketplace.register('GitHub') + pass From 1e4af12e3daa4222a93338f0feed67f136c8aae6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 13:31:08 -0500 Subject: [PATCH 05/85] BLD: improved smart contract and catalyst commands --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 9de33328..e8288037 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -12,7 +12,7 @@ CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' + '0xaa862ddac09f6736a61e1124040fd883a6533c19' ) From 354e422be84d50bef799453fed9020c9dd0a8416 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 21:54:09 -0500 Subject: [PATCH 06/85] BLD: working on ingesting alt data sources --- catalyst/marketplace/marketplace.py | 7 ++- catalyst/marketplace/utils/__init__.py | 0 catalyst/marketplace/utils/paths.py | 87 ++++++++++++++++++++++++++ tests/marketplace/test_marketplace.py | 10 +++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 catalyst/marketplace/utils/__init__.py create mode 100644 catalyst/marketplace/utils/paths.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e8288037..e22d3f49 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,13 +6,14 @@ from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR +from catalyst.marketplace.utils.paths import get_temp_bundles_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xaa862ddac09f6736a61e1124040fd883a6533c19' + '0xe2b6cf3863240892d59664d209a28289a73ef644' ) @@ -112,7 +113,9 @@ class Marketplace: pass - def ingest(self, data_source_name, data_frequency, start, end): + def ingest(self, data_source_name, data_frequency=None, start=None, + end=None): + temp_folder = get_temp_bundles_folder(data_source_name) pass def clean(self, data_source_name): diff --git a/catalyst/marketplace/utils/__init__.py b/catalyst/marketplace/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/paths.py new file mode 100644 index 00000000..008ed2b7 --- /dev/null +++ b/catalyst/marketplace/utils/paths.py @@ -0,0 +1,87 @@ +import os +import tarfile + +from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.paths import data_root, ensure_directory + + +def get_data_source_folder(data_source_name, environ=None): + """ + The root path of an data_source folder. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + data_source_folder = os.path.join(root, 'marketplace', data_source_name) + ensure_directory(data_source_folder) + + return data_source_folder + + +def get_temp_bundles_folder(data_source_name, environ=None): + """ + The temp folder for bundle downloads by algo name. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + data_source_folder = get_data_source_folder(data_source_name, environ) + + temp_bundles = os.path.join(data_source_folder, 'temp_bundles') + ensure_directory(temp_bundles) + + return temp_bundles + + +def get_data_source(data_source_name, period): + """ + Download and extract a bcolz bundle. + + Parameters + ---------- + exchange_name: str + symbol: str + data_frequency: str + period: str + + Returns + ------- + str + Filename: bitfinex-daily-neo_eth-2017-10.tar.gz + + """ + root = get_temp_bundles_folder(data_source_name) + name = '{data_source}-{period}'.format( + data_source=data_source_name, + period=period, + ) + path = os.path.join(root, name) + + if not os.path.isdir(path): + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) + + return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 1faf0648..3feeb9a6 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,5 +1,6 @@ from catalyst.marketplace.marketplace import Marketplace from catalyst.testing.fixtures import WithLogger, ZiplineTestCase +import pandas as pd class TestMarketplace(WithLogger, ZiplineTestCase): @@ -12,3 +13,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.register('GitHub') pass + + def test_ingest(self): + marketplace = Marketplace() + marketplace.ingest( + data_source_name='GitHub', + data_frequency='finest', + start=pd.Timestamp.utcnow(), + ) + pass From 50ded59c3ce1a201080a5c8281f18ac36291d4a3 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:01:00 -0500 Subject: [PATCH 07/85] DOC: documented the marketplace structure --- marketplace/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 marketplace/README.md diff --git a/marketplace/README.md b/marketplace/README.md new file mode 100644 index 00000000..599d8112 --- /dev/null +++ b/marketplace/README.md @@ -0,0 +1,18 @@ +This module contains smart contracts for the data marketplace. +It was generated with Truffle. Truffle can be used to compile and test +the smart contracts. It's being tested on a test blockhain using Ganache. + +Steps to test: +* Download an run Ganache +* Ensure that your Ganache endpoint matches `truffle.js` +* In this `marketplace` folder: + * `truffle compile` + * `truffle test` + * `truffle migrate --reset` + * `truffle console` + * From the console: `Marketplace.deployed()` +* The deployed method displays info about the deployed smart contract +including its address. Copy/paste the address into +`catalyst/marketplace/marketplace.py`. +* Run the catalyst marketplace unit tests: +`tests/marketplace/test_marketplace.py` \ No newline at end of file From 9361570895868f9b71b39da56af733073625da07 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:02 -0500 Subject: [PATCH 08/85] DOC: improved code comments --- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 1cc1cf75..115b6110 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -3,7 +3,7 @@ pragma solidity ^0.4.17; contract Marketplace { address[16] public subscribers; - // Adopting a pet + // Subscribing to a data source function subscribe(uint dataSourceId) public returns (uint) { require(dataSourceId >= 0 && dataSourceId <= 15); diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 5c4f6411..77a03aea 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -7,7 +7,7 @@ import "../contracts/Marketplace.sol"; contract TestMarketplace { Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - // Testing the adopt() function + // Testing the subscribe() function function testUserCanSubscribe() public { uint returnedId = marketplace.subscribe(2); @@ -16,24 +16,24 @@ contract TestMarketplace { Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } - // Testing retrieval of a single pet's owner + // Testing retrieval of a single subscriber function testGetSubscriberAddressByDataSourceId() public { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(2); + address subscriber = marketplace.subscribers(2); - Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); } - // Testing retrieval of all pet owners + // Testing retrieval of all subscribers function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected owner is this contract + // Expected subscriber is this contract address expected = this; - // Store adopters in memory rather than contract's storage + // Store subscribers in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); + Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); } } \ No newline at end of file From 3ea00e783c6d931d1a906cf729ceb7a9f0582689 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:52 -0500 Subject: [PATCH 09/85] DOC: improved code comments --- marketplace/test/TestMarketplace.sol | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 77a03aea..f169d728 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -13,7 +13,7 @@ contract TestMarketplace { uint expected = 2; - Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + Assert.equal(returnedId, expected, "Data source 2 should be recorded."); } // Testing retrieval of a single subscriber @@ -23,7 +23,7 @@ contract TestMarketplace { address subscriber = marketplace.subscribers(2); - Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); } // Testing retrieval of all subscribers From b1d63c9a5b7f34cee8f24774701aaa9aebd88f10 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 11 Jan 2018 12:15:49 -0700 Subject: [PATCH 10/85] Update README.md --- marketplace/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marketplace/README.md b/marketplace/README.md index 599d8112..61f63842 100644 --- a/marketplace/README.md +++ b/marketplace/README.md @@ -3,9 +3,9 @@ It was generated with Truffle. Truffle can be used to compile and test the smart contracts. It's being tested on a test blockhain using Ganache. Steps to test: -* Download an run Ganache +* Download and run Ganache * Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder: +* In this `marketplace` folder, run the following commands: * `truffle compile` * `truffle test` * `truffle migrate --reset` @@ -15,4 +15,4 @@ Steps to test: including its address. Copy/paste the address into `catalyst/marketplace/marketplace.py`. * Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` \ No newline at end of file +`tests/marketplace/test_marketplace.py` From 88aa7154db5b57cef9b6fe8d27571bd68e5761b4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 20:04:53 -0500 Subject: [PATCH 11/85] BLD: ingesting and cleaning marketplace data sources --- catalyst/exchange/utils/bundle_utils.py | 3 ++ catalyst/marketplace/marketplace.py | 35 ++++++++++++++-- catalyst/marketplace/utils/bundle_utils.py | 36 ++++++++++++++++ .../utils/{paths.py => path_utils.py} | 42 ++++++++++++++----- tests/marketplace/test_marketplace.py | 3 +- 5 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 catalyst/marketplace/utils/bundle_utils.py rename catalyst/marketplace/utils/{paths.py => path_utils.py} (61%) diff --git a/catalyst/exchange/utils/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py index e9207511..992cabbb 100644 --- a/catalyst/exchange/utils/bundle_utils.py +++ b/catalyst/exchange/utils/bundle_utils.py @@ -1,6 +1,8 @@ import calendar import os import tarfile + +import bcolz from datetime import timedelta, datetime, date import numpy as np @@ -355,3 +357,4 @@ def get_assets(exchange, include_symbols, exclude_symbols): else: return all_assets + diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e22d3f49..c71cef1c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,12 +1,17 @@ import json import os + +import bcolz import pandas as pd +import shutil from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR -from catalyst.marketplace.utils.paths import get_temp_bundles_folder +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ + get_data_source, get_bundle_folder, get_data_source_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( @@ -114,9 +119,31 @@ class Marketplace: pass def ingest(self, data_source_name, data_frequency=None, start=None, - end=None): - temp_folder = get_temp_bundles_folder(data_source_name) + end=None, force_download=False): + data_source_name = data_source_name.lower() + + period = start.strftime('%Y-%m-%d') + tmp_folder = get_data_source(data_source_name, period, force_download) + + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_folder, bundle_folder) + pass - def clean(self, data_source_name): + def clean(self, data_source_name, data_frequency=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + folder = get_data_source_folder(data_source_name) + + else: + forlder = get_bundle_folder(data_source_name, data_frequency) + + shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py new file mode 100644 index 00000000..916722da --- /dev/null +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -0,0 +1,36 @@ +import bcolz +import os +import pandas as pd +import shutil + + +def merge_bundles(zsource, ztarget): + """ + Merge + Parameters + ---------- + zsource + ztarget + + Returns + ------- + + """ + # TODO: find a way to do this iteratively instead of in-memory + df_source = zsource.todataframe() + df_source.set_index('last_updated', drop=False, inplace=True) + df_target = ztarget.todataframe() + df_target.set_index('last_updated', drop=False, inplace=True) + + df = df_target.merge( + right=df_source, + how='right', + ) # type: pd.DataFrame + + dirname = os.path.basename(ztarget.rootdir) + bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname)) + os.rename(ztarget.rootdir, bak_dir) + + z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir) + shutil.rmtree(bak_dir) + return z diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/path_utils.py similarity index 61% rename from catalyst/marketplace/utils/paths.py rename to catalyst/marketplace/utils/path_utils.py index 008ed2b7..6d301880 100644 --- a/catalyst/marketplace/utils/paths.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,6 +1,8 @@ import os import tarfile +import shutil + from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory @@ -29,6 +31,17 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +def get_bundle_folder(data_source_name, data_frequency, environ=None): + data_source_folder = get_data_source_folder(data_source_name, environ) + + subfolder = data_frequency if data_frequency is not None else 'data' + bundle_folder = os.path.join(data_source_folder, subfolder) + + ensure_directory(bundle_folder) + + return bundle_folder + + def get_temp_bundles_folder(data_source_name, environ=None): """ The temp folder for bundle downloads by algo name. @@ -51,7 +64,7 @@ def get_temp_bundles_folder(data_source_name, environ=None): return temp_bundles -def get_data_source(data_source_name, period): +def get_data_source(data_source_name, period, force_download=False): """ Download and extract a bcolz bundle. @@ -65,23 +78,30 @@ def get_data_source(data_source_name, period): Returns ------- str - Filename: bitfinex-daily-neo_eth-2017-10.tar.gz """ root = get_temp_bundles_folder(data_source_name) - name = '{data_source}-{period}'.format( + name = '{data_source}_{period}'.format( data_source=data_source_name, period=period, ) path = os.path.join(root, name) - if not os.path.isdir(path): - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) + if os.path.isdir(path): + if force_download: + shutil.rmtree(path) + + else: + return path + + ensure_directory(path) + + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 3feeb9a6..68990791 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -17,8 +17,9 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() marketplace.ingest( - data_source_name='GitHub', + data_source_name='Marketcap', data_frequency='finest', start=pd.Timestamp.utcnow(), + force_download=True, ) pass From 70b9fd8e03903a576a3422ad6a8946e1a49cab96 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 12 Jan 2018 00:39:14 -0500 Subject: [PATCH 12/85] BLD: completed data market place first integration and created an algo to test it --- catalyst/api.pyi | 76 +++++- catalyst/constants.py | 5 +- .../examples/mean_reversion_by_marketcap.py | 233 ++++++++++++++++++ catalyst/exchange/exchange_algorithm.py | 13 + catalyst/marketplace/marketplace.py | 60 ++++- catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/path_utils.py | 3 +- tests/marketplace/test_marketplace.py | 11 +- 8 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 catalyst/examples/mean_reversion_by_marketcap.py diff --git a/catalyst/api.pyi b/catalyst/api.pyi index bf104664..6e46aba9 100644 --- a/catalyst/api.pyi +++ b/catalyst/api.pyi @@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None): :func:`catalyst.api.pipeline_output` """ + def batch_market_order(share_counts): """Place a batch market order for multiple assets. @@ -48,6 +49,7 @@ def batch_market_order(share_counts): Index of ids for newly-created orders. """ + def cancel_order(order_param): """Cancel an open order. @@ -57,7 +59,9 @@ def cancel_order(order_param): The order_id or order object to cancel. """ -def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'): + +def continuous_future(root_symbol_str, offset=0, roll='volume', + adjustment='mul'): """Create a specifier for a continuous contract. Parameters @@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul' The continuous future specifier. """ -def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs): + +def fetch_csv(url, pre_func=None, post_func=None, date_column='date', + date_format=None, timezone='UTC', symbol=None, mask=True, + symbol_column=None, special_params_checker=None, **kwargs): """Fetch a csv from a remote url and register the data so that it is queryable from the ``data`` object. @@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma A requests source that will pull data from the url specified. """ + def future_symbol(symbol): """Lookup a futures contract with a given symbol. @@ -144,6 +152,7 @@ def future_symbol(symbol): Raised when no contract named 'symbol' is found. """ + def get_datetime(tz=None): """ Returns the current simulation datetime. @@ -159,6 +168,7 @@ dt : datetime The current simulation datetime converted to ``tz``. """ + def get_environment(field='platform'): """Query the execution environment. @@ -198,6 +208,7 @@ def get_environment(field='platform'): Raised when ``field`` is not a valid option. """ + def get_order(order_id): """Lookup an order based on the order id returned from one of the order functions. @@ -213,10 +224,12 @@ def get_order(order_id): The order object. """ + def history(bar_count, frequency, field, ffill=True): """DEPRECATED: use ``data.history`` instead. """ + def order(asset, amount, limit_price=None, stop_price=None, style=None): """Place an order. @@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ -def order_percent(asset, percent, limit_price=None, stop_price=None, style=None): + +def order_percent(asset, percent, limit_price=None, stop_price=None, + style=None): """Place an order in the specified asset corresponding to the given percent of the current portfolio value. @@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None) :func:`catalyst.api.order_value` """ + def order_target(asset, target, limit_price=None, stop_price=None, style=None): """Place an order to adjust a position to a target number of shares. If the position doesn't already exist, this is equivalent to placing a new @@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_target_value` """ -def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_percent(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target percent of the current portfolio value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is @@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style :func:`catalyst.api.order_target_value` """ -def order_target_value(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_value(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is equivalent to placing an @@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N :func:`catalyst.api.order_target_percent` """ + def order_value(asset, value, limit_price=None, stop_price=None, style=None): """Place an order by desired value rather than desired number of shares. @@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ + def pipeline_output(name): """Get the results of the pipeline that was attached with the name: ``name``. @@ -514,6 +536,7 @@ def pipeline_output(name): :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline` """ + def record(*args, **kwargs): """Track and record values each day. @@ -529,7 +552,9 @@ def record(*args, **kwargs): :func:`~catalyst.run_algorithm`. """ -def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None): + +def schedule_function(func, date_rule=None, time_rule=None, half_days=True, + calendar=None): """Schedules a function to be called according to some timed rules. Parameters @@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale :class:`catalyst.api.time_rules` """ + def set_asset_restrictions(restrictions, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'): catalyst.finance.asset_restrictions.Restrictions """ + def set_benchmark(benchmark): """Set the benchmark asset. @@ -576,6 +603,7 @@ def set_benchmark(benchmark): automatically reinvested. """ + def set_cancel_policy(cancel_policy): """Sets the order cancellation policy for the simulation. @@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy): :class:`catalyst.api.NeverCancel` """ + def set_commission(commission): """Sets the commission model for the simulation. @@ -605,6 +634,7 @@ def set_commission(commission): :class:`catalyst.finance.commission.PerDollar` """ + def set_do_not_order_list(restricted_list, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'): The assets that cannot be ordered. """ + def set_long_only(on_error='fail'): """Set a rule specifying that this algorithm cannot take short positions. """ + def set_max_leverage(max_leverage): """Set a limit on the maximum leverage of the algorithm. @@ -629,6 +661,7 @@ def set_max_leverage(max_leverage): be no maximum. """ + def set_max_order_count(max_count, on_error='fail'): """Set a limit on the number of orders that can be placed in a single day. @@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'): The maximum number of orders that can be placed on any single day. """ -def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_order_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value of any single order placed for sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. @@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error= The maximum value that can be ordered at one time. """ -def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_position_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value held for the given sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. This means @@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err The maximum value to hold for an asset. """ + def set_slippage(slippage): """Set the slippage model for the simulation. @@ -694,6 +732,7 @@ def set_slippage(slippage): :class:`catalyst.finance.slippage.SlippageModel` """ + def set_symbol_lookup_date(dt): """Set the date for which symbols will be resolved to their assets (symbols may map to different firms or underlying assets at @@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt): The new symbol lookup date. """ + def sid(sid): """Lookup an Asset by its unique asset identifier. @@ -724,6 +764,7 @@ def sid(sid): When a requested ``sid`` does not map to any asset. """ + def symbol(symbol_str): """Lookup an Equity by its ticker symbol. @@ -748,6 +789,7 @@ def symbol(symbol_str): :func:`catalyst.api.set_symbol_lookup_date` """ + def symbols(*args): """Lookup multuple Equities as a list. @@ -773,3 +815,21 @@ def symbols(*args): :func:`catalyst.api.set_symbol_lookup_date` """ + +def get_data_source(data_source_name, data_frequency=None, + start=None, end=None): + """ + Lookup a data source from the marketplace + + Parameters + ---------- + self + data_source_name + data_frequency + start + end + + Returns + ------- + + """ diff --git a/catalyst/constants.py b/catalyst/constants.py index 6b3fc235..5668bc83 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,6 +15,9 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +try: + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +except Exception as e: + print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py new file mode 100644 index 00000000..6d31dab7 --- /dev/null +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -0,0 +1,233 @@ +# For this example, we're going to write a simple momentum script. When the +# stock goes up quickly, we're going to buy; when it goes down quickly, we're +# going to sell. Hopefully we'll ride the waves. +import os +import tempfile +import time + +import pandas as pd +import talib +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import symbol, record, order_target_percent, get_data_source +from catalyst.exchange.utils.stats_utils import set_print_settings, \ + get_pretty_stats +# We give a name to the algorithm which Catalyst will use to persist its state. +# In this example, Catalyst will create the `.catalyst/data/live_algos` +# directory. If we stop and start the algorithm, Catalyst will resume its +# state using the files included in the folder. +from catalyst.utils.paths import ensure_directory + +NAMESPACE = 'mean_reversion_simple' +log = Logger(NAMESPACE) + + +# To run an algorithm in Catalyst, you need two functions: initialize and +# handle_data. + +def initialize(context): + # This initialize function sets any data or variables that you'll use in + # your algorithm. For instance, you'll want to define the trading pair (or + # trading pairs) you want to backtest. You'll also want to define any + # parameters or values you're going to use. + + # In our example, we're looking at Neo in Ether. + df = get_data_source( + 'marketcap', start=context.datetime + ) # type: pd.DataFrame + + # Keep only the top coins by market cap + df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))] + + set_print_settings() + + df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True) + print('the marketplace data:\n{}'.format(df)) + + # Pick the 5 assets with the lowest market cap for trading + quote_currency = 'eth' + exchange = context.exchanges[next(iter(context.exchanges))] + symbols = [a.symbol for a in exchange.assets + if a.start_date < context.datetime] + context.assets = [] + for currency, price in df['market_cap_usd'].iteritems(): + if len(context.assets) >= 5: + break + + s = '{}_{}'.format(currency.decode('utf-8'), quote_currency) + if s in symbols: + context.assets.append(symbol(s)) + + context.base_price = None + context.current_day = None + + context.RSI_OVERSOLD = 55 + context.RSI_OVERBOUGHT = 60 + context.CANDLE_SIZE = '5T' + + context.start_time = time.time() + + +def handle_data(context, data): + # This handle_data function is where the real work is done. Our data is + # minute-level tick data, and each minute is called a frame. This function + # runs on each frame of the data. + + # We flag the first period of each day. + # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle + # would only execute once. This method works with minute and daily + # frequencies. + today = data.current_dt.floor('1D') + if today != context.current_day: + context.traded_today = dict() + context.current_day = today + + # Preparing dictionaries for asset-level data points + volumes = dict() + rsis = dict() + price_values = dict() + cash = context.portfolio.cash + + for asset in context.assets: + # We're computing the volume-weighted-average-price of the security + # defined above, in the context.assets variable. For this example, + # we're using three bars on the 15 min bars. + + # The frequency attribute determine the bar size. We use this + # convention for the frequency alias: + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + prices = data.history( + asset, + fields='close', + bar_count=50, + frequency=context.CANDLE_SIZE + ) + + # Ta-lib calculates various technical indicator based on price and + # volume arrays. + + # In this example, we are comp + rsi = talib.RSI(prices.values, timeperiod=14) + + # We need a variable for the current price of the security to compare + # to the average. Since we are requesting two fields, data.current() + # returns a DataFrame with + current = data.current(asset, fields=['close', 'volume']) + price = current['close'] + + # If base_price is not set, we use the current value. This is the + # price at the first bar which we reference to calculate price_change. + # if asset not in context.base_price: + # context.base_price[asset] = price + # + # base_price = context.base_price[asset] + # price_change = (price - base_price) / base_price + + # Tracking the relevant data + volumes[asset] = current['volume'] + rsis[asset] = rsi[-1] + price_values[asset] = price + # price_changes[asset] = price_change + + # We are trying to avoid over-trading by limiting our trades to + # one per day. + if asset in context.traded_today: + continue + + # Exit if we cannot trade + if not data.can_trade(asset): + continue + + # Another powerful built-in feature of the Catalyst backtester is the + # portfolio object. The portfolio object tracks your positions, cash, + # cost basis of specific holdings, and more. In this line, we + # calculate how long or short our position is at this minute. + pos_amount = context.portfolio.positions[asset].amount + + if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: + log.info( + '{}: buying - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + # Set a style for limit orders, + limit_price = price * 1.005 + target = 1.0 / len(context.assets) + order_target_percent( + asset, target, limit_price=limit_price + ) + context.traded_today[asset] = True + + elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: + log.info( + '{}: selling - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + limit_price = price * 0.995 + order_target_percent( + asset, 0, limit_price=limit_price + ) + context.traded_today[asset] = True + + # Now that we've collected all current data for this frame, we use + # the record() method to save it. This data will be available as + # a parameter of the analyze() function for further analysis. + record( + current_price=price_values, + volume=volumes, + rsi=rsis, + cash=cash, + ) + + +def analyze(context=None, perf=None): + stats = get_pretty_stats(perf) + print('the algo stats:\n{}'.format(stats)) + pass + + +if __name__ == '__main__': + # The execution mode: backtest or live + live = False + + if live: + run_algorithm( + capital_base=0.1, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + live=True, + algo_namespace=NAMESPACE, + base_currency='btc', + live_graph=False, + simulate_orders=False, + stats_output=None, + ) + + else: + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', NAMESPACE + ) + ensure_directory(folder) + + timestr = time.strftime('%Y%m%d-%H%M%S') + out = os.path.join(folder, '{}.p'.format(timestr)) + # catalyst run -f catalyst/examples/mean_reversion_simple.py \ + # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ + # --data-frequency minute --capital-base 10000 + run_algorithm( + capital_base=100, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace=NAMESPACE, + base_currency='eth', + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-10-15', utc=True), + ) + log.info('saved perf stats: {}'.format(out)) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 782b7ec8..3ba1b5ea 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -43,6 +43,7 @@ from catalyst.finance.execution import MarketOrder from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator +from catalyst.marketplace.marketplace import Marketplace from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest @@ -92,6 +93,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): attempts=self.attempts, ) + self._marketplace = None + @staticmethod def __convert_order_params_for_blotter(limit_price, stop_price, style): """ @@ -167,6 +170,16 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): """ return round_nearest(amount, asset.min_trade_size) + @api_method + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + if self._marketplace is None: + self._marketplace = Marketplace() + + return self._marketplace.get_data_source( + data_source_name, data_frequency, start, end, + ) + @api_method @preprocess(symbol_str=ensure_upper_case) def symbol(self, symbol_str, exchange_name=None): diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index c71cef1c..041fb004 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,19 +1,22 @@ import json import os - -import bcolz -import pandas as pd import shutil +import bcolz +import logbook +import pandas as pd +import six from web3 import Web3, HTTPProvider +from catalyst.constants import ROOT_DIR, LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.constants import ROOT_DIR from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ - get_data_source, get_bundle_folder, get_data_source_folder +from catalyst.marketplace.utils.path_utils import get_data_source, \ + get_bundle_folder, get_data_source_folder +# TODO: host our own node on aws? REMOTE_NODE = 'http://localhost:7545' +# TODO: read from GitHub CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) @@ -21,6 +24,8 @@ CONTRACT_ADDRESS = Web3.toChecksumAddress( '0xe2b6cf3863240892d59664d209a28289a73ef644' ) +log = logbook.Logger('Marketplace', level=LOG_LEVEL) + class Marketplace: def __init__(self): @@ -43,21 +48,33 @@ class Marketplace: desc='The marketcap value in USD.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily'], ), dict( name='GitHub', desc='The rate of development activity on GitHub.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour'], ), dict( name='Influencers', desc='Tweets and related sentiments by selected influencers.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour', 'minute'], ), ] + def get_data_source_def(self, data_source_name): + data_source_name = data_source_name.lower() + dsm = self.get_data_sources_map() + + ds = six.next( + (d for d in dsm if d['name'].lower() == data_source_name), None + ) + return ds + def list(self): subscribers = self.contract.call( {'from': self.default_account} @@ -136,6 +153,35 @@ class Marketplace: pass + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + ds_def = self.get_data_source_def(data_source_name) + freqs = ds_def['data_frequencies'] + data_frequency = freqs[0] + + if len(freqs) > 1: + log.warn( + 'no data frequencies specified for data source {}, ' + 'selected the first one by default: {}'.format( + data_source_name, data_frequency + ) + ) + + # TODO: filter ctable by start and end date + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + z = bcolz.ctable(rootdir=bundle_folder, mode='r') + + df = z.todataframe() # type: pd.DataFrame + df.set_index(['date', 'symbol'], drop=False, inplace=True) + + if start and end is None: + df = df.xs(start, level=0) + + return df + def clean(self, data_source_name, data_frequency=None): data_source_name = data_source_name.lower() @@ -143,7 +189,7 @@ class Marketplace: folder = get_data_source_folder(data_source_name) else: - forlder = get_bundle_folder(data_source_name, data_frequency) + folder = get_bundle_folder(data_source_name, data_frequency) shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 916722da..532f46ce 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -18,9 +18,9 @@ def merge_bundles(zsource, ztarget): """ # TODO: find a way to do this iteratively instead of in-memory df_source = zsource.todataframe() - df_source.set_index('last_updated', drop=False, inplace=True) + df_source.set_index('date', drop=False, inplace=True) df_target = ztarget.todataframe() - df_target.set_index('last_updated', drop=False, inplace=True) + df_target.set_index('date', drop=False, inplace=True) df = df_target.merge( right=df_source, diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 6d301880..890eaa20 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -34,8 +34,7 @@ def get_data_source_folder(data_source_name, environ=None): def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) - subfolder = data_frequency if data_frequency is not None else 'data' - bundle_folder = os.path.join(data_source_folder, subfolder) + bundle_folder = os.path.join(data_source_folder, data_frequency) ensure_directory(bundle_folder) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 68990791..9292b3a0 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,10 +16,17 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() + ds_def = marketplace.get_data_source_def('Marketcap') + marketplace.ingest( data_source_name='Marketcap', - data_frequency='finest', - start=pd.Timestamp.utcnow(), + data_frequency=ds_def['data_frequencies'][0], + start=pd.to_datetime('2017-10-01'), force_download=True, ) pass + + def test_clean(self): + marketplace = Marketplace() + marketplace.clean('marketcap') + pass From df4dd0f5a496d42f7080660d966201a1d59ca37f Mon Sep 17 00:00:00 2001 From: Alex Iribarren Date: Sun, 14 Jan 2018 13:55:16 +0100 Subject: [PATCH 13/85] Update example-algos.rst exchange_utils was moved. --- docs/source/example-algos.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/example-algos.rst b/docs/source/example-algos.rst index 217acf06..98d9a9f7 100644 --- a/docs/source/example-algos.rst +++ b/docs/source/example-algos.rst @@ -805,7 +805,7 @@ Credits: This code was originally submitted by `Abner Ayala-Acevedo import pandas as pd from catalyst import run_algorithm - from catalyst.exchange.exchange_utils import get_exchange_symbols + from catalyst.exchange.utils.exchange_utils import get_exchange_symbols from catalyst.api import (symbols, ) From b47884a504146dbb49c6b215a8c4325303c820eb Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 22 Jan 2018 17:27:59 -0500 Subject: [PATCH 14/85] BLD: code formatting --- catalyst/exchange/ccxt/ccxt_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 5f5f7a06..1579aaa9 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1030,7 +1030,7 @@ class CCXT(Exchange): ticker['volume'] = ticker['baseVolume'] elif 'info' in ticker and 'bidQty' in ticker['info'] \ - and 'askQty' in ticker['info']: + and 'askQty' in ticker['info']: ticker['volume'] = float(ticker['info']['bidQty']) + \ float(ticker['info']['askQty']) From 8223d06d98d29bb09dfac0fe5a08cbbc4b6be41d Mon Sep 17 00:00:00 2001 From: Cam Sweeney Date: Tue, 23 Jan 2018 02:37:01 -0800 Subject: [PATCH 15/85] BUG: Use empyrical patches for persisting issue #126 The referenced issue was addressed via importing a set of patches for empyrical. However the same error occurs occasionally when calling the empyrical functions inside "/catalyst/finance/risk/period.py". This PR simply applies 2 of the same patches in period.py. I have only experienced problems with cum_returns and max_drawdown thus far, but it is likely the other patches may be needed. --- catalyst/finance/risk/period.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/catalyst/finance/risk/period.py b/catalyst/finance/risk/period.py index 987b8d72..2b072ea9 100644 --- a/catalyst/finance/risk/period.py +++ b/catalyst/finance/risk/period.py @@ -29,13 +29,15 @@ from .risk import check_entry from empyrical import ( alpha_beta_aligned, annual_volatility, - cum_returns, downside_risk, information_ratio, - max_drawdown, sharpe_ratio, sortino_ratio ) +from catalyst.patches.stats import ( + max_drawdown, + cum_returns, +) from catalyst.constants import LOG_LEVEL From 76e183b5f7fc6fb6548ff19da4245a701564a90b Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 12:48:42 -0700 Subject: [PATCH 16/85] BLD: contract address+abi on testnet --- catalyst/marketplace/contract_abi.json | 1 + catalyst/marketplace/contract_address.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 catalyst/marketplace/contract_abi.json create mode 100644 catalyst/marketplace/contract_address.txt diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_abi.json new file mode 100644 index 00000000..a3dda75f --- /dev/null +++ b/catalyst/marketplace/contract_abi.json @@ -0,0 +1 @@ +[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_address.txt new file mode 100644 index 00000000..ec7bf029 --- /dev/null +++ b/catalyst/marketplace/contract_address.txt @@ -0,0 +1 @@ +0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file From 04fed4140c9e3e37ca1c80b38c0a92f0665bfe12 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 14:12:59 -0700 Subject: [PATCH 17/85] BLD: sourcing contract address+abi from github --- catalyst/marketplace/marketplace.py | 50 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 041fb004..a28d1c9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,7 @@ import json import os import shutil +import urllib import bcolz import logbook @@ -15,31 +16,44 @@ from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder # TODO: host our own node on aws? -REMOTE_NODE = 'http://localhost:7545' -# TODO: read from GitHub -CONTRACT_PATH = os.path.join( - ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' -) -CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xe2b6cf3863240892d59664d209a28289a73ef644' -) +# TODO: switch to mainnet +REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_address.txt' + +CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - with open(CONTRACT_PATH) as handle: - json_interface = json.load(handle) - w3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, - abi=json_interface['abi'], - ) # Type: Contract - self.default_account = w3.eth.accounts[1] + contract_url = urllib.urlopen(CONTRACT_PATH) + CONTRACT_ADDRESS = Web3.toChecksumAddress( + contract_url.readline().strip()) - pass + abi_url = urllib.urlopen(CONTRACT_ABI) + abi = json.load(abi_url) + + w3 = Web3(HTTPProvider(REMOTE_NODE)) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=abi, + ) # Type: Contract + + # TODO: Set default address correctly from user-provided config + DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) + if DEFAULT_ETH_ADDRESS is None: + raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' + 'environment variable. Quitting...') + + self.default_account = DEFAULT_ETH_ADDRESS + pass def get_data_sources_map(self): return [ @@ -93,7 +107,7 @@ class Marketplace: dict( id=index, subscribed=index in subscribed, - **data_source, + **data_source ) ) From f34a66e9c68ed4ba2a7e5a760c65d4680e618812 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 24 Jan 2018 21:54:05 -0500 Subject: [PATCH 18/85] BUG: trying to fix issue #178 with Binance lot sizes --- catalyst/examples/mean_reversion_simple.py | 4 ++-- catalyst/exchange/ccxt/ccxt_exchange.py | 27 +++++++++++----------- etc/requirements.txt | 2 +- tests/exchange/test_ccxt.py | 10 ++++---- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index b215f5cf..6909dbc4 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -248,7 +248,7 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=0.01, + capital_base=100, initialize=initialize, handle_data=handle_data, analyze=analyze, @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='btc', live_graph=False, - simulate_orders=False, + simulate_orders=True, stats_output=None, # auth_aliases=dict(poloniex='auth2') ) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 1579aaa9..5f330b9c 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -760,25 +760,24 @@ class CCXT(Exchange): side = 'buy' if amount > 0 else 'sell' if hasattr(self.api, 'amount_to_lots'): - adj_amount = self.api.amount_to_lots( - symbol=symbol, - amount=abs(amount), - ) - if adj_amount != abs(amount): - log.info( - 'adjusted order amount {} to {} based on lot size'.format( - abs(amount), adj_amount, + # TODO: is this right? + if self.api.markets is None: + self.api.load_markets() + + # https://github.com/ccxt/ccxt/issues/1483 + adj_amount = abs(amount) + market = self.api.markets[symbol] + if 'lots' in market and market['lots'] > amount: + raise CreateOrderError( + exchange=self.name, + e='order amount lower than the smallest lot: {}'.format( + amount ) ) + else: adj_amount = abs(amount) - if adj_amount == 0: - raise CreateOrderError( - exchange=self.name, - e='order amount lower than the smallest lot: {}'.format(amount) - ) - try: result = self.api.create_order( symbol=symbol, diff --git a/etc/requirements.txt b/etc/requirements.txt index b3f17d0e..8138f6c3 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.774 +ccxt==1.10.837 boto3==1.4.8 redo==1.6 diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 8bffe616..4e33ae3e 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -15,23 +15,23 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'bitfinex' + exchange_name = 'binance' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, key=auth['key'], secret=auth['secret'], - base_currency='bnb', + base_currency='usdt', ) self.exchange.init() def test_order(self): log.info('creating order') - asset = self.exchange.get_asset('neo_bnb') + asset = self.exchange.get_asset('eth_usdt') order_id = self.exchange.order( asset=asset, - style=ExchangeLimitOrder(limit_price=10), - amount=1, + style=ExchangeLimitOrder(limit_price=1000), + amount=1.01, ) log.info('order created {}'.format(order_id)) assert order_id is not None From c09f53449a690944b5d4905ac040d366b9f064c1 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 24 Jan 2018 22:26:20 -0500 Subject: [PATCH 19/85] BUG: fixed issue #176 with ignoring ticker errors instead of raising --- catalyst/exchange/ccxt/ccxt_exchange.py | 8 ++++++-- catalyst/exchange/exchange.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 5f330b9c..3227361f 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -984,7 +984,7 @@ class CCXT(Exchange): ) raise ExchangeRequestError(error=e) - def tickers(self, assets): + def tickers(self, assets, on_ticker_error='raise'): """ Retrieve current tick data for the given assets @@ -1016,7 +1016,11 @@ class CCXT(Exchange): self.name, asset.symbol, e ) ) - continue + if on_ticker_error == 'warn': + continue + + else: + raise ExchangeRequestError(error=e) ticker['last_traded'] = from_ms_timestamp(ticker['timestamp']) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 71ae0689..c66121bc 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -972,13 +972,15 @@ class Exchange: pass @abc.abstractmethod - def tickers(self, assets): + def tickers(self, assets, on_ticker_error='raise'): """ Retrieve current tick data for the given assets Parameters ---------- assets: list[TradingPair] + on_ticker_error: str [raise|warn] + How to handle an error when retrieving a single ticker. Returns ------- From 0aa8c91577aab693ad94ce0152fd6df04ee9f754 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 24 Jan 2018 22:43:08 -0500 Subject: [PATCH 20/85] BLD: for issue #174, re-implemented the fetch_tickers approach --- catalyst/exchange/ccxt/ccxt_exchange.py | 36 ++++++++++++------------- tests/exchange/test_ccxt.py | 3 ++- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 3227361f..543466aa 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -997,31 +997,31 @@ class CCXT(Exchange): list[dict[str, float] """ - tickers = {} + symbols = self.get_symbols(assets) + try: + results = self.api.fetch_tickers(symbols=symbols) + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch tickers {} / {}: {}'.format( + self.name, symbols, e + ) + ) + raise ExchangeRequestError(error=e) + + tickers = dict() for asset in assets: symbol = self.get_symbol(asset) - - # Test the CCXT throttling further to see if we need this - self.ask_request() - - # TODO: use fetch_tickers() for efficiency - # I tried using fetch_tickers() but noticed some - # inconsistencies, see issue: - # https://github.com/ccxt/ccxt/issues/870 - try: - ticker = self.api.fetch_ticker(symbol=symbol) - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch ticker {} / {}: {}'.format( - self.name, asset.symbol, e - ) + if symbol not in results: + msg = 'ticker not found {} / {}'.format( + self.name, symbol ) + log.warn(msg) if on_ticker_error == 'warn': continue - else: - raise ExchangeRequestError(error=e) + raise ExchangeRequestError(error=msg) + ticker = results[symbol] ticker['last_traded'] = from_ms_timestamp(ticker['timestamp']) if 'last_price' not in ticker: diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 4e33ae3e..3683ca4a 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -72,7 +72,8 @@ class TestCCXT(BaseExchangeTestCase): def test_tickers(self): log.info('retrieving tickers') assets = [ - self.exchange.get_asset('iot_usd'), + self.exchange.get_asset('ada_eth'), + self.exchange.get_asset('zrx_eth'), ] tickers = self.exchange.tickers(assets) assert len(tickers) == 1 From c6dfd502d28a3967ff202205ed3dc708594dbd7a Mon Sep 17 00:00:00 2001 From: Avishai Weingarten <33716232+AvishaiW@users.noreply.github.com> Date: Thu, 25 Jan 2018 16:02:44 +0200 Subject: [PATCH 21/85] fix for click.echo added sys.stdout to click.echo to prevent errors on jupyter --- catalyst/__main__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 2c531e15..1c6d0870 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -3,6 +3,7 @@ import os from functools import wraps import click +import sys import logbook import pandas as pd from six import text_type @@ -257,7 +258,7 @@ def run(ctx, if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") - click.echo('Running in backtesting mode.') + click.echo('Running in backtesting mode.', sys.stdout) perf = _run( initialize=None, @@ -290,7 +291,7 @@ def run(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -460,10 +461,10 @@ def live(ctx, ctx.fail("must specify a capital base with '--capital-base'") if simulate_orders: - click.echo('Running in paper trading mode.') + click.echo('Running in paper trading mode.', sys.stdout) else: - click.echo('Running in live trading mode.') + click.echo('Running in live trading mode.', sys.stdout) perf = _run( initialize=None, @@ -496,7 +497,7 @@ def live(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -578,7 +579,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -601,10 +602,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def clean_algo(ctx, algo_namespace): click.echo( - 'Cleaning algo state: {}'.format(algo_namespace) + 'Cleaning algo state: {}'.format(algo_namespace), + sys.stdout ) delete_algo_folder(algo_namespace) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command(name='clean-exchange') @@ -631,11 +633,11 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command() @@ -756,7 +758,7 @@ def bundles(): # because there were no entries, print a single message indicating that # no ingestions have yet been made. for timestamp in ingestions or [""]: - click.echo("%s %s" % (bundle, timestamp)) + click.echo("%s %s" % (bundle, timestamp), sys.stdout) if __name__ == '__main__': From 2ec9aa2ca96c8121b36d53868c5c68529ee7180e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 11:55:25 -0700 Subject: [PATCH 22/85] BLD: CLI implementation for the marketplace --- catalyst/__main__.py | 197 ++++++++++++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 69 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f70b985..10c6c6ac 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -574,75 +574,6 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) -@main.command(name='ls-data') -@click.pass_context -def ls_data(ctx): - click.echo( - 'The alternative data sources:' - ) - marketplace = Marketplace() - marketplace.list() - click.echo('Done') - - -@main.command(name='register-data') -@click.argument('data_source_name') -@click.pass_context -def register_data(ctx, data_source_name): - click.echo( - 'Registering to data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.register(data_source_name) - click.echo('Done') - - -@main.command(name='ingest-data') -@click.argument('data_source_name') -@click.option( - '-f', - '--data-frequency', - type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), - default='daily', - show_default=True, - help='The data frequency of the desired OHLCV bars.', -) -@click.option( - '-s', - '--start', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The start date of the data range. (default: one year from end date)', -) -@click.option( - '-e', - '--end', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The end date of the data range. (default: today)', -) -@click.pass_context -def ingest_data(ctx, data_source_name, data_frequency, start, end): - click.echo( - 'Ingesting data: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.ingest(data_source_name, data_frequency, start, end) - click.echo('Done') - - -@main.command(name='clean-data') -@click.argument('data_source_name') -@click.pass_context -def clean_data(ctx, data_source_name): - click.echo( - 'Cleaning data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.clean(data_source_name) - click.echo('Done') - - @main.command(name='clean-algo') @click.option( '-n', @@ -810,5 +741,133 @@ def bundles(): click.echo("%s %s" % (bundle, timestamp)) +@main.group() +@click.pass_context +def marketplace(ctx): + pass + + +@marketplace.command() +@click.pass_context +def ls(ctx): + click.echo('Listing of available data sources on the marketplace:') + marketplace = Marketplace() + marketplace.list() + click.echo('Functionality not yet implemented.') + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def subscribe(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + marketplace = Marketplace() + marketplace.subscribe(dataset) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest(ctx, dataset, data_frequency, start, end): + if dataset is None: + ctx.fail("must specify a dataset to clean with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Ingesting data: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.ingest(dataset, data_frequency, start, end) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def clean(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to ingest with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Cleaning data source: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.clean(dataset) + click.echo('Done') + + +@marketplace.command() +@click.pass_context +def register(ctx): + marketplace = Marketplace() + marketplace.register() + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the Marketplace dataset to publish data for.', +) +@click.option( + '--datadir', + default=None, + help='The folder that contains the CSV data files to publish.', +) +@click.option( + '--watch/--no-watch', + is_flag=True, + default=False, + help='Whether to watch the datadir for live data.', +) +@click.pass_context +def publish(ctx, dataset, datadir, watch): + marketplace = Marketplace() + if dataset is None: + ctx.fail("must specify a dataset to publish data for " + " with '--dataset'\n") + if datadir is None: + ctx.fail("must specify a datadir where to find the files to publish " + " with '--datadir'\n") + marketplace.publish(dataset, datadir, watch) + + if __name__ == '__main__': main() From 940f625ec1fe37995193e30a783844cb16f3db11 Mon Sep 17 00:00:00 2001 From: Avishai Weingarten <33716232+AvishaiW@users.noreply.github.com> Date: Thu, 25 Jan 2018 16:02:44 +0200 Subject: [PATCH 23/85] fix for click.echo added sys.stdout to click.echo to prevent errors on jupyter --- catalyst/__main__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 10c6c6ac..411f712f 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -3,6 +3,7 @@ import os from functools import wraps import click +import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -258,7 +259,7 @@ def run(ctx, if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") - click.echo('Running in backtesting mode.') + click.echo('Running in backtesting mode.', sys.stdout) perf = _run( initialize=None, @@ -290,7 +291,7 @@ def run(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -443,10 +444,10 @@ def live(ctx, ctx.fail("must specify a capital base with '--capital-base'") if simulate_orders: - click.echo('Running in paper trading mode.') + click.echo('Running in paper trading mode.', sys.stdout) else: - click.echo('Running in live trading mode.') + click.echo('Running in live trading mode.', sys.stdout) perf = _run( initialize=None, @@ -478,7 +479,7 @@ def live(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -560,7 +561,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -583,10 +584,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def clean_algo(ctx, algo_namespace): click.echo( - 'Cleaning algo state: {}'.format(algo_namespace) + 'Cleaning algo state: {}'.format(algo_namespace), + sys.stdout ) delete_algo_folder(algo_namespace) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command(name='clean-exchange') @@ -613,11 +615,11 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command() @@ -738,7 +740,7 @@ def bundles(): # because there were no entries, print a single message indicating that # no ingestions have yet been made. for timestamp in ingestions or [""]: - click.echo("%s %s" % (bundle, timestamp)) + click.echo("%s %s" % (bundle, timestamp), sys.stdout) @main.group() From 7ae23e23403df5f515230a72b955bd12b0cb3ee8 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 25 Jan 2018 17:40:45 -0500 Subject: [PATCH 24/85] BLD: conditionally fetching single or multi tickers for performance reasons --- catalyst/exchange/ccxt/ccxt_exchange.py | 38 +++++++++++++++++++------ catalyst/exchange/exchange.py | 2 +- tests/exchange/test_ccxt.py | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 543466aa..3e5d5b29 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -997,16 +997,36 @@ class CCXT(Exchange): list[dict[str, float] """ - symbols = self.get_symbols(assets) - try: - results = self.api.fetch_tickers(symbols=symbols) - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch tickers {} / {}: {}'.format( - self.name, symbols, e + if len(assets) == 1: + symbol = self.get_symbol(assets[0]) + try: + log.debug('fetching single ticker: {}'.format(symbol)) + results = dict() + results[symbol] = self.api.fetch_ticker(symbol=symbol) + + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch ticker {} / {}: {}'.format( + self.name, symbol, e + ) ) - ) - raise ExchangeRequestError(error=e) + raise ExchangeRequestError(error=e) + + elif len(assets) > 1: + symbols = self.get_symbols(assets) + try: + log.debug('fetching multiple tickers: {}'.format(symbols)) + results = self.api.fetch_tickers(symbols=symbols) + + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch tickers {} / {}: {}'.format( + self.name, symbols, e + ) + ) + raise ExchangeRequestError(error=e) + else: + raise ValueError('Cannot request tickers with not assets.') tickers = dict() for asset in assets: diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c66121bc..1d96d886 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -701,7 +701,7 @@ class Exchange: ) positions_value = 0.0 - if positions is not None: + if positions: assets = set([position.asset for position in positions]) tickers = self.tickers(assets) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 3683ca4a..af455cc4 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -76,7 +76,7 @@ class TestCCXT(BaseExchangeTestCase): self.exchange.get_asset('zrx_eth'), ] tickers = self.exchange.tickers(assets) - assert len(tickers) == 1 + assert len(tickers) == 2 pass def test_my_trades(self): From 8ff8e6458eb9a402da24bc74b64b34b2e27e4e87 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 25 Jan 2018 17:43:00 -0500 Subject: [PATCH 25/85] BUG: fixed stats output issue #171 by adding orders and transactions in the header --- catalyst/exchange/utils/stats_utils.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index 6ca58d13..6e2aab0b 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -44,7 +44,7 @@ def crossover(source, target): """ if isinstance(target, numbers.Number): if source[-1] is np.nan or source[-2] is np.nan \ - or target is np.nan: + or target is np.nan: return False if source[-1] >= target > source[-2]: @@ -54,7 +54,7 @@ def crossover(source, target): else: if source[-1] is np.nan or source[-2] is np.nan \ - or target[-1] is np.nan or target[-2] is np.nan: + or target[-1] is np.nan or target[-2] is np.nan: return False if source[-1] > target[-1] and source[-2] < target[-2]: @@ -81,7 +81,7 @@ def crossunder(source, target): """ if isinstance(target, numbers.Number): if source[-1] is np.nan or source[-2] is np.nan \ - or target is np.nan: + or target is np.nan: return False if source[-1] < target <= source[-2]: @@ -90,7 +90,7 @@ def crossunder(source, target): return False else: if source[-1] is np.nan or source[-2] is np.nan \ - or target[-1] is np.nan or target[-2] is np.nan: + or target[-1] is np.nan or target[-2] is np.nan: return False if source[-1] < target[-1] and source[-2] >= target[-2]: @@ -229,7 +229,10 @@ def prepare_stats(stats, recorded_cols=list()): asset_values) df = pd.DataFrame(stats) - + df['orders'] = df['orders'].apply(lambda orders: len(orders)) + df['transactions'] = df['transactions'].apply( + lambda transactions: len(transactions) + ) index_cols = [ 'period_close', 'starting_cash', 'ending_cash', 'portfolio_value', 'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions', @@ -241,11 +244,6 @@ def prepare_stats(stats, recorded_cols=list()): for column in recorded_cols: index_cols.append(column) - df['orders'] = df['orders'].apply(lambda orders: len(orders)) - df['transactions'] = df['transactions'].apply( - lambda transactions: len(transactions) - ) - if asset_cols: columns = asset_cols df.set_index(index_cols, drop=True, inplace=True) From fa60457a0f4d7ff03acf2228b0c47a9d30320644 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 25 Jan 2018 23:38:37 -0500 Subject: [PATCH 26/85] BLD: misc adjustments to fetch all candles on the Binance exchange --- catalyst/exchange/exchange.py | 1 + catalyst/exchange/exchange_algorithm.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 1d96d886..84cac446 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -178,6 +178,7 @@ class Exchange: if symbols is None: # Make a distinct list of all symbols symbols = list(set([asset.symbol for asset in self.assets])) + symbols.sort() if quote_currency is not None: for symbol in symbols[:]: diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 857d44ac..5c83a74a 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -67,7 +67,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): self.current_day = None if self.simulate_orders is None \ - and self.sim_params.arena == 'backtest': + and self.sim_params.arena == 'backtest': self.simulate_orders = True # Operations with retry features @@ -115,7 +115,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # be in-line with CXXT and many exchanges. We'll consider # adding more order types in the future. if not isinstance(style, ExchangeLimitOrder) or \ - not isinstance(style, MarketOrder): + not isinstance(style, MarketOrder): raise OrderTypeNotSupported( order_type=style.__class__.__name__ ) @@ -901,7 +901,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): sleeptime=self.attempts['retry_sleeptime'], retry_exceptions=(ExchangeRequestError,), cleanup=lambda: log.warn('Fetching open orders again.'), - args=(asset,)) + args=(asset,) + ) @api_method def get_order(self, order_id, exchange_name): From ff989ba52405e344c5365d9b6430afeb4490f425 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 22:39:54 -0700 Subject: [PATCH 27/85] BLD: register dataset in marketplace --- catalyst/__main__.py | 17 ++- catalyst/marketplace/marketplace.py | 140 +++++++++++++++++---- catalyst/marketplace/marketplace_errors.py | 24 ++++ catalyst/marketplace/utils/path_utils.py | 56 +++++++++ 4 files changed, 205 insertions(+), 32 deletions(-) create mode 100644 catalyst/marketplace/marketplace_errors.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 411f712f..3833f3e4 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,9 +1,9 @@ import errno import os +import sys from functools import wraps import click -import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -752,10 +752,11 @@ def marketplace(ctx): @marketplace.command() @click.pass_context def ls(ctx): - click.echo('Listing of available data sources on the marketplace:') + click.echo('Listing of available data sources on the marketplace:', + sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.') + click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() @@ -808,9 +809,7 @@ def ingest(ctx, dataset, data_frequency, start, end): ctx.fail("must specify a dataset to clean with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Ingesting data: {}'.format(dataset) - ) + click.echo('Ingesting data: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.ingest(dataset, data_frequency, start, end) @@ -827,12 +826,10 @@ def clean(ctx, dataset): ctx.fail("must specify a dataset to ingest with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Cleaning data source: {}'.format(dataset) - ) + click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.clean(dataset) - click.echo('Done') + click.echo('Done', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a28d1c9f..4b1cfe8b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,23 +1,31 @@ import json import os import shutil -import urllib +import sys +import binascii import bcolz import logbook import pandas as pd import six from web3 import Web3, HTTPProvider -from catalyst.constants import ROOT_DIR, LOG_LEVEL +from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder + get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr + +if sys.version_info.major < 3: + import urllib +else: + import urllib.request as urllib # TODO: host our own node on aws? # TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' +REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ @@ -32,28 +40,28 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - contract_url = urllib.urlopen(CONTRACT_PATH) - CONTRACT_ADDRESS = Web3.toChecksumAddress( + self.addresses = get_user_pubaddr() + + if self.addresses[0]['pubAddr'] == '': + raise MarketplacePubAddressEmpty( + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) + self.default_account = self.addresses[0]['pubAddr'] + + contract_url = urllib.urlopen(CONTRACT_PATH) + self.contract_address = Web3.toChecksumAddress( contract_url.readline().strip()) abi_url = urllib.urlopen(CONTRACT_ABI) abi = json.load(abi_url) - w3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, + self.contract = self.web3.eth.contract( + self.contract_address, abi=abi, - ) # Type: Contract - - # TODO: Set default address correctly from user-provided config - DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) - if DEFAULT_ETH_ADDRESS is None: - raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' - 'environment variable. Quitting...') - - self.default_account = DEFAULT_ETH_ADDRESS - pass + ) def get_data_sources_map(self): return [ @@ -122,11 +130,11 @@ class Marketplace: pass - def register(self, data_source_name): + def subscribe(self, dataset): data_sources = self.get_data_sources_map() index = next( (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == data_source_name.lower()), + d['name'].lower() == dataset.lower()), None ) if index is None: @@ -140,7 +148,7 @@ class Marketplace: ).subscribe(index) print( 'Subscribed to data source {} successfully'.format( - data_source_name + dataset ) ) @@ -207,3 +215,91 @@ class Marketplace: shutil.rmtree(folder) pass + + def register(self): + dataset = input('Enter the name of the dataset to register: ') + price = int(input('Enter the price for a monthly subscription to ' + 'this dataset in ENG: ')) + + # while True: + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): + # print("Not a valid frequency.") + # else: + # break + + # while True: + # reg_pub = input('Can data be published every hour at a regular ' + # 'time? [default: Y]: ') or 'y' + # if reg_pub.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if reg_pub.lower() == 'y': + # reg_pub = True + # else: + # reg_pub = False + # break + + # while True: + # hist = input('Does it include historical data? ' + # '[default: Y]') or 'y' + # if hist.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if hist.lower() == 'y': + # hist = True + # else: + # hist = False + # break + + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose the your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + tx = self.contract.functions.register( + binascii.hexlify(dataset.encode('utf-8')), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') + + tx_hash = '0x{}'.format(binascii.hexlify( + self.web3.eth.sendRawTransaction(signed_tx) + ).decode('utf-8')) + + print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py new file mode 100644 index 00000000..04fca091 --- /dev/null +++ b/catalyst/marketplace/marketplace_errors.py @@ -0,0 +1,24 @@ +import sys +import traceback + +from catalyst.errors import ZiplineError + + +def silent_except_hook(exctype, excvalue, exctraceback): + if exctype in [MarketplacePubAddressEmpty]: + fn = traceback.extract_tb(exctraceback)[-1][0] + ln = traceback.extract_tb(exctraceback)[-1][1] + print("Error traceback: {1} (line {2})\n" + "{0.__name__}: {3}".format(exctype, fn, ln, excvalue)) + else: + sys.__excepthook__(exctype, excvalue, exctraceback) + + +sys.excepthook = silent_except_hook + + +class MarketplacePubAddressEmpty(ZiplineError): + msg = ( + 'Please enter your public address to use in the Data Marketplace ' + 'in the following file: {filename}' + ).strip() diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 890eaa20..2cee9303 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,4 +1,5 @@ import os +import json import tarfile import shutil @@ -7,6 +8,29 @@ from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory +def get_marketplace_folder(environ=None): + """ + The root path of the marketplace folder. + + Parameters + ---------- + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + marketplace_folder = os.path.join(root, 'marketplace') + ensure_directory(marketplace_folder) + + return marketplace_folder + + def get_data_source_folder(data_source_name, environ=None): """ The root path of an data_source folder. @@ -104,3 +128,35 @@ def get_data_source(data_source_name, period, force_download=False): tar.extractall(path) return path + + +def get_user_pubaddr(environ=None): + """ + The de-serialized contend of the user's addresses.json file. + Parameters + ---------- + environ: + + Returns + ------- + Object + + """ + marketplace_folder = get_marketplace_folder(environ) + + filename = os.path.join(marketplace_folder, 'addresses.json') + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + try: + d = data[0]['pubAddr'] + except Exception as e: + return [data, ] + return data + else: + data = dict(pubAddr='', desc='') + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + return data From 68faad40989bd5b37e56eb9d45bca9083feb72fb Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 26 Jan 2018 15:32:53 -0700 Subject: [PATCH 28/85] BLD: publish dataset in marketplace --- catalyst/marketplace/marketplace.py | 84 +++++++++++++++++++--- catalyst/marketplace/marketplace_errors.py | 29 +++++++- catalyst/marketplace/utils/auth_utils.py | 47 ++++++++++++ catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/eth_utils.py | 33 +++++++++ catalyst/marketplace/utils/path_utils.py | 26 ++++++- 6 files changed, 210 insertions(+), 13 deletions(-) create mode 100644 catalyst/marketplace/utils/auth_utils.py create mode 100644 catalyst/marketplace/utils/eth_utils.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4b1cfe8b..af4c4aef 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,22 +1,31 @@ -import json import os -import shutil import sys +import json +import hmac +import glob +import time +import shutil +import hashlib -import binascii import bcolz import logbook import pandas as pd import six +import requests from web3 import Web3, HTTPProvider from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty +from catalyst.marketplace.marketplace_errors import ( + MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: import urllib @@ -34,6 +43,8 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' +AUTH_SERVER = 'http://localhost:5000' + log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -259,7 +270,7 @@ class Marketplace: self.addresses[i]['pubAddr'], self.addresses[i]['desc']) ) - address_i = int(input('Choose the your address associated with ' + address_i = int(input('Choose your address associated with ' 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( @@ -270,7 +281,7 @@ class Marketplace: break tx = self.contract.functions.register( - binascii.hexlify(dataset.encode('utf-8')), + bytes32(dataset), price, address, ).buildTransaction( @@ -298,8 +309,63 @@ class Marketplace: signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') - tx_hash = '0x{}'.format(binascii.hexlify( - self.web3.eth.sendRawTransaction(signed_tx) - ).decode('utf-8')) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + def publish(self, dataset, datadir, watch): + + datasource = self.contract.functions.getDataSource( + bytes32(dataset)).call() + + if not datasource[4]: + raise MarketplaceDatasetNotFound(dataset=dataset) + + match = next((l for l in self.addresses if + l['pubAddr'] == datasource[0]), None) + + if not match: + raise MarketplaceNoAddressMatch( + dataset=dataset, + address=datasource[0]) + + print('Using address: {} to publish this dataset.'.format( + datasource[0])) + + if 'key' in match: + key = match['key'] + secret = match['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + nonce.encode('utf-8'), + hashlib.sha512).hexdigest() + headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + filenames = glob.glob(os.path.join(datadir, '*.csv')) + + if not filenames: + raise MarketplaceNoCSVFiles(datadir=datadir) + + files = [] + for file in filenames: + files.append(('file', open(file, 'rb'))) + + r = requests.post('{}/publish'.format(AUTH_SERVER), + files=files, + headers=headers) + + if r.status_code != 200: + raise MarketplaceHTTPRequest(request='upload file', + error=r.status_code) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest(request='upload file', + error=r.json()['error']) + + print('Dataset {} published successfully.'.format(dataset)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 04fca091..3992158d 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -5,7 +5,9 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): - if exctype in [MarketplacePubAddressEmpty]: + if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -22,3 +24,28 @@ class MarketplacePubAddressEmpty(ZiplineError): 'Please enter your public address to use in the Data Marketplace ' 'in the following file: {filename}' ).strip() + + +class MarketplaceDatasetNotFound(ZiplineError): + msg = ( + 'The dataset "{dataset}" is not registered in the Data Marketplace.' + ).strip() + + +class MarketplaceNoAddressMatch(ZiplineError): + msg = ( + 'The address registered with the dataset {dataset}: {address} ' + 'does not match any of your addresses.' + ).strip() + + +class MarketplaceHTTPRequest(ZiplineError): + msg = ( + 'Request to remote server to {request} failed: {error}' + ).strip() + + +class MarketplaceNoCSVFiles(ZiplineError): + msg = ( + 'No CSV files found on {datadir} to upload.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py new file mode 100644 index 00000000..f1193758 --- /dev/null +++ b/catalyst/marketplace/utils/auth_utils.py @@ -0,0 +1,47 @@ +import requests + +from catalyst.marketplace.marketplace_errors import ( + MarketplaceHTTPRequest) +from catalyst.marketplace.utils.path_utils import ( + get_user_pubaddr, save_user_pubaddr) + +AUTH_SERVER = 'http://localhost:5000' + + +def get_key_secret(pubAddr, dataset): + """ + Obtain a new key/secret pair from authentication server + + Parameters + ---------- + pubAddr: str + dataset: str + + Returns + ------- + key: str + secret: str + + """ + session = requests.Session() + response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + headers={ + 'pubAddr': pubAddr, + 'dataset': dataset}) + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + + return match['key'], match['secret'] diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 532f46ce..b58595ac 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -1,8 +1,8 @@ -import bcolz import os -import pandas as pd import shutil +import bcolz + def merge_bundles(zsource, ztarget): """ diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py new file mode 100644 index 00000000..c2be4bd6 --- /dev/null +++ b/catalyst/marketplace/utils/eth_utils.py @@ -0,0 +1,33 @@ +import binascii + + +def bytes32(string): + """ + Convert string to bytes32 data type for smart contract + + Parameters + ---------- + string: str + + Returns + ------- + list + + """ + return binascii.hexlify(string.encode('utf-8')) + + +def b32_str(bytes32): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(bytes32.decode('utf-8')) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 2cee9303..62f8beac 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,6 +133,7 @@ def get_data_source(data_source_name, period, force_download=False): def get_user_pubaddr(environ=None): """ The de-serialized contend of the user's addresses.json file. + Parameters ---------- environ: @@ -143,7 +144,6 @@ def get_user_pubaddr(environ=None): """ marketplace_folder = get_marketplace_folder(environ) - filename = os.path.join(marketplace_folder, 'addresses.json') if os.path.isfile(filename): @@ -160,3 +160,27 @@ def get_user_pubaddr(environ=None): json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) return data + + +def save_user_pubaddr(data, environ=None): + """ + Saves the user's public addresses and their related metadata in + the corresponding addresses.json file. + + Parameters + ---------- + data: dict + + Returns + ------- + True + + """ + marketplace_folder = get_marketplace_folder(environ) + filename = os.path.join(marketplace_folder, 'addresses.json') + + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + + return True From 7f021acb2efdee761f6c0f2ca1b0dc761e605bd2 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 15:58:34 -0500 Subject: [PATCH 29/85] BUG: fixed catalyst import --- tests/exchange/test_suites/test_suite_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index 4088a675..44e445bf 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -15,7 +15,7 @@ from catalyst.exchange.utils.test_utils import select_random_exchanges, \ handle_exchange_error, select_random_assets from catalyst.testing import ZiplineTestCase from catalyst.testing.fixtures import WithLogger -from exchange.utils.factory import get_exchanges +from catalyst.exchange.utils.factory import get_exchanges log = Logger('TestSuiteExchange') From 5c236a65f7ed0906bfc29cf2a08c920831327f79 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 19:20:34 -0500 Subject: [PATCH 30/85] BUG: for issue #178, checking the order status instead of relying on the open amount --- catalyst/exchange/exchange_blotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index d4957e31..6ded5adf 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -214,7 +214,7 @@ class ExchangeBlotter(Blotter): # that this is safer until we have a robust way to track # the trades already processed by the algo. We can't loose # them if the algo shuts down. - if transactions and order.open_amount == 0: + if transactions and order.status == ORDER_STATUS.FILLED: avg_price = np.average( a=[t.price for t in transactions], weights=[t.amount for t in transactions], From 18c96303a67d607b7fb4e0a2344edacbef9ffb1f Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 19:20:55 -0500 Subject: [PATCH 31/85] BLD: improved unit test --- tests/exchange/test_suites/test_suite_exchange.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index 44e445bf..ca0bd666 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -138,7 +138,6 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): assets=assets, bar_count=bar_count, start_dt=dt_range[0], - end_dt=dt_range[-1], ) assert len(candles) == asset_population From c26d78cf0556a5807409deec1e55f20cd2ab001d Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 30 Jan 2018 11:20:06 -0700 Subject: [PATCH 32/85] BLD: set AUTH_SERVER as a Catalyst constant --- catalyst/constants.py | 2 ++ catalyst/marketplace/marketplace.py | 4 +--- catalyst/marketplace/utils/auth_utils.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 5668bc83..abfa1974 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -21,3 +21,5 @@ except Exception as e: print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False + +AUTH_SERVER = 'https://data.enigma.co' diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index af4c4aef..bbd0be9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,7 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL +from catalyst.constants import LOG_LEVEL, AUTH_SERVER from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -43,8 +43,6 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' -AUTH_SERVER = 'http://localhost:5000' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index f1193758..35fa3504 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -4,8 +4,7 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) - -AUTH_SERVER = 'http://localhost:5000' +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): From 0f55f0e9a6db6439fc6436f31f1bb3a215bbd6ad Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 10:50:10 -0700 Subject: [PATCH 33/85] BLD: passing dataset to marketplace publish request --- catalyst/marketplace/marketplace.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index bbd0be9f..f276d997 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -341,9 +341,13 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - nonce.encode('utf-8'), + '{}{}'.format(dataset,nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} filenames = glob.glob(os.path.join(datadir, '*.csv')) From d1cd95c492032877cb1db5e94a3fea73ac72bc06 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 13:39:59 -0700 Subject: [PATCH 34/85] BLD: marketplace subscribe + refactoring --- catalyst/marketplace/marketplace.py | 144 ++++++++++++++---------- catalyst/marketplace/utils/eth_utils.py | 2 +- 2 files changed, 88 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index f276d997..e0c6b6b0 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,6 +6,7 @@ import glob import time import shutil import hashlib +import binascii import bcolz import logbook @@ -97,6 +98,52 @@ class Marketplace: ), ] + def choose_pubaddr(self): + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + return address + + def sign_transaction(self, from_address, tx): + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + + if signed_tx.startswith('0x'): + signed_tx = signed_tx[2:] + + return signed_tx + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -137,35 +184,51 @@ class Marketplace: ) print(df.to_string(formatters=formatters)) - pass - def subscribe(self, dataset): - data_sources = self.get_data_sources_map() - index = next( - (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == dataset.lower()), - None - ) - if index is None: - raise ValueError( - 'Data source not found.' - ) + + # data_sources = self.get_data_sources_map() + # index = next( + # (index for (index, d) in enumerate(data_sources) if + # d['name'].lower() == dataset.lower()), + # None + # ) + + # Check if dataset exists, if not throw exception + # if index is None: + # raise ValueError( + # 'Data source not found.' + # ) + + address = self.choose_pubaddr() + + tx = self.contract.functions.subscribe( + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = 4700000 + + signed_tx = self.sign_transaction(address, tx) try: - self.contract.transact( - {'from': self.default_account} - ).subscribe(index) - print( - 'Subscribed to data source {} successfully'.format( - dataset - ) - ) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) - pass - def ingest(self, data_source_name, data_frequency=None, start=None, end=None, force_download=False): data_source_name = data_source_name.lower() @@ -261,22 +324,7 @@ class Marketplace: # hist = False # break - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + address = self.choose_pubaddr() tx = self.contract.functions.register( bytes32(dataset), @@ -287,25 +335,7 @@ class Marketplace: tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas - print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' - 'enter the following parameters:\n\n' - 'From Address:\t\t{_from}\n' - 'To Address:\t\t{to}\n' - 'Value / Amount to Send:\t{value}\n' - 'Gas Limit:\t\t{gas}\n' - 'Nonce:\t\t\t{nonce}\n' - 'Data:\t\t\t{data}\n'.format( - _from=address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) - - signed_tx = input('Copy and Paste the "Signed Transaction" ' - 'field here:\n') + signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(b32_str( self.web3.eth.sendRawTransaction(signed_tx))) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index c2be4bd6..fd6ba1e1 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,4 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32.decode('utf-8')) + return binascii.hexlify(bytes32).decode('utf-8') From 311e357451f4ca661513bf5bfc705c07a6c6f099 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 16:10:39 -0500 Subject: [PATCH 35/85] BLD: updated CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 24d0bfa1..795c320d 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.774 + - ccxt==1.10.944 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index 8138f6c3..0789cdef 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.837 +ccxt==1.10.944 boto3==1.4.8 redo==1.6 From 69507d1b004ed4a13bcdccb5b3fc143d445aa615 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 16:32:22 -0500 Subject: [PATCH 36/85] BLD: for issue #178, fixed the "set" issue when fetching a ticker from positions --- catalyst/exchange/ccxt/ccxt_exchange.py | 4 ++-- catalyst/exchange/exchange.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 3e5d5b29..2d770c8b 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -998,13 +998,13 @@ class CCXT(Exchange): """ if len(assets) == 1: - symbol = self.get_symbol(assets[0]) try: + symbol = self.get_symbol(assets[0]) log.debug('fetching single ticker: {}'.format(symbol)) results = dict() results[symbol] = self.api.fetch_ticker(symbol=symbol) - except (ExchangeError, NetworkError) as e: + except (ExchangeError, NetworkError, Exception) as e: log.warn( 'unable to fetch ticker {} / {}: {}'.format( self.name, symbol, e diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 84cac446..f32d9a2b 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -703,7 +703,7 @@ class Exchange: positions_value = 0.0 if positions: - assets = set([position.asset for position in positions]) + assets = list(set([position.asset for position in positions])) tickers = self.tickers(assets) for position in positions: From c0ba8b2ebb59acb74bee6816143f91fcf4d432ae Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 17:12:05 -0500 Subject: [PATCH 37/85] BUG: for issue #178, adjusted the fallback processing of orders for exchanges lacking a "my trades" api --- catalyst/examples/mean_reversion_simple.py | 6 ++--- catalyst/exchange/ccxt/ccxt_exchange.py | 30 ++++++++++++---------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 6909dbc4..79b2f60b 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -248,7 +248,7 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=100, + capital_base=0.035, initialize=initialize, handle_data=handle_data, analyze=analyze, @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='btc', live_graph=False, - simulate_orders=True, + simulate_orders=False, stats_output=None, # auth_aliases=dict(poloniex='auth2') ) @@ -274,7 +274,7 @@ if __name__ == '__main__': # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ # --data-frequency minute --capital-base 10000 run_algorithm( - capital_base=0.1, + capital_base=0.035, data_frequency='minute', initialize=initialize, handle_data=handle_data, diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 2d770c8b..5edd86f2 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -765,7 +765,7 @@ class CCXT(Exchange): self.api.load_markets() # https://github.com/ccxt/ccxt/issues/1483 - adj_amount = abs(amount) + adj_amount = round(abs(amount), asset.decimals) market = self.api.markets[symbol] if 'lots' in market and market['lots'] > amount: raise CreateOrderError( @@ -776,7 +776,7 @@ class CCXT(Exchange): ) else: - adj_amount = abs(amount) + adj_amount = round(abs(amount), asset.decimals) try: result = self.api.create_order( @@ -858,25 +858,29 @@ class CCXT(Exchange): order.id, order.asset, return_price=True ) order.status = exc_order.status - order.commission = exc_order.commission - if order.amount != exc_order.amount: - log.warn( - 'executed order amount {} differs ' - 'from original'.format( - exc_order.amount, order.amount - ) - ) - order.amount = exc_order.amount + order.filled = exc_order.amount - if order.status == ORDER_STATUS.FILLED: + if exc_order.status == ORDER_STATUS.FILLED: + if order.amount > exc_order.amount: + log.warn( + 'executed order amount {} differs ' + 'from original'.format( + exc_order.amount, order.amount + ) + ) + + order.check_triggers( + price=price, + dt=exc_order.dt, + ) transaction = Transaction( asset=order.asset, amount=order.amount, dt=pd.Timestamp.utcnow(), price=price, order_id=order.id, - commission=order.commission + commission=order.commission, ) return [transaction] From e91448132574b358a8ba49cdac4bbc07676cd81c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 17:19:09 -0500 Subject: [PATCH 38/85] BLD: minor adjustment --- catalyst/exchange/ccxt/ccxt_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 5edd86f2..1f050f7f 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1008,7 +1008,7 @@ class CCXT(Exchange): results = dict() results[symbol] = self.api.fetch_ticker(symbol=symbol) - except (ExchangeError, NetworkError, Exception) as e: + except (ExchangeError, NetworkError,) as e: log.warn( 'unable to fetch ticker {} / {}: {}'.format( self.name, symbol, e From 857a5d8a91c0c2d872c2276198cdc910dd639722 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 21:57:33 -0500 Subject: [PATCH 39/85] BUG: for issue #178, modified logic to track adjusted order amount --- catalyst/exchange/ccxt/ccxt_exchange.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 1f050f7f..2a677cd1 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -798,6 +798,14 @@ class CCXT(Exchange): ) raise ExchangeRequestError(error=e) + if 'amount' in result and result['amount'] != adj_amount: + log.info( + 'order amount adjusted by {} from {} to {}'.format( + self.name, adj_amount, result['amount'] + ) + ) + adj_amount = result['amount'] + if 'info' not in result: raise ValueError('cannot use order without info attribute') @@ -886,7 +894,7 @@ class CCXT(Exchange): def process_order(self, order): # TODO: move to parent class after tracking features in the parent - if not self.api.hasFetchMyTrades: + if not self.api.has['fetchMyTrades']: return self._process_order_fallback(order) try: From fcbdc131ec792da0cc3b3b90ccabd93521b2dbae Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 22:51:41 -0500 Subject: [PATCH 40/85] BLD: trying to catch as many trades as possible --- catalyst/exchange/ccxt/ccxt_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 2a677cd1..377c3096 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1103,7 +1103,7 @@ class CCXT(Exchange): return result - def get_trades(self, asset, my_trades=True, start_dt=None, limit=None): + def get_trades(self, asset, my_trades=True, start_dt=None, limit=100): if not my_trades: raise NotImplemented( 'get_trades only supports "my trades"' From a360a5fe3ae81a009ca18ae76997136e4ff5aed6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 31 Jan 2018 23:57:22 -0500 Subject: [PATCH 41/85] Housekeeping --- catalyst/examples/mean_reversion_simple.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 79b2f60b..db7d5e39 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -33,7 +33,7 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - context.market = symbol('eth_btc') + context.market = symbol('bnb_eth') context.base_price = None context.current_day = None @@ -248,14 +248,14 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=0.035, + capital_base=0.1, initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='poloniex', + exchange_name='binance', live=True, algo_namespace=NAMESPACE, - base_currency='btc', + base_currency='eth', live_graph=False, simulate_orders=False, stats_output=None, From e906315969529e9a49a4baf81a8abd1f2b669012 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 22:06:52 -0700 Subject: [PATCH 42/85] BLD: marketplace: enigma contract files --- catalyst/marketplace/contract_enigma_abi.json | 302 ++++++++++++++++++ .../marketplace/contract_enigma_address.txt | 1 + ...abi.json => contract_marketplace_abi.json} | 0 ...s.txt => contract_marketplace_address.txt} | 0 4 files changed, 303 insertions(+) create mode 100644 catalyst/marketplace/contract_enigma_abi.json create mode 100644 catalyst/marketplace/contract_enigma_address.txt rename catalyst/marketplace/{contract_abi.json => contract_marketplace_abi.json} (100%) rename catalyst/marketplace/{contract_address.txt => contract_marketplace_address.txt} (100%) diff --git a/catalyst/marketplace/contract_enigma_abi.json b/catalyst/marketplace/contract_enigma_abi.json new file mode 100644 index 00000000..4dd70538 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_abi.json @@ -0,0 +1,302 @@ +[ + { + "constant": true, + "inputs": [], + "name": "name", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_from", + "type": "address" + }, + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "INITIAL_SUPPLY", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "decimals", + "outputs": [ + { + "name": "", + "type": "uint8" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_subtractedValue", + "type": "uint256" + } + ], + "name": "decreaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [], + "name": "getAfterApproveTest", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "name": "balance", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "symbol", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_addedValue", + "type": "uint256" + } + ], + "name": "increaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + }, + { + "name": "_spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "name": "testValue", + "type": "address" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "from", + "type": "address" + }, + { + "indexed": true, + "name": "to", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + } + ] \ No newline at end of file diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt new file mode 100644 index 00000000..157e2c23 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -0,0 +1 @@ +0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_marketplace_abi.json similarity index 100% rename from catalyst/marketplace/contract_abi.json rename to catalyst/marketplace/contract_marketplace_abi.json diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_marketplace_address.txt similarity index 100% rename from catalyst/marketplace/contract_address.txt rename to catalyst/marketplace/contract_marketplace_address.txt From a04a99373d6d7276301f335f655314e5f90fe6fd Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 08:07:17 -0700 Subject: [PATCH 43/85] BLD: marketplace: subscription to dataset --- catalyst/marketplace/marketplace.py | 268 ++++++++++++++++----- catalyst/marketplace/marketplace_errors.py | 19 +- catalyst/marketplace/utils/eth_utils.py | 19 +- 3 files changed, 248 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e0c6b6b0..d6bf5293 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,7 +6,6 @@ import glob import time import shutil import hashlib -import binascii import bcolz import logbook @@ -20,12 +19,13 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles) + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: @@ -38,11 +38,23 @@ else: REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github -CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_address.txt' +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' -CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -59,17 +71,31 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - contract_url = urllib.urlopen(CONTRACT_PATH) - self.contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) - - abi_url = urllib.urlopen(CONTRACT_ABI) - abi = json.load(abi_url) - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = self.web3.eth.contract( - self.contract_address, + contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) + + self.mkt_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) + abi = json.load(abi_url) + + self.mkt_contract = self.web3.eth.contract( + self.mkt_contract_address, + abi=abi, + ) + + contract_url = urllib.urlopen(ENIGMA_CONTRACT) + + self.eng_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) + abi = json.load(abi_url) + + self.eng_contract = self.web3.eth.contract( + self.eng_contract_address, abi=abi, ) @@ -137,7 +163,8 @@ class Marketplace: ) ) - signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') if signed_tx.startswith('0x'): signed_tx = signed_tx[2:] @@ -186,57 +213,185 @@ class Marketplace: def subscribe(self, dataset): - # data_sources = self.get_data_sources_map() - # index = next( - # (index for (index, d) in enumerate(data_sources) if - # d['name'].lower() == dataset.lower()), - # None - # ) - - # Check if dataset exists, if not throw exception - # if index is None: - # raise ValueError( - # 'Data source not found.' - # ) - address = self.choose_pubaddr() - tx = self.contract.functions.subscribe( + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + price = dataset_info[1] + + print('\nThe price for a monthly subscription to this dataset is' + ' {} ENG'.format(price)) + + print('Checking that the ENG balance in {} is greater than ' + '{} ENG... '.format(address, price), end='') + + balance = self.web3.eth.call({ + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) + + balance = int(balance[2:], 16) // 10 ** 8 + + if balance > price: + print('OK.') + else: + print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower ' + 'than the price of the dataset that you are trying to\n' + 'buy: {} ENG. Get enough ENG to cover the costs of the ' + 'monthly\nsubscription for what you are trying to buy, ' + 'and try again.'.format(address, balance, price)) + return + + while True: + agree_pay = input('Please confirm that you agree to pay {} ENG ' + 'for a monthly subscription to the dataset "{}" ' + 'starting today. [default: Y] '.format( + price, dataset)) or 'y' + if agree_pay.lower() not in ('y', 'n'): + print("Please answer Y or N.") + else: + if agree_pay.lower() == 'y': + break + else: + return + + print('Ready to subscribe to dataset {}.\n'.format(dataset)) + print('In order to execute the subscription, you will need to sign ' + 'two different transactions:\n' + '1. First transaction is to authorize the Marketplace contract ' + 'to spend {} ENG on your behalf.\n' + '2. Second transaction is the actual subscription for the ' + 'desired dataset'.format(price)) + + tx = self.eng_contract.functions.approve( + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) + + signed_tx = self.sign_transaction(address, tx) + + try: + tx_hash = '0x{}'.format(bin_hex( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + return + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + + print('Waiting for the first transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nFirst transaction successful!\n' + 'Now processing second transaction.') + + tx = self.mkt_contract.functions.subscribe( bytes32(dataset), ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in REMOTE_NODE: - tx['gas'] = 4700000 + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) try: - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) - if 'ropsten' in REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}'.format(etherscan)) - except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) + return - def ingest(self, data_source_name, data_frequency=None, start=None, + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) + + print('Waiting for the second transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nSecond transaction successful!\n' + 'You have successfully subscribed to dataset {} with' + 'address {}.\n' + 'You can now ingest this dataset anytime during the ' + 'next month by running the following command:\n' + 'catalyst marketplace ingest --dataset={}'.format( + dataset, address, dataset)) + + def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - data_source_name = data_source_name.lower() + + address = self.choose_pubaddr() + + check_sub = self.mkt_contract.functions.checkAddressSubscription( + address, bytes32(dataset)).call() + + if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + raise MarketplaceContractDataNoMatch( + params='address: {}, dataset: {}'.format( + address, dataset)) + + if not check_sub[5]: + raise MarketplaceSubscriptionExpired( + dataset=dataset, + date=check_sub[4]) + + # WIP + + dataset = dataset.lower() period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(data_source_name, period, force_download) + tmp_folder = get_data_source(dataset, period, force_download) - bundle_folder = get_bundle_folder(data_source_name, data_frequency) + bundle_folder = get_bundle_folder(dataset, data_frequency) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') @@ -294,10 +449,10 @@ class Marketplace: 'this dataset in ENG: ')) # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): # print("Not a valid frequency.") - # else: + # else: # break # while True: @@ -326,25 +481,26 @@ class Marketplace: address = self.choose_pubaddr() - tx = self.contract.functions.register( + tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): - datasource = self.contract.functions.getDataSource( + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() if not datasource[4]: @@ -371,11 +527,11 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset,nonce).encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, - 'Key': key, + headers = {'Sign': signature, + 'Key': key, 'Nonce': nonce, 'Dataset': dataset} diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 3992158d..ed3d0fa4 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -7,7 +7,8 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles]: + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -49,3 +50,19 @@ class MarketplaceNoCSVFiles(ZiplineError): msg = ( 'No CSV files found on {datadir} to upload.' ) + + +class MarketplaceContractDataNoMatch(ZiplineError): + msg = ( + 'The information found on the contract does not match the ' + 'requested data:\n{params}.' + ) + + +class MarketplaceSubscriptionExpired(ZiplineError): + msg = ( + 'Your subscription to dataset "{dataset}" expired on {date} ' + 'and is no longer active. You have to subscribe again running the ' + 'following command:\n' + 'catalyst marketplace subscribe --dataset={dataset}' + ) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index fd6ba1e1..f5fe8b7b 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,21 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32).decode('utf-8') + return binascii.unhexlify( + bytes32.decode('utf-8').rstrip('\0')).decode('ascii') + + +def bin_hex(binary): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(binary).decode('utf-8') From 18123d7f7e75514dd0194f0e406bc99513bfde20 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 11:47:25 -0700 Subject: [PATCH 44/85] BLD: marketplace constants in constants.py --- catalyst/constants.py | 21 +++++ catalyst/marketplace/marketplace.py | 120 ++++++++++++++-------------- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index abfa1974..df0ea82b 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -23,3 +23,24 @@ except Exception as e: AUTO_INGEST = False AUTH_SERVER = 'https://data.enigma.co' + +# TODO: switch to mainnet +ETH_REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' \ No newline at end of file diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index d6bf5293..08249775 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,9 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL, AUTH_SERVER +from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, + MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, + ENIGMA_CONTRACT_ABI ) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -33,28 +35,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib -# TODO: host our own node on aws? -# TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' - -# TODO: move to MASTER branch on github -MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_address.txt' - -MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_abi.json' - -# TODO: switch to mainnet -ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_address.txt' - -ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_abi.json' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -71,7 +51,7 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) @@ -99,30 +79,30 @@ class Marketplace: abi=abi, ) - def get_data_sources_map(self): - return [ - dict( - name='Marketcap', - desc='The marketcap value in USD.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily'], - ), - dict( - name='GitHub', - desc='The rate of development activity on GitHub.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour'], - ), - dict( - name='Influencers', - desc='Tweets and related sentiments by selected influencers.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour', 'minute'], - ), - ] + # def get_data_sources_map(self): + # return [ + # dict( + # name='Marketcap', + # desc='The marketcap value in USD.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily'], + # ), + # dict( + # name='GitHub', + # desc='The rate of development activity on GitHub.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour'], + # ), + # dict( + # name='Influencers', + # desc='Tweets and related sentiments by selected influencers.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour', 'minute'], + # ), + # ] def choose_pubaddr(self): while True: @@ -142,7 +122,7 @@ class Marketplace: self.addresses[address_i]['pubAddr']) break - return address + return address, address_i def sign_transaction(self, from_address, tx): @@ -213,7 +193,7 @@ class Marketplace: def subscribe(self, dataset): - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -272,7 +252,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -287,7 +267,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -319,7 +299,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -334,7 +314,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -369,7 +349,7 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - address = self.choose_pubaddr() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( address, bytes32(dataset)).call() @@ -384,7 +364,31 @@ class Marketplace: dataset=dataset, date=check_sub[4]) - # WIP + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), + hashlib.sha512).hexdigest() + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} + + r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + + + print(r) + exit(0) + + dataset = dataset.lower() @@ -479,7 +483,7 @@ class Marketplace: # hist = False # break - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( bytes32(dataset), @@ -488,7 +492,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) From 56bd3db7a2f8ee2ec0c765f9e81215b223251a9f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 16:54:24 -0700 Subject: [PATCH 45/85] BLD: marketplace - ingestion downloads multiple files --- catalyst/marketplace/marketplace.py | 40 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 08249775..30860551 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,4 +1,5 @@ import os +import re import sys import json import hmac @@ -11,12 +12,13 @@ import bcolz import logbook import pandas as pd import six +from requests_toolbelt import MultipartDecoder import requests from web3 import Web3, HTTPProvider -from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, - MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, - ENIGMA_CONTRACT_ABI ) +from catalyst.constants import ( + LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, + MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -97,7 +99,7 @@ class Marketplace: # ), # dict( # name='Influencers', - # desc='Tweets and related sentiments by selected influencers.', + # desc='Tweets & related sentiments by selected influencers.', # start_date=pd.to_datetime('2017-01-01'), # end_date=pd.to_datetime('2018-01-15'), # data_frequencies=['daily', 'hour', 'minute'], @@ -349,6 +351,8 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): + dataset = dataset.lower() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -369,7 +373,7 @@ class Marketplace: secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) + key, secret = get_key_secret(address, dataset) nonce = str(int(time.time())) @@ -382,15 +386,27 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} - r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, stream=True) + if r.status_code == 200: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + else: + raise MarketplaceHTTPRequest(request='ingest dataset', + error=r.status_code) - print(r) + print('Download successful, now we need to ingest...') exit(0) - - - - dataset = dataset.lower() + # TODO: sync with Fred on the below and + # exiting for now period = start.strftime('%Y-%m-%d') tmp_folder = get_data_source(dataset, period, force_download) @@ -404,8 +420,6 @@ class Marketplace: else: os.rename(tmp_folder, bundle_folder) - pass - def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): data_source_name = data_source_name.lower() From 37980f35806c63fa6c87eedb1aaef12ade913e4e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 20:43:48 -0500 Subject: [PATCH 46/85] BLD: working on marketplace integration --- catalyst/marketplace/marketplace.py | 99 +++++++++++++-------------- etc/requirements_marketplace.txt | 2 +- tests/marketplace/test_marketplace.py | 3 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 30860551..4c55b2da 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -37,7 +37,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -48,9 +47,9 @@ class Marketplace: if self.addresses[0]['pubAddr'] == '': raise MarketplacePubAddressEmpty( - filename=os.path.join( - get_marketplace_folder(), 'addresses.json') - ) + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) self.default_account = self.addresses[0]['pubAddr'] self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) @@ -58,7 +57,7 @@ class Marketplace: contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) self.mkt_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -71,7 +70,7 @@ class Marketplace: contract_url = urllib.urlopen(ENIGMA_CONTRACT) self.eng_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) @@ -118,10 +117,10 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) + self.addresses[address_i]['pubAddr']) break return address, address_i @@ -136,14 +135,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -198,7 +197,7 @@ class Marketplace: address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() price = dataset_info[1] @@ -209,11 +208,11 @@ class Marketplace: '{} ENG... '.format(address, price), end='') balance = self.web3.eth.call({ - 'from': address, - 'to': self.eng_contract_address, - 'data': '0x70a08231000000000000000000000000{}'.format( - address[2:]) - }) + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) balance = int(balance[2:], 16) // 10 ** 8 @@ -231,7 +230,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -249,10 +248,10 @@ class Marketplace: 'desired dataset'.format(price)) tx = self.eng_contract.functions.approve( - self.mkt_contract_address, - price, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -261,7 +260,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -271,7 +270,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -297,9 +296,9 @@ class Marketplace: 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - bytes32(dataset), - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -308,7 +307,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -318,7 +317,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -346,7 +345,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): @@ -356,17 +355,17 @@ class Marketplace: address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(dataset)).call() if check_sub[0] != address or b32_str(check_sub[1]) != dataset: raise MarketplaceContractDataNoMatch( - params='address: {}, dataset: {}'.format( - address, dataset)) + params='address: {}, dataset: {}'.format( + address, dataset)) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=dataset, + date=check_sub[4]) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -500,11 +499,11 @@ class Marketplace: address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( - bytes32(dataset), - price, - address, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 @@ -512,14 +511,14 @@ class Marketplace: signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() if not datasource[4]: raise MarketplaceDatasetNotFound(dataset=dataset) @@ -529,11 +528,11 @@ class Marketplace: if not match: raise MarketplaceNoAddressMatch( - dataset=dataset, - address=datasource[0]) + dataset=dataset, + address=datasource[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + datasource[0])) if 'key' in match: key = match['key'] diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index cbb3a5a3..8062d192 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1 @@ -web3==4.0.0b5 \ No newline at end of file +web3==4.0.0b7 \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 9292b3a0..815ff885 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -11,7 +11,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_register(self): marketplace = Marketplace() - marketplace.register('GitHub') + marketplace.register() pass def test_ingest(self): @@ -30,3 +30,4 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass + From b9130fd968507d22b6e507e48255911e674403e4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 22:03:16 -0500 Subject: [PATCH 47/85] BLD: added missing requirement --- etc/requirements_marketplace.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index 8062d192..2a56c200 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1,2 @@ -web3==4.0.0b7 \ No newline at end of file +web3==4.0.0b7 +requests-toolbelt==0.8.0 From e4602900565d2998dc95d7f5bc3a4e935e28c5b0 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 21:57:13 -0700 Subject: [PATCH 48/85] BLD: better exception handling for failed multipart download --- catalyst/marketplace/marketplace.py | 65 ++++++++++++++++++----------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4c55b2da..43c15f19 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -13,6 +13,8 @@ import logbook import pandas as pd import six from requests_toolbelt import MultipartDecoder +from requests_toolbelt.multipart.decoder import \ + NonMultipartContentTypeException import requests from web3 import Web3, HTTPProvider @@ -106,22 +108,28 @@ class Marketplace: # ] def choose_pubaddr(self): - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses) - 1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + + if len(self.addresses) == 1: + address = self.addresses[0]['pubAddr'] + address_i = 0 + print('Using {} for this transaction.'.format(address)) + else: + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break return address, address_i @@ -385,19 +393,26 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} + print('Starting download of dataset for ingestion...') + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), headers=headers, stream=True) if r.status_code == 200: - decoder = MultipartDecoder.from_response(r) - for part in decoder.parts: - h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) - with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) - f.write(part.content) + try: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + except NonMultipartContentTypeException: + response = r.json() + raise MarketplaceHTTPRequest(request='ingest dataset', + error=response) else: raise MarketplaceHTTPRequest(request='ingest dataset', error=r.status_code) From 0decdf5aab874ab56d5889ffe77766e94c4b2e9e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 22:30:24 -0700 Subject: [PATCH 49/85] BLD: check for name duplicates when registering dataset --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 43c15f19..ced5bfde 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -160,6 +160,17 @@ class Marketplace: return signed_tx + def check_transaction(self, tx_hash): + + if 'ropsten' in ETH_REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('\nYou can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -276,14 +287,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in ETH_REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}\n\n'.format(etherscan)) + self.check_transaction(tx_hash) print('Waiting for the first transaction to succeed...') @@ -476,7 +480,20 @@ class Marketplace: pass def register(self): - dataset = input('Enter the name of the dataset to register: ') + while True: + dataset = input('Enter the name of the dataset to register: ') + dataset = dataset.lower() + + datasource = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if datasource[4]: + print('There is already a dataset registered under ' + 'the name "{}". Please choose a different ' + 'name.'.format(dataset)) + else: + break + price = int(input('Enter the price for a monthly subscription to ' 'this dataset in ENG: ')) @@ -520,8 +537,8 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: - tx['gas'] = 4700000 + if 'ropsten' in ETH_REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -530,6 +547,8 @@ class Marketplace: print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + self.check_transaction(tx_hash) + def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( From db4ee4c01ec3337da1c7695747ef4f6cb0293251 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 23:42:40 -0700 Subject: [PATCH 50/85] BLD: check for existence of dataset in ingestion & subscription --- catalyst/marketplace/marketplace.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ced5bfde..a59d5ed7 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -213,11 +213,18 @@ class Marketplace: def subscribe(self, dataset): + dataset = dataset.lower() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + price = dataset_info[1] print('\nThe price for a monthly subscription to this dataset is' @@ -364,6 +371,14 @@ class Marketplace: dataset = dataset.lower() + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -551,6 +566,8 @@ class Marketplace: def publish(self, dataset, datadir, watch): + dataset = dataset.lower() + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -607,4 +624,4 @@ class Marketplace: raise MarketplaceHTTPRequest(request='upload file', error=r.json()['error']) - print('Dataset {} published successfully.'.format(dataset)) + print('Dataset {} uploaded successfully.'.format(dataset)) From 9ef000e529c0dc6af15f9199eb17c0992d8db1ef Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Feb 2018 12:06:57 -0700 Subject: [PATCH 51/85] BUG: balance returns different types --- catalyst/marketplace/marketplace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a59d5ed7..828224ab 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -240,7 +240,10 @@ class Marketplace: address[2:]) }) - balance = int(balance[2:], 16) // 10 ** 8 + try: + balance = int(balance[2:], 16) // 10 ** 8 + except ValueError: + balance = int(bin_hex(balance), 16) // 10 ** 8 if balance > price: print('OK.') From 99dece953f157570f78e96404ce92a58bad046ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 16:20:33 -0500 Subject: [PATCH 52/85] BLD: ingesting marketplace bundles --- catalyst/marketplace/marketplace.py | 147 ++++++++++++++--------- catalyst/marketplace/utils/path_utils.py | 52 +++----- tests/marketplace/test_marketplace.py | 9 +- 3 files changed, 109 insertions(+), 99 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 828224ab..14721a82 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,21 +1,21 @@ +import glob +import hashlib +import hmac +import json import os import re -import sys -import json -import hmac -import glob -import time import shutil -import hashlib +import sys +import time import bcolz import logbook import pandas as pd +import requests import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException -import requests from web3 import Web3, HTTPProvider from catalyst.constants import ( @@ -27,12 +27,12 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ - get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex +from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ + get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr, get_temp_bundles_folder, extract_bundle if sys.version_info.major < 3: import urllib @@ -125,7 +125,7 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( self.addresses[address_i]['pubAddr']) @@ -369,92 +369,119 @@ class Marketplace: 'catalyst marketplace ingest --dataset={}'.format( dataset, address, dataset)) - def ingest(self, dataset, data_frequency=None, start=None, - end=None, force_download=False): + def process_temp_bundle(self, ds_name, path): + """ + Merge the temp bundle into the main bundle for the specified + data source. - dataset = dataset.lower() + Parameters + ---------- + ds_name + path + Returns + ------- + + """ + tmp_bundle = extract_bundle(path) + bundle_folder = get_data_source_folder(ds_name) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_bundle, bundle_folder) + + pass + + def ingest(self, ds_name, start=None, end=None, force_download=False): + + ds_name = ds_name.lower() dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(ds_name) + ).call() if not dataset_info[4]: print('The requested "{}" dataset is not registered in ' - 'the Data Marketplace.'.format(dataset)) + 'the Data Marketplace.'.format(ds_name)) return address, address_i = self.choose_pubaddr() - check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(ds_name) + ).call() - if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + if check_sub[0] != address or b32_str(check_sub[1]) != ds_name: raise MarketplaceContractDataNoMatch( params='address: {}, dataset: {}'.format( - address, dataset)) + address, ds_name + ) + ) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=ds_name, + date=check_sub[4], + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, dataset) + key, secret = get_key_secret(address, ds_name) nonce = str(int(time.time())) + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} - - print('Starting download of dataset for ingestion...') - - r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), - headers=headers, stream=True) - + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + log.debug('Starting download of dataset for ingestion...') + r = requests.post( + '{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, + stream=True, + ) if r.status_code == 200: + target_path = get_temp_bundles_folder() try: decoder = MultipartDecoder.from_response(r) for part in decoder.parts: h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) + # Extracting the filename from the header + name = re.search(r'filename="(.*)"', h).group(1) + + filename = os.path.join(target_path, name) with open(filename, 'wb') as f: # for chunk in part.content.iter_content(chunk_size=1024): # if chunk: # filter out keep-alive new chunks # f.write(chunk) f.write(part.content) + + self.process_temp_bundle(ds_name, filename) + except NonMultipartContentTypeException: response = r.json() - raise MarketplaceHTTPRequest(request='ingest dataset', - error=response) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=response, + ) else: - raise MarketplaceHTTPRequest(request='ingest dataset', - error=r.status_code) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=r.status_code, + ) - print('Download successful, now we need to ingest...') - exit(0) - # TODO: sync with Fred on the below and - # exiting for now - - period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(dataset, period, force_download) - - bundle_folder = get_bundle_folder(dataset, data_frequency) - if os.listdir(bundle_folder): - zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') - ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') - merge_bundles(zsource, ztarget) - - else: - os.rename(tmp_folder, bundle_folder) + log.info('{} ingested successfully'.format(ds_name)) def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 62f8beac..b19e1921 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -5,6 +5,7 @@ import tarfile import shutil from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory @@ -55,6 +56,7 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +@deprecated def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) @@ -65,13 +67,13 @@ def get_bundle_folder(data_source_name, data_frequency, environ=None): return bundle_folder -def get_temp_bundles_folder(data_source_name, environ=None): +def get_temp_bundles_folder(environ=None): """ The temp folder for bundle downloads by algo name. Parameters ---------- - data_source_name: str + ds_name: str environ: Returns @@ -79,55 +81,31 @@ def get_temp_bundles_folder(data_source_name, environ=None): str """ - data_source_folder = get_data_source_folder(data_source_name, environ) + root = data_root(environ) + folder = os.path.join(root, 'marketplace', 'temp_bundles') + ensure_directory(folder) - temp_bundles = os.path.join(data_source_folder, 'temp_bundles') - ensure_directory(temp_bundles) - - return temp_bundles + return folder -def get_data_source(data_source_name, period, force_download=False): +def extract_bundle(tar_filename): """ - Download and extract a bcolz bundle. + Extract a bcolz bundle. Parameters ---------- - exchange_name: str - symbol: str - data_frequency: str - period: str + ds_name Returns ------- str """ - root = get_temp_bundles_folder(data_source_name) - name = '{data_source}_{period}'.format( - data_source=data_source_name, - period=period, - ) - path = os.path.join(root, name) + target_path = tar_filename.replace('.tar.gz', '') + with tarfile.open(tar_filename, 'r') as tar: + tar.extractall(target_path) - if os.path.isdir(path): - if force_download: - shutil.rmtree(path) - - else: - return path - - ensure_directory(path) - - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) - - return path + return target_path def get_user_pubaddr(environ=None): diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 815ff885..7315629a 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -14,9 +14,15 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace.register() pass + def test_subscribe(self): + marketplace = Marketplace() + marketplace.subscribe('Victor2') + pass + def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.get_data_source_def('Marketcap') + ds_def = marketplace.ingest('Victor2') + pass marketplace.ingest( data_source_name='Marketcap', @@ -30,4 +36,3 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass - From d1529afeb318f9dcd0e5d4aa5b0cb20d9f4cb411 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:10:10 -0500 Subject: [PATCH 53/85] BLD: enhancing registration features --- catalyst/marketplace/marketplace.py | 78 ++++++++++++++------------- tests/marketplace/test_marketplace.py | 5 ++ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 14721a82..18294072 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -539,57 +539,61 @@ class Marketplace: else: break - price = int(input('Enter the price for a monthly subscription to ' - 'this dataset in ENG: ')) + price = int( + input( + 'Enter the price for a monthly subscription to ' + 'this dataset in ENG: ' + ) + ) + while True: + freq = input('Enter the data frequency [daily, hourly, minute]: ') + if freq.lower() not in ('daily', 'hourly', 'minute'): + print('Not a valid frequency.') + else: + break - # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): - # print("Not a valid frequency.") - # else: - # break + while True: + reg_pub = input( + 'Doest it include historical data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_history = True + else: + has_history = False + break - # while True: - # reg_pub = input('Can data be published every hour at a regular ' - # 'time? [default: Y]: ') or 'y' - # if reg_pub.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if reg_pub.lower() == 'y': - # reg_pub = True - # else: - # reg_pub = False - # break - - # while True: - # hist = input('Does it include historical data? ' - # '[default: Y]') or 'y' - # if hist.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if hist.lower() == 'y': - # hist = True - # else: - # hist = False - # break + while True: + reg_pub = input( + 'Doest it include live data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_live = True + else: + has_live = False + break address = self.choose_pubaddr()[0] - tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + {'nonce': self.web3.eth.getTransactionCount(address)} + ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - - tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) - + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) self.check_transaction(tx_hash) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 7315629a..c7c7c467 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -32,6 +32,11 @@ class TestMarketplace(WithLogger, ZiplineTestCase): ) pass + def test_publish(self): + marketplace = Marketplace() + datadir = '/Users/fredfortier/Downloads/marketcap_test_single' + marketplace.publish('marketcap1', datadir, False) + def test_clean(self): marketplace = Marketplace() marketplace.clean('marketcap') From b9bedfda21045d074abdba3cf8b1a2cfea599aba Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:23:13 -0500 Subject: [PATCH 54/85] BLD: enhancing registration features --- tests/marketplace/test_marketplace.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index c7c7c467..d2511ce8 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,20 +16,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('Victor2') + marketplace.subscribe('marketcap1') pass def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('Victor2') - pass - - marketplace.ingest( - data_source_name='Marketcap', - data_frequency=ds_def['data_frequencies'][0], - start=pd.to_datetime('2017-10-01'), - force_download=True, - ) + ds_def = marketplace.ingest('marketcap1') pass def test_publish(self): From 9f88e7a003d37a110246b20ac10a0f35a90b1e2b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 3 Feb 2018 18:24:28 -0500 Subject: [PATCH 55/85] BUG: for issue #183, added more logic to catch order amount adjustments --- catalyst/examples/mean_reversion_simple.py | 4 ++-- catalyst/exchange/ccxt/ccxt_exchange.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index db7d5e39..db787612 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,8 +37,8 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 55 - context.RSI_OVERBOUGHT = 60 + context.RSI_OVERSOLD = 40 + context.RSI_OVERBOUGHT = 50 context.CANDLE_SIZE = '15T' context.start_time = time.time() diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 377c3096..7a118a52 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -798,13 +798,21 @@ class CCXT(Exchange): ) raise ExchangeRequestError(error=e) + exchange_amount = None if 'amount' in result and result['amount'] != adj_amount: + exchange_amount = result['amount'] + + elif 'info' in result: + if 'origQty' in result['info']: + exchange_amount = float(result['info']['origQty']) + + if exchange_amount: log.info( 'order amount adjusted by {} from {} to {}'.format( - self.name, adj_amount, result['amount'] + self.name, adj_amount, exchange_amount ) ) - adj_amount = result['amount'] + adj_amount = exchange_amount if 'info' not in result: raise ValueError('cannot use order without info attribute') From e94c9db2ec41c45583c0a126e0ada83cebcfb756 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Feb 2018 18:29:11 +0200 Subject: [PATCH 56/85] Updated contract --- catalyst/marketplace/contract_enigma_address.txt | 2 +- catalyst/marketplace/contract_marketplace_abi.json | 2 +- catalyst/marketplace/contract_marketplace_address.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt index 157e2c23..90fcd06a 100644 --- a/catalyst/marketplace/contract_enigma_address.txt +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -1 +1 @@ -0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file +0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10 \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_abi.json b/catalyst/marketplace/contract_marketplace_abi.json index a3dda75f..4f0e2460 100644 --- a/catalyst/marketplace/contract_marketplace_abi.json +++ b/catalyst/marketplace/contract_marketplace_abi.json @@ -1 +1 @@ -[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file +[{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"isActive","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mBegin","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isPunished","type":"bool"}],"name":"setPunishProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataProviderInfo","outputs":[{"name":"owner","type":"address"},{"name":"price","type":"uint256"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"isPunished","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mProvidersSize","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"withdrawProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getAllProviders","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getMarketplaceTotalBalance","outputs":[{"name":"totalBalance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"refundSubscriber","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mCurrent","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getWithdrawAmount","outputs":[{"name":"withdrawAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"subscriber","type":"address"},{"name":"dataSourceName","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isUnExpired","type":"bool"},{"name":"isPaid","type":"bool"},{"name":"isPunishedProvider","type":"bool"},{"name":"isOrder","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isActive","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"mProviders","outputs":[{"name":"owner","type":"address"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"name","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"isPunished","type":"bool"},{"name":"punishTimeStamp","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"nextProvider","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"},{"name":"","type":"uint256"}],"name":"mOrders","outputs":[{"name":"dataSourceName","type":"bytes32"},{"name":"subscriber","type":"address"},{"name":"provider","type":"address"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isPaid","type":"bool"},{"name":"isOrder","type":"bool"},{"name":"isRefundPaid","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"owner","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"getRefundAmount","outputs":[{"name":"refundAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"MARKETPLACE_VERSION","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"isExpired","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"previousOwner","type":"address"},{"indexed":true,"name":"newOwner","type":"address"}],"name":"OwnershipTransferred","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionDeposited","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"TransferToProvider","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"ProviderWithdraw","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"isPunished","type":"bool"}],"name":"ProviderPunishStatus","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"refundAmount","type":"uint256"}],"name":"SubscriberRefund","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_address.txt b/catalyst/marketplace/contract_marketplace_address.txt index ec7bf029..86810a5f 100644 --- a/catalyst/marketplace/contract_marketplace_address.txt +++ b/catalyst/marketplace/contract_marketplace_address.txt @@ -1 +1 @@ -0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file +0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482 \ No newline at end of file From 444fcbb2b33f6eecc336c568b8fd49973b7ffa5f Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:04:23 -0500 Subject: [PATCH 57/85] BLD: adjustments for the new contract and developing register --- catalyst/marketplace/marketplace.py | 130 ++++++++++++++--------- catalyst/marketplace/utils/auth_utils.py | 46 +++++++- 2 files changed, 118 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 18294072..5de8f538 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -27,7 +27,8 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.auth_utils import get_key_secret, \ + get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ @@ -212,20 +213,19 @@ class Marketplace: print(df.to_string(formatters=formatters)) def subscribe(self, dataset): - dataset = dataset.lower() address = self.choose_pubaddr()[0] + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(dataset)) return - price = dataset_info[1] + price = provider_info[1] print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -239,7 +239,6 @@ class Marketplace: 'data': '0x70a08231000000000000000000000000{}'.format( address[2:]) }) - try: balance = int(balance[2:], 16) // 10 ** 8 except ValueError: @@ -398,11 +397,11 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): ds_name = ds_name.lower() - dataset_info = self.mkt_contract.functions.getDataSource( + provider_info = self.mkt_contract.functions.getDataProviderInfo( bytes32(ds_name) ).call() - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(ds_name)) return @@ -432,19 +431,7 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, ds_name) - nonce = str(int(time.time())) - signature = hmac.new( - secret.encode('utf-8'), - '{}{}'.format(ds_name, nonce).encode('utf-8'), - hashlib.sha512 - ).hexdigest() - - headers = { - 'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': ds_name, - } + headers = get_signed_headers(ds_name, key, secret) log.debug('Starting download of dataset for ingestion...') r = requests.post( '{}/marketplace/ingest'.format(AUTH_SERVER), @@ -524,15 +511,46 @@ class Marketplace: shutil.rmtree(folder) pass + def create_metadata(self, key, secret, ds_name, data_frequency, desc, + has_history=True, has_live=True): + """ + + Returns + ------- + + """ + headers = get_signed_headers(ds_name, key, secret) + r = requests.post( + '{}/marketplace/register'.format(AUTH_SERVER), + data=dict( + name=ds_name, + desc=desc, + data_frequency=data_frequency, + has_history=has_history, + has_live=has_live, + ), + headers=headers, + ) + + if r.status_code != 200: + raise MarketplaceHTTPRequest( + request='register', error=r.status_code + ) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest( + request='upload file', error=r.json()['error'] + ) + def register(self): while True: - dataset = input('Enter the name of the dataset to register: ') - dataset = dataset.lower() + desc = input('Enter the name of the dataset to register: ') + dataset = desc.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if datasource[4]: + if provider_info[4]: print('There is already a dataset registered under ' 'the name "{}". Please choose a different ' 'name.'.format(dataset)) @@ -554,7 +572,7 @@ class Marketplace: while True: reg_pub = input( - 'Doest it include historical data? [default: Y]: ' + 'Does it include historical data? [default: Y]: ' ) or 'y' if reg_pub.lower() not in ('y', 'n'): print('Please answer Y or N.') @@ -578,7 +596,23 @@ class Marketplace: has_live = False break - address = self.choose_pubaddr()[0] + address, address_i = self.choose_pubaddr() + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(address, dataset) + + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -599,44 +633,34 @@ class Marketplace: self.check_transaction(tx_hash) def publish(self, dataset, datadir, watch): - dataset = dataset.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not datasource[4]: + if not provider_info[4]: raise MarketplaceDatasetNotFound(dataset=dataset) - match = next((l for l in self.addresses if - l['pubAddr'] == datasource[0]), None) - + match = next( + (l for l in self.addresses if l['pubAddr'] == provider_info[0]), + None + ) if not match: raise MarketplaceNoAddressMatch( dataset=dataset, - address=datasource[0]) + address=provider_info[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + provider_info[0])) if 'key' in match: key = match['key'] secret = match['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) - - nonce = str(int(time.time())) - - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} + key, secret = get_key_secret(provider_info[0], dataset) + headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) if not filenames: diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 35fa3504..452dadb7 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -1,10 +1,14 @@ +import hashlib +import hmac + import requests +import time from catalyst.marketplace.marketplace_errors import ( - MarketplaceHTTPRequest) + MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( - get_user_pubaddr, save_user_pubaddr) -from catalyst.constants import AUTH_SERVER + get_user_pubaddr, save_user_pubaddr) +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): @@ -25,8 +29,8 @@ def get_key_secret(pubAddr, dataset): session = requests.Session() response = session.get('{}/getkeysecret'.format(AUTH_SERVER), headers={ - 'pubAddr': pubAddr, - 'dataset': dataset}) + 'pubAddr': pubAddr, + 'dataset': dataset}) if 'error' in response.json(): raise MarketplaceHTTPRequest(request=str('obtain key/secret'), @@ -44,3 +48,35 @@ def get_key_secret(pubAddr, dataset): save_user_pubaddr(addresses) return match['key'], match['secret'] + + +def get_signed_headers(ds_name, key, secret): + """ + Return a new request header including the key / secret signature + + Parameters + ---------- + ds_name + key + secret + + Returns + ------- + + """ + nonce = str(int(time.time())) + + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() + + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + + return headers From 3bc54a6c2ebebeb8c09458a6f9a522de1107f6b5 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:46:09 -0500 Subject: [PATCH 58/85] BLD: finalized the register implementation --- catalyst/marketplace/marketplace.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 5de8f538..363171c1 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -522,8 +522,8 @@ class Marketplace: headers = get_signed_headers(ds_name, key, secret) r = requests.post( '{}/marketplace/register'.format(AUTH_SERVER), - data=dict( - name=ds_name, + json=dict( + ds_name=ds_name, desc=desc, data_frequency=data_frequency, has_history=has_history, @@ -604,15 +604,6 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, dataset) - self.create_metadata( - key=key, - secret=secret, - ds_name=dataset, - data_frequency=freq, - desc=desc, - has_history=has_history, - has_live=has_live, - ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -632,6 +623,18 @@ class Marketplace: self.check_transaction(tx_hash) + print('\nWarming up the {} dataset'.format(dataset)) + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) + print('\n{} registered successfully'.format(dataset)) + def publish(self, dataset, datadir, watch): dataset = dataset.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( From 2168fb0d5b9534c5eb901305deca3905b9d0a6db Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:17:21 -0700 Subject: [PATCH 59/85] BUG: address json initialized with array of 1 dict, instead of dict --- catalyst/marketplace/utils/path_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index b19e1921..4bbf2854 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,7 +133,8 @@ def get_user_pubaddr(environ=None): return [data, ] return data else: - data = dict(pubAddr='', desc='') + data = [] + data.append(dict(pubAddr='', desc='')) with open(filename, 'w') as f: json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) From 4c1a9b1dd7f0f8ebb08b9d719d4d1494a3f91362 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:58:40 -0700 Subject: [PATCH 60/85] BLD: marketplace added listing of datasets --- catalyst/__main__.py | 1 - catalyst/marketplace/marketplace.py | 57 +++++++++--------------- catalyst/marketplace/utils/path_utils.py | 3 -- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 3833f3e4..ecbe7c23 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -756,7 +756,6 @@ def ls(ctx): sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 363171c1..a5e7d4f5 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,4 @@ import glob -import hashlib -import hmac import json import os import re @@ -144,14 +142,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -182,35 +180,21 @@ class Marketplace: return ds def list(self): - subscribers = self.contract.call( - {'from': self.default_account} - ).getSubscribers() - subscribed = [] - for index, address in enumerate(subscribers): - if address == self.default_account: - subscribed.append(index) - - data_sources = self.get_data_sources_map() + data_sources = self.mkt_contract.functions.getAllProviders().call() data = [] for index, data_source in enumerate(data_sources): - data.append( - dict( - id=index, - subscribed=index in subscribed, - **data_source + if index > 0: + data.append( + dict( + dataset=data_source.decode('utf-8').rstrip('\0') + ) ) - ) df = pd.DataFrame(data) - df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) set_print_settings() - - formatters = dict( - subscribed=lambda s: u'\u2713' if s else '', - ) - print(df.to_string(formatters=formatters)) + print(df) def subscribe(self, dataset): dataset = dataset.lower() @@ -258,7 +242,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -366,7 +350,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -449,9 +433,10 @@ class Marketplace: filename = os.path.join(target_path, name) with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) + # for chunk in part.content.iter_content( + # chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) f.write(part.content) self.process_temp_bundle(ds_name, filename) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 4bbf2854..6565b49d 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -2,9 +2,6 @@ import os import json import tarfile -import shutil - -from catalyst.data.bundles.core import download_without_progress from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory From d97890fc4e779aa683f91da68df45225ea6dff2f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 11:43:41 -0700 Subject: [PATCH 61/85] BLD: marketplace: moving AUTH_SERVER paths to /marketplace/* --- catalyst/marketplace/marketplace.py | 2 +- catalyst/marketplace/utils/auth_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a5e7d4f5..61ed7fec 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -658,7 +658,7 @@ class Marketplace: for file in filenames: files.append(('file', open(file, 'rb'))) - r = requests.post('{}/publish'.format(AUTH_SERVER), + r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER), files=files, headers=headers) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 452dadb7..d761cf8d 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -27,7 +27,7 @@ def get_key_secret(pubAddr, dataset): """ session = requests.Session() - response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), headers={ 'pubAddr': pubAddr, 'dataset': dataset}) From 5ccddf6d2178bd59a6115d870616beb8a975ea10 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 14:17:48 -0500 Subject: [PATCH 62/85] BLD: removed dummy smart contract --- marketplace/README.md | 18 --------- marketplace/contracts/Marketplace.sol | 19 --------- marketplace/contracts/Migrations.sol | 23 ----------- marketplace/marketplace.iml | 14 ------- marketplace/migrations/1_initial_migration.js | 5 --- marketplace/migrations/2_deploy_contracts.js | 5 --- marketplace/test/TestMarketplace.sol | 39 ------------------- marketplace/truffle-config.js | 4 -- marketplace/truffle.js | 11 ------ 9 files changed, 138 deletions(-) delete mode 100644 marketplace/README.md delete mode 100644 marketplace/contracts/Marketplace.sol delete mode 100644 marketplace/contracts/Migrations.sol delete mode 100644 marketplace/marketplace.iml delete mode 100644 marketplace/migrations/1_initial_migration.js delete mode 100644 marketplace/migrations/2_deploy_contracts.js delete mode 100644 marketplace/test/TestMarketplace.sol delete mode 100644 marketplace/truffle-config.js delete mode 100644 marketplace/truffle.js diff --git a/marketplace/README.md b/marketplace/README.md deleted file mode 100644 index 61f63842..00000000 --- a/marketplace/README.md +++ /dev/null @@ -1,18 +0,0 @@ -This module contains smart contracts for the data marketplace. -It was generated with Truffle. Truffle can be used to compile and test -the smart contracts. It's being tested on a test blockhain using Ganache. - -Steps to test: -* Download and run Ganache -* Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder, run the following commands: - * `truffle compile` - * `truffle test` - * `truffle migrate --reset` - * `truffle console` - * From the console: `Marketplace.deployed()` -* The deployed method displays info about the deployed smart contract -including its address. Copy/paste the address into -`catalyst/marketplace/marketplace.py`. -* Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol deleted file mode 100644 index 115b6110..00000000 --- a/marketplace/contracts/Marketplace.sol +++ /dev/null @@ -1,19 +0,0 @@ -pragma solidity ^0.4.17; - -contract Marketplace { - address[16] public subscribers; - - // Subscribing to a data source - function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 15); - - subscribers[dataSourceId] = msg.sender; - - return dataSourceId; - } - - // Retrieving the subscribers - function getSubscribers() public view returns (address[16]) { - return subscribers; - } -} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol deleted file mode 100644 index f170cb4f..00000000 --- a/marketplace/contracts/Migrations.sol +++ /dev/null @@ -1,23 +0,0 @@ -pragma solidity ^0.4.17; - -contract Migrations { - address public owner; - uint public last_completed_migration; - - modifier restricted() { - if (msg.sender == owner) _; - } - - function Migrations() public { - owner = msg.sender; - } - - function setCompleted(uint completed) public restricted { - last_completed_migration = completed; - } - - function upgrade(address new_address) public restricted { - Migrations upgraded = Migrations(new_address); - upgraded.setCompleted(last_completed_migration); - } -} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml deleted file mode 100644 index cf4ef5ed..00000000 --- a/marketplace/marketplace.iml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js deleted file mode 100644 index 4d5f3f9b..00000000 --- a/marketplace/migrations/1_initial_migration.js +++ /dev/null @@ -1,5 +0,0 @@ -var Migrations = artifacts.require("./Migrations.sol"); - -module.exports = function(deployer) { - deployer.deploy(Migrations); -}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js deleted file mode 100644 index e3935400..00000000 --- a/marketplace/migrations/2_deploy_contracts.js +++ /dev/null @@ -1,5 +0,0 @@ -var Marketplace = artifacts.require ("Marketplace"); - -module.exports = function (deployer) { - deployer.deploy (Marketplace); -}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol deleted file mode 100644 index f169d728..00000000 --- a/marketplace/test/TestMarketplace.sol +++ /dev/null @@ -1,39 +0,0 @@ -pragma solidity ^0.4.17; - -import "truffle/Assert.sol"; -import "truffle/DeployedAddresses.sol"; -import "../contracts/Marketplace.sol"; - -contract TestMarketplace { - Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - - // Testing the subscribe() function - function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(2); - - uint expected = 2; - - Assert.equal(returnedId, expected, "Data source 2 should be recorded."); - } - - // Testing retrieval of a single subscriber - function testGetSubscriberAddressByDataSourceId() public { - // Expected owner is this contract - address expected = this; - - address subscriber = marketplace.subscribers(2); - - Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); - } - - // Testing retrieval of all subscribers - function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected subscriber is this contract - address expected = this; - - // Store subscribers in memory rather than contract's storage - address[16] memory subscribers = marketplace.getSubscribers(); - - Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); - } -} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js deleted file mode 100644 index a6330d6d..00000000 --- a/marketplace/truffle-config.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! -}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js deleted file mode 100644 index 67f2dd44..00000000 --- a/marketplace/truffle.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! - networks: { - development: { - host: "localhost", - port: 7545, - network_id: "*" // Match any network id - } - } -}; From 2320a1432ad58fbc4d1cdf68b17e52bdccc7c593 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 14:34:07 -0500 Subject: [PATCH 63/85] BLD: Merge remote-tracking branch 'remotes/origin/data-marketplace' into develop --- catalyst/__main__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 6f3f97ab..1b08f69d 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,6 +1,5 @@ import errno import os -import sys from functools import wraps import click @@ -581,7 +580,8 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), + sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -635,7 +635,8 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), + sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) @@ -773,7 +774,7 @@ def marketplace(ctx): @click.pass_context def ls(ctx): click.echo('Listing of available data sources on the marketplace:', - sys.stdout) + sys.stdout) marketplace = Marketplace() marketplace.list() @@ -885,7 +886,6 @@ def publish(ctx, dataset, datadir, watch): ctx.fail("must specify a datadir where to find the files to publish " " with '--datadir'\n") marketplace.publish(dataset, datadir, watch) - click.echo("%s %s" % (bundle, timestamp), sys.stdout) if __name__ == '__main__': From c5bed6e8c4e8a1b00c1eb362a3c9bd0ea577baee Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 21:15:30 -0500 Subject: [PATCH 64/85] BLD: made some adjustments during testing --- catalyst/examples/mean_reversion_simple.py | 2 +- catalyst/marketplace/marketplace.py | 46 +++++++++++++--------- tests/marketplace/test_marketplace.py | 7 ++-- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index db787612..3b94d925 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='eth', live_graph=False, - simulate_orders=False, + simulate_orders=True, stats_output=None, # auth_aliases=dict(poloniex='auth2') ) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 61ed7fec..8f03145c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -142,14 +142,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -197,6 +197,7 @@ class Marketplace: print(df) def subscribe(self, dataset): + # TODO: what happens if we are already subscribed? dataset = dataset.lower() address = self.choose_pubaddr()[0] @@ -217,11 +218,13 @@ class Marketplace: print('Checking that the ENG balance in {} is greater than ' '{} ENG... '.format(address, price), end='') + wallet_address = address[2:] balance = self.web3.eth.call({ 'from': address, 'to': self.eng_contract_address, 'data': '0x70a08231000000000000000000000000{}'.format( - address[2:]) + wallet_address + ) }) try: balance = int(balance[2:], 16) // 10 ** 8 @@ -242,7 +245,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -259,22 +262,25 @@ class Marketplace: '2. Second transaction is the actual subscription for the ' 'desired dataset'.format(price)) + grains = price * 10 ** 8 tx = self.eng_contract.functions.approve( self.mkt_contract_address, - price, + grains, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + {'nonce': self.web3.eth.getTransactionCount(address)} + ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - try: - tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) - print('\nThis is the TxHash for this transaction: ' - '{}'.format(tx_hash)) + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) + print( + '\nThis is the TxHash for this transaction: {}'.format(tx_hash) + ) except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) @@ -350,7 +356,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -381,6 +387,7 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): ds_name = ds_name.lower() + # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( bytes32(ds_name) ).call() @@ -391,7 +398,8 @@ class Marketplace: return address, address_i = self.choose_pubaddr() - check_sub = self.mkt_contract.functions.checkAddressSubscription( + fns = self.mkt_contract.functions + check_sub = fns.checkAddressSubscription( address, bytes32(ds_name) ).call() diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index d2511ce8..c45a727d 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,18 +16,19 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('marketcap1') + marketplace.subscribe('marketcap1234') pass def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('marketcap1') + ds_def = marketplace.ingest('marketcap1234') pass def test_publish(self): marketplace = Marketplace() datadir = '/Users/fredfortier/Downloads/marketcap_test_single' - marketplace.publish('marketcap1', datadir, False) + marketplace.publish('marketcap1234', datadir, False) + pass def test_clean(self): marketplace = Marketplace() From 2d43955abf472332b7dd78ea61d04ca1944d6766 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 00:45:33 -0500 Subject: [PATCH 65/85] BUG: fixed Python 2 issue --- catalyst/marketplace/marketplace.py | 6 ++++-- tests/marketplace/test_marketplace.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 8f03145c..61ac2a97 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -215,8 +215,10 @@ class Marketplace: print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) - print('Checking that the ENG balance in {} is greater than ' - '{} ENG... '.format(address, price), end='') + print( + 'Checking that the ENG balance in {} is greater than {} ' + 'ENG... '.format(address, price) + ) wallet_address = address[2:] balance = self.web3.eth.call({ diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index c45a727d..59564a5e 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,7 +16,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('marketcap1234') + marketplace.subscribe('marketcap2222') pass def test_ingest(self): From eee2e1be885a0fd8ec4e46fb7ecc7df8cf48838d Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 00:48:17 -0500 Subject: [PATCH 66/85] BUG: fixed Python 2 issue --- catalyst/marketplace/marketplace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 61ac2a97..1ea2ea00 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -302,7 +302,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - print('.', end='', flush=True) + # print('.', end='', flush=True) time.sleep(1) print('\nFirst transaction successful!\n' @@ -349,7 +349,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - print('.', end='', flush=True) + # print('.', end='', flush=True) time.sleep(1) print('\nSecond transaction successful!\n' From 461a5942fb0cd09f2241347170400eb19bc82a71 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 01:17:00 -0500 Subject: [PATCH 67/85] BUG: fixed Python 2 issue EXPERIMENTAL --- catalyst/marketplace/marketplace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 1ea2ea00..71620b66 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,6 @@ import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException -from web3 import Web3, HTTPProvider from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, @@ -43,6 +42,8 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): + global Web3 + from web3 import Web3, HTTPProvider self.addresses = get_user_pubaddr() @@ -107,7 +108,6 @@ class Marketplace: # ] def choose_pubaddr(self): - if len(self.addresses) == 1: address = self.addresses[0]['pubAddr'] address_i = 0 From fb658390327e1b6694e0ddc913709740d90b8151 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 23:33:34 -0700 Subject: [PATCH 68/85] BLD: marketplace: getkeysecret is authenticated --- catalyst/marketplace/marketplace.py | 9 +-- catalyst/marketplace/marketplace_errors.py | 15 ++++- catalyst/marketplace/utils/auth_utils.py | 77 ++++++++++++++++++---- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 71620b66..2eced9ee 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -422,8 +422,7 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, ds_name) + key, secret = get_key_secret(address) headers = get_signed_headers(ds_name, key, secret) log.debug('Starting download of dataset for ingestion...') @@ -596,8 +595,7 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, dataset) + key, secret = get_key_secret(address) tx = self.mkt_contract.functions.register( bytes32(dataset), @@ -655,8 +653,7 @@ class Marketplace: key = match['key'] secret = match['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(provider_info[0], dataset) + key, secret = get_key_secret(provider_info[0]) headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index ed3d0fa4..953941fc 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -8,7 +8,8 @@ def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired]: + MarketplaceSubscriptionExpired, + MarketplaceWalletNotSupported, MarketplaceEmptySignature]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -66,3 +67,15 @@ class MarketplaceSubscriptionExpired(ZiplineError): 'following command:\n' 'catalyst marketplace subscribe --dataset={dataset}' ) + + +class MarketplaceWalletNotSupported(ZiplineError): + msg = ( + 'Wallet {wallet} is not supported.' + ) + + +class MarketplaceEmptySignature(ZiplineError): + msg = ( + 'Signature cannot be empty.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index d761cf8d..4979b6f3 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -5,13 +5,14 @@ import requests import time from catalyst.marketplace.marketplace_errors import ( - MarketplaceHTTPRequest) + MarketplaceHTTPRequest, MarketplaceWalletNotSupported, + MarketplaceEmptySignature) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) from catalyst.constants import AUTH_SERVER -def get_key_secret(pubAddr, dataset): +def get_key_secret(pubAddr, wallet='mew'): """ Obtain a new key/secret pair from authentication server @@ -29,25 +30,73 @@ def get_key_secret(pubAddr, dataset): session = requests.Session() response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), headers={ - 'pubAddr': pubAddr, - 'dataset': dataset}) + 'Authorization': 'Digest username="{0}"'.format( + pubAddr)}) - if 'error' in response.json(): + if response.status_code != 401: raise MarketplaceHTTPRequest(request=str('obtain key/secret'), - error=str(response.json()['error'])) + error='Unexpected response code: ' + '{}'.format(response.status_code)) - addresses = get_user_pubaddr() + header = response.headers.get('WWW-Authenticate') + auth_type, auth_info = header.split(None, 1) + d = requests.utils.parse_dict_header(auth_info) - match = next((l for l in addresses if - l['pubAddr'] == pubAddr), None) - match['key'] = response.json()['key'] - match['secret'] = response.json()['secret'] + nonce = '0x{}'.format(d['nonce']) - addresses[addresses.index(match)] = match + if wallet == 'mew': + print('\nObtaining a key/secret pair to streamline all future ' + 'requests with the authentication server.\n' + 'Visit https://www.myetherwallet.com/signmsg.html and sign the' + 'following message:\n{}'.format(nonce)) + signature = input('Copy and Paste the "sig" field from ' + 'the signature here (without the double quotes, ' + 'only the HEX value:\n') + else: + raise MarketplaceWalletNotSupported(wallet=wallet) - save_user_pubaddr(addresses) + if signature is None: + raise MarketplaceEmptySignature() - return match['key'], match['secret'] + signature = signature[2:] + r = int(signature[0:64], base=16) + s = int(signature[64:128], base=16) + v = int(signature[128:130], base=16) + vrs = [v, r, s] + + response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), + headers={ + 'Authorization': 'Digest username="{0}",realm="{1}",' + 'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",' + 'opaque="{4}"'.format(pubAddr, + d['realm'], + d['nonce'], + ','.join(str(e) for e in vrs+[wallet]), + d['opaque'])}) + + if response.status_code == 200: + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + else: + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + print('Key/secret pair retrieved successfully from server.') + + return match['key'], match['secret'] + + else: + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=response.status_code) def get_signed_headers(ds_name, key, secret): From 093660ab1a3e625f4a2719226fd46ca45978527e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 01:46:33 -0500 Subject: [PATCH 69/85] BLD: simplified unit tests until the next release --- .../test_suites/test_suite_exchange.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index ca0bd666..79f87a2a 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -15,7 +15,7 @@ from catalyst.exchange.utils.test_utils import select_random_exchanges, \ handle_exchange_error, select_random_assets from catalyst.testing import ZiplineTestCase from catalyst.testing.fixtures import WithLogger -from catalyst.exchange.utils.factory import get_exchanges +from catalyst.exchange.utils.factory import get_exchanges, get_exchange log = Logger('TestSuiteExchange') @@ -90,7 +90,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): # exchange_population, # features=['fetchTickers'], # ) # Type: list[Exchange] - exchanges = list(get_exchanges(['bitfinex']).values()) + exchanges = list(get_exchanges(['binance']).values()) for exchange in exchanges: exchange.init() @@ -113,10 +113,11 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): exchange_population = 3 asset_population = 3 - exchanges = select_random_exchanges( - population=exchange_population, - features=['fetchOHLCV'], - ) # Type: list[Exchange] + # exchanges = select_random_exchanges( + # population=exchange_population, + # features=['fetchOHLCV'], + # ) # Type: list[Exchange] + exchanges = list(get_exchanges(['binance']).values()) for exchange in exchanges: exchange.init() @@ -154,13 +155,20 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): quote_currency = 'eth' order_amount = 0.1 - exchanges = select_random_exchanges( - population=population, - features=['fetchOrder'], - is_authenticated=True, - base_currency=quote_currency, - ) # Type: list[Exchange] + # exchanges = select_random_exchanges( + # population=population, + # features=['fetchOrder'], + # is_authenticated=True, + # base_currency=quote_currency, + # ) # Type: list[Exchange] + exchanges = [ + get_exchange( + 'binance', + base_currency=quote_currency, + must_authenticate=True, + ) + ] log_catcher = TestHandler() with log_catcher: for exchange in exchanges: From c39e075766b60c9f23145946826dbf6659ec08b2 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 23:50:05 -0700 Subject: [PATCH 70/85] MAINT: constants points to contract address/abi in develop branch --- catalyst/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index df0ea82b..1ddc7710 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -29,18 +29,18 @@ ETH_REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_marketplace_address.txt' MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_marketplace_abi.json' # TODO: switch to mainnet ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/' \ + 'develop/catalyst/marketplace/' \ 'contract_enigma_address.txt' ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_enigma_abi.json' \ No newline at end of file From 0e3be98d24bffd3f98cc64d7ccfdb4d22ffdd62a Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 01:44:16 -0700 Subject: [PATCH 71/85] BLD: mmarketplace: switched encoding to Web3.toHex() --- catalyst/marketplace/marketplace.py | 61 ++++++++++++++----------- catalyst/marketplace/utils/eth_utils.py | 46 +++++++++---------- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 2eced9ee..99385171 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import glob import json import os @@ -27,7 +29,7 @@ from catalyst.marketplace.marketplace_errors import ( from catalyst.marketplace.utils.auth_utils import get_key_secret, \ get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex +from catalyst.marketplace.utils.eth_utils import bin_hex from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr, get_temp_bundles_folder, extract_bundle @@ -142,14 +144,13 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'],) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -188,7 +189,7 @@ class Marketplace: if index > 0: data.append( dict( - dataset=data_source.decode('utf-8').rstrip('\0') + dataset=Web3.toText(data_source) ) ) @@ -197,12 +198,12 @@ class Marketplace: print(df) def subscribe(self, dataset): - # TODO: what happens if we are already subscribed? + dataset = dataset.lower() address = self.choose_pubaddr()[0] provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if not provider_info[4]: @@ -212,6 +213,15 @@ class Marketplace: price = provider_info[1] + subscribed = self.mkt_contract.functions.checkAddressSubscription( + address, Web3.toHex(dataset) + ).call() + + if subscribed[5]: + print('You are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {}, and is valid until' + '{}'.format(dataset, subscribed[3], subscribed[4])) + print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -228,10 +238,7 @@ class Marketplace: wallet_address ) }) - try: - balance = int(balance[2:], 16) // 10 ** 8 - except ValueError: - balance = int(bin_hex(balance), 16) // 10 ** 8 + balance = Web3.toInt(hexstr=balance) // 10 ** 8 if balance > price: print('OK.') @@ -247,7 +254,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -302,14 +309,14 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - # print('.', end='', flush=True) + print('.', end='', flush=True) time.sleep(1) print('\nFirst transaction successful!\n' 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - bytes32(dataset), + Web3.toHex(dataset), ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) @@ -349,7 +356,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - # print('.', end='', flush=True) + print('.', end='', flush=True) time.sleep(1) print('\nSecond transaction successful!\n' @@ -358,7 +365,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -391,7 +398,7 @@ class Marketplace: ds_name = ds_name.lower() # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(ds_name) + Web3.toHex(ds_name) ).call() if not provider_info[4]: @@ -402,10 +409,10 @@ class Marketplace: address, address_i = self.choose_pubaddr() fns = self.mkt_contract.functions check_sub = fns.checkAddressSubscription( - address, bytes32(ds_name) + address, Web3.toHex(ds_name) ).call() - if check_sub[0] != address or b32_str(check_sub[1]) != ds_name: + if check_sub[0] != address or Web3.toText(check_sub[1]) != ds_name: raise MarketplaceContractDataNoMatch( params='address: {}, dataset: {}'.format( address, ds_name @@ -541,7 +548,7 @@ class Marketplace: desc = input('Enter the name of the dataset to register: ') dataset = desc.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if provider_info[4]: @@ -598,7 +605,7 @@ class Marketplace: key, secret = get_key_secret(address) tx = self.mkt_contract.functions.register( - bytes32(dataset), + Web3.toHex(dataset), price, address, ).buildTransaction( @@ -631,7 +638,7 @@ class Marketplace: def publish(self, dataset, datadir, watch): dataset = dataset.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if not provider_info[4]: diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index f5fe8b7b..35e8b300 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -1,37 +1,37 @@ import binascii -def bytes32(string): - """ - Convert string to bytes32 data type for smart contract +# def bytes32(string): +# """ +# Convert string to bytes32 data type for smart contract - Parameters - ---------- - string: str +# Parameters +# ---------- +# string: str - Returns - ------- - list +# Returns +# ------- +# list - """ - return binascii.hexlify(string.encode('utf-8')) +# """ +# return binascii.hexlify(string.encode('utf-8')) -def b32_str(bytes32): - """ - Convert bytes32 to string +# def b32_str(bytes32): +# """ +# Convert bytes32 to string - Parameters - ---------- - input: bytes object +# Parameters +# ---------- +# input: bytes object - Returns - ------- - str +# Returns +# ------- +# str - """ - return binascii.unhexlify( - bytes32.decode('utf-8').rstrip('\0')).decode('ascii') +# """ +# return binascii.unhexlify( +# bytes32.decode('utf-8').rstrip('\0')).decode('ascii') def bin_hex(binary): From 540358d0f9accdba0ab4bd8df9e4ce0e5bf33bb6 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Wed, 7 Feb 2018 12:48:24 +0200 Subject: [PATCH 72/85] BLD: new marketplace contract address --- catalyst/marketplace/contract_marketplace_address.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/contract_marketplace_address.txt b/catalyst/marketplace/contract_marketplace_address.txt index 86810a5f..577adaa0 100644 --- a/catalyst/marketplace/contract_marketplace_address.txt +++ b/catalyst/marketplace/contract_marketplace_address.txt @@ -1 +1 @@ -0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482 \ No newline at end of file +0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4 \ No newline at end of file From e071f6ec8d10f5e8668f727e63437f85b7c0e0ff Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 10:57:07 -0700 Subject: [PATCH 73/85] BLD: implemented grains, catch JSON malformed in addresses.json --- catalyst/marketplace/marketplace.py | 72 +++++++++++++--------- catalyst/marketplace/marketplace_errors.py | 9 ++- catalyst/marketplace/utils/eth_utils.py | 32 ++++++++++ catalyst/marketplace/utils/path_utils.py | 6 +- 4 files changed, 89 insertions(+), 30 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 99385171..ab35fe65 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -24,12 +24,12 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired) + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.auth_utils import get_key_secret, \ get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.eth_utils import bin_hex +from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \ + to_grains from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr, get_temp_bundles_folder, extract_bundle @@ -109,6 +109,9 @@ class Marketplace: # ), # ] + def to_text(self, hex): + return Web3.toText(hex).rstrip('\0') + def choose_pubaddr(self): if len(self.addresses) == 1: address = self.addresses[0]['pubAddr'] @@ -187,15 +190,19 @@ class Marketplace: data = [] for index, data_source in enumerate(data_sources): if index > 0: - data.append( - dict( - dataset=Web3.toText(data_source) + # if 'test' not in Web3.toText(data_source).lower(): + data.append( + dict( + dataset=self.to_text(data_source) + ) ) - ) df = pd.DataFrame(data) set_print_settings() - print(df) + if df.empty: + print('There are no datasets available yet.') + else: + print(df) def subscribe(self, dataset): @@ -211,16 +218,20 @@ class Marketplace: 'the Data Marketplace.'.format(dataset)) return - price = provider_info[1] + grains = provider_info[1] + price = from_grains(grains) subscribed = self.mkt_contract.functions.checkAddressSubscription( address, Web3.toHex(dataset) ).call() if subscribed[5]: - print('You are already subscribed to the "{}" dataset.\n' - 'Your subscription started on {}, and is valid until' - '{}'.format(dataset, subscribed[3], subscribed[4])) + print('\nYou are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {} UTC, and is valid until ' + '{} UTC.'.format( + dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True))) + return print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -238,16 +249,18 @@ class Marketplace: wallet_address ) }) - balance = Web3.toInt(hexstr=balance) // 10 ** 8 - if balance > price: + balance = Web3.toInt(hexstr=balance) + + if balance > grains: print('OK.') else: print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower ' 'than the price of the dataset that you are trying to\n' 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' - 'and try again.'.format(address, balance, price)) + 'and try again.'.format( + address, from_grains(balance), price)) return while True: @@ -271,7 +284,6 @@ class Marketplace: '2. Second transaction is the actual subscription for the ' 'desired dataset'.format(price)) - grains = price * 10 ** 8 tx = self.eng_contract.functions.approve( self.mkt_contract_address, grains, @@ -395,7 +407,8 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): - ds_name = ds_name.lower() + # ds_name = ds_name.lower() + # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( Web3.toHex(ds_name) @@ -412,18 +425,19 @@ class Marketplace: address, Web3.toHex(ds_name) ).call() - if check_sub[0] != address or Web3.toText(check_sub[1]) != ds_name: - raise MarketplaceContractDataNoMatch( - params='address: {}, dataset: {}'.format( - address, ds_name - ) - ) + if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name: + print('You are not subscribed to dataset "{}" with address {}. ' + 'Plese subscribe first.'.format(ds_name, address)) + return if not check_sub[5]: - raise MarketplaceSubscriptionExpired( - dataset=ds_name, - date=check_sub[4], - ) + print('Your subscription to dataset "{}" expired on {} UTC.' + 'Please renew your subscription by running:\n' + 'catalyst marketplace subscribe --dataset={}'. format( + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -604,9 +618,11 @@ class Marketplace: else: key, secret = get_key_secret(address) + grains = to_grains(price) + tx = self.mkt_contract.functions.register( Web3.toHex(dataset), - price, + grains, address, ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)} diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 953941fc..b6be1c3b 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -8,7 +8,7 @@ def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired, + MarketplaceSubscriptionExpired, MarketplaceJSONError, MarketplaceWalletNotSupported, MarketplaceEmptySignature]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] @@ -79,3 +79,10 @@ class MarketplaceEmptySignature(ZiplineError): msg = ( 'Signature cannot be empty.' ) + + +class MarketplaceJSONError(ZiplineError): + msg = ( + 'The configuration file {file} is malformed. Please correct ' + 'the following error:\n{error}' + ) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index 35e8b300..9a630496 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -48,3 +48,35 @@ def bin_hex(binary): """ return binascii.hexlify(binary).decode('utf-8') + + +def from_grains(amount): + """ + Convert from grains to cryptocurrency + + Parameters + ---------- + input: amount + + Returns + ------- + int + + """ + return amount // 10 ** 8 + + +def to_grains(amount): + """ + Convert from cryptocurrency to grains + + Parameters + ---------- + input: amount + + Returns + ------- + int + + """ + return amount * 10 ** 8 diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 6565b49d..fd4ee663 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -4,6 +4,7 @@ import tarfile from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory +from catalyst.marketplace.marketplace_errors import MarketplaceJSONError def get_marketplace_folder(environ=None): @@ -123,7 +124,10 @@ def get_user_pubaddr(environ=None): if os.path.isfile(filename): with open(filename) as data_file: - data = json.load(data_file) + try: + data = json.load(data_file) + except json.decoder.JSONDecodeError as e: + raise MarketplaceJSONError(file=filename, error=e) try: d = data[0]['pubAddr'] except Exception as e: From 866b92910fc1bd45931931a845a9c8a97eaf9dfb Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 13:10:07 -0500 Subject: [PATCH 74/85] BLD: fixed the marketplace api in the algo runtime --- catalyst/api.pyi | 11 ++- .../examples/mean_reversion_by_marketcap.py | 6 +- catalyst/exchange/exchange_algorithm.py | 7 +- catalyst/marketplace/marketplace.py | 71 +++++++------------ 4 files changed, 34 insertions(+), 61 deletions(-) diff --git a/catalyst/api.pyi b/catalyst/api.pyi index 6e46aba9..57dd75a7 100644 --- a/catalyst/api.pyi +++ b/catalyst/api.pyi @@ -816,18 +816,15 @@ def symbols(*args): """ -def get_data_source(data_source_name, data_frequency=None, - start=None, end=None): +def get_dataset(ds_name, start=None, end=None): """ Lookup a data source from the marketplace Parameters ---------- - self - data_source_name - data_frequency - start - end + ds_name: str + start: pd.Timestamp + end: pd.Timestamp Returns ------- diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py index 6d31dab7..b8fb94d1 100644 --- a/catalyst/examples/mean_reversion_by_marketcap.py +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -10,7 +10,7 @@ import talib from logbook import Logger from catalyst import run_algorithm -from catalyst.api import symbol, record, order_target_percent, get_data_source +from catalyst.api import symbol, record, order_target_percent, get_dataset from catalyst.exchange.utils.stats_utils import set_print_settings, \ get_pretty_stats # We give a name to the algorithm which Catalyst will use to persist its state. @@ -33,8 +33,8 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - df = get_data_source( - 'marketcap', start=context.datetime + df = get_dataset( + 'marketcap1234', start=context.datetime ) # type: pd.DataFrame # Keep only the top coins by market cap diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 901f4ac3..da4aee11 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -171,13 +171,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): return round_nearest(amount, asset.min_trade_size) @api_method - def get_data_source(self, data_source_name, data_frequency=None, - start=None, end=None): + def get_dataset(self, data_source_name, start=None, end=None): if self._marketplace is None: self._marketplace = Marketplace() - return self._marketplace.get_data_source( - data_source_name, data_frequency, start, end, + return self._marketplace.get_dataset( + data_source_name, start, end, ) @api_method diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ab35fe65..474aa545 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -147,13 +147,13 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'],) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -174,15 +174,6 @@ class Marketplace: print('\nYou can check the outcome of your transaction here:\n' '{}\n\n'.format(etherscan)) - def get_data_source_def(self, data_source_name): - data_source_name = data_source_name.lower() - dsm = self.get_data_sources_map() - - ds = six.next( - (d for d in dsm if d['name'].lower() == data_source_name), None - ) - return ds - def list(self): data_sources = self.mkt_contract.functions.getAllProviders().call() @@ -191,11 +182,11 @@ class Marketplace: for index, data_source in enumerate(data_sources): if index > 0: # if 'test' not in Web3.toText(data_source).lower(): - data.append( - dict( - dataset=self.to_text(data_source) - ) + data.append( + dict( + dataset=self.to_text(data_source) ) + ) df = pd.DataFrame(data) set_print_settings() @@ -229,8 +220,8 @@ class Marketplace: print('\nYou are already subscribed to the "{}" dataset.\n' 'Your subscription started on {} UTC, and is valid until ' '{} UTC.'.format( - dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), - pd.to_datetime(subscribed[4], unit='s', utc=True))) + dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True))) return print('\nThe price for a monthly subscription to this dataset is' @@ -260,14 +251,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -377,7 +368,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -433,11 +424,11 @@ class Marketplace: if not check_sub[5]: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' - 'catalyst marketplace subscribe --dataset={}'. format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + 'catalyst marketplace subscribe --dataset={}'.format( + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -485,25 +476,11 @@ class Marketplace: log.info('{} ingested successfully'.format(ds_name)) - def get_data_source(self, data_source_name, data_frequency=None, - start=None, end=None): - data_source_name = data_source_name.lower() - - if data_frequency is None: - ds_def = self.get_data_source_def(data_source_name) - freqs = ds_def['data_frequencies'] - data_frequency = freqs[0] - - if len(freqs) > 1: - log.warn( - 'no data frequencies specified for data source {}, ' - 'selected the first one by default: {}'.format( - data_source_name, data_frequency - ) - ) + def get_dataset(self, ds_name, start=None, end=None): + ds_name = ds_name.lower() # TODO: filter ctable by start and end date - bundle_folder = get_bundle_folder(data_source_name, data_frequency) + bundle_folder = get_data_source_folder(ds_name) z = bcolz.ctable(rootdir=bundle_folder, mode='r') df = z.todataframe() # type: pd.DataFrame From 9f9bfc9df094d91c917141408d4f1cabd194eec6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 16:40:43 -0500 Subject: [PATCH 75/85] BLD: dropping dataset index cols to avoid duplicates --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 474aa545..e87ace9e 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -484,7 +484,7 @@ class Marketplace: z = bcolz.ctable(rootdir=bundle_folder, mode='r') df = z.todataframe() # type: pd.DataFrame - df.set_index(['date', 'symbol'], drop=False, inplace=True) + df.set_index(['date', 'symbol'], drop=True, inplace=True) if start and end is None: df = df.xs(start, level=0) From 6e642f45d81dd1fbe59d8ae8e6e5f1b33a73fbb8 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 14:44:06 -0700 Subject: [PATCH 76/85] MAINT: marketplace prompts --- catalyst/marketplace/marketplace.py | 44 ++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e87ace9e..7086b903 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -142,9 +142,11 @@ class Marketplace: print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' 'enter the following parameters:\n\n' 'From Address:\t\t{_from}\n' + '\n\tClick the "Generate Information" button\n\n' 'To Address:\t\t{to}\n' 'Value / Amount to Send:\t{value}\n' 'Gas Limit:\t\t{gas}\n' + 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( _from=from_address, @@ -229,7 +231,7 @@ class Marketplace: print( 'Checking that the ENG balance in {} is greater than {} ' - 'ENG... '.format(address, price) + 'ENG... '.format(address, price), end='' ) wallet_address = address[2:] @@ -338,14 +340,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in ETH_REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}'.format(etherscan)) + self.check_transaction(tx_hash) print('Waiting for the second transaction to succeed...') @@ -609,13 +604,36 @@ class Marketplace: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - tx_hash = '0x{}'.format( - bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) - ) - print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + try: + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) + print( + '\nThis is the TxHash for this transaction: {}'.format(tx_hash) + ) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + return self.check_transaction(tx_hash) + print('Waiting for the transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + print('\nWarming up the {} dataset'.format(dataset)) self.create_metadata( key=key, From e1abecb556390ea505a78690bc0ec32bc474d790 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 17:09:49 -0700 Subject: [PATCH 77/85] MAINT: updated catalyst logo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6c395b45..a39137fd 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg +.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png :target: https://enigmampc.github.io/catalyst :align: center :alt: Enigma | Catalyst From 1ad26b0c3983c65bff8ddd78529dbc73de8c68ae Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 19:29:38 -0500 Subject: [PATCH 78/85] BLD: trying to format a line --- catalyst/marketplace/marketplace.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 7086b903..b9f05151 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -219,11 +219,15 @@ class Marketplace: ).call() if subscribed[5]: - print('\nYou are already subscribed to the "{}" dataset.\n' - 'Your subscription started on {} UTC, and is valid until ' - '{} UTC.'.format( - dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), - pd.to_datetime(subscribed[4], unit='s', utc=True))) + print( + '\nYou are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {} UTC, and is valid until ' + '{} UTC.'.format( + dataset, + pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True) + ) + ) return print('\nThe price for a monthly subscription to this dataset is' @@ -604,7 +608,7 @@ class Marketplace: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - + try: tx_hash = '0x{}'.format( bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) From db7b7639a06191be69b99438e63bb996b7144c88 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 17:32:28 -0700 Subject: [PATCH 79/85] MAINT: not listing test datasets --- catalyst/marketplace/marketplace.py | 39 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index b9f05151..498c44b1 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -12,7 +12,6 @@ import bcolz import logbook import pandas as pd import requests -import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException @@ -149,13 +148,13 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -183,12 +182,12 @@ class Marketplace: data = [] for index, data_source in enumerate(data_sources): if index > 0: - # if 'test' not in Web3.toText(data_source).lower(): - data.append( - dict( - dataset=self.to_text(data_source) + if 'test' not in Web3.toText(data_source).lower(): + data.append( + dict( + dataset=self.to_text(data_source) + ) ) - ) df = pd.DataFrame(data) set_print_settings() @@ -257,14 +256,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -367,7 +366,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -424,10 +423,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] From 18cdd5168015dd39b3cd7cb46219f45744b8a492 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 22:16:32 -0500 Subject: [PATCH 80/85] BUG: fixed issue with order processing --- catalyst/exchange/ccxt/ccxt_exchange.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 7a118a52..9cf6c6f4 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -877,6 +877,7 @@ class CCXT(Exchange): order.commission = exc_order.commission order.filled = exc_order.amount + transactions = [] if exc_order.status == ORDER_STATUS.FILLED: if order.amount > exc_order.amount: log.warn( @@ -898,7 +899,9 @@ class CCXT(Exchange): order_id=order.id, commission=order.commission, ) - return [transaction] + transactions.append(transaction) + + return transactions def process_order(self, order): # TODO: move to parent class after tracking features in the parent From 64e22ba27dcabfaaa1efef5506845d2799506c36 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 22:28:08 -0500 Subject: [PATCH 81/85] BLD: Updated release notes for 5.0 --- docs/source/releases.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 635f2e95..ccce3da0 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,22 @@ Release Notes ============= +Version 0.5.0 +^^^^^^^^^^^^^ +**Release Date**: 2018-02-07 + +Bug Fixes +~~~~~~~~~ +- Fixed an issue with orders that stay open :issue:`211` +- Fixed Jupyter issues :issue:`179` +- Fetching multiple tickers in one call to minimize rate limit risks :issue:`174` +- Improved live state presentation :issue:`171` + + +Build +~~~~~ +- Introducing the Enigma Marketplace + Version 0.4.7 ^^^^^^^^^^^^^ **Release Date**: 2018-01-19 From 7a89cfc02b09223a7598e94bf4814b87a61ed00a Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 21:01:14 -0700 Subject: [PATCH 82/85] BUG: marketplace: balance returns different types --- catalyst/marketplace/marketplace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 498c44b1..50bacb08 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -246,7 +246,10 @@ class Marketplace: ) }) - balance = Web3.toInt(hexstr=balance) + try: + balance = Web3.toInt(balance) # web3 >= 4.0.0b7 + except TypeError: + balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 if balance > grains: print('OK.') From d2f9762fbf062e569b213edc99240e8de5083380 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:02:40 -0500 Subject: [PATCH 83/85] BLD: adjusting the marketplace sample algo --- .../examples/mean_reversion_by_marketcap.py | 10 +++-- catalyst/marketplace/marketplace.py | 37 ++++++++++--------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py index b8fb94d1..95253a87 100644 --- a/catalyst/examples/mean_reversion_by_marketcap.py +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -33,10 +33,14 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - df = get_dataset( - 'marketcap1234', start=context.datetime - ) # type: pd.DataFrame + df = get_dataset('testmarketcap2') # type: pd.DataFrame + # Picking a specific date in our DataFrame + first_dt = df.index.get_level_values(0)[0] + # Since we use a MultiIndex with date / symbol, picking a date will + # result in a new DataFrame for the selected date with a single + # symbol index + df = df.xs(first_dt, level=0) # Keep only the top coins by market cap df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))] diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 50bacb08..a1ac263c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -148,13 +148,13 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -247,9 +247,9 @@ class Marketplace: }) try: - balance = Web3.toInt(balance) # web3 >= 4.0.0b7 + balance = Web3.toInt(balance) # web3 >= 4.0.0b7 except TypeError: - balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 + balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 if balance > grains: print('OK.') @@ -259,14 +259,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -369,7 +369,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -426,10 +426,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -487,8 +487,9 @@ class Marketplace: df = z.todataframe() # type: pd.DataFrame df.set_index(['date', 'symbol'], drop=True, inplace=True) - if start and end is None: - df = df.xs(start, level=0) + # TODO: implement the filter more carefully + # if start and end is None: + # df = df.xs(start, level=0) return df From 25f1f6e641c6c2dd415ffa144379a64f1e28ab3c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:18:48 -0500 Subject: [PATCH 84/85] BLD: upgraded CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 795c320d..4232bdbb 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.944 + - ccxt==1.10.1049 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index 0789cdef..7c5f2e62 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.944 +ccxt==1.10.1049 boto3==1.4.8 redo==1.6 From e2488317192d47cf8cf276772d4e5a768ec46643 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:27:56 -0500 Subject: [PATCH 85/85] BLD: adjusted sample algo --- catalyst/examples/mean_reversion_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 3b94d925..81a3b182 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -38,7 +38,7 @@ def initialize(context): context.current_day = None context.RSI_OVERSOLD = 40 - context.RSI_OVERBOUGHT = 50 + context.RSI_OVERBOUGHT = 60 context.CANDLE_SIZE = '15T' context.start_time = time.time() @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='eth', live_graph=False, - simulate_orders=True, + simulate_orders=False, stats_output=None, # auth_aliases=dict(poloniex='auth2') )