From 94d5b4a4d564697e4aa25b51ff2f9b8ac05ed7f2 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 17:16:54 -0500 Subject: [PATCH 001/117] BLD: defined first version of commands and marketplace class --- catalyst/__main__.py | 70 ++++++++++++++++++++++++++++++++ catalyst/alt_data/__init__.py | 0 catalyst/alt_data/marketplace.py | 15 +++++++ 3 files changed, 85 insertions(+) create mode 100644 catalyst/alt_data/__init__.py create mode 100644 catalyst/alt_data/marketplace.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f8f9136..d86276ba 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,6 +5,7 @@ from functools import wraps import click import logbook import pandas as pd +from catalyst.alt_data.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -573,6 +574,75 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) +@main.command(name='ls-data') +@click.pass_context +def ls_data(ctx): + click.echo( + 'Listing available alternative data sources' + ) + marketplace = Marketplace() + marketplace.list() + click.echo('Done') + + +@main.command(name='register-data') +@click.argument('data_source_name') +@click.pass_context +def register_data(ctx, data_source_name): + click.echo( + 'Registering to data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.register(data_source_name) + click.echo('Done') + + +@main.command(name='ingest-data') +@click.argument('data_source_name') +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest_data(ctx, data_source_name, data_frequency, start, end): + click.echo( + 'Ingesting data: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.ingest(data_source_name, data_frequency, start, end) + click.echo('Done') + + +@main.command(name='clean-data') +@click.argument('data_source_name') +@click.pass_context +def clean_data(ctx, data_source_name): + click.echo( + 'Cleaning data source: {}'.format(data_source_name) + ) + marketplace = Marketplace() + marketplace.clean(data_source_name) + click.echo('Done') + + @main.command(name='clean-algo') @click.option( '-n', diff --git a/catalyst/alt_data/__init__.py b/catalyst/alt_data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/alt_data/marketplace.py b/catalyst/alt_data/marketplace.py new file mode 100644 index 00000000..8f4610cc --- /dev/null +++ b/catalyst/alt_data/marketplace.py @@ -0,0 +1,15 @@ +class Marketplace: + def __init__(self): + pass + + def list(self): + pass + + def register(self, data_source_name): + pass + + def ingest(self, data_source_name, data_frequency, start, end): + pass + + def clean(self, data_source_name): + pass \ No newline at end of file From aab501aa1791c8b14f398ebd0cb1c75808b50f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 19:08:53 -0500 Subject: [PATCH 002/117] BLD: first test of the marketplace smart contract --- .../{alt_data => marketplace}/__init__.py | 0 .../{alt_data => marketplace}/marketplace.py | 0 etc/requirements_marketplace.txt | 1 + marketplace/contracts/Marketplace.sol | 19 +++++++++ marketplace/contracts/Migrations.sol | 23 +++++++++++ marketplace/marketplace.iml | 14 +++++++ marketplace/migrations/1_initial_migration.js | 5 +++ marketplace/migrations/2_deploy_contracts.js | 5 +++ marketplace/test/TestMarketplace.sol | 39 +++++++++++++++++++ marketplace/truffle-config.js | 4 ++ marketplace/truffle.js | 11 ++++++ 11 files changed, 121 insertions(+) rename catalyst/{alt_data => marketplace}/__init__.py (100%) rename catalyst/{alt_data => marketplace}/marketplace.py (100%) create mode 100644 etc/requirements_marketplace.txt create mode 100644 marketplace/contracts/Marketplace.sol create mode 100644 marketplace/contracts/Migrations.sol create mode 100644 marketplace/marketplace.iml create mode 100644 marketplace/migrations/1_initial_migration.js create mode 100644 marketplace/migrations/2_deploy_contracts.js create mode 100644 marketplace/test/TestMarketplace.sol create mode 100644 marketplace/truffle-config.js create mode 100644 marketplace/truffle.js diff --git a/catalyst/alt_data/__init__.py b/catalyst/marketplace/__init__.py similarity index 100% rename from catalyst/alt_data/__init__.py rename to catalyst/marketplace/__init__.py diff --git a/catalyst/alt_data/marketplace.py b/catalyst/marketplace/marketplace.py similarity index 100% rename from catalyst/alt_data/marketplace.py rename to catalyst/marketplace/marketplace.py diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt new file mode 100644 index 00000000..cbb3a5a3 --- /dev/null +++ b/etc/requirements_marketplace.txt @@ -0,0 +1 @@ +web3==4.0.0b5 \ No newline at end of file diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol new file mode 100644 index 00000000..54a14d1a --- /dev/null +++ b/marketplace/contracts/Marketplace.sol @@ -0,0 +1,19 @@ +pragma solidity ^0.4.17; + +contract Marketplace { + address[16] public subscribers; + + // Adopting a pet + function subscribe(uint dataSourceId) public returns (uint) { + require(dataSourceId >= 0 && dataSourceId <= 1); + + subscribers[dataSourceId] = msg.sender; + + return dataSourceId; + } + + // Retrieving the subscribers + function getSubscribers() public view returns (address[16]) { + return subscribers; + } +} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol new file mode 100644 index 00000000..f170cb4f --- /dev/null +++ b/marketplace/contracts/Migrations.sol @@ -0,0 +1,23 @@ +pragma solidity ^0.4.17; + +contract Migrations { + address public owner; + uint public last_completed_migration; + + modifier restricted() { + if (msg.sender == owner) _; + } + + function Migrations() public { + owner = msg.sender; + } + + function setCompleted(uint completed) public restricted { + last_completed_migration = completed; + } + + function upgrade(address new_address) public restricted { + Migrations upgraded = Migrations(new_address); + upgraded.setCompleted(last_completed_migration); + } +} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml new file mode 100644 index 00000000..cf4ef5ed --- /dev/null +++ b/marketplace/marketplace.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js new file mode 100644 index 00000000..4d5f3f9b --- /dev/null +++ b/marketplace/migrations/1_initial_migration.js @@ -0,0 +1,5 @@ +var Migrations = artifacts.require("./Migrations.sol"); + +module.exports = function(deployer) { + deployer.deploy(Migrations); +}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js new file mode 100644 index 00000000..e3935400 --- /dev/null +++ b/marketplace/migrations/2_deploy_contracts.js @@ -0,0 +1,5 @@ +var Marketplace = artifacts.require ("Marketplace"); + +module.exports = function (deployer) { + deployer.deploy (Marketplace); +}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol new file mode 100644 index 00000000..ea1c7fb3 --- /dev/null +++ b/marketplace/test/TestMarketplace.sol @@ -0,0 +1,39 @@ +pragma solidity ^0.4.17; + +import "truffle/Assert.sol"; +import "truffle/DeployedAddresses.sol"; +import "../contracts/Marketplace.sol"; + +contract TestMarketplace { + Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); + + // Testing the adopt() function + function testUserCanSubscribe() public { + uint returnedId = marketplace.subscribe(0); + + uint expected = 0; + + Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + } + + // Testing retrieval of a single pet's owner + function testGetSubscriberAddressByDataSourceId() public { + // Expected owner is this contract + address expected = this; + + address adopter = marketplace.subscribers(0); + + Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + } + + // Testing retrieval of all pet owners + function testGetSubscriberAddressByDataSourceIdInArray() public { + // Expected owner is this contract + address expected = this; + + // Store adopters in memory rather than contract's storage + address[16] memory subscribers = marketplace.getSubscribers(); + + Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + } +} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js new file mode 100644 index 00000000..a6330d6d --- /dev/null +++ b/marketplace/truffle-config.js @@ -0,0 +1,4 @@ +module.exports = { + // See + // to customize your Truffle configuration! +}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js new file mode 100644 index 00000000..67f2dd44 --- /dev/null +++ b/marketplace/truffle.js @@ -0,0 +1,11 @@ +module.exports = { + // See + // to customize your Truffle configuration! + networks: { + development: { + host: "localhost", + port: 7545, + network_id: "*" // Match any network id + } + } +}; From 22a850ab141556a194c5aa8648a5bdefe6aae6ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 20:41:03 -0500 Subject: [PATCH 003/117] BLD: testing basic smart contract --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++++++-- marketplace/contracts/Marketplace.sol | 2 +- tests/marketplace/__init__.py | 0 tests/marketplace/test_marketplace.py | 4 +++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/marketplace/__init__.py create mode 100644 tests/marketplace/test_marketplace.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 8f4610cc..a6c0ff0b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,15 +1,52 @@ +import json +import os + +from web3 import Web3, HTTPProvider + +CONTRACT_PATH = os.path.join( + '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' +) +CONTRACT_ADDRESS = Web3.toChecksumAddress( + '0x345ca3e014aaf5dca488057592ee47305d9b3e10' +) + + +# we'll use one of our default accounts to deploy from. every write to the chain requires a +# payment of ethereum called "gas". if we were running an actual test ethereum node locally, +# then we'd have to go on the test net and get some free ethereum to play with. that is beyond +# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and +# the only chain we're using is our own local one + + class Marketplace: def __init__(self): - pass + with open(CONTRACT_PATH) as handle: + json_interface = json.load(handle) + + w3 = Web3(HTTPProvider('http://localhost:7545')) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=json_interface['abi'], + ) # Type: Contract + self.default_account = w3.eth.accounts[1] + + pass def list(self): + subscribers = self.contract.call( + {'from': self.default_account} + ).getSubscribers() pass def register(self, data_source_name): + test = self.contract.transact( + {'from': self.default_account} + ).subscribe(0) pass def ingest(self, data_source_name, data_frequency, start, end): pass def clean(self, data_source_name): - pass \ No newline at end of file + pass diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 54a14d1a..30401e5b 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 1); + require(dataSourceId >= 0 && dataSourceId <= 5); subscribers[dataSourceId] = msg.sender; diff --git a/tests/marketplace/__init__.py b/tests/marketplace/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py new file mode 100644 index 00000000..e03efc43 --- /dev/null +++ b/tests/marketplace/test_marketplace.py @@ -0,0 +1,4 @@ +from catalyst.marketplace.marketplace import Marketplace + +marketplace = Marketplace() +pass \ No newline at end of file From e0c8178932a17632ca8ef7fcf0363b7f9436341a Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 22:42:09 -0500 Subject: [PATCH 004/117] BLD: improved smart contract and catalyst commands --- catalyst/__main__.py | 4 +- catalyst/constants.py | 2 + catalyst/marketplace/marketplace.py | 95 +++++++++++++++++++++++---- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 8 +-- tests/marketplace/test_marketplace.py | 14 +++- 6 files changed, 102 insertions(+), 23 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index d86276ba..5f70b985 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -5,7 +5,7 @@ from functools import wraps import click import logbook import pandas as pd -from catalyst.alt_data.marketplace import Marketplace +from catalyst.marketplace.marketplace import Marketplace from six import text_type from catalyst.data import bundles as bundles_module @@ -578,7 +578,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def ls_data(ctx): click.echo( - 'Listing available alternative data sources' + 'The alternative data sources:' ) marketplace = Marketplace() marketplace.list() diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..6b3fc235 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,4 +15,6 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + AUTO_INGEST = False diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a6c0ff0b..9de33328 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,29 +1,26 @@ import json import os +import pandas as pd from web3 import Web3, HTTPProvider +from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.constants import ROOT_DIR + +REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( - '..', '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' + ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0x345ca3e014aaf5dca488057592ee47305d9b3e10' + '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' ) -# we'll use one of our default accounts to deploy from. every write to the chain requires a -# payment of ethereum called "gas". if we were running an actual test ethereum node locally, -# then we'd have to go on the test net and get some free ethereum to play with. that is beyond -# the scope of this tutorial so we're using a mini local node that has unlimited ethereum and -# the only chain we're using is our own local one - - class Marketplace: def __init__(self): with open(CONTRACT_PATH) as handle: json_interface = json.load(handle) - - w3 = Web3(HTTPProvider('http://localhost:7545')) + w3 = Web3(HTTPProvider(REMOTE_NODE)) self.contract = w3.eth.contract( CONTRACT_ADDRESS, @@ -33,16 +30,86 @@ class Marketplace: pass + def get_data_sources_map(self): + return [ + dict( + name='Marketcap', + desc='The marketcap value in USD.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='GitHub', + desc='The rate of development activity on GitHub.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + dict( + name='Influencers', + desc='Tweets and related sentiments by selected influencers.', + start_date=pd.to_datetime('2017-01-01'), + end_date=pd.to_datetime('2018-01-15'), + ), + ] + def list(self): subscribers = self.contract.call( {'from': self.default_account} ).getSubscribers() + + subscribed = [] + for index, address in enumerate(subscribers): + if address == self.default_account: + subscribed.append(index) + + data_sources = self.get_data_sources_map() + + data = [] + for index, data_source in enumerate(data_sources): + data.append( + dict( + id=index, + subscribed=index in subscribed, + **data_source, + ) + ) + + df = pd.DataFrame(data) + df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) + set_print_settings() + + formatters = dict( + subscribed=lambda s: u'\u2713' if s else '', + ) + print(df.to_string(formatters=formatters)) + pass def register(self, data_source_name): - test = self.contract.transact( - {'from': self.default_account} - ).subscribe(0) + data_sources = self.get_data_sources_map() + index = next( + (index for (index, d) in enumerate(data_sources) if + d['name'].lower() == data_source_name.lower()), + None + ) + if index is None: + raise ValueError( + 'Data source not found.' + ) + + try: + self.contract.transact( + {'from': self.default_account} + ).subscribe(index) + print( + 'Subscribed to data source {} successfully'.format( + data_source_name + ) + ) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + pass def ingest(self, data_source_name, data_frequency, start, end): diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 30401e5b..1cc1cf75 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -5,7 +5,7 @@ contract Marketplace { // Adopting a pet function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 5); + require(dataSourceId >= 0 && dataSourceId <= 15); subscribers[dataSourceId] = msg.sender; diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index ea1c7fb3..5c4f6411 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -9,9 +9,9 @@ contract TestMarketplace { // Testing the adopt() function function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(0); + uint returnedId = marketplace.subscribe(2); - uint expected = 0; + uint expected = 2; Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } @@ -21,7 +21,7 @@ contract TestMarketplace { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(0); + address adopter = marketplace.subscribers(2); Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); } @@ -34,6 +34,6 @@ contract TestMarketplace { // Store adopters in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[0], expected, "Owner of pet ID 0 should be recorded."); + Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); } } \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index e03efc43..1faf0648 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,4 +1,14 @@ from catalyst.marketplace.marketplace import Marketplace +from catalyst.testing.fixtures import WithLogger, ZiplineTestCase -marketplace = Marketplace() -pass \ No newline at end of file + +class TestMarketplace(WithLogger, ZiplineTestCase): + def test_list(self): + marketplace = Marketplace() + marketplace.list() + pass + + def test_register(self): + marketplace = Marketplace() + marketplace.register('GitHub') + pass From 1e4af12e3daa4222a93338f0feed67f136c8aae6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 13:31:08 -0500 Subject: [PATCH 005/117] BLD: improved smart contract and catalyst commands --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 9de33328..e8288037 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -12,7 +12,7 @@ CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xd8672a4a1bf37d36bef74e36edb4f17845e76f4e' + '0xaa862ddac09f6736a61e1124040fd883a6533c19' ) From 354e422be84d50bef799453fed9020c9dd0a8416 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 21:54:09 -0500 Subject: [PATCH 006/117] BLD: working on ingesting alt data sources --- catalyst/marketplace/marketplace.py | 7 ++- catalyst/marketplace/utils/__init__.py | 0 catalyst/marketplace/utils/paths.py | 87 ++++++++++++++++++++++++++ tests/marketplace/test_marketplace.py | 10 +++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 catalyst/marketplace/utils/__init__.py create mode 100644 catalyst/marketplace/utils/paths.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e8288037..e22d3f49 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,13 +6,14 @@ from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR +from catalyst.marketplace.utils.paths import get_temp_bundles_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xaa862ddac09f6736a61e1124040fd883a6533c19' + '0xe2b6cf3863240892d59664d209a28289a73ef644' ) @@ -112,7 +113,9 @@ class Marketplace: pass - def ingest(self, data_source_name, data_frequency, start, end): + def ingest(self, data_source_name, data_frequency=None, start=None, + end=None): + temp_folder = get_temp_bundles_folder(data_source_name) pass def clean(self, data_source_name): diff --git a/catalyst/marketplace/utils/__init__.py b/catalyst/marketplace/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/paths.py new file mode 100644 index 00000000..008ed2b7 --- /dev/null +++ b/catalyst/marketplace/utils/paths.py @@ -0,0 +1,87 @@ +import os +import tarfile + +from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.paths import data_root, ensure_directory + + +def get_data_source_folder(data_source_name, environ=None): + """ + The root path of an data_source folder. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + data_source_folder = os.path.join(root, 'marketplace', data_source_name) + ensure_directory(data_source_folder) + + return data_source_folder + + +def get_temp_bundles_folder(data_source_name, environ=None): + """ + The temp folder for bundle downloads by algo name. + + Parameters + ---------- + data_source_name: str + environ: + + Returns + ------- + str + + """ + data_source_folder = get_data_source_folder(data_source_name, environ) + + temp_bundles = os.path.join(data_source_folder, 'temp_bundles') + ensure_directory(temp_bundles) + + return temp_bundles + + +def get_data_source(data_source_name, period): + """ + Download and extract a bcolz bundle. + + Parameters + ---------- + exchange_name: str + symbol: str + data_frequency: str + period: str + + Returns + ------- + str + Filename: bitfinex-daily-neo_eth-2017-10.tar.gz + + """ + root = get_temp_bundles_folder(data_source_name) + name = '{data_source}-{period}'.format( + data_source=data_source_name, + period=period, + ) + path = os.path.join(root, name) + + if not os.path.isdir(path): + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) + + return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 1faf0648..3feeb9a6 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -1,5 +1,6 @@ from catalyst.marketplace.marketplace import Marketplace from catalyst.testing.fixtures import WithLogger, ZiplineTestCase +import pandas as pd class TestMarketplace(WithLogger, ZiplineTestCase): @@ -12,3 +13,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.register('GitHub') pass + + def test_ingest(self): + marketplace = Marketplace() + marketplace.ingest( + data_source_name='GitHub', + data_frequency='finest', + start=pd.Timestamp.utcnow(), + ) + pass From 50ded59c3ce1a201080a5c8281f18ac36291d4a3 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:01:00 -0500 Subject: [PATCH 007/117] DOC: documented the marketplace structure --- marketplace/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 marketplace/README.md diff --git a/marketplace/README.md b/marketplace/README.md new file mode 100644 index 00000000..599d8112 --- /dev/null +++ b/marketplace/README.md @@ -0,0 +1,18 @@ +This module contains smart contracts for the data marketplace. +It was generated with Truffle. Truffle can be used to compile and test +the smart contracts. It's being tested on a test blockhain using Ganache. + +Steps to test: +* Download an run Ganache +* Ensure that your Ganache endpoint matches `truffle.js` +* In this `marketplace` folder: + * `truffle compile` + * `truffle test` + * `truffle migrate --reset` + * `truffle console` + * From the console: `Marketplace.deployed()` +* The deployed method displays info about the deployed smart contract +including its address. Copy/paste the address into +`catalyst/marketplace/marketplace.py`. +* Run the catalyst marketplace unit tests: +`tests/marketplace/test_marketplace.py` \ No newline at end of file From 9361570895868f9b71b39da56af733073625da07 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:02 -0500 Subject: [PATCH 008/117] DOC: improved code comments --- marketplace/contracts/Marketplace.sol | 2 +- marketplace/test/TestMarketplace.sol | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol index 1cc1cf75..115b6110 100644 --- a/marketplace/contracts/Marketplace.sol +++ b/marketplace/contracts/Marketplace.sol @@ -3,7 +3,7 @@ pragma solidity ^0.4.17; contract Marketplace { address[16] public subscribers; - // Adopting a pet + // Subscribing to a data source function subscribe(uint dataSourceId) public returns (uint) { require(dataSourceId >= 0 && dataSourceId <= 15); diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 5c4f6411..77a03aea 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -7,7 +7,7 @@ import "../contracts/Marketplace.sol"; contract TestMarketplace { Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - // Testing the adopt() function + // Testing the subscribe() function function testUserCanSubscribe() public { uint returnedId = marketplace.subscribe(2); @@ -16,24 +16,24 @@ contract TestMarketplace { Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); } - // Testing retrieval of a single pet's owner + // Testing retrieval of a single subscriber function testGetSubscriberAddressByDataSourceId() public { // Expected owner is this contract address expected = this; - address adopter = marketplace.subscribers(2); + address subscriber = marketplace.subscribers(2); - Assert.equal(adopter, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); } - // Testing retrieval of all pet owners + // Testing retrieval of all subscribers function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected owner is this contract + // Expected subscriber is this contract address expected = this; - // Store adopters in memory rather than contract's storage + // Store subscribers in memory rather than contract's storage address[16] memory subscribers = marketplace.getSubscribers(); - Assert.equal(subscribers[2], expected, "Owner of pet ID 2 should be recorded."); + Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); } } \ No newline at end of file From 3ea00e783c6d931d1a906cf729ceb7a9f0582689 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 14:08:52 -0500 Subject: [PATCH 009/117] DOC: improved code comments --- marketplace/test/TestMarketplace.sol | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol index 77a03aea..f169d728 100644 --- a/marketplace/test/TestMarketplace.sol +++ b/marketplace/test/TestMarketplace.sol @@ -13,7 +13,7 @@ contract TestMarketplace { uint expected = 2; - Assert.equal(returnedId, expected, "Adoption of pet ID 8 should be recorded."); + Assert.equal(returnedId, expected, "Data source 2 should be recorded."); } // Testing retrieval of a single subscriber @@ -23,7 +23,7 @@ contract TestMarketplace { address subscriber = marketplace.subscribers(2); - Assert.equal(subscriber, expected, "Owner of data source ID 0 should be recorded."); + Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); } // Testing retrieval of all subscribers From b1d63c9a5b7f34cee8f24774701aaa9aebd88f10 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 11 Jan 2018 12:15:49 -0700 Subject: [PATCH 010/117] Update README.md --- marketplace/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/marketplace/README.md b/marketplace/README.md index 599d8112..61f63842 100644 --- a/marketplace/README.md +++ b/marketplace/README.md @@ -3,9 +3,9 @@ It was generated with Truffle. Truffle can be used to compile and test the smart contracts. It's being tested on a test blockhain using Ganache. Steps to test: -* Download an run Ganache +* Download and run Ganache * Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder: +* In this `marketplace` folder, run the following commands: * `truffle compile` * `truffle test` * `truffle migrate --reset` @@ -15,4 +15,4 @@ Steps to test: including its address. Copy/paste the address into `catalyst/marketplace/marketplace.py`. * Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` \ No newline at end of file +`tests/marketplace/test_marketplace.py` From 88aa7154db5b57cef9b6fe8d27571bd68e5761b4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 11 Jan 2018 20:04:53 -0500 Subject: [PATCH 011/117] BLD: ingesting and cleaning marketplace data sources --- catalyst/exchange/utils/bundle_utils.py | 3 ++ catalyst/marketplace/marketplace.py | 35 ++++++++++++++-- catalyst/marketplace/utils/bundle_utils.py | 36 ++++++++++++++++ .../utils/{paths.py => path_utils.py} | 42 ++++++++++++++----- tests/marketplace/test_marketplace.py | 3 +- 5 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 catalyst/marketplace/utils/bundle_utils.py rename catalyst/marketplace/utils/{paths.py => path_utils.py} (61%) diff --git a/catalyst/exchange/utils/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py index e9207511..992cabbb 100644 --- a/catalyst/exchange/utils/bundle_utils.py +++ b/catalyst/exchange/utils/bundle_utils.py @@ -1,6 +1,8 @@ import calendar import os import tarfile + +import bcolz from datetime import timedelta, datetime, date import numpy as np @@ -355,3 +357,4 @@ def get_assets(exchange, include_symbols, exclude_symbols): else: return all_assets + diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e22d3f49..c71cef1c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,12 +1,17 @@ import json import os + +import bcolz import pandas as pd +import shutil from web3 import Web3, HTTPProvider from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.constants import ROOT_DIR -from catalyst.marketplace.utils.paths import get_temp_bundles_folder +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ + get_data_source, get_bundle_folder, get_data_source_folder REMOTE_NODE = 'http://localhost:7545' CONTRACT_PATH = os.path.join( @@ -114,9 +119,31 @@ class Marketplace: pass def ingest(self, data_source_name, data_frequency=None, start=None, - end=None): - temp_folder = get_temp_bundles_folder(data_source_name) + end=None, force_download=False): + data_source_name = data_source_name.lower() + + period = start.strftime('%Y-%m-%d') + tmp_folder = get_data_source(data_source_name, period, force_download) + + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_folder, bundle_folder) + pass - def clean(self, data_source_name): + def clean(self, data_source_name, data_frequency=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + folder = get_data_source_folder(data_source_name) + + else: + forlder = get_bundle_folder(data_source_name, data_frequency) + + shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py new file mode 100644 index 00000000..916722da --- /dev/null +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -0,0 +1,36 @@ +import bcolz +import os +import pandas as pd +import shutil + + +def merge_bundles(zsource, ztarget): + """ + Merge + Parameters + ---------- + zsource + ztarget + + Returns + ------- + + """ + # TODO: find a way to do this iteratively instead of in-memory + df_source = zsource.todataframe() + df_source.set_index('last_updated', drop=False, inplace=True) + df_target = ztarget.todataframe() + df_target.set_index('last_updated', drop=False, inplace=True) + + df = df_target.merge( + right=df_source, + how='right', + ) # type: pd.DataFrame + + dirname = os.path.basename(ztarget.rootdir) + bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname)) + os.rename(ztarget.rootdir, bak_dir) + + z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir) + shutil.rmtree(bak_dir) + return z diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/path_utils.py similarity index 61% rename from catalyst/marketplace/utils/paths.py rename to catalyst/marketplace/utils/path_utils.py index 008ed2b7..6d301880 100644 --- a/catalyst/marketplace/utils/paths.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,6 +1,8 @@ import os import tarfile +import shutil + from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory @@ -29,6 +31,17 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +def get_bundle_folder(data_source_name, data_frequency, environ=None): + data_source_folder = get_data_source_folder(data_source_name, environ) + + subfolder = data_frequency if data_frequency is not None else 'data' + bundle_folder = os.path.join(data_source_folder, subfolder) + + ensure_directory(bundle_folder) + + return bundle_folder + + def get_temp_bundles_folder(data_source_name, environ=None): """ The temp folder for bundle downloads by algo name. @@ -51,7 +64,7 @@ def get_temp_bundles_folder(data_source_name, environ=None): return temp_bundles -def get_data_source(data_source_name, period): +def get_data_source(data_source_name, period, force_download=False): """ Download and extract a bcolz bundle. @@ -65,23 +78,30 @@ def get_data_source(data_source_name, period): Returns ------- str - Filename: bitfinex-daily-neo_eth-2017-10.tar.gz """ root = get_temp_bundles_folder(data_source_name) - name = '{data_source}-{period}'.format( + name = '{data_source}_{period}'.format( data_source=data_source_name, period=period, ) path = os.path.join(root, name) - if not os.path.isdir(path): - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) + if os.path.isdir(path): + if force_download: + shutil.rmtree(path) + + else: + return path + + ensure_directory(path) + + url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( + data_source=data_source_name, + name=name, + ) + bytes = download_without_progress(url) + with tarfile.open('r', fileobj=bytes) as tar: + tar.extractall(path) return path diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 3feeb9a6..68990791 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -17,8 +17,9 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() marketplace.ingest( - data_source_name='GitHub', + data_source_name='Marketcap', data_frequency='finest', start=pd.Timestamp.utcnow(), + force_download=True, ) pass From 70b9fd8e03903a576a3422ad6a8946e1a49cab96 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 12 Jan 2018 00:39:14 -0500 Subject: [PATCH 012/117] BLD: completed data market place first integration and created an algo to test it --- catalyst/api.pyi | 76 +++++- catalyst/constants.py | 5 +- .../examples/mean_reversion_by_marketcap.py | 233 ++++++++++++++++++ catalyst/exchange/exchange_algorithm.py | 13 + catalyst/marketplace/marketplace.py | 60 ++++- catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/path_utils.py | 3 +- tests/marketplace/test_marketplace.py | 11 +- 8 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 catalyst/examples/mean_reversion_by_marketcap.py diff --git a/catalyst/api.pyi b/catalyst/api.pyi index bf104664..6e46aba9 100644 --- a/catalyst/api.pyi +++ b/catalyst/api.pyi @@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None): :func:`catalyst.api.pipeline_output` """ + def batch_market_order(share_counts): """Place a batch market order for multiple assets. @@ -48,6 +49,7 @@ def batch_market_order(share_counts): Index of ids for newly-created orders. """ + def cancel_order(order_param): """Cancel an open order. @@ -57,7 +59,9 @@ def cancel_order(order_param): The order_id or order object to cancel. """ -def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'): + +def continuous_future(root_symbol_str, offset=0, roll='volume', + adjustment='mul'): """Create a specifier for a continuous contract. Parameters @@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul' The continuous future specifier. """ -def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs): + +def fetch_csv(url, pre_func=None, post_func=None, date_column='date', + date_format=None, timezone='UTC', symbol=None, mask=True, + symbol_column=None, special_params_checker=None, **kwargs): """Fetch a csv from a remote url and register the data so that it is queryable from the ``data`` object. @@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma A requests source that will pull data from the url specified. """ + def future_symbol(symbol): """Lookup a futures contract with a given symbol. @@ -144,6 +152,7 @@ def future_symbol(symbol): Raised when no contract named 'symbol' is found. """ + def get_datetime(tz=None): """ Returns the current simulation datetime. @@ -159,6 +168,7 @@ dt : datetime The current simulation datetime converted to ``tz``. """ + def get_environment(field='platform'): """Query the execution environment. @@ -198,6 +208,7 @@ def get_environment(field='platform'): Raised when ``field`` is not a valid option. """ + def get_order(order_id): """Lookup an order based on the order id returned from one of the order functions. @@ -213,10 +224,12 @@ def get_order(order_id): The order object. """ + def history(bar_count, frequency, field, ffill=True): """DEPRECATED: use ``data.history`` instead. """ + def order(asset, amount, limit_price=None, stop_price=None, style=None): """Place an order. @@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ -def order_percent(asset, percent, limit_price=None, stop_price=None, style=None): + +def order_percent(asset, percent, limit_price=None, stop_price=None, + style=None): """Place an order in the specified asset corresponding to the given percent of the current portfolio value. @@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None) :func:`catalyst.api.order_value` """ + def order_target(asset, target, limit_price=None, stop_price=None, style=None): """Place an order to adjust a position to a target number of shares. If the position doesn't already exist, this is equivalent to placing a new @@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_target_value` """ -def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_percent(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target percent of the current portfolio value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is @@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style :func:`catalyst.api.order_target_value` """ -def order_target_value(asset, target, limit_price=None, stop_price=None, style=None): + +def order_target_value(asset, target, limit_price=None, stop_price=None, + style=None): """Place an order to adjust a position to a target value. If the position doesn't already exist, this is equivalent to placing a new order. If the position does exist, this is equivalent to placing an @@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N :func:`catalyst.api.order_target_percent` """ + def order_value(asset, value, limit_price=None, stop_price=None, style=None): """Place an order by desired value rather than desired number of shares. @@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None): :func:`catalyst.api.order_percent` """ + def pipeline_output(name): """Get the results of the pipeline that was attached with the name: ``name``. @@ -514,6 +536,7 @@ def pipeline_output(name): :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline` """ + def record(*args, **kwargs): """Track and record values each day. @@ -529,7 +552,9 @@ def record(*args, **kwargs): :func:`~catalyst.run_algorithm`. """ -def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None): + +def schedule_function(func, date_rule=None, time_rule=None, half_days=True, + calendar=None): """Schedules a function to be called according to some timed rules. Parameters @@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale :class:`catalyst.api.time_rules` """ + def set_asset_restrictions(restrictions, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'): catalyst.finance.asset_restrictions.Restrictions """ + def set_benchmark(benchmark): """Set the benchmark asset. @@ -576,6 +603,7 @@ def set_benchmark(benchmark): automatically reinvested. """ + def set_cancel_policy(cancel_policy): """Sets the order cancellation policy for the simulation. @@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy): :class:`catalyst.api.NeverCancel` """ + def set_commission(commission): """Sets the commission model for the simulation. @@ -605,6 +634,7 @@ def set_commission(commission): :class:`catalyst.finance.commission.PerDollar` """ + def set_do_not_order_list(restricted_list, on_error='fail'): """Set a restriction on which assets can be ordered. @@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'): The assets that cannot be ordered. """ + def set_long_only(on_error='fail'): """Set a rule specifying that this algorithm cannot take short positions. """ + def set_max_leverage(max_leverage): """Set a limit on the maximum leverage of the algorithm. @@ -629,6 +661,7 @@ def set_max_leverage(max_leverage): be no maximum. """ + def set_max_order_count(max_count, on_error='fail'): """Set a limit on the number of orders that can be placed in a single day. @@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'): The maximum number of orders that can be placed on any single day. """ -def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_order_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value of any single order placed for sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. @@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error= The maximum value that can be ordered at one time. """ -def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'): + +def set_max_position_size(asset=None, max_shares=None, max_notional=None, + on_error='fail'): """Set a limit on the number of shares and/or dollar value held for the given sid. Limits are treated as absolute values and are enforced at the time that the algo attempts to place an order for sid. This means @@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err The maximum value to hold for an asset. """ + def set_slippage(slippage): """Set the slippage model for the simulation. @@ -694,6 +732,7 @@ def set_slippage(slippage): :class:`catalyst.finance.slippage.SlippageModel` """ + def set_symbol_lookup_date(dt): """Set the date for which symbols will be resolved to their assets (symbols may map to different firms or underlying assets at @@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt): The new symbol lookup date. """ + def sid(sid): """Lookup an Asset by its unique asset identifier. @@ -724,6 +764,7 @@ def sid(sid): When a requested ``sid`` does not map to any asset. """ + def symbol(symbol_str): """Lookup an Equity by its ticker symbol. @@ -748,6 +789,7 @@ def symbol(symbol_str): :func:`catalyst.api.set_symbol_lookup_date` """ + def symbols(*args): """Lookup multuple Equities as a list. @@ -773,3 +815,21 @@ def symbols(*args): :func:`catalyst.api.set_symbol_lookup_date` """ + +def get_data_source(data_source_name, data_frequency=None, + start=None, end=None): + """ + Lookup a data source from the marketplace + + Parameters + ---------- + self + data_source_name + data_frequency + start + end + + Returns + ------- + + """ diff --git a/catalyst/constants.py b/catalyst/constants.py index 6b3fc235..5668bc83 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -15,6 +15,9 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +try: + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +except Exception as e: + print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py new file mode 100644 index 00000000..6d31dab7 --- /dev/null +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -0,0 +1,233 @@ +# For this example, we're going to write a simple momentum script. When the +# stock goes up quickly, we're going to buy; when it goes down quickly, we're +# going to sell. Hopefully we'll ride the waves. +import os +import tempfile +import time + +import pandas as pd +import talib +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import symbol, record, order_target_percent, get_data_source +from catalyst.exchange.utils.stats_utils import set_print_settings, \ + get_pretty_stats +# We give a name to the algorithm which Catalyst will use to persist its state. +# In this example, Catalyst will create the `.catalyst/data/live_algos` +# directory. If we stop and start the algorithm, Catalyst will resume its +# state using the files included in the folder. +from catalyst.utils.paths import ensure_directory + +NAMESPACE = 'mean_reversion_simple' +log = Logger(NAMESPACE) + + +# To run an algorithm in Catalyst, you need two functions: initialize and +# handle_data. + +def initialize(context): + # This initialize function sets any data or variables that you'll use in + # your algorithm. For instance, you'll want to define the trading pair (or + # trading pairs) you want to backtest. You'll also want to define any + # parameters or values you're going to use. + + # In our example, we're looking at Neo in Ether. + df = get_data_source( + 'marketcap', start=context.datetime + ) # type: pd.DataFrame + + # Keep only the top coins by market cap + df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))] + + set_print_settings() + + df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True) + print('the marketplace data:\n{}'.format(df)) + + # Pick the 5 assets with the lowest market cap for trading + quote_currency = 'eth' + exchange = context.exchanges[next(iter(context.exchanges))] + symbols = [a.symbol for a in exchange.assets + if a.start_date < context.datetime] + context.assets = [] + for currency, price in df['market_cap_usd'].iteritems(): + if len(context.assets) >= 5: + break + + s = '{}_{}'.format(currency.decode('utf-8'), quote_currency) + if s in symbols: + context.assets.append(symbol(s)) + + context.base_price = None + context.current_day = None + + context.RSI_OVERSOLD = 55 + context.RSI_OVERBOUGHT = 60 + context.CANDLE_SIZE = '5T' + + context.start_time = time.time() + + +def handle_data(context, data): + # This handle_data function is where the real work is done. Our data is + # minute-level tick data, and each minute is called a frame. This function + # runs on each frame of the data. + + # We flag the first period of each day. + # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle + # would only execute once. This method works with minute and daily + # frequencies. + today = data.current_dt.floor('1D') + if today != context.current_day: + context.traded_today = dict() + context.current_day = today + + # Preparing dictionaries for asset-level data points + volumes = dict() + rsis = dict() + price_values = dict() + cash = context.portfolio.cash + + for asset in context.assets: + # We're computing the volume-weighted-average-price of the security + # defined above, in the context.assets variable. For this example, + # we're using three bars on the 15 min bars. + + # The frequency attribute determine the bar size. We use this + # convention for the frequency alias: + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + prices = data.history( + asset, + fields='close', + bar_count=50, + frequency=context.CANDLE_SIZE + ) + + # Ta-lib calculates various technical indicator based on price and + # volume arrays. + + # In this example, we are comp + rsi = talib.RSI(prices.values, timeperiod=14) + + # We need a variable for the current price of the security to compare + # to the average. Since we are requesting two fields, data.current() + # returns a DataFrame with + current = data.current(asset, fields=['close', 'volume']) + price = current['close'] + + # If base_price is not set, we use the current value. This is the + # price at the first bar which we reference to calculate price_change. + # if asset not in context.base_price: + # context.base_price[asset] = price + # + # base_price = context.base_price[asset] + # price_change = (price - base_price) / base_price + + # Tracking the relevant data + volumes[asset] = current['volume'] + rsis[asset] = rsi[-1] + price_values[asset] = price + # price_changes[asset] = price_change + + # We are trying to avoid over-trading by limiting our trades to + # one per day. + if asset in context.traded_today: + continue + + # Exit if we cannot trade + if not data.can_trade(asset): + continue + + # Another powerful built-in feature of the Catalyst backtester is the + # portfolio object. The portfolio object tracks your positions, cash, + # cost basis of specific holdings, and more. In this line, we + # calculate how long or short our position is at this minute. + pos_amount = context.portfolio.positions[asset].amount + + if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: + log.info( + '{}: buying - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + # Set a style for limit orders, + limit_price = price * 1.005 + target = 1.0 / len(context.assets) + order_target_percent( + asset, target, limit_price=limit_price + ) + context.traded_today[asset] = True + + elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: + log.info( + '{}: selling - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + limit_price = price * 0.995 + order_target_percent( + asset, 0, limit_price=limit_price + ) + context.traded_today[asset] = True + + # Now that we've collected all current data for this frame, we use + # the record() method to save it. This data will be available as + # a parameter of the analyze() function for further analysis. + record( + current_price=price_values, + volume=volumes, + rsi=rsis, + cash=cash, + ) + + +def analyze(context=None, perf=None): + stats = get_pretty_stats(perf) + print('the algo stats:\n{}'.format(stats)) + pass + + +if __name__ == '__main__': + # The execution mode: backtest or live + live = False + + if live: + run_algorithm( + capital_base=0.1, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + live=True, + algo_namespace=NAMESPACE, + base_currency='btc', + live_graph=False, + simulate_orders=False, + stats_output=None, + ) + + else: + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', NAMESPACE + ) + ensure_directory(folder) + + timestr = time.strftime('%Y%m%d-%H%M%S') + out = os.path.join(folder, '{}.p'.format(timestr)) + # catalyst run -f catalyst/examples/mean_reversion_simple.py \ + # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ + # --data-frequency minute --capital-base 10000 + run_algorithm( + capital_base=100, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace=NAMESPACE, + base_currency='eth', + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-10-15', utc=True), + ) + log.info('saved perf stats: {}'.format(out)) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 782b7ec8..3ba1b5ea 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -43,6 +43,7 @@ from catalyst.finance.execution import MarketOrder from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator +from catalyst.marketplace.marketplace import Marketplace from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest @@ -92,6 +93,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): attempts=self.attempts, ) + self._marketplace = None + @staticmethod def __convert_order_params_for_blotter(limit_price, stop_price, style): """ @@ -167,6 +170,16 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): """ return round_nearest(amount, asset.min_trade_size) + @api_method + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + if self._marketplace is None: + self._marketplace = Marketplace() + + return self._marketplace.get_data_source( + data_source_name, data_frequency, start, end, + ) + @api_method @preprocess(symbol_str=ensure_upper_case) def symbol(self, symbol_str, exchange_name=None): diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index c71cef1c..041fb004 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,19 +1,22 @@ import json import os - -import bcolz -import pandas as pd import shutil +import bcolz +import logbook +import pandas as pd +import six from web3 import Web3, HTTPProvider +from catalyst.constants import ROOT_DIR, LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.constants import ROOT_DIR from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_temp_bundles_folder, \ - get_data_source, get_bundle_folder, get_data_source_folder +from catalyst.marketplace.utils.path_utils import get_data_source, \ + get_bundle_folder, get_data_source_folder +# TODO: host our own node on aws? REMOTE_NODE = 'http://localhost:7545' +# TODO: read from GitHub CONTRACT_PATH = os.path.join( ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' ) @@ -21,6 +24,8 @@ CONTRACT_ADDRESS = Web3.toChecksumAddress( '0xe2b6cf3863240892d59664d209a28289a73ef644' ) +log = logbook.Logger('Marketplace', level=LOG_LEVEL) + class Marketplace: def __init__(self): @@ -43,21 +48,33 @@ class Marketplace: desc='The marketcap value in USD.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily'], ), dict( name='GitHub', desc='The rate of development activity on GitHub.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour'], ), dict( name='Influencers', desc='Tweets and related sentiments by selected influencers.', start_date=pd.to_datetime('2017-01-01'), end_date=pd.to_datetime('2018-01-15'), + data_frequencies=['daily', 'hour', 'minute'], ), ] + def get_data_source_def(self, data_source_name): + data_source_name = data_source_name.lower() + dsm = self.get_data_sources_map() + + ds = six.next( + (d for d in dsm if d['name'].lower() == data_source_name), None + ) + return ds + def list(self): subscribers = self.contract.call( {'from': self.default_account} @@ -136,6 +153,35 @@ class Marketplace: pass + def get_data_source(self, data_source_name, data_frequency=None, + start=None, end=None): + data_source_name = data_source_name.lower() + + if data_frequency is None: + ds_def = self.get_data_source_def(data_source_name) + freqs = ds_def['data_frequencies'] + data_frequency = freqs[0] + + if len(freqs) > 1: + log.warn( + 'no data frequencies specified for data source {}, ' + 'selected the first one by default: {}'.format( + data_source_name, data_frequency + ) + ) + + # TODO: filter ctable by start and end date + bundle_folder = get_bundle_folder(data_source_name, data_frequency) + z = bcolz.ctable(rootdir=bundle_folder, mode='r') + + df = z.todataframe() # type: pd.DataFrame + df.set_index(['date', 'symbol'], drop=False, inplace=True) + + if start and end is None: + df = df.xs(start, level=0) + + return df + def clean(self, data_source_name, data_frequency=None): data_source_name = data_source_name.lower() @@ -143,7 +189,7 @@ class Marketplace: folder = get_data_source_folder(data_source_name) else: - forlder = get_bundle_folder(data_source_name, data_frequency) + folder = get_bundle_folder(data_source_name, data_frequency) shutil.rmtree(folder) pass diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 916722da..532f46ce 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -18,9 +18,9 @@ def merge_bundles(zsource, ztarget): """ # TODO: find a way to do this iteratively instead of in-memory df_source = zsource.todataframe() - df_source.set_index('last_updated', drop=False, inplace=True) + df_source.set_index('date', drop=False, inplace=True) df_target = ztarget.todataframe() - df_target.set_index('last_updated', drop=False, inplace=True) + df_target.set_index('date', drop=False, inplace=True) df = df_target.merge( right=df_source, diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 6d301880..890eaa20 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -34,8 +34,7 @@ def get_data_source_folder(data_source_name, environ=None): def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) - subfolder = data_frequency if data_frequency is not None else 'data' - bundle_folder = os.path.join(data_source_folder, subfolder) + bundle_folder = os.path.join(data_source_folder, data_frequency) ensure_directory(bundle_folder) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 68990791..9292b3a0 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,10 +16,17 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() + ds_def = marketplace.get_data_source_def('Marketcap') + marketplace.ingest( data_source_name='Marketcap', - data_frequency='finest', - start=pd.Timestamp.utcnow(), + data_frequency=ds_def['data_frequencies'][0], + start=pd.to_datetime('2017-10-01'), force_download=True, ) pass + + def test_clean(self): + marketplace = Marketplace() + marketplace.clean('marketcap') + pass From 55521b87b919014d13e2ce23690a2bf5ed53c076 Mon Sep 17 00:00:00 2001 From: Guy Thouret Date: Mon, 22 Jan 2018 11:11:37 +0000 Subject: [PATCH 013/117] Allow jupyter to run as root inside Docker image Signed-off-by: Guy Thouret --- etc/docker_cmd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/docker_cmd.sh b/etc/docker_cmd.sh index 30308a61..b4c9f348 100755 --- a/etc/docker_cmd.sh +++ b/etc/docker_cmd.sh @@ -16,4 +16,4 @@ fi jupyter notebook -y --no-browser --notebook-dir=${PROJECT_DIR} \ --certfile=${SSL_CERT_PEM} --keyfile=${SSL_CERT_KEY} --ip='*' \ - --config=${CONFIG_PATH} + --config=${CONFIG_PATH} --allow-root From 76e183b5f7fc6fb6548ff19da4245a701564a90b Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 12:48:42 -0700 Subject: [PATCH 014/117] BLD: contract address+abi on testnet --- catalyst/marketplace/contract_abi.json | 1 + catalyst/marketplace/contract_address.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 catalyst/marketplace/contract_abi.json create mode 100644 catalyst/marketplace/contract_address.txt diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_abi.json new file mode 100644 index 00000000..a3dda75f --- /dev/null +++ b/catalyst/marketplace/contract_abi.json @@ -0,0 +1 @@ +[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_address.txt new file mode 100644 index 00000000..ec7bf029 --- /dev/null +++ b/catalyst/marketplace/contract_address.txt @@ -0,0 +1 @@ +0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file From 04fed4140c9e3e37ca1c80b38c0a92f0665bfe12 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 24 Jan 2018 14:12:59 -0700 Subject: [PATCH 015/117] BLD: sourcing contract address+abi from github --- catalyst/marketplace/marketplace.py | 50 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 041fb004..a28d1c9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,7 @@ import json import os import shutil +import urllib import bcolz import logbook @@ -15,31 +16,44 @@ from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder # TODO: host our own node on aws? -REMOTE_NODE = 'http://localhost:7545' -# TODO: read from GitHub -CONTRACT_PATH = os.path.join( - ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json' -) -CONTRACT_ADDRESS = Web3.toChecksumAddress( - '0xe2b6cf3863240892d59664d209a28289a73ef644' -) +# TODO: switch to mainnet +REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_address.txt' + +CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - with open(CONTRACT_PATH) as handle: - json_interface = json.load(handle) - w3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, - abi=json_interface['abi'], - ) # Type: Contract - self.default_account = w3.eth.accounts[1] + contract_url = urllib.urlopen(CONTRACT_PATH) + CONTRACT_ADDRESS = Web3.toChecksumAddress( + contract_url.readline().strip()) - pass + abi_url = urllib.urlopen(CONTRACT_ABI) + abi = json.load(abi_url) + + w3 = Web3(HTTPProvider(REMOTE_NODE)) + + self.contract = w3.eth.contract( + CONTRACT_ADDRESS, + abi=abi, + ) # Type: Contract + + # TODO: Set default address correctly from user-provided config + DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) + if DEFAULT_ETH_ADDRESS is None: + raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' + 'environment variable. Quitting...') + + self.default_account = DEFAULT_ETH_ADDRESS + pass def get_data_sources_map(self): return [ @@ -93,7 +107,7 @@ class Marketplace: dict( id=index, subscribed=index in subscribed, - **data_source, + **data_source ) ) From 2ec9aa2ca96c8121b36d53868c5c68529ee7180e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 11:55:25 -0700 Subject: [PATCH 016/117] BLD: CLI implementation for the marketplace --- catalyst/__main__.py | 197 ++++++++++++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 69 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 5f70b985..10c6c6ac 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -574,75 +574,6 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, ) -@main.command(name='ls-data') -@click.pass_context -def ls_data(ctx): - click.echo( - 'The alternative data sources:' - ) - marketplace = Marketplace() - marketplace.list() - click.echo('Done') - - -@main.command(name='register-data') -@click.argument('data_source_name') -@click.pass_context -def register_data(ctx, data_source_name): - click.echo( - 'Registering to data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.register(data_source_name) - click.echo('Done') - - -@main.command(name='ingest-data') -@click.argument('data_source_name') -@click.option( - '-f', - '--data-frequency', - type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), - default='daily', - show_default=True, - help='The data frequency of the desired OHLCV bars.', -) -@click.option( - '-s', - '--start', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The start date of the data range. (default: one year from end date)', -) -@click.option( - '-e', - '--end', - default=None, - type=Date(tz='utc', as_timestamp=True), - help='The end date of the data range. (default: today)', -) -@click.pass_context -def ingest_data(ctx, data_source_name, data_frequency, start, end): - click.echo( - 'Ingesting data: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.ingest(data_source_name, data_frequency, start, end) - click.echo('Done') - - -@main.command(name='clean-data') -@click.argument('data_source_name') -@click.pass_context -def clean_data(ctx, data_source_name): - click.echo( - 'Cleaning data source: {}'.format(data_source_name) - ) - marketplace = Marketplace() - marketplace.clean(data_source_name) - click.echo('Done') - - @main.command(name='clean-algo') @click.option( '-n', @@ -810,5 +741,133 @@ def bundles(): click.echo("%s %s" % (bundle, timestamp)) +@main.group() +@click.pass_context +def marketplace(ctx): + pass + + +@marketplace.command() +@click.pass_context +def ls(ctx): + click.echo('Listing of available data sources on the marketplace:') + marketplace = Marketplace() + marketplace.list() + click.echo('Functionality not yet implemented.') + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def subscribe(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + marketplace = Marketplace() + marketplace.subscribe(dataset) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.option( + '-f', + '--data-frequency', + type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}), + default='daily', + show_default=True, + help='The data frequency of the desired OHLCV bars.', +) +@click.option( + '-s', + '--start', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The start date of the data range. (default: one year from end date)', +) +@click.option( + '-e', + '--end', + default=None, + type=Date(tz='utc', as_timestamp=True), + help='The end date of the data range. (default: today)', +) +@click.pass_context +def ingest(ctx, dataset, data_frequency, start, end): + if dataset is None: + ctx.fail("must specify a dataset to clean with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Ingesting data: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.ingest(dataset, data_frequency, start, end) + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the dataset to ingest from the Data Marketplace.', +) +@click.pass_context +def clean(ctx, dataset): + if dataset is None: + ctx.fail("must specify a dataset to ingest with '--dataset'\n" + "List available dataset on the marketplace with " + "'catalyst marketplace ls'") + click.echo( + 'Cleaning data source: {}'.format(dataset) + ) + marketplace = Marketplace() + marketplace.clean(dataset) + click.echo('Done') + + +@marketplace.command() +@click.pass_context +def register(ctx): + marketplace = Marketplace() + marketplace.register() + + +@marketplace.command() +@click.option( + '--dataset', + default=None, + help='The name of the Marketplace dataset to publish data for.', +) +@click.option( + '--datadir', + default=None, + help='The folder that contains the CSV data files to publish.', +) +@click.option( + '--watch/--no-watch', + is_flag=True, + default=False, + help='Whether to watch the datadir for live data.', +) +@click.pass_context +def publish(ctx, dataset, datadir, watch): + marketplace = Marketplace() + if dataset is None: + ctx.fail("must specify a dataset to publish data for " + " with '--dataset'\n") + if datadir is None: + ctx.fail("must specify a datadir where to find the files to publish " + " with '--datadir'\n") + marketplace.publish(dataset, datadir, watch) + + if __name__ == '__main__': main() From 940f625ec1fe37995193e30a783844cb16f3db11 Mon Sep 17 00:00:00 2001 From: Avishai Weingarten <33716232+AvishaiW@users.noreply.github.com> Date: Thu, 25 Jan 2018 16:02:44 +0200 Subject: [PATCH 017/117] fix for click.echo added sys.stdout to click.echo to prevent errors on jupyter --- catalyst/__main__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 10c6c6ac..411f712f 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -3,6 +3,7 @@ import os from functools import wraps import click +import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -258,7 +259,7 @@ def run(ctx, if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") - click.echo('Running in backtesting mode.') + click.echo('Running in backtesting mode.', sys.stdout) perf = _run( initialize=None, @@ -290,7 +291,7 @@ def run(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -443,10 +444,10 @@ def live(ctx, ctx.fail("must specify a capital base with '--capital-base'") if simulate_orders: - click.echo('Running in paper trading mode.') + click.echo('Running in paper trading mode.', sys.stdout) else: - click.echo('Running in live trading mode.') + click.echo('Running in live trading mode.', sys.stdout) perf = _run( initialize=None, @@ -478,7 +479,7 @@ def live(ctx, ) if output == '-': - click.echo(str(perf)) + click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -560,7 +561,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -583,10 +584,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, @click.pass_context def clean_algo(ctx, algo_namespace): click.echo( - 'Cleaning algo state: {}'.format(algo_namespace) + 'Cleaning algo state: {}'.format(algo_namespace), + sys.stdout ) delete_algo_folder(algo_namespace) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command(name='clean-exchange') @@ -613,11 +615,11 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) - click.echo('Done') + click.echo('Done', sys.stdout) @main.command() @@ -738,7 +740,7 @@ def bundles(): # because there were no entries, print a single message indicating that # no ingestions have yet been made. for timestamp in ingestions or [""]: - click.echo("%s %s" % (bundle, timestamp)) + click.echo("%s %s" % (bundle, timestamp), sys.stdout) @main.group() From ff989ba52405e344c5365d9b6430afeb4490f425 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 25 Jan 2018 22:39:54 -0700 Subject: [PATCH 018/117] BLD: register dataset in marketplace --- catalyst/__main__.py | 17 ++- catalyst/marketplace/marketplace.py | 140 +++++++++++++++++---- catalyst/marketplace/marketplace_errors.py | 24 ++++ catalyst/marketplace/utils/path_utils.py | 56 +++++++++ 4 files changed, 205 insertions(+), 32 deletions(-) create mode 100644 catalyst/marketplace/marketplace_errors.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 411f712f..3833f3e4 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,9 +1,9 @@ import errno import os +import sys from functools import wraps import click -import sys import logbook import pandas as pd from catalyst.marketplace.marketplace import Marketplace @@ -752,10 +752,11 @@ def marketplace(ctx): @marketplace.command() @click.pass_context def ls(ctx): - click.echo('Listing of available data sources on the marketplace:') + click.echo('Listing of available data sources on the marketplace:', + sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.') + click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() @@ -808,9 +809,7 @@ def ingest(ctx, dataset, data_frequency, start, end): ctx.fail("must specify a dataset to clean with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Ingesting data: {}'.format(dataset) - ) + click.echo('Ingesting data: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.ingest(dataset, data_frequency, start, end) @@ -827,12 +826,10 @@ def clean(ctx, dataset): ctx.fail("must specify a dataset to ingest with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") - click.echo( - 'Cleaning data source: {}'.format(dataset) - ) + click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.clean(dataset) - click.echo('Done') + click.echo('Done', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a28d1c9f..4b1cfe8b 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,23 +1,31 @@ import json import os import shutil -import urllib +import sys +import binascii import bcolz import logbook import pandas as pd import six from web3 import Web3, HTTPProvider -from catalyst.constants import ROOT_DIR, LOG_LEVEL +from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings +from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder + get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr + +if sys.version_info.major < 3: + import urllib +else: + import urllib.request as urllib # TODO: host our own node on aws? # TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' +REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ @@ -32,28 +40,28 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): - contract_url = urllib.urlopen(CONTRACT_PATH) - CONTRACT_ADDRESS = Web3.toChecksumAddress( + self.addresses = get_user_pubaddr() + + if self.addresses[0]['pubAddr'] == '': + raise MarketplacePubAddressEmpty( + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) + self.default_account = self.addresses[0]['pubAddr'] + + contract_url = urllib.urlopen(CONTRACT_PATH) + self.contract_address = Web3.toChecksumAddress( contract_url.readline().strip()) abi_url = urllib.urlopen(CONTRACT_ABI) abi = json.load(abi_url) - w3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = w3.eth.contract( - CONTRACT_ADDRESS, + self.contract = self.web3.eth.contract( + self.contract_address, abi=abi, - ) # Type: Contract - - # TODO: Set default address correctly from user-provided config - DEFAULT_ETH_ADDRESS = os.environ.get('DEFAULT_ETH_ADDRESS', None) - if DEFAULT_ETH_ADDRESS is None: - raise ValueError('DEFAULT_ETH_ADDRESS is not set. Export as an ' - 'environment variable. Quitting...') - - self.default_account = DEFAULT_ETH_ADDRESS - pass + ) def get_data_sources_map(self): return [ @@ -122,11 +130,11 @@ class Marketplace: pass - def register(self, data_source_name): + def subscribe(self, dataset): data_sources = self.get_data_sources_map() index = next( (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == data_source_name.lower()), + d['name'].lower() == dataset.lower()), None ) if index is None: @@ -140,7 +148,7 @@ class Marketplace: ).subscribe(index) print( 'Subscribed to data source {} successfully'.format( - data_source_name + dataset ) ) @@ -207,3 +215,91 @@ class Marketplace: shutil.rmtree(folder) pass + + def register(self): + dataset = input('Enter the name of the dataset to register: ') + price = int(input('Enter the price for a monthly subscription to ' + 'this dataset in ENG: ')) + + # while True: + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): + # print("Not a valid frequency.") + # else: + # break + + # while True: + # reg_pub = input('Can data be published every hour at a regular ' + # 'time? [default: Y]: ') or 'y' + # if reg_pub.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if reg_pub.lower() == 'y': + # reg_pub = True + # else: + # reg_pub = False + # break + + # while True: + # hist = input('Does it include historical data? ' + # '[default: Y]') or 'y' + # if hist.lower() not in ('y', 'n'): + # print("Please answer Y or N.") + # else: + # if hist.lower() == 'y': + # hist = True + # else: + # hist = False + # break + + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose the your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + tx = self.contract.functions.register( + binascii.hexlify(dataset.encode('utf-8')), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') + + tx_hash = '0x{}'.format(binascii.hexlify( + self.web3.eth.sendRawTransaction(signed_tx) + ).decode('utf-8')) + + print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py new file mode 100644 index 00000000..04fca091 --- /dev/null +++ b/catalyst/marketplace/marketplace_errors.py @@ -0,0 +1,24 @@ +import sys +import traceback + +from catalyst.errors import ZiplineError + + +def silent_except_hook(exctype, excvalue, exctraceback): + if exctype in [MarketplacePubAddressEmpty]: + fn = traceback.extract_tb(exctraceback)[-1][0] + ln = traceback.extract_tb(exctraceback)[-1][1] + print("Error traceback: {1} (line {2})\n" + "{0.__name__}: {3}".format(exctype, fn, ln, excvalue)) + else: + sys.__excepthook__(exctype, excvalue, exctraceback) + + +sys.excepthook = silent_except_hook + + +class MarketplacePubAddressEmpty(ZiplineError): + msg = ( + 'Please enter your public address to use in the Data Marketplace ' + 'in the following file: {filename}' + ).strip() diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 890eaa20..2cee9303 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -1,4 +1,5 @@ import os +import json import tarfile import shutil @@ -7,6 +8,29 @@ from catalyst.data.bundles.core import download_without_progress from catalyst.utils.paths import data_root, ensure_directory +def get_marketplace_folder(environ=None): + """ + The root path of the marketplace folder. + + Parameters + ---------- + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + marketplace_folder = os.path.join(root, 'marketplace') + ensure_directory(marketplace_folder) + + return marketplace_folder + + def get_data_source_folder(data_source_name, environ=None): """ The root path of an data_source folder. @@ -104,3 +128,35 @@ def get_data_source(data_source_name, period, force_download=False): tar.extractall(path) return path + + +def get_user_pubaddr(environ=None): + """ + The de-serialized contend of the user's addresses.json file. + Parameters + ---------- + environ: + + Returns + ------- + Object + + """ + marketplace_folder = get_marketplace_folder(environ) + + filename = os.path.join(marketplace_folder, 'addresses.json') + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + try: + d = data[0]['pubAddr'] + except Exception as e: + return [data, ] + return data + else: + data = dict(pubAddr='', desc='') + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + return data From 68faad40989bd5b37e56eb9d45bca9083feb72fb Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 26 Jan 2018 15:32:53 -0700 Subject: [PATCH 019/117] BLD: publish dataset in marketplace --- catalyst/marketplace/marketplace.py | 84 +++++++++++++++++++--- catalyst/marketplace/marketplace_errors.py | 29 +++++++- catalyst/marketplace/utils/auth_utils.py | 47 ++++++++++++ catalyst/marketplace/utils/bundle_utils.py | 4 +- catalyst/marketplace/utils/eth_utils.py | 33 +++++++++ catalyst/marketplace/utils/path_utils.py | 26 ++++++- 6 files changed, 210 insertions(+), 13 deletions(-) create mode 100644 catalyst/marketplace/utils/auth_utils.py create mode 100644 catalyst/marketplace/utils/eth_utils.py diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4b1cfe8b..af4c4aef 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,22 +1,31 @@ -import json import os -import shutil import sys +import json +import hmac +import glob +import time +import shutil +import hashlib -import binascii import bcolz import logbook import pandas as pd import six +import requests from web3 import Web3, HTTPProvider from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.stats_utils import set_print_settings -from catalyst.marketplace.marketplace_errors import MarketplacePubAddressEmpty +from catalyst.marketplace.marketplace_errors import ( + MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: import urllib @@ -34,6 +43,8 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' +AUTH_SERVER = 'http://localhost:5000' + log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -259,7 +270,7 @@ class Marketplace: self.addresses[i]['pubAddr'], self.addresses[i]['desc']) ) - address_i = int(input('Choose the your address associated with ' + address_i = int(input('Choose your address associated with ' 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( @@ -270,7 +281,7 @@ class Marketplace: break tx = self.contract.functions.register( - binascii.hexlify(dataset.encode('utf-8')), + bytes32(dataset), price, address, ).buildTransaction( @@ -298,8 +309,63 @@ class Marketplace: signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') - tx_hash = '0x{}'.format(binascii.hexlify( - self.web3.eth.sendRawTransaction(signed_tx) - ).decode('utf-8')) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + def publish(self, dataset, datadir, watch): + + datasource = self.contract.functions.getDataSource( + bytes32(dataset)).call() + + if not datasource[4]: + raise MarketplaceDatasetNotFound(dataset=dataset) + + match = next((l for l in self.addresses if + l['pubAddr'] == datasource[0]), None) + + if not match: + raise MarketplaceNoAddressMatch( + dataset=dataset, + address=datasource[0]) + + print('Using address: {} to publish this dataset.'.format( + datasource[0])) + + if 'key' in match: + key = match['key'] + secret = match['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + nonce.encode('utf-8'), + hashlib.sha512).hexdigest() + headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + filenames = glob.glob(os.path.join(datadir, '*.csv')) + + if not filenames: + raise MarketplaceNoCSVFiles(datadir=datadir) + + files = [] + for file in filenames: + files.append(('file', open(file, 'rb'))) + + r = requests.post('{}/publish'.format(AUTH_SERVER), + files=files, + headers=headers) + + if r.status_code != 200: + raise MarketplaceHTTPRequest(request='upload file', + error=r.status_code) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest(request='upload file', + error=r.json()['error']) + + print('Dataset {} published successfully.'.format(dataset)) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 04fca091..3992158d 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -5,7 +5,9 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): - if exctype in [MarketplacePubAddressEmpty]: + if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, + MarketplaceNoAddressMatch, MarketplaceHTTPRequest, + MarketplaceNoCSVFiles]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -22,3 +24,28 @@ class MarketplacePubAddressEmpty(ZiplineError): 'Please enter your public address to use in the Data Marketplace ' 'in the following file: {filename}' ).strip() + + +class MarketplaceDatasetNotFound(ZiplineError): + msg = ( + 'The dataset "{dataset}" is not registered in the Data Marketplace.' + ).strip() + + +class MarketplaceNoAddressMatch(ZiplineError): + msg = ( + 'The address registered with the dataset {dataset}: {address} ' + 'does not match any of your addresses.' + ).strip() + + +class MarketplaceHTTPRequest(ZiplineError): + msg = ( + 'Request to remote server to {request} failed: {error}' + ).strip() + + +class MarketplaceNoCSVFiles(ZiplineError): + msg = ( + 'No CSV files found on {datadir} to upload.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py new file mode 100644 index 00000000..f1193758 --- /dev/null +++ b/catalyst/marketplace/utils/auth_utils.py @@ -0,0 +1,47 @@ +import requests + +from catalyst.marketplace.marketplace_errors import ( + MarketplaceHTTPRequest) +from catalyst.marketplace.utils.path_utils import ( + get_user_pubaddr, save_user_pubaddr) + +AUTH_SERVER = 'http://localhost:5000' + + +def get_key_secret(pubAddr, dataset): + """ + Obtain a new key/secret pair from authentication server + + Parameters + ---------- + pubAddr: str + dataset: str + + Returns + ------- + key: str + secret: str + + """ + session = requests.Session() + response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + headers={ + 'pubAddr': pubAddr, + 'dataset': dataset}) + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + + return match['key'], match['secret'] diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 532f46ce..b58595ac 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -1,8 +1,8 @@ -import bcolz import os -import pandas as pd import shutil +import bcolz + def merge_bundles(zsource, ztarget): """ diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py new file mode 100644 index 00000000..c2be4bd6 --- /dev/null +++ b/catalyst/marketplace/utils/eth_utils.py @@ -0,0 +1,33 @@ +import binascii + + +def bytes32(string): + """ + Convert string to bytes32 data type for smart contract + + Parameters + ---------- + string: str + + Returns + ------- + list + + """ + return binascii.hexlify(string.encode('utf-8')) + + +def b32_str(bytes32): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(bytes32.decode('utf-8')) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 2cee9303..62f8beac 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,6 +133,7 @@ def get_data_source(data_source_name, period, force_download=False): def get_user_pubaddr(environ=None): """ The de-serialized contend of the user's addresses.json file. + Parameters ---------- environ: @@ -143,7 +144,6 @@ def get_user_pubaddr(environ=None): """ marketplace_folder = get_marketplace_folder(environ) - filename = os.path.join(marketplace_folder, 'addresses.json') if os.path.isfile(filename): @@ -160,3 +160,27 @@ def get_user_pubaddr(environ=None): json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) return data + + +def save_user_pubaddr(data, environ=None): + """ + Saves the user's public addresses and their related metadata in + the corresponding addresses.json file. + + Parameters + ---------- + data: dict + + Returns + ------- + True + + """ + marketplace_folder = get_marketplace_folder(environ) + filename = os.path.join(marketplace_folder, 'addresses.json') + + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + + return True From 15aadbb10119294573c6c025a567e6dd4e3a24fc Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:14:15 -0500 Subject: [PATCH 020/117] documentation for PyCharm --- docs/source/beginner-tutorial.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 3bcfc4cf..2ca5d283 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16607,7 +16607,30 @@ NaN +Catalyst using PyCharm +~~~~~~~~~~~~~~~~~~~~~~ +Install +^^^^^^^ +Install PyCharm from their 'website '_ . + +Setup +^^^^^ + +1. Create a new project folder for your scripts or open your existing folder in PyCharm. + +2. Once your project is open, go to File -> Settings -> Project:'NAME_OF_PROJECT' -> Project Interpreter. + Click the gear box next to the project interpreter and select 'add local'. + + Then select 'Conda Environment' -> 'Existing environment'. Click the '...' button and + navigate to your catalyst env located in the Anaconda2 folder to select the 'python.exe' file + (for example: 'C:\Users\user\Anaconda2\envs\catalyst\python.exe'). Select OK, then apply and click OK again. + +3. Next, click on the dropdown menu on the top right of PyCharm and select 'Edit Configurations'. + Select the '+' button. + Set the script Path to the location of your script and make sure the interpreter is correct, then hit ok. + +You should now be able to run your script in PyCharm. Next steps ~~~~~~~~~~ From 929e0a9a016235aa71b86d08b286134064625df9 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:18:16 -0500 Subject: [PATCH 021/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 2ca5d283..7a918478 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -Install PyCharm from their 'website '_ . +Install PyCharm from their ('website ') . Setup ^^^^^ From 1cadaf1ed0936de4489c9a6a7e6657e6bdd76254 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:19:44 -0500 Subject: [PATCH 022/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 7a918478..a77b9d1b 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -Install PyCharm from their ('website ') . +('Install PyCharm from their website '_ .) Setup ^^^^^ From 5417881a721182d160cea5995000190e659b1ca7 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:21:53 -0500 Subject: [PATCH 023/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index a77b9d1b..43084cd3 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -('Install PyCharm from their website '_ .) +Install PyCharm from their 'website'_ . Setup ^^^^^ From 5b77f5eebaa7fe16894fe8eb4cc4ca51ea19b50f Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:23:00 -0500 Subject: [PATCH 024/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 43084cd3..c02febe4 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -Install PyCharm from their 'website'_ . +Install PyCharm from their 'website ' . Setup ^^^^^ From e9a3bcf3e9002203f361eea6edbe0f08bcb991ec Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:23:22 -0500 Subject: [PATCH 025/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index c02febe4..e8e0252e 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -Install PyCharm from their 'website ' . +Install PyCharm from their 'website '__ . Setup ^^^^^ From 426fde40b1b3331ba52905744474f05c4660805b Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:24:21 -0500 Subject: [PATCH 026/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index e8e0252e..82da1900 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16612,7 +16612,7 @@ Catalyst using PyCharm Install ^^^^^^^ -Install PyCharm from their 'website '__ . +Install PyCharm from their `Website `__. Setup ^^^^^ From 4a4e32846b7c3ddc5eca0b7ed4a2d3b265ae41a5 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:27:14 -0500 Subject: [PATCH 027/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 82da1900..0c2a8564 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16624,7 +16624,7 @@ Setup Then select 'Conda Environment' -> 'Existing environment'. Click the '...' button and navigate to your catalyst env located in the Anaconda2 folder to select the 'python.exe' file - (for example: 'C:\Users\user\Anaconda2\envs\catalyst\python.exe'). Select OK, then apply and click OK again. + (for example: 'C:"\\"Users"\\"user"\\"Anaconda2"\\"envs"\\"catalyst"\\"python.exe'). Select OK, then apply and click OK again. 3. Next, click on the dropdown menu on the top right of PyCharm and select 'Edit Configurations'. Select the '+' button. From cbae2465d2de5651c96892657598f4cb263dc733 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:27:58 -0500 Subject: [PATCH 028/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 0c2a8564..8cad3596 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16624,7 +16624,7 @@ Setup Then select 'Conda Environment' -> 'Existing environment'. Click the '...' button and navigate to your catalyst env located in the Anaconda2 folder to select the 'python.exe' file - (for example: 'C:"\\"Users"\\"user"\\"Anaconda2"\\"envs"\\"catalyst"\\"python.exe'). Select OK, then apply and click OK again. + (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). Select OK, then apply and click OK again. 3. Next, click on the dropdown menu on the top right of PyCharm and select 'Edit Configurations'. Select the '+' button. From 1f29f7fdf142057df7a9fee55b938c6806067381 Mon Sep 17 00:00:00 2001 From: izokay Date: Mon, 29 Jan 2018 17:31:32 -0500 Subject: [PATCH 029/117] Update beginner-tutorial.rst --- docs/source/beginner-tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 8cad3596..c2a037d2 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16628,7 +16628,7 @@ Setup 3. Next, click on the dropdown menu on the top right of PyCharm and select 'Edit Configurations'. Select the '+' button. - Set the script Path to the location of your script and make sure the interpreter is correct, then hit ok. + Set the script Path to the path of your script and make sure the interpreter is correct, then hit ok. You should now be able to run your script in PyCharm. From c26d78cf0556a5807409deec1e55f20cd2ab001d Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 30 Jan 2018 11:20:06 -0700 Subject: [PATCH 030/117] BLD: set AUTH_SERVER as a Catalyst constant --- catalyst/constants.py | 2 ++ catalyst/marketplace/marketplace.py | 4 +--- catalyst/marketplace/utils/auth_utils.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 5668bc83..abfa1974 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -21,3 +21,5 @@ except Exception as e: print('unable to get catalyst path: {}'.format(e)) AUTO_INGEST = False + +AUTH_SERVER = 'https://data.enigma.co' diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index af4c4aef..bbd0be9f 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,7 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL +from catalyst.constants import LOG_LEVEL, AUTH_SERVER from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -43,8 +43,6 @@ CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'data-marketplace/catalyst/marketplace/contract_abi.json' -AUTH_SERVER = 'http://localhost:5000' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index f1193758..35fa3504 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -4,8 +4,7 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) - -AUTH_SERVER = 'http://localhost:5000' +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): From 0f55f0e9a6db6439fc6436f31f1bb3a215bbd6ad Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 10:50:10 -0700 Subject: [PATCH 031/117] BLD: passing dataset to marketplace publish request --- catalyst/marketplace/marketplace.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index bbd0be9f..f276d997 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -341,9 +341,13 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - nonce.encode('utf-8'), + '{}{}'.format(dataset,nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, 'Key': key, 'Nonce': nonce} + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} filenames = glob.glob(os.path.join(datadir, '*.csv')) From d1cd95c492032877cb1db5e94a3fea73ac72bc06 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 13:39:59 -0700 Subject: [PATCH 032/117] BLD: marketplace subscribe + refactoring --- catalyst/marketplace/marketplace.py | 144 ++++++++++++++---------- catalyst/marketplace/utils/eth_utils.py | 2 +- 2 files changed, 88 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index f276d997..e0c6b6b0 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,6 +6,7 @@ import glob import time import shutil import hashlib +import binascii import bcolz import logbook @@ -97,6 +98,52 @@ class Marketplace: ), ] + def choose_pubaddr(self): + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break + + return address + + def sign_transaction(self, from_address, tx): + + print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' + 'enter the following parameters:\n\n' + 'From Address:\t\t{_from}\n' + 'To Address:\t\t{to}\n' + 'Value / Amount to Send:\t{value}\n' + 'Gas Limit:\t\t{gas}\n' + 'Nonce:\t\t\t{nonce}\n' + 'Data:\t\t\t{data}\n'.format( + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) + + signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + + if signed_tx.startswith('0x'): + signed_tx = signed_tx[2:] + + return signed_tx + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -137,35 +184,51 @@ class Marketplace: ) print(df.to_string(formatters=formatters)) - pass - def subscribe(self, dataset): - data_sources = self.get_data_sources_map() - index = next( - (index for (index, d) in enumerate(data_sources) if - d['name'].lower() == dataset.lower()), - None - ) - if index is None: - raise ValueError( - 'Data source not found.' - ) + + # data_sources = self.get_data_sources_map() + # index = next( + # (index for (index, d) in enumerate(data_sources) if + # d['name'].lower() == dataset.lower()), + # None + # ) + + # Check if dataset exists, if not throw exception + # if index is None: + # raise ValueError( + # 'Data source not found.' + # ) + + address = self.choose_pubaddr() + + tx = self.contract.functions.subscribe( + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = 4700000 + + signed_tx = self.sign_transaction(address, tx) try: - self.contract.transact( - {'from': self.default_account} - ).subscribe(index) - print( - 'Subscribed to data source {} successfully'.format( - dataset - ) - ) + tx_hash = '0x{}'.format(b32_str( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) - pass - def ingest(self, data_source_name, data_frequency=None, start=None, end=None, force_download=False): data_source_name = data_source_name.lower() @@ -261,22 +324,7 @@ class Marketplace: # hist = False # break - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + address = self.choose_pubaddr() tx = self.contract.functions.register( bytes32(dataset), @@ -287,25 +335,7 @@ class Marketplace: tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas - print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' - 'enter the following parameters:\n\n' - 'From Address:\t\t{_from}\n' - 'To Address:\t\t{to}\n' - 'Value / Amount to Send:\t{value}\n' - 'Gas Limit:\t\t{gas}\n' - 'Nonce:\t\t\t{nonce}\n' - 'Data:\t\t\t{data}\n'.format( - _from=address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) - - signed_tx = input('Copy and Paste the "Signed Transaction" ' - 'field here:\n') + signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(b32_str( self.web3.eth.sendRawTransaction(signed_tx))) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index c2be4bd6..fd6ba1e1 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,4 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32.decode('utf-8')) + return binascii.hexlify(bytes32).decode('utf-8') From e906315969529e9a49a4baf81a8abd1f2b669012 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 31 Jan 2018 22:06:52 -0700 Subject: [PATCH 033/117] BLD: marketplace: enigma contract files --- catalyst/marketplace/contract_enigma_abi.json | 302 ++++++++++++++++++ .../marketplace/contract_enigma_address.txt | 1 + ...abi.json => contract_marketplace_abi.json} | 0 ...s.txt => contract_marketplace_address.txt} | 0 4 files changed, 303 insertions(+) create mode 100644 catalyst/marketplace/contract_enigma_abi.json create mode 100644 catalyst/marketplace/contract_enigma_address.txt rename catalyst/marketplace/{contract_abi.json => contract_marketplace_abi.json} (100%) rename catalyst/marketplace/{contract_address.txt => contract_marketplace_address.txt} (100%) diff --git a/catalyst/marketplace/contract_enigma_abi.json b/catalyst/marketplace/contract_enigma_abi.json new file mode 100644 index 00000000..4dd70538 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_abi.json @@ -0,0 +1,302 @@ +[ + { + "constant": true, + "inputs": [], + "name": "name", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_from", + "type": "address" + }, + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "INITIAL_SUPPLY", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "decimals", + "outputs": [ + { + "name": "", + "type": "uint8" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_subtractedValue", + "type": "uint256" + } + ], + "name": "decreaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [], + "name": "getAfterApproveTest", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "name": "balance", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "symbol", + "outputs": [ + { + "name": "", + "type": "string" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_to", + "type": "address" + }, + { + "name": "_value", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_spender", + "type": "address" + }, + { + "name": "_addedValue", + "type": "uint256" + } + ], + "name": "increaseApproval", + "outputs": [ + { + "name": "success", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "name": "_owner", + "type": "address" + }, + { + "name": "_spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "name": "testValue", + "type": "address" + } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "from", + "type": "address" + }, + { + "indexed": true, + "name": "to", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + } + ] \ No newline at end of file diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt new file mode 100644 index 00000000..157e2c23 --- /dev/null +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -0,0 +1 @@ +0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file diff --git a/catalyst/marketplace/contract_abi.json b/catalyst/marketplace/contract_marketplace_abi.json similarity index 100% rename from catalyst/marketplace/contract_abi.json rename to catalyst/marketplace/contract_marketplace_abi.json diff --git a/catalyst/marketplace/contract_address.txt b/catalyst/marketplace/contract_marketplace_address.txt similarity index 100% rename from catalyst/marketplace/contract_address.txt rename to catalyst/marketplace/contract_marketplace_address.txt From a04a99373d6d7276301f335f655314e5f90fe6fd Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 08:07:17 -0700 Subject: [PATCH 034/117] BLD: marketplace: subscription to dataset --- catalyst/marketplace/marketplace.py | 268 ++++++++++++++++----- catalyst/marketplace/marketplace_errors.py | 19 +- catalyst/marketplace/utils/eth_utils.py | 19 +- 3 files changed, 248 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e0c6b6b0..d6bf5293 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -6,7 +6,6 @@ import glob import time import shutil import hashlib -import binascii import bcolz import logbook @@ -20,12 +19,13 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles) + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired) from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.path_utils import get_data_source, \ get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret if sys.version_info.major < 3: @@ -38,11 +38,23 @@ else: REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github -CONTRACT_PATH = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_address.txt' +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' -CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/contract_abi.json' log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -59,17 +71,31 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - contract_url = urllib.urlopen(CONTRACT_PATH) - self.contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) - - abi_url = urllib.urlopen(CONTRACT_ABI) - abi = json.load(abi_url) - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) - self.contract = self.web3.eth.contract( - self.contract_address, + contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) + + self.mkt_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) + abi = json.load(abi_url) + + self.mkt_contract = self.web3.eth.contract( + self.mkt_contract_address, + abi=abi, + ) + + contract_url = urllib.urlopen(ENIGMA_CONTRACT) + + self.eng_contract_address = Web3.toChecksumAddress( + contract_url.readline().strip()) + + abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) + abi = json.load(abi_url) + + self.eng_contract = self.web3.eth.contract( + self.eng_contract_address, abi=abi, ) @@ -137,7 +163,8 @@ class Marketplace: ) ) - signed_tx = input('Copy and Paste the "Signed Transaction" field here:\n') + signed_tx = input('Copy and Paste the "Signed Transaction" ' + 'field here:\n') if signed_tx.startswith('0x'): signed_tx = signed_tx[2:] @@ -186,57 +213,185 @@ class Marketplace: def subscribe(self, dataset): - # data_sources = self.get_data_sources_map() - # index = next( - # (index for (index, d) in enumerate(data_sources) if - # d['name'].lower() == dataset.lower()), - # None - # ) - - # Check if dataset exists, if not throw exception - # if index is None: - # raise ValueError( - # 'Data source not found.' - # ) - address = self.choose_pubaddr() - tx = self.contract.functions.subscribe( + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + price = dataset_info[1] + + print('\nThe price for a monthly subscription to this dataset is' + ' {} ENG'.format(price)) + + print('Checking that the ENG balance in {} is greater than ' + '{} ENG... '.format(address, price), end='') + + balance = self.web3.eth.call({ + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) + + balance = int(balance[2:], 16) // 10 ** 8 + + if balance > price: + print('OK.') + else: + print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower ' + 'than the price of the dataset that you are trying to\n' + 'buy: {} ENG. Get enough ENG to cover the costs of the ' + 'monthly\nsubscription for what you are trying to buy, ' + 'and try again.'.format(address, balance, price)) + return + + while True: + agree_pay = input('Please confirm that you agree to pay {} ENG ' + 'for a monthly subscription to the dataset "{}" ' + 'starting today. [default: Y] '.format( + price, dataset)) or 'y' + if agree_pay.lower() not in ('y', 'n'): + print("Please answer Y or N.") + else: + if agree_pay.lower() == 'y': + break + else: + return + + print('Ready to subscribe to dataset {}.\n'.format(dataset)) + print('In order to execute the subscription, you will need to sign ' + 'two different transactions:\n' + '1. First transaction is to authorize the Marketplace contract ' + 'to spend {} ENG on your behalf.\n' + '2. Second transaction is the actual subscription for the ' + 'desired dataset'.format(price)) + + tx = self.eng_contract.functions.approve( + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) + + if 'ropsten' in REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) + + signed_tx = self.sign_transaction(address, tx) + + try: + tx_hash = '0x{}'.format(bin_hex( + self.web3.eth.sendRawTransaction(signed_tx))) + print('\nThis is the TxHash for this transaction: ' + '{}'.format(tx_hash)) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + return + + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + + print('Waiting for the first transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nFirst transaction successful!\n' + 'Now processing second transaction.') + + tx = self.mkt_contract.functions.subscribe( bytes32(dataset), ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in REMOTE_NODE: - tx['gas'] = 4700000 + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) try: - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) - if 'ropsten' in REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}'.format(etherscan)) - except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) + return - def ingest(self, data_source_name, data_frequency=None, start=None, + if 'ropsten' in REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('You can check the outcome of your transaction here:\n' + '{}'.format(etherscan)) + + print('Waiting for the second transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + + print('\nSecond transaction successful!\n' + 'You have successfully subscribed to dataset {} with' + 'address {}.\n' + 'You can now ingest this dataset anytime during the ' + 'next month by running the following command:\n' + 'catalyst marketplace ingest --dataset={}'.format( + dataset, address, dataset)) + + def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - data_source_name = data_source_name.lower() + + address = self.choose_pubaddr() + + check_sub = self.mkt_contract.functions.checkAddressSubscription( + address, bytes32(dataset)).call() + + if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + raise MarketplaceContractDataNoMatch( + params='address: {}, dataset: {}'.format( + address, dataset)) + + if not check_sub[5]: + raise MarketplaceSubscriptionExpired( + dataset=dataset, + date=check_sub[4]) + + # WIP + + dataset = dataset.lower() period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(data_source_name, period, force_download) + tmp_folder = get_data_source(dataset, period, force_download) - bundle_folder = get_bundle_folder(data_source_name, data_frequency) + bundle_folder = get_bundle_folder(dataset, data_frequency) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') @@ -294,10 +449,10 @@ class Marketplace: 'this dataset in ENG: ')) # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): + # freq = input('Enter the data frequency [daily, hourly, minute]: ') + # if freq.lower() not in ('daily', 'houlry', 'minute'): # print("Not a valid frequency.") - # else: + # else: # break # while True: @@ -326,25 +481,26 @@ class Marketplace: address = self.choose_pubaddr() - tx = self.contract.functions.register( + tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - tx['gas'] = int(tx['gas'] * 1.5) # Defaults to not enough gas + if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) - tx_hash = '0x{}'.format(b32_str( + tx_hash = '0x{}'.format(bin_hex( self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): - datasource = self.contract.functions.getDataSource( + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() if not datasource[4]: @@ -371,11 +527,11 @@ class Marketplace: nonce = str(int(time.time())) signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset,nonce).encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), hashlib.sha512).hexdigest() - headers = {'Sign': signature, - 'Key': key, + headers = {'Sign': signature, + 'Key': key, 'Nonce': nonce, 'Dataset': dataset} diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 3992158d..ed3d0fa4 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -7,7 +7,8 @@ from catalyst.errors import ZiplineError def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles]: + MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, + MarketplaceSubscriptionExpired]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -49,3 +50,19 @@ class MarketplaceNoCSVFiles(ZiplineError): msg = ( 'No CSV files found on {datadir} to upload.' ) + + +class MarketplaceContractDataNoMatch(ZiplineError): + msg = ( + 'The information found on the contract does not match the ' + 'requested data:\n{params}.' + ) + + +class MarketplaceSubscriptionExpired(ZiplineError): + msg = ( + 'Your subscription to dataset "{dataset}" expired on {date} ' + 'and is no longer active. You have to subscribe again running the ' + 'following command:\n' + 'catalyst marketplace subscribe --dataset={dataset}' + ) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index fd6ba1e1..f5fe8b7b 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -30,4 +30,21 @@ def b32_str(bytes32): str """ - return binascii.hexlify(bytes32).decode('utf-8') + return binascii.unhexlify( + bytes32.decode('utf-8').rstrip('\0')).decode('ascii') + + +def bin_hex(binary): + """ + Convert bytes32 to string + + Parameters + ---------- + input: bytes object + + Returns + ------- + str + + """ + return binascii.hexlify(binary).decode('utf-8') From 18123d7f7e75514dd0194f0e406bc99513bfde20 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 11:47:25 -0700 Subject: [PATCH 035/117] BLD: marketplace constants in constants.py --- catalyst/constants.py | 21 +++++ catalyst/marketplace/marketplace.py | 120 ++++++++++++++-------------- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index abfa1974..df0ea82b 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -23,3 +23,24 @@ except Exception as e: AUTO_INGEST = False AUTH_SERVER = 'https://data.enigma.co' + +# TODO: switch to mainnet +ETH_REMOTE_NODE = 'https://ropsten.infura.io/' + +# TODO: move to MASTER branch on github +MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_address.txt' + +MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_marketplace_abi.json' + +# TODO: switch to mainnet +ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ + 'data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_address.txt' + +ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ + 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'contract_enigma_abi.json' \ No newline at end of file diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index d6bf5293..08249775 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,9 @@ import six import requests from web3 import Web3, HTTPProvider -from catalyst.constants import LOG_LEVEL, AUTH_SERVER +from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, + MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, + ENIGMA_CONTRACT_ABI ) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -33,28 +35,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib -# TODO: host our own node on aws? -# TODO: switch to mainnet -REMOTE_NODE = 'https://ropsten.infura.io/' - -# TODO: move to MASTER branch on github -MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_address.txt' - -MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_marketplace_abi.json' - -# TODO: switch to mainnet -ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_address.txt' - -ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ - 'contract_enigma_abi.json' - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -71,7 +51,7 @@ class Marketplace: ) self.default_account = self.addresses[0]['pubAddr'] - self.web3 = Web3(HTTPProvider(REMOTE_NODE)) + self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) @@ -99,30 +79,30 @@ class Marketplace: abi=abi, ) - def get_data_sources_map(self): - return [ - dict( - name='Marketcap', - desc='The marketcap value in USD.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily'], - ), - dict( - name='GitHub', - desc='The rate of development activity on GitHub.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour'], - ), - dict( - name='Influencers', - desc='Tweets and related sentiments by selected influencers.', - start_date=pd.to_datetime('2017-01-01'), - end_date=pd.to_datetime('2018-01-15'), - data_frequencies=['daily', 'hour', 'minute'], - ), - ] + # def get_data_sources_map(self): + # return [ + # dict( + # name='Marketcap', + # desc='The marketcap value in USD.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily'], + # ), + # dict( + # name='GitHub', + # desc='The rate of development activity on GitHub.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour'], + # ), + # dict( + # name='Influencers', + # desc='Tweets and related sentiments by selected influencers.', + # start_date=pd.to_datetime('2017-01-01'), + # end_date=pd.to_datetime('2018-01-15'), + # data_frequencies=['daily', 'hour', 'minute'], + # ), + # ] def choose_pubaddr(self): while True: @@ -142,7 +122,7 @@ class Marketplace: self.addresses[address_i]['pubAddr']) break - return address + return address, address_i def sign_transaction(self, from_address, tx): @@ -213,7 +193,7 @@ class Marketplace: def subscribe(self, dataset): - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -272,7 +252,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -287,7 +267,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -319,7 +299,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -334,7 +314,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in REMOTE_NODE: + if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( tx_hash) else: @@ -369,7 +349,7 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): - address = self.choose_pubaddr() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( address, bytes32(dataset)).call() @@ -384,7 +364,31 @@ class Marketplace: dataset=dataset, date=check_sub[4]) - # WIP + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(datasource[0], dataset) + + nonce = str(int(time.time())) + + signature = hmac.new(secret.encode('utf-8'), + '{}{}'.format(dataset, nonce).encode('utf-8'), + hashlib.sha512).hexdigest() + + headers = {'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': dataset} + + r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + + + print(r) + exit(0) + + dataset = dataset.lower() @@ -479,7 +483,7 @@ class Marketplace: # hist = False # break - address = self.choose_pubaddr() + address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( bytes32(dataset), @@ -488,7 +492,7 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in REMOTE_NODE and tx['gas'] > 4700000: + if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 signed_tx = self.sign_transaction(address, tx) From 56bd3db7a2f8ee2ec0c765f9e81215b223251a9f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 16:54:24 -0700 Subject: [PATCH 036/117] BLD: marketplace - ingestion downloads multiple files --- catalyst/marketplace/marketplace.py | 40 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 08249775..30860551 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,4 +1,5 @@ import os +import re import sys import json import hmac @@ -11,12 +12,13 @@ import bcolz import logbook import pandas as pd import six +from requests_toolbelt import MultipartDecoder import requests from web3 import Web3, HTTPProvider -from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, - MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, - ENIGMA_CONTRACT_ABI ) +from catalyst.constants import ( + LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, + MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI) from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -97,7 +99,7 @@ class Marketplace: # ), # dict( # name='Influencers', - # desc='Tweets and related sentiments by selected influencers.', + # desc='Tweets & related sentiments by selected influencers.', # start_date=pd.to_datetime('2017-01-01'), # end_date=pd.to_datetime('2018-01-15'), # data_frequencies=['daily', 'hour', 'minute'], @@ -349,6 +351,8 @@ class Marketplace: def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): + dataset = dataset.lower() + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -369,7 +373,7 @@ class Marketplace: secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) + key, secret = get_key_secret(address, dataset) nonce = str(int(time.time())) @@ -382,15 +386,27 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} - r = requests.post('{}/ingest'.format(AUTH_SERVER), headers=headers) + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, stream=True) + if r.status_code == 200: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + else: + raise MarketplaceHTTPRequest(request='ingest dataset', + error=r.status_code) - print(r) + print('Download successful, now we need to ingest...') exit(0) - - - - dataset = dataset.lower() + # TODO: sync with Fred on the below and + # exiting for now period = start.strftime('%Y-%m-%d') tmp_folder = get_data_source(dataset, period, force_download) @@ -404,8 +420,6 @@ class Marketplace: else: os.rename(tmp_folder, bundle_folder) - pass - def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): data_source_name = data_source_name.lower() From 37980f35806c63fa6c87eedb1aaef12ade913e4e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 20:43:48 -0500 Subject: [PATCH 037/117] BLD: working on marketplace integration --- catalyst/marketplace/marketplace.py | 99 +++++++++++++-------------- etc/requirements_marketplace.txt | 2 +- tests/marketplace/test_marketplace.py | 3 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 30860551..4c55b2da 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -37,7 +37,6 @@ if sys.version_info.major < 3: else: import urllib.request as urllib - log = logbook.Logger('Marketplace', level=LOG_LEVEL) @@ -48,9 +47,9 @@ class Marketplace: if self.addresses[0]['pubAddr'] == '': raise MarketplacePubAddressEmpty( - filename=os.path.join( - get_marketplace_folder(), 'addresses.json') - ) + filename=os.path.join( + get_marketplace_folder(), 'addresses.json') + ) self.default_account = self.addresses[0]['pubAddr'] self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) @@ -58,7 +57,7 @@ class Marketplace: contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) self.mkt_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -71,7 +70,7 @@ class Marketplace: contract_url = urllib.urlopen(ENIGMA_CONTRACT) self.eng_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + contract_url.readline().strip()) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) @@ -118,10 +117,10 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) + self.addresses[address_i]['pubAddr']) break return address, address_i @@ -136,14 +135,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -198,7 +197,7 @@ class Marketplace: address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() price = dataset_info[1] @@ -209,11 +208,11 @@ class Marketplace: '{} ENG... '.format(address, price), end='') balance = self.web3.eth.call({ - 'from': address, - 'to': self.eng_contract_address, - 'data': '0x70a08231000000000000000000000000{}'.format( - address[2:]) - }) + 'from': address, + 'to': self.eng_contract_address, + 'data': '0x70a08231000000000000000000000000{}'.format( + address[2:]) + }) balance = int(balance[2:], 16) // 10 ** 8 @@ -231,7 +230,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -249,10 +248,10 @@ class Marketplace: 'desired dataset'.format(price)) tx = self.eng_contract.functions.approve( - self.mkt_contract_address, - price, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + self.mkt_contract_address, + price, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -261,7 +260,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -271,7 +270,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -297,9 +296,9 @@ class Marketplace: 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - bytes32(dataset), - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) @@ -308,7 +307,7 @@ class Marketplace: try: tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: ' '{}'.format(tx_hash)) @@ -318,7 +317,7 @@ class Marketplace: if 'ropsten' in ETH_REMOTE_NODE: etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) + tx_hash) else: etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) @@ -346,7 +345,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def ingest(self, dataset, data_frequency=None, start=None, end=None, force_download=False): @@ -356,17 +355,17 @@ class Marketplace: address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(dataset)).call() if check_sub[0] != address or b32_str(check_sub[1]) != dataset: raise MarketplaceContractDataNoMatch( - params='address: {}, dataset: {}'.format( - address, dataset)) + params='address: {}, dataset: {}'.format( + address, dataset)) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=dataset, + date=check_sub[4]) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -500,11 +499,11 @@ class Marketplace: address = self.choose_pubaddr()[0] tx = self.mkt_contract.functions.register( - bytes32(dataset), - price, - address, - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + bytes32(dataset), + price, + address, + ).buildTransaction( + {'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: tx['gas'] = 4700000 @@ -512,14 +511,14 @@ class Marketplace: signed_tx = self.sign_transaction(address, tx) tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) + self.web3.eth.sendRawTransaction(signed_tx))) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(dataset)).call() if not datasource[4]: raise MarketplaceDatasetNotFound(dataset=dataset) @@ -529,11 +528,11 @@ class Marketplace: if not match: raise MarketplaceNoAddressMatch( - dataset=dataset, - address=datasource[0]) + dataset=dataset, + address=datasource[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + datasource[0])) if 'key' in match: key = match['key'] diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index cbb3a5a3..8062d192 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1 @@ -web3==4.0.0b5 \ No newline at end of file +web3==4.0.0b7 \ No newline at end of file diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 9292b3a0..815ff885 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -11,7 +11,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_register(self): marketplace = Marketplace() - marketplace.register('GitHub') + marketplace.register() pass def test_ingest(self): @@ -30,3 +30,4 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass + From b9130fd968507d22b6e507e48255911e674403e4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 1 Feb 2018 22:03:16 -0500 Subject: [PATCH 038/117] BLD: added missing requirement --- etc/requirements_marketplace.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt index 8062d192..2a56c200 100644 --- a/etc/requirements_marketplace.txt +++ b/etc/requirements_marketplace.txt @@ -1 +1,2 @@ -web3==4.0.0b7 \ No newline at end of file +web3==4.0.0b7 +requests-toolbelt==0.8.0 From e4602900565d2998dc95d7f5bc3a4e935e28c5b0 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 21:57:13 -0700 Subject: [PATCH 039/117] BLD: better exception handling for failed multipart download --- catalyst/marketplace/marketplace.py | 65 ++++++++++++++++++----------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 4c55b2da..43c15f19 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -13,6 +13,8 @@ import logbook import pandas as pd import six from requests_toolbelt import MultipartDecoder +from requests_toolbelt.multipart.decoder import \ + NonMultipartContentTypeException import requests from web3 import Web3, HTTPProvider @@ -106,22 +108,28 @@ class Marketplace: # ] def choose_pubaddr(self): - while True: - for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( - i, - self.addresses[i]['pubAddr'], - self.addresses[i]['desc']) - ) - address_i = int(input('Choose your address associated with ' - 'this transaction: [default: 0] ') or 0) - if not (0 <= address_i < len(self.addresses)): - print('Please choose a number between 0 and {}\n'.format( - len(self.addresses) - 1)) - else: - address = Web3.toChecksumAddress( - self.addresses[address_i]['pubAddr']) - break + + if len(self.addresses) == 1: + address = self.addresses[0]['pubAddr'] + address_i = 0 + print('Using {} for this transaction.'.format(address)) + else: + while True: + for i in range(0, len(self.addresses)): + print('{}\t{}\t{}'.format( + i, + self.addresses[i]['pubAddr'], + self.addresses[i]['desc']) + ) + address_i = int(input('Choose your address associated with ' + 'this transaction: [default: 0] ') or 0) + if not (0 <= address_i < len(self.addresses)): + print('Please choose a number between 0 and {}\n'.format( + len(self.addresses)-1)) + else: + address = Web3.toChecksumAddress( + self.addresses[address_i]['pubAddr']) + break return address, address_i @@ -385,19 +393,26 @@ class Marketplace: 'Nonce': nonce, 'Dataset': dataset} + print('Starting download of dataset for ingestion...') + r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), headers=headers, stream=True) if r.status_code == 200: - decoder = MultipartDecoder.from_response(r) - for part in decoder.parts: - h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) - with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) - f.write(part.content) + try: + decoder = MultipartDecoder.from_response(r) + for part in decoder.parts: + h = part.headers[b'Content-Disposition'].decode('utf-8') + filename = re.search(r'filename="(.*)"', h).group(1) + with open(filename, 'wb') as f: + # for chunk in part.content.iter_content(chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) + f.write(part.content) + except NonMultipartContentTypeException: + response = r.json() + raise MarketplaceHTTPRequest(request='ingest dataset', + error=response) else: raise MarketplaceHTTPRequest(request='ingest dataset', error=r.status_code) From 0decdf5aab874ab56d5889ffe77766e94c4b2e9e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 22:30:24 -0700 Subject: [PATCH 040/117] BLD: check for name duplicates when registering dataset --- catalyst/marketplace/marketplace.py | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 43c15f19..ced5bfde 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -160,6 +160,17 @@ class Marketplace: return signed_tx + def check_transaction(self, tx_hash): + + if 'ropsten' in ETH_REMOTE_NODE: + etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( + tx_hash) + else: + etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) + + print('\nYou can check the outcome of your transaction here:\n' + '{}\n\n'.format(etherscan)) + def get_data_source_def(self, data_source_name): data_source_name = data_source_name.lower() dsm = self.get_data_sources_map() @@ -276,14 +287,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in ETH_REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}\n\n'.format(etherscan)) + self.check_transaction(tx_hash) print('Waiting for the first transaction to succeed...') @@ -476,7 +480,20 @@ class Marketplace: pass def register(self): - dataset = input('Enter the name of the dataset to register: ') + while True: + dataset = input('Enter the name of the dataset to register: ') + dataset = dataset.lower() + + datasource = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if datasource[4]: + print('There is already a dataset registered under ' + 'the name "{}". Please choose a different ' + 'name.'.format(dataset)) + else: + break + price = int(input('Enter the price for a monthly subscription to ' 'this dataset in ENG: ')) @@ -520,8 +537,8 @@ class Marketplace: ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) - if 'ropsten' in ETH_REMOTE_NODE and tx['gas'] > 4700000: - tx['gas'] = 4700000 + if 'ropsten' in ETH_REMOTE_NODE: + tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) @@ -530,6 +547,8 @@ class Marketplace: print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + self.check_transaction(tx_hash) + def publish(self, dataset, datadir, watch): datasource = self.mkt_contract.functions.getDataSource( From db4ee4c01ec3337da1c7695747ef4f6cb0293251 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 1 Feb 2018 23:42:40 -0700 Subject: [PATCH 041/117] BLD: check for existence of dataset in ingestion & subscription --- catalyst/marketplace/marketplace.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ced5bfde..a59d5ed7 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -213,11 +213,18 @@ class Marketplace: def subscribe(self, dataset): + dataset = dataset.lower() + address = self.choose_pubaddr()[0] dataset_info = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + price = dataset_info[1] print('\nThe price for a monthly subscription to this dataset is' @@ -364,6 +371,14 @@ class Marketplace: dataset = dataset.lower() + dataset_info = self.mkt_contract.functions.getDataSource( + bytes32(dataset)).call() + + if not dataset_info[4]: + print('The requested "{}" dataset is not registered in ' + 'the Data Marketplace.'.format(dataset)) + return + address, address_i = self.choose_pubaddr() check_sub = self.mkt_contract.functions.checkAddressSubscription( @@ -551,6 +566,8 @@ class Marketplace: def publish(self, dataset, datadir, watch): + dataset = dataset.lower() + datasource = self.mkt_contract.functions.getDataSource( bytes32(dataset)).call() @@ -607,4 +624,4 @@ class Marketplace: raise MarketplaceHTTPRequest(request='upload file', error=r.json()['error']) - print('Dataset {} published successfully.'.format(dataset)) + print('Dataset {} uploaded successfully.'.format(dataset)) From 9ef000e529c0dc6af15f9199eb17c0992d8db1ef Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Feb 2018 12:06:57 -0700 Subject: [PATCH 042/117] BUG: balance returns different types --- catalyst/marketplace/marketplace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a59d5ed7..828224ab 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -240,7 +240,10 @@ class Marketplace: address[2:]) }) - balance = int(balance[2:], 16) // 10 ** 8 + try: + balance = int(balance[2:], 16) // 10 ** 8 + except ValueError: + balance = int(bin_hex(balance), 16) // 10 ** 8 if balance > price: print('OK.') From 99dece953f157570f78e96404ce92a58bad046ca Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 16:20:33 -0500 Subject: [PATCH 043/117] BLD: ingesting marketplace bundles --- catalyst/marketplace/marketplace.py | 147 ++++++++++++++--------- catalyst/marketplace/utils/path_utils.py | 52 +++----- tests/marketplace/test_marketplace.py | 9 +- 3 files changed, 109 insertions(+), 99 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 828224ab..14721a82 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,21 +1,21 @@ +import glob +import hashlib +import hmac +import json import os import re -import sys -import json -import hmac -import glob -import time import shutil -import hashlib +import sys +import time import bcolz import logbook import pandas as pd +import requests import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException -import requests from web3 import Web3, HTTPProvider from catalyst.constants import ( @@ -27,12 +27,12 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.path_utils import get_data_source, \ - get_bundle_folder, get_data_source_folder, get_marketplace_folder, \ - get_user_pubaddr -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.bundle_utils import merge_bundles +from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex +from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ + get_data_source_folder, get_marketplace_folder, \ + get_user_pubaddr, get_temp_bundles_folder, extract_bundle if sys.version_info.major < 3: import urllib @@ -125,7 +125,7 @@ class Marketplace: 'this transaction: [default: 0] ') or 0) if not (0 <= address_i < len(self.addresses)): print('Please choose a number between 0 and {}\n'.format( - len(self.addresses)-1)) + len(self.addresses) - 1)) else: address = Web3.toChecksumAddress( self.addresses[address_i]['pubAddr']) @@ -369,92 +369,119 @@ class Marketplace: 'catalyst marketplace ingest --dataset={}'.format( dataset, address, dataset)) - def ingest(self, dataset, data_frequency=None, start=None, - end=None, force_download=False): + def process_temp_bundle(self, ds_name, path): + """ + Merge the temp bundle into the main bundle for the specified + data source. - dataset = dataset.lower() + Parameters + ---------- + ds_name + path + Returns + ------- + + """ + tmp_bundle = extract_bundle(path) + bundle_folder = get_data_source_folder(ds_name) + if os.listdir(bundle_folder): + zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') + ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') + merge_bundles(zsource, ztarget) + + else: + os.rename(tmp_bundle, bundle_folder) + + pass + + def ingest(self, ds_name, start=None, end=None, force_download=False): + + ds_name = ds_name.lower() dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() + bytes32(ds_name) + ).call() if not dataset_info[4]: print('The requested "{}" dataset is not registered in ' - 'the Data Marketplace.'.format(dataset)) + 'the Data Marketplace.'.format(ds_name)) return address, address_i = self.choose_pubaddr() - check_sub = self.mkt_contract.functions.checkAddressSubscription( - address, bytes32(dataset)).call() + address, bytes32(ds_name) + ).call() - if check_sub[0] != address or b32_str(check_sub[1]) != dataset: + if check_sub[0] != address or b32_str(check_sub[1]) != ds_name: raise MarketplaceContractDataNoMatch( params='address: {}, dataset: {}'.format( - address, dataset)) + address, ds_name + ) + ) if not check_sub[5]: raise MarketplaceSubscriptionExpired( - dataset=dataset, - date=check_sub[4]) + dataset=ds_name, + date=check_sub[4], + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, dataset) + key, secret = get_key_secret(address, ds_name) nonce = str(int(time.time())) + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} - - print('Starting download of dataset for ingestion...') - - r = requests.post('{}/marketplace/ingest'.format(AUTH_SERVER), - headers=headers, stream=True) - + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + log.debug('Starting download of dataset for ingestion...') + r = requests.post( + '{}/marketplace/ingest'.format(AUTH_SERVER), + headers=headers, + stream=True, + ) if r.status_code == 200: + target_path = get_temp_bundles_folder() try: decoder = MultipartDecoder.from_response(r) for part in decoder.parts: h = part.headers[b'Content-Disposition'].decode('utf-8') - filename = re.search(r'filename="(.*)"', h).group(1) + # Extracting the filename from the header + name = re.search(r'filename="(.*)"', h).group(1) + + filename = os.path.join(target_path, name) with open(filename, 'wb') as f: # for chunk in part.content.iter_content(chunk_size=1024): # if chunk: # filter out keep-alive new chunks # f.write(chunk) f.write(part.content) + + self.process_temp_bundle(ds_name, filename) + except NonMultipartContentTypeException: response = r.json() - raise MarketplaceHTTPRequest(request='ingest dataset', - error=response) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=response, + ) else: - raise MarketplaceHTTPRequest(request='ingest dataset', - error=r.status_code) + raise MarketplaceHTTPRequest( + request='ingest dataset', + error=r.status_code, + ) - print('Download successful, now we need to ingest...') - exit(0) - # TODO: sync with Fred on the below and - # exiting for now - - period = start.strftime('%Y-%m-%d') - tmp_folder = get_data_source(dataset, period, force_download) - - bundle_folder = get_bundle_folder(dataset, data_frequency) - if os.listdir(bundle_folder): - zsource = bcolz.ctable(rootdir=tmp_folder, mode='r') - ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r') - merge_bundles(zsource, ztarget) - - else: - os.rename(tmp_folder, bundle_folder) + log.info('{} ingested successfully'.format(ds_name)) def get_data_source(self, data_source_name, data_frequency=None, start=None, end=None): diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 62f8beac..b19e1921 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -5,6 +5,7 @@ import tarfile import shutil from catalyst.data.bundles.core import download_without_progress +from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory @@ -55,6 +56,7 @@ def get_data_source_folder(data_source_name, environ=None): return data_source_folder +@deprecated def get_bundle_folder(data_source_name, data_frequency, environ=None): data_source_folder = get_data_source_folder(data_source_name, environ) @@ -65,13 +67,13 @@ def get_bundle_folder(data_source_name, data_frequency, environ=None): return bundle_folder -def get_temp_bundles_folder(data_source_name, environ=None): +def get_temp_bundles_folder(environ=None): """ The temp folder for bundle downloads by algo name. Parameters ---------- - data_source_name: str + ds_name: str environ: Returns @@ -79,55 +81,31 @@ def get_temp_bundles_folder(data_source_name, environ=None): str """ - data_source_folder = get_data_source_folder(data_source_name, environ) + root = data_root(environ) + folder = os.path.join(root, 'marketplace', 'temp_bundles') + ensure_directory(folder) - temp_bundles = os.path.join(data_source_folder, 'temp_bundles') - ensure_directory(temp_bundles) - - return temp_bundles + return folder -def get_data_source(data_source_name, period, force_download=False): +def extract_bundle(tar_filename): """ - Download and extract a bcolz bundle. + Extract a bcolz bundle. Parameters ---------- - exchange_name: str - symbol: str - data_frequency: str - period: str + ds_name Returns ------- str """ - root = get_temp_bundles_folder(data_source_name) - name = '{data_source}_{period}'.format( - data_source=data_source_name, - period=period, - ) - path = os.path.join(root, name) + target_path = tar_filename.replace('.tar.gz', '') + with tarfile.open(tar_filename, 'r') as tar: + tar.extractall(target_path) - if os.path.isdir(path): - if force_download: - shutil.rmtree(path) - - else: - return path - - ensure_directory(path) - - url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format( - data_source=data_source_name, - name=name, - ) - bytes = download_without_progress(url) - with tarfile.open('r', fileobj=bytes) as tar: - tar.extractall(path) - - return path + return target_path def get_user_pubaddr(environ=None): diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 815ff885..7315629a 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -14,9 +14,15 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace.register() pass + def test_subscribe(self): + marketplace = Marketplace() + marketplace.subscribe('Victor2') + pass + def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.get_data_source_def('Marketcap') + ds_def = marketplace.ingest('Victor2') + pass marketplace.ingest( data_source_name='Marketcap', @@ -30,4 +36,3 @@ class TestMarketplace(WithLogger, ZiplineTestCase): marketplace = Marketplace() marketplace.clean('marketcap') pass - From d1529afeb318f9dcd0e5d4aa5b0cb20d9f4cb411 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:10:10 -0500 Subject: [PATCH 044/117] BLD: enhancing registration features --- catalyst/marketplace/marketplace.py | 78 ++++++++++++++------------- tests/marketplace/test_marketplace.py | 5 ++ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 14721a82..18294072 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -539,57 +539,61 @@ class Marketplace: else: break - price = int(input('Enter the price for a monthly subscription to ' - 'this dataset in ENG: ')) + price = int( + input( + 'Enter the price for a monthly subscription to ' + 'this dataset in ENG: ' + ) + ) + while True: + freq = input('Enter the data frequency [daily, hourly, minute]: ') + if freq.lower() not in ('daily', 'hourly', 'minute'): + print('Not a valid frequency.') + else: + break - # while True: - # freq = input('Enter the data frequency [daily, hourly, minute]: ') - # if freq.lower() not in ('daily', 'houlry', 'minute'): - # print("Not a valid frequency.") - # else: - # break + while True: + reg_pub = input( + 'Doest it include historical data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_history = True + else: + has_history = False + break - # while True: - # reg_pub = input('Can data be published every hour at a regular ' - # 'time? [default: Y]: ') or 'y' - # if reg_pub.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if reg_pub.lower() == 'y': - # reg_pub = True - # else: - # reg_pub = False - # break - - # while True: - # hist = input('Does it include historical data? ' - # '[default: Y]') or 'y' - # if hist.lower() not in ('y', 'n'): - # print("Please answer Y or N.") - # else: - # if hist.lower() == 'y': - # hist = True - # else: - # hist = False - # break + while True: + reg_pub = input( + 'Doest it include live data? [default: Y]: ' + ) or 'y' + if reg_pub.lower() not in ('y', 'n'): + print('Please answer Y or N.') + else: + if reg_pub.lower() == 'y': + has_live = True + else: + has_live = False + break address = self.choose_pubaddr()[0] - tx = self.mkt_contract.functions.register( bytes32(dataset), price, address, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + {'nonce': self.web3.eth.getTransactionCount(address)} + ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - - tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) - + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) self.check_transaction(tx_hash) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 7315629a..c7c7c467 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -32,6 +32,11 @@ class TestMarketplace(WithLogger, ZiplineTestCase): ) pass + def test_publish(self): + marketplace = Marketplace() + datadir = '/Users/fredfortier/Downloads/marketcap_test_single' + marketplace.publish('marketcap1', datadir, False) + def test_clean(self): marketplace = Marketplace() marketplace.clean('marketcap') From b9bedfda21045d074abdba3cf8b1a2cfea599aba Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 2 Feb 2018 17:23:13 -0500 Subject: [PATCH 045/117] BLD: enhancing registration features --- tests/marketplace/test_marketplace.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index c7c7c467..d2511ce8 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,20 +16,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('Victor2') + marketplace.subscribe('marketcap1') pass def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('Victor2') - pass - - marketplace.ingest( - data_source_name='Marketcap', - data_frequency=ds_def['data_frequencies'][0], - start=pd.to_datetime('2017-10-01'), - force_download=True, - ) + ds_def = marketplace.ingest('marketcap1') pass def test_publish(self): From e94c9db2ec41c45583c0a126e0ada83cebcfb756 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Feb 2018 18:29:11 +0200 Subject: [PATCH 046/117] Updated contract --- catalyst/marketplace/contract_enigma_address.txt | 2 +- catalyst/marketplace/contract_marketplace_abi.json | 2 +- catalyst/marketplace/contract_marketplace_address.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/marketplace/contract_enigma_address.txt b/catalyst/marketplace/contract_enigma_address.txt index 157e2c23..90fcd06a 100644 --- a/catalyst/marketplace/contract_enigma_address.txt +++ b/catalyst/marketplace/contract_enigma_address.txt @@ -1 +1 @@ -0xfdA2D18921e9Bab0773f6fF904D3610972D97c48 \ No newline at end of file +0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10 \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_abi.json b/catalyst/marketplace/contract_marketplace_abi.json index a3dda75f..4f0e2460 100644 --- a/catalyst/marketplace/contract_marketplace_abi.json +++ b/catalyst/marketplace/contract_marketplace_abi.json @@ -1 +1 @@ -[{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"","type":"address"},{"name":"","type":"bytes32"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"status","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataSource","outputs":[{"name":"","type":"address"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"bool"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionPaid","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"}] \ No newline at end of file +[{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"isActiveDataSource","outputs":[{"name":"isActive","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mBegin","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"FIXED_SUBSCRIPTION_PERIOD","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isPunished","type":"bool"}],"name":"setPunishProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getDataProviderInfo","outputs":[{"name":"owner","type":"address"},{"name":"price","type":"uint256"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"isPunished","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"subscribe","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mProvidersSize","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_newPrice","type":"uint256"}],"name":"updateDataSourcePrice","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"withdrawProvider","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getAllProviders","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getMarketplaceTotalBalance","outputs":[{"name":"totalBalance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"refundSubscriber","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_price","type":"uint256"},{"name":"_dataOwner","type":"address"}],"name":"register","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mCurrent","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getWithdrawAmount","outputs":[{"name":"withdrawAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"checkAddressSubscription","outputs":[{"name":"subscriber","type":"address"},{"name":"dataSourceName","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isUnExpired","type":"bool"},{"name":"isPaid","type":"bool"},{"name":"isPunishedProvider","type":"bool"},{"name":"isOrder","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_dataSourceName","type":"bytes32"},{"name":"_isActive","type":"bool"}],"name":"changeDataSourceActivityStatus","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"mProviders","outputs":[{"name":"owner","type":"address"},{"name":"volume","type":"uint256"},{"name":"subscriptionsNum","type":"uint256"},{"name":"name","type":"bytes32"},{"name":"price","type":"uint256"},{"name":"isPunished","type":"bool"},{"name":"punishTimeStamp","type":"uint256"},{"name":"isProvider","type":"bool"},{"name":"isActive","type":"bool"},{"name":"nextProvider","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"},{"name":"","type":"uint256"}],"name":"mOrders","outputs":[{"name":"dataSourceName","type":"bytes32"},{"name":"subscriber","type":"address"},{"name":"provider","type":"address"},{"name":"price","type":"uint256"},{"name":"startTime","type":"uint256"},{"name":"endTime","type":"uint256"},{"name":"isPaid","type":"bool"},{"name":"isOrder","type":"bool"},{"name":"isRefundPaid","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_dataSourceName","type":"bytes32"}],"name":"getOwnerFromName","outputs":[{"name":"owner","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"getRefundAmount","outputs":[{"name":"refundAmount","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"MARKETPLACE_VERSION","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_subscriber","type":"address"},{"name":"_dataSourceName","type":"bytes32"}],"name":"isExpiredSubscription","outputs":[{"name":"isExpired","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_tokenAddress","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"previousOwner","type":"address"},{"indexed":true,"name":"newOwner","type":"address"}],"name":"OwnershipTransferred","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newPrice","type":"uint256"}],"name":"PriceUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"editor","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"newStatus","type":"bool"}],"name":"ActivityUpdate","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Registered","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"SubscriptionDeposited","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":false,"name":"price","type":"uint256"},{"indexed":false,"name":"success","type":"bool"}],"name":"Subscribed","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"TransferToProvider","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"ProviderWithdraw","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dataOwner","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"isPunished","type":"bool"}],"name":"ProviderPunishStatus","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"subscriber","type":"address"},{"indexed":true,"name":"dataSourceName","type":"bytes32"},{"indexed":false,"name":"refundAmount","type":"uint256"}],"name":"SubscriberRefund","type":"event"}] \ No newline at end of file diff --git a/catalyst/marketplace/contract_marketplace_address.txt b/catalyst/marketplace/contract_marketplace_address.txt index ec7bf029..86810a5f 100644 --- a/catalyst/marketplace/contract_marketplace_address.txt +++ b/catalyst/marketplace/contract_marketplace_address.txt @@ -1 +1 @@ -0x2d1aD04bF3e150f8334dEb2180D7a7B51dFfAF92 \ No newline at end of file +0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482 \ No newline at end of file From ee93b16558d9bda15f82e52e8fcf2d83f8ea107f Mon Sep 17 00:00:00 2001 From: Isan-Rivkin Date: Sun, 4 Feb 2018 08:42:42 -0800 Subject: [PATCH 047/117] BLD:Solidity contract addressed added to constants --- catalyst/constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..94d2487c 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -16,3 +16,9 @@ DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' AUTO_INGEST = False + +''' +Marketplace and Enigma contract +''' +ADDRESS_MARKETPLACE_CONTRACT = '0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10' +ADDRESS_ENIGMA_CONTRACT = '0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482' \ No newline at end of file From 5889f4c74bf5dc8ac7a33c230cffcdf40271622e Mon Sep 17 00:00:00 2001 From: Isan-Rivkin Date: Sun, 4 Feb 2018 08:45:06 -0800 Subject: [PATCH 048/117] BLD: added smart contract addresses --- catalyst/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 94d2487c..fbc973cc 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -18,7 +18,7 @@ DATE_FORMAT = '%Y-%m-%d' AUTO_INGEST = False ''' -Marketplace and Enigma contract +Marketplace and Enigma contract current network: Ropsten ''' -ADDRESS_MARKETPLACE_CONTRACT = '0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10' -ADDRESS_ENIGMA_CONTRACT = '0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482' \ No newline at end of file +ADDRESS_MARKETPLACE_CONTRACT = '0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482' +ADDRESS_ENIGMA_CONTRACT = '0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10' \ No newline at end of file From 881e3e595391dbf972d2017c64ded2b7965f0fc0 Mon Sep 17 00:00:00 2001 From: Isan-Rivkin Date: Sun, 4 Feb 2018 08:51:27 -0800 Subject: [PATCH 049/117] REV:constants.py revert --- catalyst/constants.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index fbc973cc..517b20d2 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -17,8 +17,3 @@ DATE_FORMAT = '%Y-%m-%d' AUTO_INGEST = False -''' -Marketplace and Enigma contract current network: Ropsten -''' -ADDRESS_MARKETPLACE_CONTRACT = '0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482' -ADDRESS_ENIGMA_CONTRACT = '0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10' \ No newline at end of file From 444fcbb2b33f6eecc336c568b8fd49973b7ffa5f Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:04:23 -0500 Subject: [PATCH 050/117] BLD: adjustments for the new contract and developing register --- catalyst/marketplace/marketplace.py | 130 ++++++++++++++--------- catalyst/marketplace/utils/auth_utils.py | 46 +++++++- 2 files changed, 118 insertions(+), 58 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 18294072..5de8f538 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -27,7 +27,8 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired) -from catalyst.marketplace.utils.auth_utils import get_key_secret +from catalyst.marketplace.utils.auth_utils import get_key_secret, \ + get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ @@ -212,20 +213,19 @@ class Marketplace: print(df.to_string(formatters=formatters)) def subscribe(self, dataset): - dataset = dataset.lower() address = self.choose_pubaddr()[0] + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - dataset_info = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(dataset)) return - price = dataset_info[1] + price = provider_info[1] print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -239,7 +239,6 @@ class Marketplace: 'data': '0x70a08231000000000000000000000000{}'.format( address[2:]) }) - try: balance = int(balance[2:], 16) // 10 ** 8 except ValueError: @@ -398,11 +397,11 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): ds_name = ds_name.lower() - dataset_info = self.mkt_contract.functions.getDataSource( + provider_info = self.mkt_contract.functions.getDataProviderInfo( bytes32(ds_name) ).call() - if not dataset_info[4]: + if not provider_info[4]: print('The requested "{}" dataset is not registered in ' 'the Data Marketplace.'.format(ds_name)) return @@ -432,19 +431,7 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, ds_name) - nonce = str(int(time.time())) - signature = hmac.new( - secret.encode('utf-8'), - '{}{}'.format(ds_name, nonce).encode('utf-8'), - hashlib.sha512 - ).hexdigest() - - headers = { - 'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': ds_name, - } + headers = get_signed_headers(ds_name, key, secret) log.debug('Starting download of dataset for ingestion...') r = requests.post( '{}/marketplace/ingest'.format(AUTH_SERVER), @@ -524,15 +511,46 @@ class Marketplace: shutil.rmtree(folder) pass + def create_metadata(self, key, secret, ds_name, data_frequency, desc, + has_history=True, has_live=True): + """ + + Returns + ------- + + """ + headers = get_signed_headers(ds_name, key, secret) + r = requests.post( + '{}/marketplace/register'.format(AUTH_SERVER), + data=dict( + name=ds_name, + desc=desc, + data_frequency=data_frequency, + has_history=has_history, + has_live=has_live, + ), + headers=headers, + ) + + if r.status_code != 200: + raise MarketplaceHTTPRequest( + request='register', error=r.status_code + ) + + if 'error' in r.json(): + raise MarketplaceHTTPRequest( + request='upload file', error=r.json()['error'] + ) + def register(self): while True: - dataset = input('Enter the name of the dataset to register: ') - dataset = dataset.lower() + desc = input('Enter the name of the dataset to register: ') + dataset = desc.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if datasource[4]: + if provider_info[4]: print('There is already a dataset registered under ' 'the name "{}". Please choose a different ' 'name.'.format(dataset)) @@ -554,7 +572,7 @@ class Marketplace: while True: reg_pub = input( - 'Doest it include historical data? [default: Y]: ' + 'Does it include historical data? [default: Y]: ' ) or 'y' if reg_pub.lower() not in ('y', 'n'): print('Please answer Y or N.') @@ -578,7 +596,23 @@ class Marketplace: has_live = False break - address = self.choose_pubaddr()[0] + address, address_i = self.choose_pubaddr() + if 'key' in self.addresses[address_i]: + key = self.addresses[address_i]['key'] + secret = self.addresses[address_i]['secret'] + else: + # TODO: Verify signature to obtain key/secret pair + key, secret = get_key_secret(address, dataset) + + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -599,44 +633,34 @@ class Marketplace: self.check_transaction(tx_hash) def publish(self, dataset, datadir, watch): - dataset = dataset.lower() + provider_info = self.mkt_contract.functions.getDataProviderInfo( + bytes32(dataset) + ).call() - datasource = self.mkt_contract.functions.getDataSource( - bytes32(dataset)).call() - - if not datasource[4]: + if not provider_info[4]: raise MarketplaceDatasetNotFound(dataset=dataset) - match = next((l for l in self.addresses if - l['pubAddr'] == datasource[0]), None) - + match = next( + (l for l in self.addresses if l['pubAddr'] == provider_info[0]), + None + ) if not match: raise MarketplaceNoAddressMatch( dataset=dataset, - address=datasource[0]) + address=provider_info[0]) print('Using address: {} to publish this dataset.'.format( - datasource[0])) + provider_info[0])) if 'key' in match: key = match['key'] secret = match['secret'] else: # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(datasource[0], dataset) - - nonce = str(int(time.time())) - - signature = hmac.new(secret.encode('utf-8'), - '{}{}'.format(dataset, nonce).encode('utf-8'), - hashlib.sha512).hexdigest() - - headers = {'Sign': signature, - 'Key': key, - 'Nonce': nonce, - 'Dataset': dataset} + key, secret = get_key_secret(provider_info[0], dataset) + headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) if not filenames: diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 35fa3504..452dadb7 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -1,10 +1,14 @@ +import hashlib +import hmac + import requests +import time from catalyst.marketplace.marketplace_errors import ( - MarketplaceHTTPRequest) + MarketplaceHTTPRequest) from catalyst.marketplace.utils.path_utils import ( - get_user_pubaddr, save_user_pubaddr) -from catalyst.constants import AUTH_SERVER + get_user_pubaddr, save_user_pubaddr) +from catalyst.constants import AUTH_SERVER def get_key_secret(pubAddr, dataset): @@ -25,8 +29,8 @@ def get_key_secret(pubAddr, dataset): session = requests.Session() response = session.get('{}/getkeysecret'.format(AUTH_SERVER), headers={ - 'pubAddr': pubAddr, - 'dataset': dataset}) + 'pubAddr': pubAddr, + 'dataset': dataset}) if 'error' in response.json(): raise MarketplaceHTTPRequest(request=str('obtain key/secret'), @@ -44,3 +48,35 @@ def get_key_secret(pubAddr, dataset): save_user_pubaddr(addresses) return match['key'], match['secret'] + + +def get_signed_headers(ds_name, key, secret): + """ + Return a new request header including the key / secret signature + + Parameters + ---------- + ds_name + key + secret + + Returns + ------- + + """ + nonce = str(int(time.time())) + + signature = hmac.new( + secret.encode('utf-8'), + '{}{}'.format(ds_name, nonce).encode('utf-8'), + hashlib.sha512 + ).hexdigest() + + headers = { + 'Sign': signature, + 'Key': key, + 'Nonce': nonce, + 'Dataset': ds_name, + } + + return headers From 3bc54a6c2ebebeb8c09458a6f9a522de1107f6b5 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 5 Feb 2018 23:46:09 -0500 Subject: [PATCH 051/117] BLD: finalized the register implementation --- catalyst/marketplace/marketplace.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 5de8f538..363171c1 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -522,8 +522,8 @@ class Marketplace: headers = get_signed_headers(ds_name, key, secret) r = requests.post( '{}/marketplace/register'.format(AUTH_SERVER), - data=dict( - name=ds_name, + json=dict( + ds_name=ds_name, desc=desc, data_frequency=data_frequency, has_history=has_history, @@ -604,15 +604,6 @@ class Marketplace: # TODO: Verify signature to obtain key/secret pair key, secret = get_key_secret(address, dataset) - self.create_metadata( - key=key, - secret=secret, - ds_name=dataset, - data_frequency=freq, - desc=desc, - has_history=has_history, - has_live=has_live, - ) tx = self.mkt_contract.functions.register( bytes32(dataset), price, @@ -632,6 +623,18 @@ class Marketplace: self.check_transaction(tx_hash) + print('\nWarming up the {} dataset'.format(dataset)) + self.create_metadata( + key=key, + secret=secret, + ds_name=dataset, + data_frequency=freq, + desc=desc, + has_history=has_history, + has_live=has_live, + ) + print('\n{} registered successfully'.format(dataset)) + def publish(self, dataset, datadir, watch): dataset = dataset.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( From 2168fb0d5b9534c5eb901305deca3905b9d0a6db Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:17:21 -0700 Subject: [PATCH 052/117] BUG: address json initialized with array of 1 dict, instead of dict --- catalyst/marketplace/utils/path_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index b19e1921..4bbf2854 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -133,7 +133,8 @@ def get_user_pubaddr(environ=None): return [data, ] return data else: - data = dict(pubAddr='', desc='') + data = [] + data.append(dict(pubAddr='', desc='')) with open(filename, 'w') as f: json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) From 4c1a9b1dd7f0f8ebb08b9d719d4d1494a3f91362 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 5 Feb 2018 22:58:40 -0700 Subject: [PATCH 053/117] BLD: marketplace added listing of datasets --- catalyst/__main__.py | 1 - catalyst/marketplace/marketplace.py | 57 +++++++++--------------- catalyst/marketplace/utils/path_utils.py | 3 -- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 3833f3e4..ecbe7c23 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -756,7 +756,6 @@ def ls(ctx): sys.stdout) marketplace = Marketplace() marketplace.list() - click.echo('Functionality not yet implemented.', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 363171c1..a5e7d4f5 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,6 +1,4 @@ import glob -import hashlib -import hmac import json import os import re @@ -144,14 +142,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -182,35 +180,21 @@ class Marketplace: return ds def list(self): - subscribers = self.contract.call( - {'from': self.default_account} - ).getSubscribers() - subscribed = [] - for index, address in enumerate(subscribers): - if address == self.default_account: - subscribed.append(index) - - data_sources = self.get_data_sources_map() + data_sources = self.mkt_contract.functions.getAllProviders().call() data = [] for index, data_source in enumerate(data_sources): - data.append( - dict( - id=index, - subscribed=index in subscribed, - **data_source + if index > 0: + data.append( + dict( + dataset=data_source.decode('utf-8').rstrip('\0') + ) ) - ) df = pd.DataFrame(data) - df.set_index(['id', 'name', 'desc'], drop=True, inplace=True) set_print_settings() - - formatters = dict( - subscribed=lambda s: u'\u2713' if s else '', - ) - print(df.to_string(formatters=formatters)) + print(df) def subscribe(self, dataset): dataset = dataset.lower() @@ -258,7 +242,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -366,7 +350,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -449,9 +433,10 @@ class Marketplace: filename = os.path.join(target_path, name) with open(filename, 'wb') as f: - # for chunk in part.content.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # f.write(chunk) + # for chunk in part.content.iter_content( + # chunk_size=1024): + # if chunk: # filter out keep-alive new chunks + # f.write(chunk) f.write(part.content) self.process_temp_bundle(ds_name, filename) diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 4bbf2854..6565b49d 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -2,9 +2,6 @@ import os import json import tarfile -import shutil - -from catalyst.data.bundles.core import download_without_progress from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory From d97890fc4e779aa683f91da68df45225ea6dff2f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 11:43:41 -0700 Subject: [PATCH 054/117] BLD: marketplace: moving AUTH_SERVER paths to /marketplace/* --- catalyst/marketplace/marketplace.py | 2 +- catalyst/marketplace/utils/auth_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a5e7d4f5..61ed7fec 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -658,7 +658,7 @@ class Marketplace: for file in filenames: files.append(('file', open(file, 'rb'))) - r = requests.post('{}/publish'.format(AUTH_SERVER), + r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER), files=files, headers=headers) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 452dadb7..d761cf8d 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -27,7 +27,7 @@ def get_key_secret(pubAddr, dataset): """ session = requests.Session() - response = session.get('{}/getkeysecret'.format(AUTH_SERVER), + response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), headers={ 'pubAddr': pubAddr, 'dataset': dataset}) From 5ccddf6d2178bd59a6115d870616beb8a975ea10 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 14:17:48 -0500 Subject: [PATCH 055/117] BLD: removed dummy smart contract --- marketplace/README.md | 18 --------- marketplace/contracts/Marketplace.sol | 19 --------- marketplace/contracts/Migrations.sol | 23 ----------- marketplace/marketplace.iml | 14 ------- marketplace/migrations/1_initial_migration.js | 5 --- marketplace/migrations/2_deploy_contracts.js | 5 --- marketplace/test/TestMarketplace.sol | 39 ------------------- marketplace/truffle-config.js | 4 -- marketplace/truffle.js | 11 ------ 9 files changed, 138 deletions(-) delete mode 100644 marketplace/README.md delete mode 100644 marketplace/contracts/Marketplace.sol delete mode 100644 marketplace/contracts/Migrations.sol delete mode 100644 marketplace/marketplace.iml delete mode 100644 marketplace/migrations/1_initial_migration.js delete mode 100644 marketplace/migrations/2_deploy_contracts.js delete mode 100644 marketplace/test/TestMarketplace.sol delete mode 100644 marketplace/truffle-config.js delete mode 100644 marketplace/truffle.js diff --git a/marketplace/README.md b/marketplace/README.md deleted file mode 100644 index 61f63842..00000000 --- a/marketplace/README.md +++ /dev/null @@ -1,18 +0,0 @@ -This module contains smart contracts for the data marketplace. -It was generated with Truffle. Truffle can be used to compile and test -the smart contracts. It's being tested on a test blockhain using Ganache. - -Steps to test: -* Download and run Ganache -* Ensure that your Ganache endpoint matches `truffle.js` -* In this `marketplace` folder, run the following commands: - * `truffle compile` - * `truffle test` - * `truffle migrate --reset` - * `truffle console` - * From the console: `Marketplace.deployed()` -* The deployed method displays info about the deployed smart contract -including its address. Copy/paste the address into -`catalyst/marketplace/marketplace.py`. -* Run the catalyst marketplace unit tests: -`tests/marketplace/test_marketplace.py` diff --git a/marketplace/contracts/Marketplace.sol b/marketplace/contracts/Marketplace.sol deleted file mode 100644 index 115b6110..00000000 --- a/marketplace/contracts/Marketplace.sol +++ /dev/null @@ -1,19 +0,0 @@ -pragma solidity ^0.4.17; - -contract Marketplace { - address[16] public subscribers; - - // Subscribing to a data source - function subscribe(uint dataSourceId) public returns (uint) { - require(dataSourceId >= 0 && dataSourceId <= 15); - - subscribers[dataSourceId] = msg.sender; - - return dataSourceId; - } - - // Retrieving the subscribers - function getSubscribers() public view returns (address[16]) { - return subscribers; - } -} \ No newline at end of file diff --git a/marketplace/contracts/Migrations.sol b/marketplace/contracts/Migrations.sol deleted file mode 100644 index f170cb4f..00000000 --- a/marketplace/contracts/Migrations.sol +++ /dev/null @@ -1,23 +0,0 @@ -pragma solidity ^0.4.17; - -contract Migrations { - address public owner; - uint public last_completed_migration; - - modifier restricted() { - if (msg.sender == owner) _; - } - - function Migrations() public { - owner = msg.sender; - } - - function setCompleted(uint completed) public restricted { - last_completed_migration = completed; - } - - function upgrade(address new_address) public restricted { - Migrations upgraded = Migrations(new_address); - upgraded.setCompleted(last_completed_migration); - } -} diff --git a/marketplace/marketplace.iml b/marketplace/marketplace.iml deleted file mode 100644 index cf4ef5ed..00000000 --- a/marketplace/marketplace.iml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/marketplace/migrations/1_initial_migration.js b/marketplace/migrations/1_initial_migration.js deleted file mode 100644 index 4d5f3f9b..00000000 --- a/marketplace/migrations/1_initial_migration.js +++ /dev/null @@ -1,5 +0,0 @@ -var Migrations = artifacts.require("./Migrations.sol"); - -module.exports = function(deployer) { - deployer.deploy(Migrations); -}; diff --git a/marketplace/migrations/2_deploy_contracts.js b/marketplace/migrations/2_deploy_contracts.js deleted file mode 100644 index e3935400..00000000 --- a/marketplace/migrations/2_deploy_contracts.js +++ /dev/null @@ -1,5 +0,0 @@ -var Marketplace = artifacts.require ("Marketplace"); - -module.exports = function (deployer) { - deployer.deploy (Marketplace); -}; \ No newline at end of file diff --git a/marketplace/test/TestMarketplace.sol b/marketplace/test/TestMarketplace.sol deleted file mode 100644 index f169d728..00000000 --- a/marketplace/test/TestMarketplace.sol +++ /dev/null @@ -1,39 +0,0 @@ -pragma solidity ^0.4.17; - -import "truffle/Assert.sol"; -import "truffle/DeployedAddresses.sol"; -import "../contracts/Marketplace.sol"; - -contract TestMarketplace { - Marketplace marketplace = Marketplace(DeployedAddresses.Marketplace()); - - // Testing the subscribe() function - function testUserCanSubscribe() public { - uint returnedId = marketplace.subscribe(2); - - uint expected = 2; - - Assert.equal(returnedId, expected, "Data source 2 should be recorded."); - } - - // Testing retrieval of a single subscriber - function testGetSubscriberAddressByDataSourceId() public { - // Expected owner is this contract - address expected = this; - - address subscriber = marketplace.subscribers(2); - - Assert.equal(subscriber, expected, "Subscriber of data source ID 2 should be recorded."); - } - - // Testing retrieval of all subscribers - function testGetSubscriberAddressByDataSourceIdInArray() public { - // Expected subscriber is this contract - address expected = this; - - // Store subscribers in memory rather than contract's storage - address[16] memory subscribers = marketplace.getSubscribers(); - - Assert.equal(subscribers[2], expected, "Subscriber of data source 2 should be recorded."); - } -} \ No newline at end of file diff --git a/marketplace/truffle-config.js b/marketplace/truffle-config.js deleted file mode 100644 index a6330d6d..00000000 --- a/marketplace/truffle-config.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! -}; diff --git a/marketplace/truffle.js b/marketplace/truffle.js deleted file mode 100644 index 67f2dd44..00000000 --- a/marketplace/truffle.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = { - // See - // to customize your Truffle configuration! - networks: { - development: { - host: "localhost", - port: 7545, - network_id: "*" // Match any network id - } - } -}; From 2320a1432ad58fbc4d1cdf68b17e52bdccc7c593 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 14:34:07 -0500 Subject: [PATCH 056/117] BLD: Merge remote-tracking branch 'remotes/origin/data-marketplace' into develop --- catalyst/__main__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 6f3f97ab..1b08f69d 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,6 +1,5 @@ import errno import os -import sys from functools import wraps import click @@ -581,7 +580,8 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) + click.echo('Ingesting exchange bundle {}...'.format(exchange_name), + sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, @@ -635,7 +635,8 @@ def clean_exchange(ctx, exchange_name, data_frequency): exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) + click.echo('Cleaning exchange bundle {}...'.format(exchange_name), + sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) @@ -773,7 +774,7 @@ def marketplace(ctx): @click.pass_context def ls(ctx): click.echo('Listing of available data sources on the marketplace:', - sys.stdout) + sys.stdout) marketplace = Marketplace() marketplace.list() @@ -885,7 +886,6 @@ def publish(ctx, dataset, datadir, watch): ctx.fail("must specify a datadir where to find the files to publish " " with '--datadir'\n") marketplace.publish(dataset, datadir, watch) - click.echo("%s %s" % (bundle, timestamp), sys.stdout) if __name__ == '__main__': From c5bed6e8c4e8a1b00c1eb362a3c9bd0ea577baee Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 6 Feb 2018 21:15:30 -0500 Subject: [PATCH 057/117] BLD: made some adjustments during testing --- catalyst/examples/mean_reversion_simple.py | 2 +- catalyst/marketplace/marketplace.py | 46 +++++++++++++--------- tests/marketplace/test_marketplace.py | 7 ++-- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index db787612..3b94d925 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='eth', live_graph=False, - simulate_orders=False, + simulate_orders=True, stats_output=None, # auth_aliases=dict(poloniex='auth2') ) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 61ed7fec..8f03145c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -142,14 +142,14 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], + ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -197,6 +197,7 @@ class Marketplace: print(df) def subscribe(self, dataset): + # TODO: what happens if we are already subscribed? dataset = dataset.lower() address = self.choose_pubaddr()[0] @@ -217,11 +218,13 @@ class Marketplace: print('Checking that the ENG balance in {} is greater than ' '{} ENG... '.format(address, price), end='') + wallet_address = address[2:] balance = self.web3.eth.call({ 'from': address, 'to': self.eng_contract_address, 'data': '0x70a08231000000000000000000000000{}'.format( - address[2:]) + wallet_address + ) }) try: balance = int(balance[2:], 16) // 10 ** 8 @@ -242,7 +245,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -259,22 +262,25 @@ class Marketplace: '2. Second transaction is the actual subscription for the ' 'desired dataset'.format(price)) + grains = price * 10 ** 8 tx = self.eng_contract.functions.approve( self.mkt_contract_address, - price, + grains, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + {'nonce': self.web3.eth.getTransactionCount(address)} + ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - try: - tx_hash = '0x{}'.format(bin_hex( - self.web3.eth.sendRawTransaction(signed_tx))) - print('\nThis is the TxHash for this transaction: ' - '{}'.format(tx_hash)) + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) + print( + '\nThis is the TxHash for this transaction: {}'.format(tx_hash) + ) except Exception as e: print('Unable to subscribe to data source: {}'.format(e)) @@ -350,7 +356,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -381,6 +387,7 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): ds_name = ds_name.lower() + # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( bytes32(ds_name) ).call() @@ -391,7 +398,8 @@ class Marketplace: return address, address_i = self.choose_pubaddr() - check_sub = self.mkt_contract.functions.checkAddressSubscription( + fns = self.mkt_contract.functions + check_sub = fns.checkAddressSubscription( address, bytes32(ds_name) ).call() diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index d2511ce8..c45a727d 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,18 +16,19 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('marketcap1') + marketplace.subscribe('marketcap1234') pass def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('marketcap1') + ds_def = marketplace.ingest('marketcap1234') pass def test_publish(self): marketplace = Marketplace() datadir = '/Users/fredfortier/Downloads/marketcap_test_single' - marketplace.publish('marketcap1', datadir, False) + marketplace.publish('marketcap1234', datadir, False) + pass def test_clean(self): marketplace = Marketplace() From 2d43955abf472332b7dd78ea61d04ca1944d6766 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 00:45:33 -0500 Subject: [PATCH 058/117] BUG: fixed Python 2 issue --- catalyst/marketplace/marketplace.py | 6 ++++-- tests/marketplace/test_marketplace.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 8f03145c..61ac2a97 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -215,8 +215,10 @@ class Marketplace: print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) - print('Checking that the ENG balance in {} is greater than ' - '{} ENG... '.format(address, price), end='') + print( + 'Checking that the ENG balance in {} is greater than {} ' + 'ENG... '.format(address, price) + ) wallet_address = address[2:] balance = self.web3.eth.call({ diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index c45a727d..59564a5e 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -16,7 +16,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_subscribe(self): marketplace = Marketplace() - marketplace.subscribe('marketcap1234') + marketplace.subscribe('marketcap2222') pass def test_ingest(self): From eee2e1be885a0fd8ec4e46fb7ecc7df8cf48838d Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 00:48:17 -0500 Subject: [PATCH 059/117] BUG: fixed Python 2 issue --- catalyst/marketplace/marketplace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 61ac2a97..1ea2ea00 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -302,7 +302,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - print('.', end='', flush=True) + # print('.', end='', flush=True) time.sleep(1) print('\nFirst transaction successful!\n' @@ -349,7 +349,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - print('.', end='', flush=True) + # print('.', end='', flush=True) time.sleep(1) print('\nSecond transaction successful!\n' From 461a5942fb0cd09f2241347170400eb19bc82a71 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 01:17:00 -0500 Subject: [PATCH 060/117] BUG: fixed Python 2 issue EXPERIMENTAL --- catalyst/marketplace/marketplace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 1ea2ea00..71620b66 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -14,7 +14,6 @@ import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException -from web3 import Web3, HTTPProvider from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, @@ -43,6 +42,8 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): + global Web3 + from web3 import Web3, HTTPProvider self.addresses = get_user_pubaddr() @@ -107,7 +108,6 @@ class Marketplace: # ] def choose_pubaddr(self): - if len(self.addresses) == 1: address = self.addresses[0]['pubAddr'] address_i = 0 From fb658390327e1b6694e0ddc913709740d90b8151 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 23:33:34 -0700 Subject: [PATCH 061/117] BLD: marketplace: getkeysecret is authenticated --- catalyst/marketplace/marketplace.py | 9 +-- catalyst/marketplace/marketplace_errors.py | 15 ++++- catalyst/marketplace/utils/auth_utils.py | 77 ++++++++++++++++++---- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 71620b66..2eced9ee 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -422,8 +422,7 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, ds_name) + key, secret = get_key_secret(address) headers = get_signed_headers(ds_name, key, secret) log.debug('Starting download of dataset for ingestion...') @@ -596,8 +595,7 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(address, dataset) + key, secret = get_key_secret(address) tx = self.mkt_contract.functions.register( bytes32(dataset), @@ -655,8 +653,7 @@ class Marketplace: key = match['key'] secret = match['secret'] else: - # TODO: Verify signature to obtain key/secret pair - key, secret = get_key_secret(provider_info[0], dataset) + key, secret = get_key_secret(provider_info[0]) headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index ed3d0fa4..953941fc 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -8,7 +8,8 @@ def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired]: + MarketplaceSubscriptionExpired, + MarketplaceWalletNotSupported, MarketplaceEmptySignature]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -66,3 +67,15 @@ class MarketplaceSubscriptionExpired(ZiplineError): 'following command:\n' 'catalyst marketplace subscribe --dataset={dataset}' ) + + +class MarketplaceWalletNotSupported(ZiplineError): + msg = ( + 'Wallet {wallet} is not supported.' + ) + + +class MarketplaceEmptySignature(ZiplineError): + msg = ( + 'Signature cannot be empty.' + ) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index d761cf8d..4979b6f3 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -5,13 +5,14 @@ import requests import time from catalyst.marketplace.marketplace_errors import ( - MarketplaceHTTPRequest) + MarketplaceHTTPRequest, MarketplaceWalletNotSupported, + MarketplaceEmptySignature) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) from catalyst.constants import AUTH_SERVER -def get_key_secret(pubAddr, dataset): +def get_key_secret(pubAddr, wallet='mew'): """ Obtain a new key/secret pair from authentication server @@ -29,25 +30,73 @@ def get_key_secret(pubAddr, dataset): session = requests.Session() response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), headers={ - 'pubAddr': pubAddr, - 'dataset': dataset}) + 'Authorization': 'Digest username="{0}"'.format( + pubAddr)}) - if 'error' in response.json(): + if response.status_code != 401: raise MarketplaceHTTPRequest(request=str('obtain key/secret'), - error=str(response.json()['error'])) + error='Unexpected response code: ' + '{}'.format(response.status_code)) - addresses = get_user_pubaddr() + header = response.headers.get('WWW-Authenticate') + auth_type, auth_info = header.split(None, 1) + d = requests.utils.parse_dict_header(auth_info) - match = next((l for l in addresses if - l['pubAddr'] == pubAddr), None) - match['key'] = response.json()['key'] - match['secret'] = response.json()['secret'] + nonce = '0x{}'.format(d['nonce']) - addresses[addresses.index(match)] = match + if wallet == 'mew': + print('\nObtaining a key/secret pair to streamline all future ' + 'requests with the authentication server.\n' + 'Visit https://www.myetherwallet.com/signmsg.html and sign the' + 'following message:\n{}'.format(nonce)) + signature = input('Copy and Paste the "sig" field from ' + 'the signature here (without the double quotes, ' + 'only the HEX value:\n') + else: + raise MarketplaceWalletNotSupported(wallet=wallet) - save_user_pubaddr(addresses) + if signature is None: + raise MarketplaceEmptySignature() - return match['key'], match['secret'] + signature = signature[2:] + r = int(signature[0:64], base=16) + s = int(signature[64:128], base=16) + v = int(signature[128:130], base=16) + vrs = [v, r, s] + + response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER), + headers={ + 'Authorization': 'Digest username="{0}",realm="{1}",' + 'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",' + 'opaque="{4}"'.format(pubAddr, + d['realm'], + d['nonce'], + ','.join(str(e) for e in vrs+[wallet]), + d['opaque'])}) + + if response.status_code == 200: + + if 'error' in response.json(): + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=str(response.json()['error'])) + else: + addresses = get_user_pubaddr() + + match = next((l for l in addresses if + l['pubAddr'] == pubAddr), None) + match['key'] = response.json()['key'] + match['secret'] = response.json()['secret'] + + addresses[addresses.index(match)] = match + + save_user_pubaddr(addresses) + print('Key/secret pair retrieved successfully from server.') + + return match['key'], match['secret'] + + else: + raise MarketplaceHTTPRequest(request=str('obtain key/secret'), + error=response.status_code) def get_signed_headers(ds_name, key, secret): From 093660ab1a3e625f4a2719226fd46ca45978527e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 01:46:33 -0500 Subject: [PATCH 062/117] BLD: simplified unit tests until the next release --- .../test_suites/test_suite_exchange.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index ca0bd666..79f87a2a 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -15,7 +15,7 @@ from catalyst.exchange.utils.test_utils import select_random_exchanges, \ handle_exchange_error, select_random_assets from catalyst.testing import ZiplineTestCase from catalyst.testing.fixtures import WithLogger -from catalyst.exchange.utils.factory import get_exchanges +from catalyst.exchange.utils.factory import get_exchanges, get_exchange log = Logger('TestSuiteExchange') @@ -90,7 +90,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): # exchange_population, # features=['fetchTickers'], # ) # Type: list[Exchange] - exchanges = list(get_exchanges(['bitfinex']).values()) + exchanges = list(get_exchanges(['binance']).values()) for exchange in exchanges: exchange.init() @@ -113,10 +113,11 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): exchange_population = 3 asset_population = 3 - exchanges = select_random_exchanges( - population=exchange_population, - features=['fetchOHLCV'], - ) # Type: list[Exchange] + # exchanges = select_random_exchanges( + # population=exchange_population, + # features=['fetchOHLCV'], + # ) # Type: list[Exchange] + exchanges = list(get_exchanges(['binance']).values()) for exchange in exchanges: exchange.init() @@ -154,13 +155,20 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase): quote_currency = 'eth' order_amount = 0.1 - exchanges = select_random_exchanges( - population=population, - features=['fetchOrder'], - is_authenticated=True, - base_currency=quote_currency, - ) # Type: list[Exchange] + # exchanges = select_random_exchanges( + # population=population, + # features=['fetchOrder'], + # is_authenticated=True, + # base_currency=quote_currency, + # ) # Type: list[Exchange] + exchanges = [ + get_exchange( + 'binance', + base_currency=quote_currency, + must_authenticate=True, + ) + ] log_catcher = TestHandler() with log_catcher: for exchange in exchanges: From c39e075766b60c9f23145946826dbf6659ec08b2 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 6 Feb 2018 23:50:05 -0700 Subject: [PATCH 063/117] MAINT: constants points to contract address/abi in develop branch --- catalyst/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index df0ea82b..1ddc7710 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -29,18 +29,18 @@ ETH_REMOTE_NODE = 'https://ropsten.infura.io/' # TODO: move to MASTER branch on github MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_marketplace_address.txt' MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_marketplace_abi.json' # TODO: switch to mainnet ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'data-marketplace/catalyst/marketplace/' \ + 'develop/catalyst/marketplace/' \ 'contract_enigma_address.txt' ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/data-marketplace/catalyst/marketplace/' \ + 'catalyst/develop/catalyst/marketplace/' \ 'contract_enigma_abi.json' \ No newline at end of file From 0e3be98d24bffd3f98cc64d7ccfdb4d22ffdd62a Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 01:44:16 -0700 Subject: [PATCH 064/117] BLD: mmarketplace: switched encoding to Web3.toHex() --- catalyst/marketplace/marketplace.py | 61 ++++++++++++++----------- catalyst/marketplace/utils/eth_utils.py | 46 +++++++++---------- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 2eced9ee..99385171 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import glob import json import os @@ -27,7 +29,7 @@ from catalyst.marketplace.marketplace_errors import ( from catalyst.marketplace.utils.auth_utils import get_key_secret, \ get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.eth_utils import bytes32, b32_str, bin_hex +from catalyst.marketplace.utils.eth_utils import bin_hex from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr, get_temp_bundles_folder, extract_bundle @@ -142,14 +144,13 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], - ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'],) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -188,7 +189,7 @@ class Marketplace: if index > 0: data.append( dict( - dataset=data_source.decode('utf-8').rstrip('\0') + dataset=Web3.toText(data_source) ) ) @@ -197,12 +198,12 @@ class Marketplace: print(df) def subscribe(self, dataset): - # TODO: what happens if we are already subscribed? + dataset = dataset.lower() address = self.choose_pubaddr()[0] provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if not provider_info[4]: @@ -212,6 +213,15 @@ class Marketplace: price = provider_info[1] + subscribed = self.mkt_contract.functions.checkAddressSubscription( + address, Web3.toHex(dataset) + ).call() + + if subscribed[5]: + print('You are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {}, and is valid until' + '{}'.format(dataset, subscribed[3], subscribed[4])) + print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -228,10 +238,7 @@ class Marketplace: wallet_address ) }) - try: - balance = int(balance[2:], 16) // 10 ** 8 - except ValueError: - balance = int(bin_hex(balance), 16) // 10 ** 8 + balance = Web3.toInt(hexstr=balance) // 10 ** 8 if balance > price: print('OK.') @@ -247,7 +254,7 @@ class Marketplace: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -302,14 +309,14 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - # print('.', end='', flush=True) + print('.', end='', flush=True) time.sleep(1) print('\nFirst transaction successful!\n' 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - bytes32(dataset), + Web3.toHex(dataset), ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)}) @@ -349,7 +356,7 @@ class Marketplace: except AttributeError: pass for i in range(0, 10): - # print('.', end='', flush=True) + print('.', end='', flush=True) time.sleep(1) print('\nSecond transaction successful!\n' @@ -358,7 +365,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -391,7 +398,7 @@ class Marketplace: ds_name = ds_name.lower() # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(ds_name) + Web3.toHex(ds_name) ).call() if not provider_info[4]: @@ -402,10 +409,10 @@ class Marketplace: address, address_i = self.choose_pubaddr() fns = self.mkt_contract.functions check_sub = fns.checkAddressSubscription( - address, bytes32(ds_name) + address, Web3.toHex(ds_name) ).call() - if check_sub[0] != address or b32_str(check_sub[1]) != ds_name: + if check_sub[0] != address or Web3.toText(check_sub[1]) != ds_name: raise MarketplaceContractDataNoMatch( params='address: {}, dataset: {}'.format( address, ds_name @@ -541,7 +548,7 @@ class Marketplace: desc = input('Enter the name of the dataset to register: ') dataset = desc.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if provider_info[4]: @@ -598,7 +605,7 @@ class Marketplace: key, secret = get_key_secret(address) tx = self.mkt_contract.functions.register( - bytes32(dataset), + Web3.toHex(dataset), price, address, ).buildTransaction( @@ -631,7 +638,7 @@ class Marketplace: def publish(self, dataset, datadir, watch): dataset = dataset.lower() provider_info = self.mkt_contract.functions.getDataProviderInfo( - bytes32(dataset) + Web3.toHex(dataset) ).call() if not provider_info[4]: diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index f5fe8b7b..35e8b300 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -1,37 +1,37 @@ import binascii -def bytes32(string): - """ - Convert string to bytes32 data type for smart contract +# def bytes32(string): +# """ +# Convert string to bytes32 data type for smart contract - Parameters - ---------- - string: str +# Parameters +# ---------- +# string: str - Returns - ------- - list +# Returns +# ------- +# list - """ - return binascii.hexlify(string.encode('utf-8')) +# """ +# return binascii.hexlify(string.encode('utf-8')) -def b32_str(bytes32): - """ - Convert bytes32 to string +# def b32_str(bytes32): +# """ +# Convert bytes32 to string - Parameters - ---------- - input: bytes object +# Parameters +# ---------- +# input: bytes object - Returns - ------- - str +# Returns +# ------- +# str - """ - return binascii.unhexlify( - bytes32.decode('utf-8').rstrip('\0')).decode('ascii') +# """ +# return binascii.unhexlify( +# bytes32.decode('utf-8').rstrip('\0')).decode('ascii') def bin_hex(binary): From 540358d0f9accdba0ab4bd8df9e4ce0e5bf33bb6 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Wed, 7 Feb 2018 12:48:24 +0200 Subject: [PATCH 065/117] BLD: new marketplace contract address --- catalyst/marketplace/contract_marketplace_address.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/contract_marketplace_address.txt b/catalyst/marketplace/contract_marketplace_address.txt index 86810a5f..577adaa0 100644 --- a/catalyst/marketplace/contract_marketplace_address.txt +++ b/catalyst/marketplace/contract_marketplace_address.txt @@ -1 +1 @@ -0x4fe3d3c2d8c0730d565789ddd3b63e53f342a482 \ No newline at end of file +0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4 \ No newline at end of file From e071f6ec8d10f5e8668f727e63437f85b7c0e0ff Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 10:57:07 -0700 Subject: [PATCH 066/117] BLD: implemented grains, catch JSON malformed in addresses.json --- catalyst/marketplace/marketplace.py | 72 +++++++++++++--------- catalyst/marketplace/marketplace_errors.py | 9 ++- catalyst/marketplace/utils/eth_utils.py | 32 ++++++++++ catalyst/marketplace/utils/path_utils.py | 6 +- 4 files changed, 89 insertions(+), 30 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 99385171..ab35fe65 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -24,12 +24,12 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired) + MarketplaceNoCSVFiles) from catalyst.marketplace.utils.auth_utils import get_key_secret, \ get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles -from catalyst.marketplace.utils.eth_utils import bin_hex +from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \ + to_grains from catalyst.marketplace.utils.path_utils import get_bundle_folder, \ get_data_source_folder, get_marketplace_folder, \ get_user_pubaddr, get_temp_bundles_folder, extract_bundle @@ -109,6 +109,9 @@ class Marketplace: # ), # ] + def to_text(self, hex): + return Web3.toText(hex).rstrip('\0') + def choose_pubaddr(self): if len(self.addresses) == 1: address = self.addresses[0]['pubAddr'] @@ -187,15 +190,19 @@ class Marketplace: data = [] for index, data_source in enumerate(data_sources): if index > 0: - data.append( - dict( - dataset=Web3.toText(data_source) + # if 'test' not in Web3.toText(data_source).lower(): + data.append( + dict( + dataset=self.to_text(data_source) + ) ) - ) df = pd.DataFrame(data) set_print_settings() - print(df) + if df.empty: + print('There are no datasets available yet.') + else: + print(df) def subscribe(self, dataset): @@ -211,16 +218,20 @@ class Marketplace: 'the Data Marketplace.'.format(dataset)) return - price = provider_info[1] + grains = provider_info[1] + price = from_grains(grains) subscribed = self.mkt_contract.functions.checkAddressSubscription( address, Web3.toHex(dataset) ).call() if subscribed[5]: - print('You are already subscribed to the "{}" dataset.\n' - 'Your subscription started on {}, and is valid until' - '{}'.format(dataset, subscribed[3], subscribed[4])) + print('\nYou are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {} UTC, and is valid until ' + '{} UTC.'.format( + dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True))) + return print('\nThe price for a monthly subscription to this dataset is' ' {} ENG'.format(price)) @@ -238,16 +249,18 @@ class Marketplace: wallet_address ) }) - balance = Web3.toInt(hexstr=balance) // 10 ** 8 - if balance > price: + balance = Web3.toInt(hexstr=balance) + + if balance > grains: print('OK.') else: print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower ' 'than the price of the dataset that you are trying to\n' 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' - 'and try again.'.format(address, balance, price)) + 'and try again.'.format( + address, from_grains(balance), price)) return while True: @@ -271,7 +284,6 @@ class Marketplace: '2. Second transaction is the actual subscription for the ' 'desired dataset'.format(price)) - grains = price * 10 ** 8 tx = self.eng_contract.functions.approve( self.mkt_contract_address, grains, @@ -395,7 +407,8 @@ class Marketplace: def ingest(self, ds_name, start=None, end=None, force_download=False): - ds_name = ds_name.lower() + # ds_name = ds_name.lower() + # TODO: catch error conditions provider_info = self.mkt_contract.functions.getDataProviderInfo( Web3.toHex(ds_name) @@ -412,18 +425,19 @@ class Marketplace: address, Web3.toHex(ds_name) ).call() - if check_sub[0] != address or Web3.toText(check_sub[1]) != ds_name: - raise MarketplaceContractDataNoMatch( - params='address: {}, dataset: {}'.format( - address, ds_name - ) - ) + if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name: + print('You are not subscribed to dataset "{}" with address {}. ' + 'Plese subscribe first.'.format(ds_name, address)) + return if not check_sub[5]: - raise MarketplaceSubscriptionExpired( - dataset=ds_name, - date=check_sub[4], - ) + print('Your subscription to dataset "{}" expired on {} UTC.' + 'Please renew your subscription by running:\n' + 'catalyst marketplace subscribe --dataset={}'. format( + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -604,9 +618,11 @@ class Marketplace: else: key, secret = get_key_secret(address) + grains = to_grains(price) + tx = self.mkt_contract.functions.register( Web3.toHex(dataset), - price, + grains, address, ).buildTransaction( {'nonce': self.web3.eth.getTransactionCount(address)} diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index 953941fc..b6be1c3b 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -8,7 +8,7 @@ def silent_except_hook(exctype, excvalue, exctraceback): if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, - MarketplaceSubscriptionExpired, + MarketplaceSubscriptionExpired, MarketplaceJSONError, MarketplaceWalletNotSupported, MarketplaceEmptySignature]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] @@ -79,3 +79,10 @@ class MarketplaceEmptySignature(ZiplineError): msg = ( 'Signature cannot be empty.' ) + + +class MarketplaceJSONError(ZiplineError): + msg = ( + 'The configuration file {file} is malformed. Please correct ' + 'the following error:\n{error}' + ) diff --git a/catalyst/marketplace/utils/eth_utils.py b/catalyst/marketplace/utils/eth_utils.py index 35e8b300..9a630496 100644 --- a/catalyst/marketplace/utils/eth_utils.py +++ b/catalyst/marketplace/utils/eth_utils.py @@ -48,3 +48,35 @@ def bin_hex(binary): """ return binascii.hexlify(binary).decode('utf-8') + + +def from_grains(amount): + """ + Convert from grains to cryptocurrency + + Parameters + ---------- + input: amount + + Returns + ------- + int + + """ + return amount // 10 ** 8 + + +def to_grains(amount): + """ + Convert from cryptocurrency to grains + + Parameters + ---------- + input: amount + + Returns + ------- + int + + """ + return amount * 10 ** 8 diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index 6565b49d..fd4ee663 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -4,6 +4,7 @@ import tarfile from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory +from catalyst.marketplace.marketplace_errors import MarketplaceJSONError def get_marketplace_folder(environ=None): @@ -123,7 +124,10 @@ def get_user_pubaddr(environ=None): if os.path.isfile(filename): with open(filename) as data_file: - data = json.load(data_file) + try: + data = json.load(data_file) + except json.decoder.JSONDecodeError as e: + raise MarketplaceJSONError(file=filename, error=e) try: d = data[0]['pubAddr'] except Exception as e: From 866b92910fc1bd45931931a845a9c8a97eaf9dfb Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 13:10:07 -0500 Subject: [PATCH 067/117] BLD: fixed the marketplace api in the algo runtime --- catalyst/api.pyi | 11 ++- .../examples/mean_reversion_by_marketcap.py | 6 +- catalyst/exchange/exchange_algorithm.py | 7 +- catalyst/marketplace/marketplace.py | 71 +++++++------------ 4 files changed, 34 insertions(+), 61 deletions(-) diff --git a/catalyst/api.pyi b/catalyst/api.pyi index 6e46aba9..57dd75a7 100644 --- a/catalyst/api.pyi +++ b/catalyst/api.pyi @@ -816,18 +816,15 @@ def symbols(*args): """ -def get_data_source(data_source_name, data_frequency=None, - start=None, end=None): +def get_dataset(ds_name, start=None, end=None): """ Lookup a data source from the marketplace Parameters ---------- - self - data_source_name - data_frequency - start - end + ds_name: str + start: pd.Timestamp + end: pd.Timestamp Returns ------- diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py index 6d31dab7..b8fb94d1 100644 --- a/catalyst/examples/mean_reversion_by_marketcap.py +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -10,7 +10,7 @@ import talib from logbook import Logger from catalyst import run_algorithm -from catalyst.api import symbol, record, order_target_percent, get_data_source +from catalyst.api import symbol, record, order_target_percent, get_dataset from catalyst.exchange.utils.stats_utils import set_print_settings, \ get_pretty_stats # We give a name to the algorithm which Catalyst will use to persist its state. @@ -33,8 +33,8 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - df = get_data_source( - 'marketcap', start=context.datetime + df = get_dataset( + 'marketcap1234', start=context.datetime ) # type: pd.DataFrame # Keep only the top coins by market cap diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 901f4ac3..da4aee11 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -171,13 +171,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): return round_nearest(amount, asset.min_trade_size) @api_method - def get_data_source(self, data_source_name, data_frequency=None, - start=None, end=None): + def get_dataset(self, data_source_name, start=None, end=None): if self._marketplace is None: self._marketplace = Marketplace() - return self._marketplace.get_data_source( - data_source_name, data_frequency, start, end, + return self._marketplace.get_dataset( + data_source_name, start, end, ) @api_method diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ab35fe65..474aa545 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -147,13 +147,13 @@ class Marketplace: 'Gas Limit:\t\t{gas}\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'],) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -174,15 +174,6 @@ class Marketplace: print('\nYou can check the outcome of your transaction here:\n' '{}\n\n'.format(etherscan)) - def get_data_source_def(self, data_source_name): - data_source_name = data_source_name.lower() - dsm = self.get_data_sources_map() - - ds = six.next( - (d for d in dsm if d['name'].lower() == data_source_name), None - ) - return ds - def list(self): data_sources = self.mkt_contract.functions.getAllProviders().call() @@ -191,11 +182,11 @@ class Marketplace: for index, data_source in enumerate(data_sources): if index > 0: # if 'test' not in Web3.toText(data_source).lower(): - data.append( - dict( - dataset=self.to_text(data_source) - ) + data.append( + dict( + dataset=self.to_text(data_source) ) + ) df = pd.DataFrame(data) set_print_settings() @@ -229,8 +220,8 @@ class Marketplace: print('\nYou are already subscribed to the "{}" dataset.\n' 'Your subscription started on {} UTC, and is valid until ' '{} UTC.'.format( - dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), - pd.to_datetime(subscribed[4], unit='s', utc=True))) + dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True))) return print('\nThe price for a monthly subscription to this dataset is' @@ -260,14 +251,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -377,7 +368,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -433,11 +424,11 @@ class Marketplace: if not check_sub[5]: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' - 'catalyst marketplace subscribe --dataset={}'. format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + 'catalyst marketplace subscribe --dataset={}'.format( + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -485,25 +476,11 @@ class Marketplace: log.info('{} ingested successfully'.format(ds_name)) - def get_data_source(self, data_source_name, data_frequency=None, - start=None, end=None): - data_source_name = data_source_name.lower() - - if data_frequency is None: - ds_def = self.get_data_source_def(data_source_name) - freqs = ds_def['data_frequencies'] - data_frequency = freqs[0] - - if len(freqs) > 1: - log.warn( - 'no data frequencies specified for data source {}, ' - 'selected the first one by default: {}'.format( - data_source_name, data_frequency - ) - ) + def get_dataset(self, ds_name, start=None, end=None): + ds_name = ds_name.lower() # TODO: filter ctable by start and end date - bundle_folder = get_bundle_folder(data_source_name, data_frequency) + bundle_folder = get_data_source_folder(ds_name) z = bcolz.ctable(rootdir=bundle_folder, mode='r') df = z.todataframe() # type: pd.DataFrame From 9f9bfc9df094d91c917141408d4f1cabd194eec6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 16:40:43 -0500 Subject: [PATCH 068/117] BLD: dropping dataset index cols to avoid duplicates --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 474aa545..e87ace9e 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -484,7 +484,7 @@ class Marketplace: z = bcolz.ctable(rootdir=bundle_folder, mode='r') df = z.todataframe() # type: pd.DataFrame - df.set_index(['date', 'symbol'], drop=False, inplace=True) + df.set_index(['date', 'symbol'], drop=True, inplace=True) if start and end is None: df = df.xs(start, level=0) From 6e642f45d81dd1fbe59d8ae8e6e5f1b33a73fbb8 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 14:44:06 -0700 Subject: [PATCH 069/117] MAINT: marketplace prompts --- catalyst/marketplace/marketplace.py | 44 ++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e87ace9e..7086b903 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -142,9 +142,11 @@ class Marketplace: print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' 'enter the following parameters:\n\n' 'From Address:\t\t{_from}\n' + '\n\tClick the "Generate Information" button\n\n' 'To Address:\t\t{to}\n' 'Value / Amount to Send:\t{value}\n' 'Gas Limit:\t\t{gas}\n' + 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( _from=from_address, @@ -229,7 +231,7 @@ class Marketplace: print( 'Checking that the ENG balance in {} is greater than {} ' - 'ENG... '.format(address, price) + 'ENG... '.format(address, price), end='' ) wallet_address = address[2:] @@ -338,14 +340,7 @@ class Marketplace: print('Unable to subscribe to data source: {}'.format(e)) return - if 'ropsten' in ETH_REMOTE_NODE: - etherscan = 'https://ropsten.etherscan.io/tx/{}'.format( - tx_hash) - else: - etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash) - - print('You can check the outcome of your transaction here:\n' - '{}'.format(etherscan)) + self.check_transaction(tx_hash) print('Waiting for the second transaction to succeed...') @@ -609,13 +604,36 @@ class Marketplace: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - tx_hash = '0x{}'.format( - bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) - ) - print('\nThis is the TxHash for this transaction: {}'.format(tx_hash)) + + try: + tx_hash = '0x{}'.format( + bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) + ) + print( + '\nThis is the TxHash for this transaction: {}'.format(tx_hash) + ) + + except Exception as e: + print('Unable to subscribe to data source: {}'.format(e)) + return self.check_transaction(tx_hash) + print('Waiting for the transaction to succeed...') + + while True: + try: + if self.web3.eth.getTransactionReceipt(tx_hash).status: + break + else: + print('\nTransaction failed. Aborting...') + return + except AttributeError: + pass + for i in range(0, 10): + print('.', end='', flush=True) + time.sleep(1) + print('\nWarming up the {} dataset'.format(dataset)) self.create_metadata( key=key, From e1abecb556390ea505a78690bc0ec32bc474d790 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 17:09:49 -0700 Subject: [PATCH 070/117] MAINT: updated catalyst logo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6c395b45..a39137fd 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg +.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png :target: https://enigmampc.github.io/catalyst :align: center :alt: Enigma | Catalyst From 1ad26b0c3983c65bff8ddd78529dbc73de8c68ae Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 19:29:38 -0500 Subject: [PATCH 071/117] BLD: trying to format a line --- catalyst/marketplace/marketplace.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 7086b903..b9f05151 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -219,11 +219,15 @@ class Marketplace: ).call() if subscribed[5]: - print('\nYou are already subscribed to the "{}" dataset.\n' - 'Your subscription started on {} UTC, and is valid until ' - '{} UTC.'.format( - dataset, pd.to_datetime(subscribed[3], unit='s', utc=True), - pd.to_datetime(subscribed[4], unit='s', utc=True))) + print( + '\nYou are already subscribed to the "{}" dataset.\n' + 'Your subscription started on {} UTC, and is valid until ' + '{} UTC.'.format( + dataset, + pd.to_datetime(subscribed[3], unit='s', utc=True), + pd.to_datetime(subscribed[4], unit='s', utc=True) + ) + ) return print('\nThe price for a monthly subscription to this dataset is' @@ -604,7 +608,7 @@ class Marketplace: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) signed_tx = self.sign_transaction(address, tx) - + try: tx_hash = '0x{}'.format( bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) From db7b7639a06191be69b99438e63bb996b7144c88 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 17:32:28 -0700 Subject: [PATCH 072/117] MAINT: not listing test datasets --- catalyst/marketplace/marketplace.py | 39 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index b9f05151..498c44b1 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -12,7 +12,6 @@ import bcolz import logbook import pandas as pd import requests -import six from requests_toolbelt import MultipartDecoder from requests_toolbelt.multipart.decoder import \ NonMultipartContentTypeException @@ -149,13 +148,13 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -183,12 +182,12 @@ class Marketplace: data = [] for index, data_source in enumerate(data_sources): if index > 0: - # if 'test' not in Web3.toText(data_source).lower(): - data.append( - dict( - dataset=self.to_text(data_source) + if 'test' not in Web3.toText(data_source).lower(): + data.append( + dict( + dataset=self.to_text(data_source) + ) ) - ) df = pd.DataFrame(data) set_print_settings() @@ -257,14 +256,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -367,7 +366,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -424,10 +423,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] From 18cdd5168015dd39b3cd7cb46219f45744b8a492 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 22:16:32 -0500 Subject: [PATCH 073/117] BUG: fixed issue with order processing --- catalyst/exchange/ccxt/ccxt_exchange.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 7a118a52..9cf6c6f4 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -877,6 +877,7 @@ class CCXT(Exchange): order.commission = exc_order.commission order.filled = exc_order.amount + transactions = [] if exc_order.status == ORDER_STATUS.FILLED: if order.amount > exc_order.amount: log.warn( @@ -898,7 +899,9 @@ class CCXT(Exchange): order_id=order.id, commission=order.commission, ) - return [transaction] + transactions.append(transaction) + + return transactions def process_order(self, order): # TODO: move to parent class after tracking features in the parent From 64e22ba27dcabfaaa1efef5506845d2799506c36 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 7 Feb 2018 22:28:08 -0500 Subject: [PATCH 074/117] BLD: Updated release notes for 5.0 --- docs/source/releases.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 635f2e95..ccce3da0 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,22 @@ Release Notes ============= +Version 0.5.0 +^^^^^^^^^^^^^ +**Release Date**: 2018-02-07 + +Bug Fixes +~~~~~~~~~ +- Fixed an issue with orders that stay open :issue:`211` +- Fixed Jupyter issues :issue:`179` +- Fetching multiple tickers in one call to minimize rate limit risks :issue:`174` +- Improved live state presentation :issue:`171` + + +Build +~~~~~ +- Introducing the Enigma Marketplace + Version 0.4.7 ^^^^^^^^^^^^^ **Release Date**: 2018-01-19 From 7a89cfc02b09223a7598e94bf4814b87a61ed00a Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 7 Feb 2018 21:01:14 -0700 Subject: [PATCH 075/117] BUG: marketplace: balance returns different types --- catalyst/marketplace/marketplace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 498c44b1..50bacb08 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -246,7 +246,10 @@ class Marketplace: ) }) - balance = Web3.toInt(hexstr=balance) + try: + balance = Web3.toInt(balance) # web3 >= 4.0.0b7 + except TypeError: + balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 if balance > grains: print('OK.') From d2f9762fbf062e569b213edc99240e8de5083380 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:02:40 -0500 Subject: [PATCH 076/117] BLD: adjusting the marketplace sample algo --- .../examples/mean_reversion_by_marketcap.py | 10 +++-- catalyst/marketplace/marketplace.py | 37 ++++++++++--------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/mean_reversion_by_marketcap.py index b8fb94d1..95253a87 100644 --- a/catalyst/examples/mean_reversion_by_marketcap.py +++ b/catalyst/examples/mean_reversion_by_marketcap.py @@ -33,10 +33,14 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - df = get_dataset( - 'marketcap1234', start=context.datetime - ) # type: pd.DataFrame + df = get_dataset('testmarketcap2') # type: pd.DataFrame + # Picking a specific date in our DataFrame + first_dt = df.index.get_level_values(0)[0] + # Since we use a MultiIndex with date / symbol, picking a date will + # result in a new DataFrame for the selected date with a single + # symbol index + df = df.xs(first_dt, level=0) # Keep only the top coins by market cap df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))] diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 50bacb08..a1ac263c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -148,13 +148,13 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -247,9 +247,9 @@ class Marketplace: }) try: - balance = Web3.toInt(balance) # web3 >= 4.0.0b7 + balance = Web3.toInt(balance) # web3 >= 4.0.0b7 except TypeError: - balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 + balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6 if balance > grains: print('OK.') @@ -259,14 +259,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -369,7 +369,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -426,10 +426,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -487,8 +487,9 @@ class Marketplace: df = z.todataframe() # type: pd.DataFrame df.set_index(['date', 'symbol'], drop=True, inplace=True) - if start and end is None: - df = df.xs(start, level=0) + # TODO: implement the filter more carefully + # if start and end is None: + # df = df.xs(start, level=0) return df From 25f1f6e641c6c2dd415ffa144379a64f1e28ab3c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:18:48 -0500 Subject: [PATCH 077/117] BLD: upgraded CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 795c320d..4232bdbb 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.944 + - ccxt==1.10.1049 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index 0789cdef..7c5f2e62 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.944 +ccxt==1.10.1049 boto3==1.4.8 redo==1.6 From e2488317192d47cf8cf276772d4e5a768ec46643 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:27:56 -0500 Subject: [PATCH 078/117] BLD: adjusted sample algo --- catalyst/examples/mean_reversion_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 3b94d925..81a3b182 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -38,7 +38,7 @@ def initialize(context): context.current_day = None context.RSI_OVERSOLD = 40 - context.RSI_OVERBOUGHT = 50 + context.RSI_OVERBOUGHT = 60 context.CANDLE_SIZE = '15T' context.start_time = time.time() @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='eth', live_graph=False, - simulate_orders=True, + simulate_orders=False, stats_output=None, # auth_aliases=dict(poloniex='auth2') ) From 793b6e92d8c1d58aca9838b1aba579ad1fcdaa6c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 00:47:25 -0500 Subject: [PATCH 079/117] BLD: included marketplace dependencies --- etc/python2.7-environment.yml | 2 ++ etc/requirements.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 4232bdbb..4f06bbae 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -21,6 +21,8 @@ dependencies: - bottleneck==1.2.1 - chardet==3.0.4 - ccxt==1.10.1049 + - web3==4.0.0b7 + - requests-toolbelt==0.8.0 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index 7c5f2e62..5d890d2e 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -84,3 +84,5 @@ tables==3.3.0 ccxt==1.10.1049 boto3==1.4.8 redo==1.6 +web3==4.0.0b7 +requests-toolbelt==0.8.0 From 43a5f8858b6e743ec2bdd2c0d15e20ee5f382810 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 01:11:09 -0500 Subject: [PATCH 080/117] DOC: updated release number --- docs/source/releases.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index ccce3da0..9289d56a 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,7 +2,7 @@ Release Notes ============= -Version 0.5.0 +Version 0.5.1 ^^^^^^^^^^^^^ **Release Date**: 2018-02-07 From 82d318a1c0237216c32ac06c579d1ccebd276f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 15:51:55 -0500 Subject: [PATCH 081/117] BUG: fixed issue #216 with bad candle data --- catalyst/exchange/ccxt/ccxt_exchange.py | 20 +- catalyst/support/issue_216.py | 376 ++++++++++++++++++++++++ tests/exchange/test_ccxt.py | 9 +- 3 files changed, 392 insertions(+), 13 deletions(-) create mode 100644 catalyst/support/issue_216.py diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 9cf6c6f4..c5fee495 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -425,15 +425,12 @@ class CCXT(Exchange): 'Please provide either start_dt or end_dt, not both.' ) - elif end_dt is not None: - # Make sure that end_dt really wants data in the past - # if it's close to now, we skip the 'since' parameters to - # lower the probability of error - bars_to_now = pd.date_range( - end_dt, pd.Timestamp.utcnow(), freq=freq - ) - # See: https://github.com/ccxt/ccxt/issues/1360 - if len(bars_to_now) > 1 or self.name in ['poloniex']: + if start_dt is None: + # TODO: determine why binance is failing + if end_dt is None and self.name not in ['binance']: + end_dt = pd.Timestamp.utcnow() + + if end_dt is not None: dt_range = get_periods_range( end_dt=end_dt, periods=bar_count, @@ -441,10 +438,13 @@ class CCXT(Exchange): ) start_dt = dt_range[0] - since = None if start_dt is not None: + # Convert out start date to a UNIX timestamp, then translate to + # milliseconds delta = start_dt - get_epoch() since = int(delta.total_seconds()) * 1000 + else: + since = None candles = dict() for index, asset in enumerate(assets): diff --git a/catalyst/support/issue_216.py b/catalyst/support/issue_216.py new file mode 100644 index 00000000..8a038da0 --- /dev/null +++ b/catalyst/support/issue_216.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python2 + +import sys +import os +import pandas as pd +import signal +# import talib + +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import ( + symbol, + record, + order, + order_target, + order_target_percent, + get_open_orders +) +from catalyst.finance import commission + + +# from base.telegrambot import TelegramBot + + +class GracefulKiller: + # Source: https://stackoverflow.com/a/31464349 + def __init__(self, context): + self.kill_now = False + self.signal = 0 + self.context = context + signal.signal(signal.SIGINT, self.exit_gracefully) + + def exit_gracefully(self, signum, frame): + self.kill_now = True + self.signal = signum + if hasattr(self.context, + 'telegram_bot') and self.context.telegram_bot is not None: + self.context.telegram_bot.updater.stop() + sys.exit(0) + + def exit(self): + return self.kill_now + + +class SimulationParameters: + MODE = 'paper' + CAPITAL_BASE = 1000 + """ + Capital base used on this simulation + """ + + DATA_FREQUECY = 'minute' + + EXCHANGE_NAME = 'bitfinex' + # EXCHANGE_NAME = 'binance' + """ + Exchange used on this simulation + """ + + DATA_DIR = '/home/av/Dropbox/simulations/data' + ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0] + ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images', + ALGO_NAMESPACE) + ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables', + ALGO_NAMESPACE + '_results') + ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, + 'tables', + ALGO_NAMESPACE + '_transactions') + BASE_CURRENCY = 'usd' + # BASE_CURRENCY = 'usdt' + + # SHORT PERIOD + START_DATE = '2017-09-07' + """ + Start date used on this simulation + """ + END_DATE = '2017-12-12' + """ + End date used on this simulation + """ + + SKIP_FIRST_CANDLES = 0 + + # CANDLES_SAMPLE_RATE = 60 + # CANDLES_SAMPLE_RATE = 30 + CANDLES_SAMPLE_RATE = 1 + """ + Candle interval used on this simulation (in minutes) + """ + + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + # 30 minute interval ohlcv data (the standard data required for candlestick or + # indicators/signals) + # 30T means 30 minutes re-sampling of one minute data. + # CANDLES_FREQUENCY = '60T' + # CANDLES_FREQUENCY = '30T' + CANDLES_FREQUENCY = '1T' + CANDLES_BUFFER_SIZE = 48 + COIN_PAIR = 'btc_usd' + # COIN_PAIR = 'btc_usdt' + """ + Coin pair used on this simulation + """ + + # TRANSACTIONS + COMMISSION_FEE = 0.0030 + BUY_MIN_AMOUNT = 5 # i.e: USD + SELL_MIN_AMOUNT = 0.001 # i.e: USD + BUY_SELL_PERCENTAGE = 1 # 0.50 + BUY_PERCENTAGE = BUY_SELL_PERCENTAGE + SELL_PERCENTAGE = BUY_SELL_PERCENTAGE + + BASE_PRICE = 'close' + """ + Base price used (close / Heiken Ashi) + """ + + +log = None +parameters = None + + +def print_facts(context): + context.log.info(""" +Index: {} +Date: {} +Candle: +O: {} +H: {} +L: {} +C: {} +V: {} +Metrics: +... +Portfolio: +Base price: {} +Base coin (coin2/usd): {} +Amount (coin1/btc): {} +""".format( + # Facts + context.i, + context.curr_minute, + context.candles_open[-1], + context.candles_high[-1], + context.candles_low[-1], + context.candles_close[-1], + context.candles_volume[-1], + # Metrics + # ... + # Portfolio + context.curr_base_price, + context.portfolio.cash, + context.portfolio.positions[context.coin_pair].amount, + )) + + +def print_facts_telegram(context): + price = context.curr_base_price + amount = context.portfolio.positions[context.coin_pair].amount + pnl = context.portfolio.pnl + capital_used = context.portfolio.capital_used + portfolio_value = context.portfolio.portfolio_value + portfolio_returns = context.portfolio.returns + starting_cash = context.portfolio.starting_cash + cash = context.portfolio.cash + + msg = """ +Status... +Price: {} +Starting cash: {} +Cash: {} +Capital used: {} +Amount: {} +Portfolio value: {} +Returns: {} +PnL: {} + """.format( + price, + starting_cash, + cash, + capital_used, + amount, + portfolio_value, + portfolio_returns, + pnl, + ) + if hasattr(context, 'telegram_bot') and context.telegram_bot is not None: + context.telegram_bot.msg(msg) + + +def default_initialize(context): + # FIXME: set_benchmark + # set_benchmark(symbol(context.parameters.COIN_PAIR)) + + context.coin_pair = symbol(context.parameters.COIN_PAIR) + context.base_price = None + context.current_day = None + context.counter = -1 + context.i = 0 + + context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE + context.candles_frequency = context.parameters.CANDLES_FREQUENCY + context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE + context.set_commission( + commission.PerShare(cost=context.parameters.COMMISSION_FEE)) + + +def default_handle_data(context, data): + context.curr_minute = data.current_dt + context.counter += 1 + + if context.candles_sample_rate == 1: + context.i += 1 + elif context.counter % context.candles_sample_rate != 0: + context.i += 1 + return + + if context.i < context.parameters.SKIP_FIRST_CANDLES: + return + + context.candles_open = data.history( + context.coin_pair, + 'open', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_high = data.history( + context.coin_pair, + 'high', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_low = data.history( + context.coin_pair, + 'low', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_close = data.history( + context.coin_pair, + 'price', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_volume = data.history( + context.coin_pair, + 'volume', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + + # FIXME: Here is the error! + # The candles_close frame shows more or less always a value of 94, while + # bitcoin price is very different from that + print(context.candles_close) + + context.base_prices = context.candles_close + cash = context.portfolio.cash + amount = context.portfolio.positions[context.coin_pair].amount + price = data.current(context.coin_pair, 'price') + order_id = None + context.last_base_price = context.base_prices[-2] + context.curr_base_price = context.base_prices[-1] + + # TA calculations + # ... + + # Sanity checks + # assert cash >= 0 + if cash < 0: + import ipdb; + ipdb.set_trace() # BREAKPOINT + + print_facts(context) + print_facts_telegram(context) + + # Order management + net_shares = 0 + if context.counter == 2: + brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = brute_shares - share_commission_fee + buy_order_id = order(context.coin_pair, net_shares) + + if context.counter == 3: + brute_shares = amount * context.parameters.SELL_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = -(brute_shares - share_commission_fee) + sell_order_id = order(context.coin_pair, net_shares) + + # Record + record( + price=price, + foo='bar', + # volume=current['volume'], + # price_change=price_change, + # Metrics + cash=cash, + # buy=context.buy, + # sell=context.sell + ) + + +def default_analyze(context=None, perf=None): + pass + + +def initialize(context): + global log + context.parameters = parameters + context.log = Logger(context.parameters.ALGO_NAMESPACE) + log = context.log + default_initialize(context) + context.killer = GracefulKiller(context) + context.telegram_bot = None + + # TELEGRAM_TOKEN='token' + # context.telegram_bot = TelegramBot() + # context.telegram_bot.initialize(TELEGRAM_TOKEN, context) + + +if __name__ == '__main__': + # Parameters: + parameters = SimulationParameters() + start_date = pd.to_datetime(parameters.START_DATE, utc=True) + end_date = pd.to_datetime(parameters.END_DATE, utc=True) + + if parameters.MODE == 'backtest': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + start=start_date, + end=end_date, + live=False, + live_graph=False + ) + + returns_daily = results + results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE)) + + # returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1) + + # FIXME: pyfolio integration + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results) + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01']) + # pyfolio.create_full_tear_sheet(*pf_data) + + elif parameters.MODE == 'paper': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + simulate_orders=True, + live_graph=False + ) + + elif parameters.MODE == 'live': + results = run_algorithm( + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + live_graph=True + ) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index af455cc4..38323347 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -1,8 +1,7 @@ import pandas as pd from logbook import Logger -from catalyst.testing import ZiplineTestCase -from catalyst.testing.fixtures import WithLogger +from catalyst.exchange.utils.stats_utils import set_print_settings from .base import BaseExchangeTestCase from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange_execution import ExchangeLimitOrder @@ -61,12 +60,16 @@ class TestCCXT(BaseExchangeTestCase): freq='30T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, - start_dt=pd.to_datetime('2017-09-01', utc=True) + # start_dt=pd.to_datetime('2017-09-01', utc=True), ) for asset in candles: df = pd.DataFrame(candles[asset]) df.set_index('last_traded', drop=True, inplace=True) + + set_print_settings() + print(df.head(10)) + print(df.tail(10)) pass def test_tickers(self): From a820f66bdc3e032675b624b5cb81143a64a2f99c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 16:57:23 -0500 Subject: [PATCH 082/117] BLD: adjusted unit test --- tests/exchange/test_ccxt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 38323347..16fe944f 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -14,7 +14,7 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'binance' + exchange_name = 'bittrex' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, @@ -57,7 +57,7 @@ class TestCCXT(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') candles = self.exchange.get_candles( - freq='30T', + freq='1T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, # start_dt=pd.to_datetime('2017-09-01', utc=True), @@ -68,6 +68,7 @@ class TestCCXT(BaseExchangeTestCase): df.set_index('last_traded', drop=True, inplace=True) set_print_settings() + print('got {} candles'.format(len(df))) print(df.head(10)) print(df.tail(10)) pass From 00f232e2d7426b8593295ee3e4e6488a39a4866b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:10:41 -0500 Subject: [PATCH 083/117] BUG: fixed sample algo --- catalyst/examples/dual_moving_average.py | 63 ++++++++++++++---------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index ff5dfc5e..363edba1 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -20,8 +20,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 50 - long_window = 200 + short_window = 2 + long_window = 2 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -32,16 +32,18 @@ def handle_data(context, data): # moving average with the appropriate parameters. We choose to use # minute bars for this simulation -> freq="1m" # Returns a pandas dataframe. - short_mavg = data.history(context.asset, + short_data = data.history(context.asset, 'price', bar_count=short_window, - frequency="1m", - ).mean() - long_mavg = data.history(context.asset, + frequency="1T", + ) + short_mavg = short_data.mean() + long_data = data.history(context.asset, 'price', bar_count=long_window, - frequency="1m", - ).mean() + frequency="1T", + ) + long_mavg = long_data.mean() # Let's keep the price of our asset in a more handy variable price = data.current(context.asset, 'price') @@ -82,7 +84,6 @@ def handle_data(context, data): def analyze(context, perf): - # Get the base_currency that was passed as a parameter to the simulation exchange = list(context.exchanges.values())[0] base_currency = exchange.base_currency.upper() @@ -93,7 +94,7 @@ def analyze(context, perf): ax1.legend_.remove() ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) # Second chart: Plot asset price, moving averages and buys/sells ax2 = plt.subplot(412, sharex=ax1) @@ -104,9 +105,9 @@ def analyze(context, perf): ax2.set_ylabel('{asset}\n({base})'.format( asset=context.asset.symbol, base=base_currency - )) + )) start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) transaction_df = extract_transactions(perf) if not transaction_df.empty: @@ -136,28 +137,40 @@ def analyze(context, perf): ax3.legend_.remove() ax3.set_ylabel('Percent Change') start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) # Fourth chart: Plot our cash ax4 = plt.subplot(414, sharex=ax1) perf.cash.plot(ax=ax4) ax4.set_ylabel('Cash\n({})'.format(base_currency)) start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) + ax4.yaxis.set_ticks(np.arange(0, end, end / 5)) plt.show() if __name__ == '__main__': run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + simulate_orders=True, + live=True, + ) + # run_algorithm( + # capital_base=1000, + # data_frequency='minute', + # initialize=initialize, + # handle_data=handle_data, + # analyze=analyze, + # exchange_name='bitfinex', + # algo_namespace=NAMESPACE, + # base_currency='usd', + # start=pd.to_datetime('2017-9-22', utc=True), + # end=pd.to_datetime('2017-9-23', utc=True), + # ) From e56e1f8e21188df373182a8afcec5cdad97d0cfa Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:26:48 -0500 Subject: [PATCH 084/117] BUG: fixed an issue with open orders --- catalyst/examples/buy_low_sell_high.py | 6 +++--- catalyst/exchange/exchange_algorithm.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index 7dc95b5b..075e2f71 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -60,7 +60,7 @@ def _handle_data(context, data): rsi=rsi, ) - orders = get_open_orders(context.asset) + orders = context.blotter.open_orders if orders: log.info('skipping bar until all open orders execute') return @@ -146,11 +146,11 @@ if __name__ == '__main__': live = True if live: run_algorithm( - capital_base=0.001, + capital_base=1000, initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='binance', + exchange_name='bittrex', live=True, algo_namespace=algo_namespace, base_currency='btc', diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index da4aee11..aeff9e4e 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -391,8 +391,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.warn("Can't initialize signal handler inside another thread." "Exit should be handled by the user.") - log.info('initialized trading algorithm in live mode') - def interrupt_algorithm(self): self.is_running = False @@ -874,6 +872,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): raise NotImplementedError() def _get_open_orders(self, asset=None): + if self.simulate_orders: + raise ValueError( + 'The get_open_orders() method only works in live mode. ' + 'The purpose is to list open orders on the exchange ' + 'regardless who placed them. To list the open orders of ' + 'this algo, use `context.blotter.open_orders`.' + ) if asset: exchange = self.exchanges[asset.exchange] return exchange.get_open_orders(asset) @@ -907,6 +912,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): If an asset is passed then this will return a list of the open orders for this asset. """ + # TODO: should this be a shortcut to the open orders in the blotter? return retry( action=self._get_open_orders, attempts=self.attempts['get_open_orders_attempts'], From 09ad397ee6c711b7531ab24ce642ba8b78159707 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:31:09 -0500 Subject: [PATCH 085/117] DOC: adjusted the release notes --- docs/source/releases.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 9289d56a..7c5dca1d 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,14 @@ Release Notes ============= +Version 0.5.2 +^^^^^^^^^^^^^ +**Release Date**: 2018-02-08 + +Bug Fixes +~~~~~~~~~ +- Fixed an issue with live candle values :issue:`216` and :issue:`199` + Version 0.5.1 ^^^^^^^^^^^^^ **Release Date**: 2018-02-07 From bebebc46dccca405037f981f9ad21e413e03b156 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 8 Feb 2018 22:07:41 -0700 Subject: [PATCH 086/117] DOC: small edits, eliminating sphinx warnings --- README.rst | 4 +--- docs/source/videos.rst | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index a39137fd..7bb7361c 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,7 @@ insights regarding a particular strategy's performance. Catalyst also supports live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, and Poloniex) with more being added over time. Catalyst empowers users to share and curate data and build profitable, data-driven investment strategies. Please -visit `enigma.co `_ to learn more about Catalyst, or -refer to the `whitepaper `_ for -further technical details. +visit `enigma.co `_ to learn more about Catalyst. Catalyst builds on top of the well-established `Zipline `_ project. We did our best to diff --git a/docs/source/videos.rst b/docs/source/videos.rst index 1db8ff28..0beb291b 100644 --- a/docs/source/videos.rst +++ b/docs/source/videos.rst @@ -11,6 +11,7 @@ Installation: MacOS | | + Installation: Windows --------------------- @@ -21,6 +22,7 @@ Where things go smoothly: | + Where things don't: .. raw:: html @@ -29,6 +31,7 @@ Where things don't: | | + Backtesting a Strategy ---------------------- @@ -44,6 +47,7 @@ sell. Hopefully, we’ll ride the waves. | | + Live Trading a Strategy ----------------------- @@ -54,5 +58,6 @@ in the previous video, we now take it to trade live against the Bittrex exchange .. raw:: html + +| | -| \ No newline at end of file From 44e4e66f5cb7774ac279d0d240401f6d5dc06d03 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Feb 2018 00:56:35 -0500 Subject: [PATCH 087/117] BLD: adding more log messages --- .../exchange/test_suites/test_suite_bundle.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 557d0744..ada0dcfd 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -12,6 +12,7 @@ from catalyst.exchange.utils.exchange_utils import get_candles_df from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.utils.test_utils import output_df, \ select_random_assets +from catalyst.exchange.utils.stats_utils import set_print_settings pd.set_option('display.expand_frame_repr', False) pd.set_option('precision', 8) @@ -58,6 +59,12 @@ class TestSuiteBundle: log_catcher = TestHandler() with log_catcher: + symbols = [asset.symbol for asset in assets] + print( + 'comparing data for {}/{} with {} timeframe until {}'.format( + exchange.name, symbols, freq, end_dt + ) + ) data['bundle'] = data_portal.get_history_window( assets=assets, end_dt=end_dt, @@ -66,6 +73,12 @@ class TestSuiteBundle: field='close', data_frequency=data_frequency, ) + set_print_settings() + print( + 'the bundle first / last row:\n{}'.format( + data['bundle'].iloc[[-1, 0]] + ) + ) candles = exchange.get_candles( end_dt=end_dt, freq=freq, @@ -79,6 +92,11 @@ class TestSuiteBundle: bar_count=bar_count, end_dt=end_dt, ) + print( + 'the exchange first / last row:\n{}'.format( + data['exchange'].iloc[[-1, 0]] + ) + ) for source in data: df = data[source] path, folder = output_df( @@ -126,6 +144,8 @@ class TestSuiteBundle: frequencies = exchange.get_candle_frequencies(data_frequency) freq = random.sample(frequencies, 1)[0] + freq = '15T' + bar_count = random.randint(1, 10) assets = select_random_assets( From 5c77cfc68d2dc153067f56faa9205adb5b84c030 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Feb 2018 01:38:22 -0500 Subject: [PATCH 088/117] BUG: for issue #219, fixed a resampling issue --- catalyst/exchange/exchange_bundle.py | 41 ++++++++----------- catalyst/exchange/exchange_data_portal.py | 2 - catalyst/exchange/utils/exchange_utils.py | 2 +- .../exchange/test_suites/test_suite_bundle.py | 2 + 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 38aed7c7..569fa6e5 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -458,7 +458,7 @@ class ExchangeBundle: last_entry = None if start is None or \ - (earliest_trade is not None and earliest_trade > start): + (earliest_trade is not None and earliest_trade > start): start = earliest_trade if last_entry is not None and (end is None or end > last_entry): @@ -600,14 +600,14 @@ class ExchangeBundle: if show_breakdown: for asset in chunks: with maybe_show_progress( - chunks[asset], - show_progress, - label='Ingesting {frequency} price data for ' - '{symbol} on {exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - symbol=asset.symbol - )) as it: + chunks[asset], + show_progress, + label='Ingesting {frequency} price data for ' + '{symbol} on {exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + symbol=asset.symbol + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -625,13 +625,13 @@ class ExchangeBundle: key=lambda chunk: pd.to_datetime(chunk['period']) ) with maybe_show_progress( - all_chunks, - show_progress, - label='Ingesting {frequency} price data on ' - '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - )) as it: + all_chunks, + show_progress, + label='Ingesting {frequency} price data on ' + '{exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -830,7 +830,6 @@ class ExchangeBundle: field, data_frequency, algo_end_dt=None, - trailing_bar_count=None, force_auto_ingest=False ): """ @@ -858,7 +857,6 @@ class ExchangeBundle: bar_count=bar_count, field=field, data_frequency=data_frequency, - trailing_bar_count=trailing_bar_count, ) return pd.DataFrame(series) @@ -887,7 +885,6 @@ class ExchangeBundle: field=field, data_frequency=data_frequency, reset_reader=True, - trailing_bar_count=trailing_bar_count, ) return series @@ -898,7 +895,6 @@ class ExchangeBundle: bar_count=bar_count, field=field, data_frequency=data_frequency, - trailing_bar_count=trailing_bar_count, ) return pd.DataFrame(series) @@ -962,12 +958,7 @@ class ExchangeBundle: bar_count, field, data_frequency, - trailing_bar_count=None, reset_reader=False): - if trailing_bar_count: - delta = get_delta(trailing_bar_count, data_frequency) - end_dt += delta - start_dt = get_start_dt(end_dt, bar_count, data_frequency, False) start_dt, _ = self.get_adj_dates( start_dt, end_dt, assets, data_frequency diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index 511bba69..8f9665dc 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -298,7 +298,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): frequency, data_frequency ) adj_bar_count = candle_size * bar_count - trailing_bar_count = candle_size - 1 if data_frequency == 'minute' and adj_data_frequency == 'daily': end_dt = end_dt.floor('1D') @@ -310,7 +309,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): field=field, data_frequency=adj_data_frequency, algo_end_dt=self._last_available_session, - trailing_bar_count=trailing_bar_count, ) df = resample_history_df(pd.DataFrame(series), freq, field) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 3c87b510..132845bb 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -540,7 +540,7 @@ def resample_history_df(df, freq, field): else: raise ValueError('Invalid field.') - resampled_df = df.resample(freq).agg(agg) + resampled_df = df.resample(freq, closed='left', label='left').agg(agg) return resampled_df diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index ada0dcfd..33ed7c2f 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -2,6 +2,7 @@ import random import os import pandas as pd +from datetime import timedelta from logbook import TestHandler from pandas.util.testing import assert_frame_equal @@ -159,6 +160,7 @@ class TestSuiteBundle: if end_dt is None or asset_end_dt < end_dt: end_dt = asset_end_dt + end_dt = end_dt + timedelta(minutes=3) dt_range = pd.date_range( end=end_dt, periods=bar_count, freq=freq ) From 262dffd4bf9101571458c561a73d4f5a70ef5e90 Mon Sep 17 00:00:00 2001 From: westurner <@westurner> Date: Fri, 9 Feb 2018 01:46:44 -0500 Subject: [PATCH 089/117] DOC: live-trading.rst: add newline before ul --- docs/source/live-trading.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index a2898d61..d0d7f721 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -38,6 +38,7 @@ that you should test any new algorithm. Once you are confident with the simulations that you have obtained with your algorithm in backtesting, you may switch to live trading, where you have two different modes: + * *Paper Trading*: The simulated algorithm runs in real time, and fetches pricing data in real time from the exchange, but the orders never reach the exchange, and are instead kept within Catalyst and simulated. No real currency From dcdb3d9d7ae608ead3029703da81a5279d18121e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 00:07:32 -0700 Subject: [PATCH 090/117] MAINT: removing many warnings from building docs --- catalyst/algorithm.py | 10 +- catalyst/utils/run_algo.py | 14 +- docs/source/appendix.rst | 352 ++++++++++++------------- docs/source/beginner-tutorial.rst | 12 +- docs/source/conf.py | 7 +- docs/source/development-guidelines.rst | 24 +- docs/source/example-algos.rst | 1 + docs/source/index.rst | 2 + docs/source/install.rst | 3 +- docs/source/live-trading.rst | 2 +- etc/requirements_dev.txt | 2 +- etc/requirements_docs.txt | 2 +- 12 files changed, 211 insertions(+), 220 deletions(-) diff --git a/catalyst/algorithm.py b/catalyst/algorithm.py index 1cf9476a..2a3cfbf6 100644 --- a/catalyst/algorithm.py +++ b/catalyst/algorithm.py @@ -939,7 +939,7 @@ class TradingAlgorithm(object): The field to query. The options have the following meanings: arena : str The arena from the simulation parameters. This will normally - be ``'backtest'`` but some systems may use this distinguish + be ``backtest`` but some systems may use this distinguish live trading from backtesting. data_frequency : {'daily', 'minute'} data_frequency tells the algorithm if it is running with @@ -954,7 +954,7 @@ class TradingAlgorithm(object): The platform that the code is running on. By default this will be the string 'catalyst'. This can allow algorithms to know if they are running on the Quantopian platform instead. - * : dict[str -> any] + \* : dict[str -> any] Returns all of the fields in a dictionary. Returns @@ -1032,7 +1032,7 @@ class TradingAlgorithm(object): argument is the name of the column in the preprocessed dataframe containing the symbols. This will be used along with the date information to map the sids in the asset finder. - **kwargs + \*\*kwargs Forwarded to :func:`pandas.read_csv`. Returns @@ -1156,7 +1156,7 @@ class TradingAlgorithm(object): Parameters ---------- - **kwargs + \*\*kwargs The names and values to record. Notes @@ -1273,7 +1273,7 @@ class TradingAlgorithm(object): Parameters ---------- - *args : iterable[str] + \*args : iterable[str] The ticker symbols to lookup. Returns diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index a327e8a5..2e9981fa 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -55,6 +55,7 @@ class _RunAlgoError(click.ClickException, ValueError): ---------- pyfunc_msg : str The message that will be shown when called as a python function. + cmdline_msg : str The message that will be shown on the command line. """ @@ -416,7 +417,8 @@ def run_algorithm(initialize, auth_aliases=None, stats_output=None, output=os.devnull): - """Run a trading algorithm. + """ + Run a trading algorithm. Parameters ---------- @@ -458,7 +460,7 @@ def run_algorithm(initialize, This argument is mutually exclusive with ``data``. default_extension : bool, optional Should the default catalyst extension be loaded. This is found at - ``$ZIPLINE_ROOT/extension.py`` + ``$CATALYST_ROOT/extension.py`` extensions : iterable[str], optional The names of any other extensions to load. Each element may either be a dotted module path like ``a.b.c`` or a path to a python file ending @@ -469,12 +471,8 @@ def run_algorithm(initialize, environ : mapping[str -> str], optional The os environment to use. Many extensions use this to get parameters. This defaults to ``os.environ``. - live: execute live trading - exchange_conn: The exchange connection parameters - - Supported Exchanges - ------------------- - bitfinex + live : bool, optional + Execute algorithm in live trading mode. Returns ------- diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst index f5ad6501..6b3590a7 100644 --- a/docs/source/appendix.rst +++ b/docs/source/appendix.rst @@ -4,7 +4,7 @@ API Reference Running a Backtest ~~~~~~~~~~~~~~~~~~ -.. autofunction:: zipline.run_algorithm(...) +.. autofunction:: catalyst.run_algorithm(...) Algorithm API ~~~~~~~~~~~~~ @@ -18,341 +18,335 @@ currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance. Data Object ``````````` -.. autoclass:: zipline.protocol.BarData +.. autoclass:: catalyst.protocol.BarData :members: Scheduling Functions ```````````````````` -.. autofunction:: zipline.api.schedule_function +.. autofunction:: catalyst.api.schedule_function -.. autoclass:: zipline.api.date_rules +.. autoclass:: catalyst.api.date_rules :members: :undoc-members: -.. autoclass:: zipline.api.time_rules +.. autoclass:: catalyst.api.time_rules :members: Orders `````` -.. autofunction:: zipline.api.order +.. autofunction:: catalyst.api.order -.. autofunction:: zipline.api.order_value +.. autofunction:: catalyst.api.order_value -.. autofunction:: zipline.api.order_percent +.. autofunction:: catalyst.api.order_percent -.. autofunction:: zipline.api.order_target +.. autofunction:: catalyst.api.order_target -.. autofunction:: zipline.api.order_target_value +.. autofunction:: catalyst.api.order_target_value -.. autofunction:: zipline.api.order_target_percent +.. autofunction:: catalyst.api.order_target_percent -.. autoclass:: zipline.finance.execution.ExecutionStyle +.. autoclass:: catalyst.finance.execution.ExecutionStyle :members: -.. autoclass:: zipline.finance.execution.MarketOrder +.. autoclass:: catalyst.finance.execution.MarketOrder -.. autoclass:: zipline.finance.execution.LimitOrder +.. autoclass:: catalyst.finance.execution.LimitOrder -.. autoclass:: zipline.finance.execution.StopOrder +.. autoclass:: catalyst.finance.execution.StopOrder -.. autoclass:: zipline.finance.execution.StopLimitOrder +.. autoclass:: catalyst.finance.execution.StopLimitOrder -.. autofunction:: zipline.api.get_order +.. autofunction:: catalyst.api.get_order -.. autofunction:: zipline.api.get_open_orders +.. autofunction:: catalyst.api.get_open_orders -.. autofunction:: zipline.api.cancel_order +.. autofunction:: catalyst.api.cancel_order Order Cancellation Policies ''''''''''''''''''''''''''' -.. autofunction:: zipline.api.set_cancel_policy +.. autofunction:: catalyst.api.set_cancel_policy -.. autoclass:: zipline.finance.cancel_policy.CancelPolicy +.. autoclass:: catalyst.finance.cancel_policy.CancelPolicy :members: -.. autofunction:: zipline.api.EODCancel +.. autofunction:: catalyst.api.EODCancel -.. autofunction:: zipline.api.NeverCancel +.. autofunction:: catalyst.api.NeverCancel Assets `````` -.. autofunction:: zipline.api.symbol +.. autofunction:: catalyst.api.symbol -.. autofunction:: zipline.api.symbols +.. autofunction:: catalyst.api.symbols -.. autofunction:: zipline.api.future_symbol +.. autofunction:: catalyst.api.set_symbol_lookup_date -.. autofunction:: zipline.api.set_symbol_lookup_date - -.. autofunction:: zipline.api.sid +.. autofunction:: catalyst.api.sid Trading Controls ```````````````` -Zipline provides trading controls to help ensure that the algorithm is +zipline provides trading controls to help ensure that the algorithm is performing as expected. The functions help protect the algorithm from certian bugs that could cause undesirable behavior when trading with real money. -.. autofunction:: zipline.api.set_do_not_order_list +.. autofunction:: catalyst.api.set_do_not_order_list -.. autofunction:: zipline.api.set_long_only +.. autofunction:: catalyst.api.set_long_only -.. autofunction:: zipline.api.set_max_leverage +.. autofunction:: catalyst.api.set_max_leverage -.. autofunction:: zipline.api.set_max_order_count +.. autofunction:: catalyst.api.set_max_order_count -.. autofunction:: zipline.api.set_max_order_size +.. autofunction:: catalyst.api.set_max_order_size -.. autofunction:: zipline.api.set_max_position_size +.. autofunction:: catalyst.api.set_max_position_size Simulation Parameters ````````````````````` -.. autofunction:: zipline.api.set_benchmark +.. autofunction:: catalyst.api.set_benchmark Commission Models ''''''''''''''''' -.. autofunction:: zipline.api.set_commission +.. autofunction:: catalyst.api.set_commission -.. autoclass:: zipline.finance.commission.CommissionModel +.. autoclass:: catalyst.finance.commission.CommissionModel :members: -.. autoclass:: zipline.finance.commission.PerShare +.. autoclass:: catalyst.finance.commission.PerShare -.. autoclass:: zipline.finance.commission.PerTrade +.. autoclass:: catalyst.finance.commission.PerTrade -.. autoclass:: zipline.finance.commission.PerDollar +.. autoclass:: catalyst.finance.commission.PerDollar Slippage Models ''''''''''''''' -.. autofunction:: zipline.api.set_slippage +.. autofunction:: catalyst.api.set_slippage -.. autoclass:: zipline.finance.slippage.SlippageModel +.. autoclass:: catalyst.finance.slippage.SlippageModel :members: -.. autoclass:: zipline.finance.slippage.FixedSlippage +.. autoclass:: catalyst.finance.slippage.FixedSlippage -.. autoclass:: zipline.finance.slippage.VolumeShareSlippage +.. autoclass:: catalyst.finance.slippage.VolumeShareSlippage Pipeline ```````` -For more information, see :ref:`pipeline-api` +Not supported yet. -.. autofunction:: zipline.api.attach_pipeline +.. For more information, see :ref:`pipeline-api` -.. autofunction:: zipline.api.pipeline_output +.. .. autofunction:: catalyst.api.attach_pipeline + +.. .. autofunction:: catalyst.api.pipeline_output Miscellaneous ````````````` -.. autofunction:: zipline.api.record +.. autofunction:: catalyst.api.record -.. autofunction:: zipline.api.get_environment +.. autofunction:: catalyst.api.get_environment -.. autofunction:: zipline.api.fetch_csv +.. autofunction:: catalyst.api.fetch_csv .. _pipeline-api: -Pipeline API -~~~~~~~~~~~~ +.. Pipeline API +.. ~~~~~~~~~~~~ -.. autoclass:: zipline.pipeline.Pipeline - :members: - :member-order: groupwise +.. .. autoclass:: zipline.pipeline.Pipeline +.. :members: +.. :member-order: groupwise -.. autoclass:: zipline.pipeline.CustomFactor - :members: - :member-order: groupwise +.. .. autoclass:: zipline.pipeline.CustomFactor +.. :members: +.. :member-order: groupwise -.. autoclass:: zipline.pipeline.filters.Filter - :members: __and__, __or__ - :exclude-members: dtype +.. .. autoclass:: zipline.pipeline.filters.Filter +.. :members: __and__, __or__ +.. :exclude-members: dtype -.. autoclass:: zipline.pipeline.factors.Factor - :members: bottom, deciles, demean, linear_regression, pearsonr, - percentile_between, quantiles, quartiles, quintiles, rank, - spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq, - __add__, __sub__, __mul__, __div__, __mod__, __pow__, __lt__, - __le__, __ne__, __ge__, __gt__ - :exclude-members: dtype - :member-order: bysource +.. .. autoclass:: zipline.pipeline.factors.Factor +.. :members: bottom, deciles, demean, linear_regression, pearsonr, +.. percentile_between, quantiles, quartiles, quintiles, rank, +.. spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq, +.. \__add__, \__sub__, \__mul__, \__div__, \__mod__, \__pow__, +.. \__lt__, \__le__, \__ne__, \__ge__, \__gt__ +.. :exclude-members: dtype +.. :member-order: bysource -.. autoclass:: zipline.pipeline.term.Term - :members: - :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed +.. .. autoclass:: zipline.pipeline.term.Term +.. :members: +.. :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed -.. autoclass:: zipline.pipeline.data.USEquityPricing - :members: open, high, low, close, volume - :undoc-members: +.. .. autoclass:: zipline.pipeline.data.USEquityPricing +.. :members: open, high, low, close, volume +.. :undoc-members: -Built-in Factors -```````````````` +.. Built-in Factors +.. ```````````````` -.. autoclass:: zipline.pipeline.factors.AverageDollarVolume - :members: +.. .. autoclass:: zipline.pipeline.factors.AverageDollarVolume +.. :members: -.. autoclass:: zipline.pipeline.factors.BollingerBands - :members: +.. .. autoclass:: zipline.pipeline.factors.BollingerBands +.. :members: -.. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent - :members: +.. .. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent +.. :members: -.. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent - :members: +.. .. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent +.. :members: -.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage - :members: +.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage +.. :members: -.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev - :members: +.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev +.. :members: -.. autoclass:: zipline.pipeline.factors.Latest - :members: +.. .. autoclass:: zipline.pipeline.factors.Latest +.. :members: -.. autoclass:: zipline.pipeline.factors.MaxDrawdown - :members: +.. .. autoclass:: zipline.pipeline.factors.MaxDrawdown +.. :members: -.. autoclass:: zipline.pipeline.factors.Returns - :members: +.. .. autoclass:: zipline.pipeline.factors.Returns +.. :members: -.. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns - :members: +.. .. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns +.. :members: -.. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns - :members: +.. .. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns +.. :members: -.. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns - :members: +.. .. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns +.. :members: -.. autoclass:: zipline.pipeline.factors.RSI - :members: +.. .. autoclass:: zipline.pipeline.factors.RSI +.. :members: -.. autoclass:: zipline.pipeline.factors.SimpleMovingAverage - :members: +.. .. autoclass:: zipline.pipeline.factors.SimpleMovingAverage +.. :members: -.. autoclass:: zipline.pipeline.factors.VWAP - :members: +.. .. autoclass:: zipline.pipeline.factors.VWAP +.. :members: -.. autoclass:: zipline.pipeline.factors.WeightedAverageValue - :members: +.. .. autoclass:: zipline.pipeline.factors.WeightedAverageValue +.. :members: -Pipeline Engine -``````````````` +.. Pipeline Engine +.. ``````````````` -.. autoclass:: zipline.pipeline.engine.PipelineEngine - :members: run_pipeline, run_chunked_pipeline - :member-order: bysource +.. .. autoclass:: zipline.pipeline.engine.PipelineEngine +.. :members: run_pipeline, run_chunked_pipeline +.. :member-order: bysource -.. autoclass:: zipline.pipeline.engine.SimplePipelineEngine - :members: __init__, run_pipeline, run_chunked_pipeline - :member-order: bysource +.. .. autoclass:: zipline.pipeline.engine.SimplePipelineEngine +.. :members: __init__, run_pipeline, run_chunked_pipeline +.. :member-order: bysource -.. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace +.. .. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace -Data Loaders -```````````` +.. Data Loaders +.. ```````````` -.. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader - :members: __init__, from_files, load_adjusted_array - :member-order: bysource +.. .. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader +.. :members: __init__, from_files, load_adjusted_array +.. :member-order: bysource Asset Metadata ~~~~~~~~~~~~~~ -.. autoclass:: zipline.assets.Asset +.. autoclass:: catalyst.assets.Asset :members: -.. autoclass:: zipline.assets.Equity - :members: - -.. autoclass:: zipline.assets.Future - :members: - -.. autoclass:: zipline.assets.AssetConvertible +.. autoclass:: catalyst.assets.AssetConvertible :members: Trading Calendar API ~~~~~~~~~~~~~~~~~~~~ -.. autofunction:: zipline.utils.calendars.get_calendar +.. autofunction:: catalyst.utils.calendars.get_calendar -.. autoclass:: zipline.utils.calendars.TradingCalendar +.. autoclass:: catalyst.utils.calendars.TradingCalendar :members: -.. autofunction:: zipline.utils.calendars.register_calendar +.. autofunction:: catalyst.utils.calendars.register_calendar -.. autofunction:: zipline.utils.calendars.register_calendar_type +.. autofunction:: catalyst.utils.calendars.register_calendar_type -.. autofunction:: zipline.utils.calendars.deregister_calendar +.. autofunction:: catalyst.utils.calendars.deregister_calendar -.. autofunction:: zipline.utils.calendars.clear_calendars +.. autofunction:: catalyst.utils.calendars.clear_calendars Data API ~~~~~~~~ -Writers -``````` -.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter - :members: +.. Writers +.. ``````` +.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter +.. :members: -.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter - :members: +.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter +.. :members: -.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter - :members: +.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter +.. :members: -.. autoclass:: zipline.assets.AssetDBWriter - :members: +.. .. autoclass:: zipline.assets.AssetDBWriter +.. :members: -Readers -``````` -.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader - :members: +.. Readers +.. ``````` +.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader +.. :members: -.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader - :members: +.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader +.. :members: -.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader - :members: +.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader +.. :members: -.. autoclass:: zipline.assets.AssetFinder - :members: +.. .. autoclass:: zipline.assets.AssetFinder +.. :members: -.. autoclass:: zipline.data.data_portal.DataPortal - :members: +.. .. autoclass:: zipline.data.data_portal.DataPortal +.. :members: -Bundles -``````` -.. autofunction:: zipline.data.bundles.register +.. Bundles +.. ``````` +.. .. autofunction:: zipline.data.bundles.register -.. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True) +.. .. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True) -.. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None) +.. .. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None) -.. autofunction:: zipline.data.bundles.unregister +.. .. autofunction:: zipline.data.bundles.unregister -.. data:: zipline.data.bundles.bundles +.. .. data:: zipline.data.bundles.bundles - The bundles that have been registered as a mapping from bundle name to bundle - data. This mapping is immutable and should only be updated through - :func:`~zipline.data.bundles.register` or - :func:`~zipline.data.bundles.unregister`. +.. The bundles that have been registered as a mapping from bundle name to bundle +.. data. This mapping is immutable and should only be updated through +.. :func:`~zipline.data.bundles.register` or +.. :func:`~zipline.data.bundles.unregister`. -.. autofunction:: zipline.data.bundles.yahoo_equities +.. .. autofunction:: zipline.data.bundles.yahoo_equities @@ -362,16 +356,16 @@ Utilities Caching ``````` -.. autoclass:: zipline.utils.cache.CachedObject +.. autoclass:: catalyst.utils.cache.CachedObject -.. autoclass:: zipline.utils.cache.ExpiringCache +.. autoclass:: catalyst.utils.cache.ExpiringCache -.. autoclass:: zipline.utils.cache.dataframe_cache +.. autoclass:: catalyst.utils.cache.dataframe_cache -.. autoclass:: zipline.utils.cache.working_file +.. autoclass:: catalyst.utils.cache.working_file -.. autoclass:: zipline.utils.cache.working_dir +.. autoclass:: catalyst.utils.cache.working_dir Command Line ```````````` -.. autofunction:: zipline.utils.cli.maybe_show_progress +.. autofunction:: catalyst.utils.cli.maybe_show_progress diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 3bcfc4cf..129fe90c 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -168,7 +168,7 @@ We'll start with the CLI, and introduce the ``run_algorithm()`` in the last example of this tutorial. Some of the :doc:`example algorithms ` provide instructions on how to run them both from the CLI, and using the :func:`~catalyst.run_algorithm` function. For the third method, refer to the -corresponding section on :doc:`Catalyst & Jupyter Notebook ` after you +corresponding section on :ref:`Catalyst & Jupyter Notebook ` after you have assimilated the contents of this tutorial. Command line interface @@ -473,6 +473,7 @@ Which we execute by running: | + There is a row for each trading day, starting on the first day of our simulation Jan 1st, 2016. In the columns you can find various information about the state of your algorithm. The column @@ -518,7 +519,7 @@ alongside enigma-catalyst (with the exception of the ``Conda`` install, where it was included by default inside the conda environment we created). If for any reason you don't have it installed, you can add it by running: -.. code-block:: python +.. code-block:: bash (catalyst)$ pip install matplotlib @@ -806,6 +807,7 @@ the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than ``pandas.DataFrame``\ s, so you can simply pass the underlying ``ndarray`` of a ``DataFrame`` via ``.values``). +.. _jupyter: Jupyter Notebook ~~~~~~~~~~~~~~~~ @@ -826,13 +828,13 @@ In order to use Jupyter Notebook, you first have to install it inside your environment. It's available as ``pip`` package, so regardless of how you installed Catalyst, go inside your catalyst environemnt and run: -.. code:: bash +.. code-block:: bash (catalyst)$ pip install jupyter Once you have Jupyter Notebook installed, every time you want to use it run: -.. code:: bash +.. code-block:: bash (catalyst)$ jupyter notebook @@ -846,7 +848,7 @@ Before running your algorithms inside the Jupyter Notebook, remember to ingest the data from the command line interface (CLI). In the example below, you would need to run first: -.. code:: bash +.. code-block:: bash catalyst ingest-exchange -x bitfinex -i btc_usd diff --git a/docs/source/conf.py b/docs/source/conf.py index 3dc91ef9..33892914 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,8 +27,8 @@ extlinks = { # -- Docstrings --------------------------------------------------------------- -#extensions += ['numpydoc'] -#numpydoc_show_class_members = False +extensions += ['numpydoc'] +numpydoc_show_class_members = False # Add any paths that contain templates here, relative to this directory. templates_path = ['.templates'] @@ -97,3 +97,6 @@ intersphinx_mapping = { doctest_global_setup = "import catalyst" todo_include_todos = True + +suppress_warnings = ['image.nonlocal_uri'] + diff --git a/docs/source/development-guidelines.rst b/docs/source/development-guidelines.rst index 677246ec..3adf2a43 100644 --- a/docs/source/development-guidelines.rst +++ b/docs/source/development-guidelines.rst @@ -36,25 +36,15 @@ Finally, you can build the C extensions by running: $ python setup.py build_ext --inplace -.. To finish, make sure `tests`__ pass. +Development with Docker +----------------------- -.. __ #style-guide-running-tests +If you want to work with zipline using a `Docker`__ container, you'll need to +build the ``Dockerfile`` in the Zipline root directory, and then build +``Dockerfile-dev``. Instructions for building both containers can be found in +``Dockerfile`` and ``Dockerfile-dev``, respectively. -.. If you get an error running nosetests after setting up a fresh virtualenv, please try running - -.. code-block - -.. # where zipline is the name of your virtualenv -.. $ deactivate zipline -.. $ workon zipline - - -.. Development with Docker -.. ----------------------- - -..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively. - -.. __ https://docs.docker.com/get-started/ +__ https://docs.docker.com/get-started/ Git Branching Structure ----------------------- diff --git a/docs/source/example-algos.rst b/docs/source/example-algos.rst index 98d9a9f7..0136b899 100644 --- a/docs/source/example-algos.rst +++ b/docs/source/example-algos.rst @@ -1,4 +1,5 @@ | + Example Algorithms ================== diff --git a/docs/source/index.rst b/docs/source/index.rst index c500ba29..5681e3b6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,6 +1,8 @@ .. include:: ../../README.rst + | | + Table of Contents ----------------- diff --git a/docs/source/install.rst b/docs/source/install.rst index e54e92c8..7459f97a 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -298,7 +298,7 @@ Troubleshooting ``pip`` Install .. _pipenv: Installing with ``pipenv`` -------------------------- +-------------------------- Installing Catalyst via ``pipenv`` is perhaps easier that installing it via ``pip`` itself but you need to install ``pipenv`` first via ``pip``. @@ -476,6 +476,7 @@ mentioned above are as follows: default you get 0 as the Value Data) | + - **The installer has encountered an unexpected error installing this package. This may indicate a problem with this package. The error code is 2503.** diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index a2898d61..d503b7eb 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -113,7 +113,7 @@ Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention. Here are some examples: -.. code-block:: json +.. code:: python # With Bitfinex bitcoin_usd_asset = symbol('btc_usd') diff --git a/etc/requirements_dev.txt b/etc/requirements_dev.txt index 7c457e50..194c57ed 100644 --- a/etc/requirements_dev.txt +++ b/etc/requirements_dev.txt @@ -16,7 +16,7 @@ babel==1.3 docutils==0.12 snowballstemmer==1.2.0 sphinx-rtd-theme==0.1.8 -sphinx==1.3.4 +sphinx==1.6.7 pbr==1.10.0 mock==2.0.0 diff --git a/etc/requirements_docs.txt b/etc/requirements_docs.txt index e89e00e8..d6087f5a 100644 --- a/etc/requirements_docs.txt +++ b/etc/requirements_docs.txt @@ -1,4 +1,4 @@ -Sphinx>=1.3.2 +Sphinx==1.6.7 numpydoc>=0.5.0 sphinx-autobuild==0.6.0 docutils==0.12 From 511af7946ec33b38488859e1b7f78fb4c77b4a11 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Feb 2018 02:40:53 -0500 Subject: [PATCH 091/117] BUG: for issue #219, created another unit test which compares current price against last candle --- .../exchange/test_suites/test_suite_bundle.py | 103 +++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 33ed7c2f..15d9cbcd 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -125,6 +125,65 @@ class TestSuiteBundle: pass + def compare_current_with_last_candle(self, exchange, assets, end_dt, + freq, data_frequency, data_portal): + """ + Creates DataFrames from the bundle and exchange for the specified + data set. + + Parameters + ---------- + exchange: Exchange + assets + end_dt + bar_count + freq + data_frequency + data_portal + + Returns + ------- + + """ + data = dict() + + assets = sorted(assets, key=lambda a: a.symbol) + log_catcher = TestHandler() + with log_catcher: + symbols = [asset.symbol for asset in assets] + print( + 'comparing data for {}/{} with {} timeframe on {}'.format( + exchange.name, symbols, freq, end_dt + ) + ) + data['candle'] = data_portal.get_history_window( + assets=assets, + end_dt=end_dt, + bar_count=1, + frequency=freq, + field='close', + data_frequency=data_frequency, + ) + set_print_settings() + print( + 'the bundle first / last row:\n{}'.format( + data['candle'].iloc[[-1]] + ) + ) + current = data_portal.get_spot_value( + assets=assets, + field='close', + dt=end_dt, + data_frequency=data_frequency, + ) + data['current'] = pd.Series(data=current, index=assets) + print( + 'the current price:\n{}'.format( + data['current'] + ) + ) + pass + def test_validate_bundles(self): # exchange_population = 3 asset_population = 3 @@ -145,8 +204,6 @@ class TestSuiteBundle: frequencies = exchange.get_candle_frequencies(data_frequency) freq = random.sample(frequencies, 1)[0] - freq = '15T' - bar_count = random.randint(1, 10) assets = select_random_assets( @@ -174,3 +231,45 @@ class TestSuiteBundle: data_portal=data_portal, ) pass + + def test_validate_last_candle(self): + # exchange_population = 3 + asset_population = 3 + data_frequency = random.choice(['minute']) + + # bundle = 'dailyBundle' if data_frequency + # == 'daily' else 'minuteBundle' + # exchanges = select_random_exchanges( + # population=exchange_population, + # features=[bundle], + # ) # Type: list[Exchange] + exchanges = [get_exchange('poloniex', skip_init=True)] + + data_portal = TestSuiteBundle.get_data_portal(exchanges) + for exchange in exchanges: + exchange.init() + + frequencies = exchange.get_candle_frequencies(data_frequency) + freq = random.sample(frequencies, 1)[0] + + assets = select_random_assets( + exchange.assets, asset_population + ) + end_dt = None + for asset in assets: + attribute = 'end_{}'.format(data_frequency) + asset_end_dt = getattr(asset, attribute) + + if end_dt is None or asset_end_dt < end_dt: + end_dt = asset_end_dt + + end_dt = end_dt + timedelta(minutes=3) + self.compare_current_with_last_candle( + exchange=exchange, + assets=assets, + end_dt=end_dt, + freq=freq, + data_frequency=data_frequency, + data_portal=data_portal, + ) + pass From 6b1982274e66728a88e223a140dd16d57df96ae4 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Feb 2018 02:44:45 -0500 Subject: [PATCH 092/117] DOC: updated release notes for 0.5.3 --- docs/source/releases.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 7c5dca1d..e49ef423 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,14 @@ Release Notes ============= +Version 0.5.3 +^^^^^^^^^^^^^ +**Release Date**: 2018-02-09 + +Bug Fixes +~~~~~~~~~ +- Fixed an issue with last candle in backtesting :issue:`219` + Version 0.5.2 ^^^^^^^^^^^^^ **Release Date**: 2018-02-08 From a8cfcead70cab6d276adfefacbdfd0c9af1a3a81 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 10:13:52 -0800 Subject: [PATCH 093/117] DOC: live-trading: improved upon #221 --- docs/source/live-trading.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index 7576b798..cb881a76 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -30,8 +30,8 @@ Paper Trading vs Live Trading modes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Catalyst currently supports three different modes in which you can execute your -trading algorithm. The first is backtesting, which is covered extensively in the -tutorial, and uses historical data to run your algorithm. There is no +trading algorithm. The first is **backtesting**, which is covered extensively in +the tutorial, and uses historical data to run your algorithm. There is no interaction with the exchange in backtesting mode, and this is the first mode that you should test any new algorithm. @@ -39,14 +39,15 @@ Once you are confident with the simulations that you have obtained with your algorithm in backtesting, you may switch to live trading, where you have two different modes: -* *Paper Trading*: The simulated algorithm runs in real time, and fetches -pricing data in real time from the exchange, but the orders never reach the -exchange, and are instead kept within Catalyst and simulated. No real currency -is bought or sold. Think of it as a `backtesting happening in real time`. -* *Live Trading*: This is the proper live trading mode in which an algorithm -runs in real time, fetching pricing data from live exchanges and placing orders -against the exchange. Real currency is transacted on the exchange driven by the -algorithm. +* **Paper Trading**: The simulated algorithm runs in real time, and fetches + pricing data in real time from the exchange, but the orders never reach the + exchange, and are instead kept within Catalyst and simulated. No real currency + is bought or sold. Think of it as a `backtesting happening in real time`. + +* **Live Trading**: This is the proper live trading mode in which an algorithm + runs in real time, fetching pricing data from live exchanges and placing + orders against the exchange. Real currency is transacted on the exchange + driven by the algorithm. These three modes are controlled by the following variables: From 2c1f81016366a1541c9eafe9d470d3a1b3037c2d Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 11:10:32 -0800 Subject: [PATCH 094/117] DOC: pycharm: improved upon #195 --- docs/source/beginner-tutorial.rst | 45 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 43cda572..41afe3c0 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -16609,30 +16609,49 @@ NaN -Catalyst using PyCharm -~~~~~~~~~~~~~~~~~~~~~~ +PyCharm IDE +~~~~~~~~~~~ + +PyCharm is an Integrated Development Environment (IDE) used in computer +programming, specifically for the Python language. It streamlines the continuos +development of Python code, and among other things includes a debugger that +comes in handy to see the inner workings of Catalyst, and your trading +algorithms. Install ^^^^^^^ -Install PyCharm from their `Website `__. +Install PyCharm from their `Website `__. +There is a free and open-source **Community** version. Setup ^^^^^ -1. Create a new project folder for your scripts or open your existing folder in PyCharm. +1. When creating a new project in PyCharm, right under you specify the Location, + click on **Project Interpreter** to display a drop down menu -2. Once your project is open, go to File -> Settings -> Project:'NAME_OF_PROJECT' -> Project Interpreter. - Click the gear box next to the project interpreter and select 'add local'. +2. Select **Existing interpreter**, click the gear box right next to it and + select 'add local'. Depending on your installation, select either + "*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to + navigate to your catalyst env and select the Python binary file: + ``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows + installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). + Select OK. You may want to click on *Make available to all projects* for your + future reference. Click OK again, and create your new environment using the + set up of your virtual environment. - Then select 'Conda Environment' -> 'Existing environment'. Click the '...' button and - navigate to your catalyst env located in the Anaconda2 folder to select the 'python.exe' file - (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). Select OK, then apply and click OK again. +Alternatively, if you already have your project created, in Windows do: -3. Next, click on the dropdown menu on the top right of PyCharm and select 'Edit Configurations'. - Select the '+' button. - Set the script Path to the path of your script and make sure the interpreter is correct, then hit ok. +1. File -> Default Settings -> Project Interpreter. Click the gear box next to + the project interpreter and select ‘add local’, and follow the steps from the + second step above. -You should now be able to run your script in PyCharm. +On MacOS: + +1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ -> + Project Interpreter. Click the gear box next to the project interpreter + and select ‘add local’, and follow the steps from the second step above. + +You should now be able to run your project/scripts in PyCharm. Next steps ~~~~~~~~~~ From f35444fabc4d1c6763e0eb3371fe23bd565398a0 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 15:16:14 -0800 Subject: [PATCH 095/117] DOC: updated algo as per #201 --- catalyst/examples/portfolio_optimization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/examples/portfolio_optimization.py b/catalyst/examples/portfolio_optimization.py index 37f8a55d..26c957b6 100644 --- a/catalyst/examples/portfolio_optimization.py +++ b/catalyst/examples/portfolio_optimization.py @@ -146,4 +146,5 @@ if __name__ == '__main__': start=start, end=end, exchange_name='poloniex', - capital_base=100000, ) + capital_base=100000, + base_currency='usdt', ) From e9745be0e883157964c9ba816fff532e2cefc1f2 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 11 Feb 2018 13:53:39 +0200 Subject: [PATCH 096/117] DOC: added use of end_date & start_date in live mode. issues #225 & #213 --- docs/source/live-trading.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index cb881a76..f2b6873a 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -176,6 +176,13 @@ Here is the breakdown of the new arguments: - ``simulate_orders``: Enables the paper trading mode, in which orders are simulated in Catalyst instead of processed on the exchange. It defaults to ``True``. +- ``end_date``: When setting the end_date to a time in the **future**, + it will schedule the live algo to finish gracefully at the specified date. +- ``start_date``: (**Will be implemented in the future**) + The live algo starts by default in the present, as mentioned above. + by setting the start_date to a time in the future, the algorithm would + essentially sleep and when the predefined time comes, it would start executing. + Here is a complete algorithm for reference: `Buy Low and Sell High `_ From 5345bfc43d3eb57b64a4c547794ffbc66e8b0bcf Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 12 Feb 2018 13:59:19 -0500 Subject: [PATCH 097/117] BLD: remove extra candle at the beginning of history bars after resampling --- catalyst/exchange/exchange_data_portal.py | 8 +++-- catalyst/exchange/utils/exchange_utils.py | 9 +++--- .../exchange/test_suites/test_suite_bundle.py | 30 +++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index 8f9665dc..c6523326 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -9,8 +9,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( ExchangeRequestError, PricingDataNotLoadedError) -from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange -from catalyst.exchange.utils.datetime_utils import get_frequency +from catalyst.exchange.utils.exchange_utils import resample_history_df, \ + group_assets_by_exchange +from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt from logbook import Logger from redo import retry @@ -311,7 +312,8 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): algo_end_dt=self._last_available_session, ) - df = resample_history_df(pd.DataFrame(series), freq, field) + start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) + df = resample_history_df(pd.DataFrame(series), freq, field, start_dt) return df def get_exchange_spot_value(self, diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 132845bb..ac48728e 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -126,8 +126,8 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): filename = get_exchange_symbols_filename(exchange_name, is_local) if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): + pd.Timestamp('now', tz='UTC') - last_modified_time( + filename)).days > 1): try: download_exchange_symbols(exchange_name, environ) except Exception as e: @@ -512,7 +512,7 @@ def get_common_assets(exchanges): return assets -def resample_history_df(df, freq, field): +def resample_history_df(df, freq, field, start_dt=None): """ Resample the OHCLV DataFrame using the specified frequency. @@ -541,6 +541,7 @@ def resample_history_df(df, freq, field): raise ValueError('Invalid field.') resampled_df = df.resample(freq, closed='left', label='left').agg(agg) + resampled_df = resampled_df[resampled_df.index >= start_dt] return resampled_df @@ -567,7 +568,7 @@ def mixin_market_params(exchange_name, params, market): params['taker'] = 0.002 elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None and market['taker'] is not None: + and market['maker'] is not None and market['taker'] is not None: params['maker'] = market['maker'] params['taker'] = market['taker'] diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 15d9cbcd..0bf8b1d2 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -37,7 +37,7 @@ class TestSuiteBundle: return data_portal def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count, - freq, data_frequency, data_portal): + freq, data_frequency, data_portal, field): """ Creates DataFrames from the bundle and exchange for the specified data set. @@ -62,8 +62,8 @@ class TestSuiteBundle: with log_catcher: symbols = [asset.symbol for asset in assets] print( - 'comparing data for {}/{} with {} timeframe until {}'.format( - exchange.name, symbols, freq, end_dt + 'comparing {} for {}/{} with {} timeframe until {}'.format( + field, exchange.name, symbols, freq, end_dt ) ) data['bundle'] = data_portal.get_history_window( @@ -71,13 +71,13 @@ class TestSuiteBundle: end_dt=end_dt, bar_count=bar_count, frequency=freq, - field='close', + field=field, data_frequency=data_frequency, ) set_print_settings() print( - 'the bundle first / last row:\n{}'.format( - data['bundle'].iloc[[-1, 0]] + 'the bundle data:\n{}'.format( + data['bundle'] ) ) candles = exchange.get_candles( @@ -88,14 +88,14 @@ class TestSuiteBundle: ) data['exchange'] = get_candles_df( candles=candles, - field='close', + field=field, freq=freq, bar_count=bar_count, end_dt=end_dt, ) print( - 'the exchange first / last row:\n{}'.format( - data['exchange'].iloc[[-1, 0]] + 'the exchange data:\n{}'.format( + data['exchange'] ) ) for source in data: @@ -118,8 +118,10 @@ class TestSuiteBundle: check_less_precise=min([a.decimals for a in assets]), ) except Exception as e: - print('Some differences were found within a 1 decimal point ' - 'interval of confidence: {}'.format(e)) + print( + 'Some differences were found within a 1 decimal point ' + 'interval of confidence: {}'.format(e) + ) with open(os.path.join(folder, 'compare.txt'), 'w+') as handle: handle.write(e.args[0]) @@ -203,8 +205,11 @@ class TestSuiteBundle: frequencies = exchange.get_candle_frequencies(data_frequency) freq = random.sample(frequencies, 1)[0] + rnd = random.SystemRandom() + # field = rnd.choice(['open', 'high', 'low', 'close', 'volume']) + field = rnd.choice(['close']) - bar_count = random.randint(1, 10) + bar_count = random.randint(3, 6) assets = select_random_assets( exchange.assets, asset_population @@ -229,6 +234,7 @@ class TestSuiteBundle: freq=freq, data_frequency=data_frequency, data_portal=data_portal, + field=field, ) pass From d64d6f191f5a7c7165ef626c6d1460377bdad1ac Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 12 Feb 2018 16:44:25 -0500 Subject: [PATCH 098/117] BLD: upgraded CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- tests/exchange/test_suites/test_suite_bundle.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 4f06bbae..0aa916d6 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.1049 + - ccxt==1.10.1094 - web3==4.0.0b7 - requests-toolbelt==0.8.0 - click==6.7 diff --git a/etc/requirements.txt b/etc/requirements.txt index 5d890d2e..bac9666d 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,7 +81,7 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.1049 +ccxt==1.10.1094 boto3==1.4.8 redo==1.6 web3==4.0.0b7 diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 0bf8b1d2..d338045d 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -207,7 +207,7 @@ class TestSuiteBundle: freq = random.sample(frequencies, 1)[0] rnd = random.SystemRandom() # field = rnd.choice(['open', 'high', 'low', 'close', 'volume']) - field = rnd.choice(['close']) + field = rnd.choice(['volume']) bar_count = random.randint(3, 6) From 67400f048bc717ad3b5e14430c6be16da41b1b43 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 12 Feb 2018 22:39:03 -0500 Subject: [PATCH 099/117] BLD: fix issue with removing extra candle in resampling --- catalyst/exchange/utils/exchange_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index ac48728e..3f8d6121 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -540,8 +540,16 @@ def resample_history_df(df, freq, field, start_dt=None): else: raise ValueError('Invalid field.') - resampled_df = df.resample(freq, closed='left', label='left').agg(agg) - resampled_df = resampled_df[resampled_df.index >= start_dt] + resampled_df = df.resample( + freq, closed='left', label='left' + ).agg(agg) # type: pd.DataFrame + + # Because the samples are closed left, we get one more candle at + # the beginning then the requested number for bars. Removing this + # candle to avoid confusion. + if start_dt and not resampled_df.empty: + resampled_df = resampled_df[resampled_df.index >= start_dt] + return resampled_df From f45d673d94d0c19ed583998a0a80415978ec950b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 13 Feb 2018 00:11:20 -0500 Subject: [PATCH 100/117] BUG: fixed issue #227 by allowing H frequency --- catalyst/exchange/utils/datetime_utils.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index 2a8cb886..5f3a2e04 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -302,16 +302,12 @@ def get_frequency(freq, data_frequency=None): elif unit.lower() == 'm' or unit == 'T': unit = 'T' alias = '{}T'.format(candle_size) + data_frequency = 'minute' - if data_frequency == 'daily': - data_frequency = 'minute' - - # elif unit.lower() == 'h': - # candle_size = candle_size * 60 - # - # alias = '{}T'.format(candle_size) - # if data_frequency == 'daily': - # data_frequency = 'minute' + elif unit.lower() == 'h': + candle_size = candle_size * 60 + alias = '{}T'.format(candle_size) + data_frequency = 'minute' else: raise InvalidHistoryFrequencyAlias(freq=freq) From 89bf6742e36f72d1195f5e18081c9d147aad7c5b Mon Sep 17 00:00:00 2001 From: lenak25 Date: Tue, 13 Feb 2018 16:57:03 +0200 Subject: [PATCH 101/117] MAINT: remove unnecessary Binance exclusion code --- catalyst/exchange/ccxt/ccxt_exchange.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index c5fee495..28a90a69 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -426,25 +426,18 @@ class CCXT(Exchange): ) if start_dt is None: - # TODO: determine why binance is failing - if end_dt is None and self.name not in ['binance']: + if end_dt is None: end_dt = pd.Timestamp.utcnow() - if end_dt is not None: - dt_range = get_periods_range( - end_dt=end_dt, - periods=bar_count, - freq=freq, - ) - start_dt = dt_range[0] + dt_range = get_periods_range( + end_dt=end_dt, + periods=bar_count, + freq=freq, + ) + start_dt = dt_range[0] - if start_dt is not None: - # Convert out start date to a UNIX timestamp, then translate to - # milliseconds - delta = start_dt - get_epoch() - since = int(delta.total_seconds()) * 1000 - else: - since = None + delta = start_dt - get_epoch() + since = int(delta.total_seconds()) * 1000 candles = dict() for index, asset in enumerate(assets): From 6812aa0c5bf31c1341432560d1e0eb10a2980916 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 13 Feb 2018 19:39:38 +0200 Subject: [PATCH 102/117] BLD: added password into credentials needed for exchanges such as 'gdax' --- catalyst/exchange/utils/factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py index 77b2d708..67294f95 100644 --- a/catalyst/exchange/utils/factory.py +++ b/catalyst/exchange/utils/factory.py @@ -33,6 +33,8 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False, exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], + password=exchange_auth['password'] if 'password' + in exchange_auth.keys() else '', base_currency=base_currency, ) exchange_cache[key] = exchange From f3183b267aa4fcaeaa3ff48d38e26751a433ff80 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 13 Feb 2018 19:41:37 +0200 Subject: [PATCH 103/117] BLD: added password into credentials needed for exchanges such as 'gdax' --- catalyst/exchange/ccxt/ccxt_exchange.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index c5fee495..1c81c90f 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -43,7 +43,8 @@ SUPPORTED_EXCHANGES = dict( class CCXT(Exchange): - def __init__(self, exchange_name, key, secret, base_currency): + def __init__(self, exchange_name, key, + secret, password, base_currency): log.debug( 'finding {} in CCXT exchanges:\n{}'.format( exchange_name, ccxt.exchanges @@ -60,6 +61,7 @@ class CCXT(Exchange): self.api = exchange_attr({ 'apiKey': key, 'secret': secret, + 'password': password, }) self.api.enableRateLimit = True From d494de7551e43ad0797a52265c73d02d338e59ee Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 13 Feb 2018 12:04:49 -0700 Subject: [PATCH 104/117] MAINT: conda env for Python3 --- etc/python2.7-environment.yml | 2 +- etc/python3.6-environment.yml | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 etc/python3.6-environment.yml diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 0aa916d6..2037515b 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -59,4 +59,4 @@ dependencies: - tables==3.4.2 - toolz==0.8.2 - urllib3==1.22 - - enigma-catalyst>=0.3 + - enigma-catalyst>=0.5 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml new file mode 100644 index 00000000..446198e0 --- /dev/null +++ b/etc/python3.6-environment.yml @@ -0,0 +1,95 @@ +name: catalyst +channels: +- defaults +dependencies: +- ca-certificates=2017.08.26=ha1e5d58_0 +- certifi=2018.1.18=py36_0 +- intel-openmp=2018.0.0=h8158457_8 +- libcxx=4.0.1=h579ed51_0 +- libcxxabi=4.0.1=hebd6815_0 +- libedit=3.1=hb4e282d_0 +- libffi=3.2.1=h475c297_4 +- libgfortran=3.0.1=h93005f0_2 +- mkl=2018.0.1=hfbd8650_4 +- ncurses=6.0=hd04f020_2 +- numpy=1.14.0=py36h8a80b8c_1 +- openssl=1.0.2n=hdbc3d79_0 +- pip=9.0.1=py36h1555ced_4 +- python=3.6.4=hc167b69_1 +- readline=7.0=hc1231fa_4 +- scipy=1.0.0=py36h1de22e9_0 +- setuptools=38.4.0=py36_0 +- sqlite=3.22.0=h3efe00b_0 +- tk=8.6.7=h35a86e2_3 +- wheel=0.30.0=py36h5eb2c71_1 +- xz=5.2.3=h0278029_2 +- zlib=1.2.11=hf3cbc9b_2 +- pip: + - aiodns==1.1.1 + - aiohttp==3.0.1 + - alembic==0.9.7 + - async-timeout==2.0.0 + - attrdict==2.0.0 + - attrs==17.4.0 + - bcolz==0.12.1 + - boto3==1.5.27 + - botocore==1.8.41 + - bottleneck==1.2.1 + - cchardet==2.1.1 + - ccxt==1.10.1102 + - chardet==3.0.4 + - click==6.7 + - contextlib2==0.5.5 + - cyordereddict==1.0.0 + - cython==0.27.3 + - cytoolz==0.9.0 + - decorator==4.2.1 + - docutils==0.14 + - empyrical==0.2.1 + - enigma-catalyst>=0.5.3 + - eth-abi==1.0.0b0 + - eth-account==0.1.0a2 + - eth-keyfile==0.5.1 + - eth-keys==0.2.0b1 + - eth-rlp==0.1.0a2 + - eth-utils==1.0.0b1 + - hexbytes==0.1.0b0 + - idna==2.6 + - idna-ssl==1.0.0 + - intervaltree==2.1.0 + - jmespath==0.9.3 + - logbook==1.2.1 + - lru-dict==1.1.6 + - lxml==4.1.1 + - mako==1.0.7 + - markupsafe==1.0 + - multidict==4.1.0 + - multipledispatch==0.4.9 + - networkx==2.1 + - numexpr==2.6.4 + - pandas==0.19.2 + - pandas-datareader==0.6.0 + - patsy==0.5.0 + - pycares==2.3.0 + - pycryptodome==3.4.11 + - pysha3==1.0.2 + - python-dateutil==2.6.1 + - python-editor==1.0.3 + - pytz==2018.3 + - redo==1.6 + - requests==2.18.4 + - requests-file==1.4.3 + - requests-ftp==0.3.1 + - requests-toolbelt==0.8.0 + - rlp==0.6.0 + - s3transfer==0.1.12 + - six==1.11.0 + - sortedcontainers==1.5.9 + - sqlalchemy==1.2.2 + - statsmodels==0.8.0 + - tables==3.4.2 + - toolz==0.9.0 + - urllib3==1.22 + - web3==4.0.0b9 + - wrapt==1.10.11 + - yarl==1.1.0 From 92d3b98448a1c2d804479d11bb9329ef1b38fd00 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 13 Feb 2018 12:14:11 -0700 Subject: [PATCH 105/117] MAINT: updated conda install instructions for Python3 --- docs/source/install.rst | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/source/install.rst b/docs/source/install.rst index 7459f97a..59279683 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -47,8 +47,10 @@ you can install MiniConda, which is a smaller footprint (fewer packages and smaller size) than its big brother Anaconda, but it still contains all the main packages needed. To install MiniConda, you can follow these steps: -1. Download `MiniConda `_. Select Python 2.7 - for your Operating System. +1. Download `MiniConda `_. Select either + Python 3.6 (recommended) or Python 2.7 for your Operating System. The + `Enigma Data Marketplace `_ will + require Python3, that's why we are recommending to opt for the newer version. 2. Install MiniConda. See the `Installation Instructions `_ if you need help. 3. Ensure the correct installation by running ``conda list`` in a Terminal @@ -64,21 +66,30 @@ main packages needed. To install MiniConda, you can follow these steps: Once either Conda or MiniConda has been set up you can install Catalyst: -1. Download the file `python2.7-environment.yml - `_. +1. Download the file `python3.6-environment.yml + `_ + (recommended) or `python2.7-environment.yml + `_ + matching your Conda installation from step #1 above. To download, simply click on the 'Raw' button and save the file locally to a folder you can remember. Make sure that the file gets saved with the ``.yml`` extension, and nothing like a ``.txt`` file or anything else. 2. Open a Terminal window and enter [``cd/dir``] into the directory where you - saved the above ``python2.7-environment.yml`` file. + saved the above ``.yml`` file. 3. Install using this file. This step can take about 5-10 minutes to install. .. code-block:: bash - conda env create -f python2.7-environment.yml + conda env create -f python3.6-environment.yml + + or + + .. code-block:: bash + + conda env create -f python2.7-environment.yml 4. Activate the environment (which you need to do every time you start a new session to run Catalyst): From 8072ded532afcc3a82c3a96e54fafe30a37a41a0 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 14 Feb 2018 12:25:44 -0500 Subject: [PATCH 106/117] BLD: clarified open order message --- catalyst/exchange/exchange_blotter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index 6ded5adf..c82b2f00 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -238,9 +238,12 @@ class ExchangeBlotter(Blotter): else: delta = pd.Timestamp.utcnow() - order.dt log.info( - 'order {order_id} still open after {delta}'.format( + '{exchange} order {order_id} for {symbol} still open ' + 'after {delta}'.format( + exchange=exchange.name, order_id=order.id, - delta=delta + delta=delta, + symbol=order.asset.symbol, ) ) From 041665dee1ede9ea3ccccb2c00444675994870e7 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 14 Feb 2018 13:37:51 -0500 Subject: [PATCH 107/117] BUG: fixed issue with incremental ingestion --- catalyst/marketplace/utils/bundle_utils.py | 10 +++++----- tests/marketplace/test_marketplace.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index b58595ac..014887a6 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -2,6 +2,7 @@ import os import shutil import bcolz +import pandas as pd def merge_bundles(zsource, ztarget): @@ -18,14 +19,13 @@ def merge_bundles(zsource, ztarget): """ # TODO: find a way to do this iteratively instead of in-memory df_source = zsource.todataframe() - df_source.set_index('date', drop=False, inplace=True) df_target = ztarget.todataframe() - df_target.set_index('date', drop=False, inplace=True) - df = df_target.merge( - right=df_source, - how='right', + df = pd.concat( + [df_source, df_target], ignore_index=True ) # type: pd.DataFrame + df.drop_duplicates(inplace=True) + df.set_index(['date', 'symbol'], drop=False, inplace=True) dirname = os.path.basename(ztarget.rootdir) bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname)) diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py index 59564a5e..c0895b5a 100644 --- a/tests/marketplace/test_marketplace.py +++ b/tests/marketplace/test_marketplace.py @@ -21,7 +21,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase): def test_ingest(self): marketplace = Marketplace() - ds_def = marketplace.ingest('marketcap1234') + ds_def = marketplace.ingest('github') pass def test_publish(self): From 2ba825db9bf8a375e61434f55cd7e0ff6e0edfcf Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 14 Feb 2018 20:26:41 -0500 Subject: [PATCH 108/117] BUG: for issue #227, made more mappings for hourly frequency --- catalyst/examples/simple_loop.py | 4 ++-- catalyst/exchange/exchange.py | 5 +++-- catalyst/exchange/utils/datetime_utils.py | 15 ++++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index 0de91d3d..c95726fa 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -26,7 +26,7 @@ def handle_data(context, data): context.asset, fields='price', bar_count=20, - frequency='30T' + frequency='1H' ) last_traded = prices.index[-1] log.info('last candle date: {}'.format(last_traded)) @@ -114,7 +114,7 @@ def analyze(context, perf): if __name__ == '__main__': - mode = 'backtest' + mode = 'live' if mode == 'backtest': run_algorithm( diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index f32d9a2b..c1b64f7c 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -502,7 +502,7 @@ class Exchange: """ freq, candle_size, unit, data_frequency = get_frequency( - frequency, data_frequency + frequency, data_frequency, supported_freqs=['T', 'D', 'H'] ) # The get_history method supports multiple asset candles = self.get_candles( @@ -523,8 +523,9 @@ class Exchange: field=field, ) + delta_candle_size = candle_size * 60 if unit == 'H' else candle_size # Checking to make sure that the dates match - delta = get_delta(candle_size, data_frequency) + delta = get_delta(delta_candle_size, data_frequency) adj_end_dt = end_dt - delta last_traded = asset_series.index[-1] diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index 5f3a2e04..c34b0c69 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -92,7 +92,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None): adj_periods = periods * unit_periods # TODO: standardize time aliases to avoid any mapping - unit = 'd' if unit == 'D' else 'm' + unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm' delta = pd.Timedelta(adj_periods, unit) if start_dt is not None: @@ -248,7 +248,7 @@ def get_year_start_end(dt, first_day=None, last_day=None): return year_start, year_end -def get_frequency(freq, data_frequency=None): +def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']): """ Get the frequency parameters. @@ -305,9 +305,14 @@ def get_frequency(freq, data_frequency=None): data_frequency = 'minute' elif unit.lower() == 'h': - candle_size = candle_size * 60 - alias = '{}T'.format(candle_size) - data_frequency = 'minute' + if 'H' in supported_freqs: + unit = 'H' + alias = '{}H'.format(candle_size) + + else: + candle_size = candle_size * 60 + alias = '{}T'.format(candle_size) + data_frequency = 'minute' else: raise InvalidHistoryFrequencyAlias(freq=freq) From 866b0a215c7321b7ce9f371fa04ecf8f32861ae5 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 14 Feb 2018 23:30:53 -0500 Subject: [PATCH 109/117] BUG: for issue #227, made more mappings for hourly frequency --- catalyst/examples/simple_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index c95726fa..99822b70 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -26,7 +26,7 @@ def handle_data(context, data): context.asset, fields='price', bar_count=20, - frequency='1H' + frequency='2H' ) last_traded = prices.index[-1] log.info('last candle date: {}'.format(last_traded)) From a61857b37d87de8c029857cdfd5899feb2f9597c Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 15 Feb 2018 20:22:15 +0200 Subject: [PATCH 110/117] BUG: changed the data, analyze gets in live - fixed the bug following #229 - at the end of each day the stores the daily stats to local directory - removes the stats folder at the begining of each run to avoid overloading the disk. - removes old data (over a month) during the run to avoid overloading the disk --- catalyst/exchange/exchange_algorithm.py | 99 ++++++++++++++++++----- catalyst/exchange/utils/exchange_utils.py | 70 +++++++++++++++- 2 files changed, 148 insertions(+), 21 deletions(-) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index aeff9e4e..5c9f8771 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -16,7 +16,7 @@ import signal import sys from datetime import timedelta from os import listdir -from os.path import isfile, join +from os.path import isfile, join, exists import catalyst.protocol as zp import logbook @@ -36,9 +36,11 @@ from catalyst.exchange.utils.exchange_utils import ( get_algo_folder, get_algo_df, save_algo_df, + clear_frame_stats_directory, + remove_old_files, group_assets_by_exchange, ) -from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \ - stats_to_algo_folder +from catalyst.exchange.utils.stats_utils import \ + get_pretty_stats, stats_to_s3, stats_to_algo_folder from catalyst.finance.execution import MarketOrder from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats @@ -67,8 +69,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): self.current_day = None - if self.simulate_orders is None \ - and self.sim_params.arena == 'backtest': + if self.simulate_orders is None and \ + self.sim_params.arena == 'backtest': self.simulate_orders = True # Operations with retry features @@ -118,7 +120,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # be in-line with CXXT and many exchanges. We'll consider # adding more order types in the future. if not isinstance(style, ExchangeLimitOrder) or \ - not isinstance(style, MarketOrder): + not isinstance(style, MarketOrder): raise OrderTypeNotSupported( order_type=style.__class__.__name__ ) @@ -368,6 +370,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self._clock = None self.frame_stats = list() + # erase the frame_stats folder to avoid overloading the disk + error = clear_frame_stats_directory(self.algo_namespace) + if error: + log.warning(error) + self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats') self.custom_signals_stats = \ @@ -392,6 +399,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): "Exit should be handled by the user.") def interrupt_algorithm(self): + """ + + when algorithm comes to an end this function is called. + extracts the stats and calls analyze. + after finishing, it exits the run. + + Parameters + ---------- + + Returns + ------- + + """ self.is_running = False if self._analyze is None: @@ -401,21 +421,31 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.info('Exiting the algorithm. Calling `analyze()` ' 'before exiting the algorithm.') + # add the last day stats which is not saved in the directory + current_stats = pd.DataFrame(self.frame_stats) + current_stats.set_index('period_close', drop=False, inplace=True) + + # get the location of the directory algo_folder = get_algo_folder(self.algo_namespace) - folder = join(algo_folder, 'daily_performance') - files = [f for f in listdir(folder) if isfile(join(folder, f))] + folder = join(algo_folder, 'frame_stats') - daily_perf_list = [] - for item in files: - filename = join(folder, item) + if exists(folder): + files = [f for f in listdir(folder) if isfile(join(folder, f))] - with open(filename, 'rb') as handle: - perf_period = pickle.load(handle) - perf_period_dict = perf_period.to_dict() - daily_perf_list.append(perf_period_dict) + period_stats_list = [] + for item in files: + filename = join(folder, item) - stats = pd.DataFrame(daily_perf_list) - stats.set_index('period_close', drop=False, inplace=True) + with open(filename, 'rb') as handle: + perf_period = pickle.load(handle) + period_stats_list.extend(perf_period) + + stats = pd.DataFrame(period_stats_list) + stats.set_index('period_close', drop=False, inplace=True) + + stats = pd.concat([stats, current_stats]) + else: + stats = current_stats self.analyze(stats) @@ -709,6 +739,37 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.algo_namespace, 'exposure_stats', self.exposure_stats ) + def nullify_frame_stats(self, now): + """ + + Save all period_stats to local directory + erase old files from the folder and nullify + self.frame_stats + + Parameters + ---------- + now: Timestamp + + Returns + ------- + + """ + save_algo_object( + algo_name=self.algo_namespace, + key=now.floor('1D').strftime('%Y-%m-%d'), + obj=self.frame_stats, + rel_path='frame_stats' + ) + error = remove_old_files( + algo_name=self.algo_namespace, + today=now, + rel_path='frame_stats' + ) + if error: + log.warning(error) + + self.frame_stats = list() + def handle_data(self, data): """ Wrapper around the handle_data method of each algo. @@ -728,7 +789,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Resetting the frame stats every day to minimize memory footprint today = data.current_dt.floor('1D') if self.current_day is not None and today > self.current_day: - self.frame_stats = list() + self.nullify_frame_stats(now=data.current_dt) self.performance_needs_update = False orders = list(self.perf_tracker.todays_performance.orders_by_id.keys()) @@ -808,6 +869,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Saving the last hour in memory self.frame_stats.append(frame_stats) + # creating and saving the pnl_stats into the local + # directory self.add_pnl_stats(frame_stats) if self.recorded_vars: self.add_custom_signals_stats(frame_stats) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 3f8d6121..fb24f1c8 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -130,7 +130,7 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): filename)).days > 1): try: download_exchange_symbols(exchange_name, environ) - except Exception as e: + except Exception: pass if os.path.isfile(filename): @@ -273,6 +273,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'): key: str environ: rel_path: str + how: str Returns ------- @@ -316,6 +317,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None, obj: Object environ: rel_path: str + how: str """ folder = get_algo_folder(algo_name, environ) @@ -392,6 +394,67 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None): df.to_csv(handle, encoding='UTF_8') +def clear_frame_stats_directory(algo_name): + """ + remove the outdated directory + to avoid overloading the disk + + Parameters + ---------- + algo_name: str + + Returns + ------- + error: str + + """ + error = None + algo_folder = get_algo_folder(algo_name) + folder = os.path.join(algo_folder, 'frame_stats') + if os.path.exists(folder): + try: + shutil.rmtree(folder) + except OSError: + error = 'unable to remove {}, the analyze ' \ + 'data will be inconsistent'.format(folder) + return error + + +def remove_old_files(algo_name, today, rel_path): + """ + remove old files from a directory + to avoid overloading the disk + + Parameters + ---------- + algo_name: str + today: Timestamp + rel_path: str + + Returns + ------- + error: str + + """ + error = None + algo_folder = get_algo_folder(algo_name) + folder = os.path.join(algo_folder, rel_path) + + # run on all files in the folder + for f in os.listdir(folder): + creation_unix = os.path.getctime(f) + creation_time = pd.to_datetime(creation_unix, unit='s', ) + + # if the file is older than 30 days erase it + if today - pd.DateOffset(30) > creation_time: + try: + os.unlink(f) + except OSError: + error = 'unable to erase files in {}'.format(folder) + + return error + + def get_exchange_minute_writer_root(exchange_name, environ=None): """ The minute writer folder for the exchange. @@ -575,8 +638,9 @@ def mixin_market_params(exchange_name, params, market): params['maker'] = 0.001 params['taker'] = 0.002 - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None and market['taker'] is not None: + elif 'maker' in market and 'taker' in market and \ + market['maker'] is not None and market['taker'] is not None: + params['maker'] = market['maker'] params['taker'] = market['taker'] From bd88ba2277f6067a41ac00f388c754a9091b68b2 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 16 Feb 2018 11:48:52 -0700 Subject: [PATCH 111/117] DOC: marketplace code examples --- .../examples/marketplace/github-research.py | 70 +++++++++++++++++++ .../mean_reversion_by_marketcap.py | 0 2 files changed, 70 insertions(+) create mode 100644 catalyst/examples/marketplace/github-research.py rename catalyst/examples/{ => marketplace}/mean_reversion_by_marketcap.py (100%) diff --git a/catalyst/examples/marketplace/github-research.py b/catalyst/examples/marketplace/github-research.py new file mode 100644 index 00000000..b2443597 --- /dev/null +++ b/catalyst/examples/marketplace/github-research.py @@ -0,0 +1,70 @@ +import pandas as pd +import matplotlib.pyplot as plt + +from catalyst import run_algorithm +from catalyst.api import symbol, get_dataset + +START = '2017-01-01' +END = '2017-12-31' + + +def initialize(context): + pass + + +def handle_data(context, data): + context.github = get_dataset('github') + context.github.sort_index(level=0, inplace=True) + + context.zec = data.history(symbol('zec_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + context.xmr = data.history(symbol('xmr_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + + +def analyze(context=None, results=None): + ax1 = plt.subplot(211) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'ZEC']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax1, color='blue') + ax1.legend(loc=2) + ax1.set_title('Zcash') + ax2 = ax1.twinx() + context.zec['price'].loc[START:END].plot(ax=ax2, color='green') + ax2.legend(loc=1) + + ax3 = plt.subplot(212) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'XMR']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax3, color='blue') + ax3.legend(loc=2) + ax3.set_title('Monero') + ax4 = ax3.twinx() + context.xmr['price'].loc[START:END].plot(ax=ax4, color='green') + ax4.legend(loc=1) + + plt.show() + + +if __name__ == '__main__': + run_algorithm( + capital_base=1000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace='algo-github', + base_currency='usdt', + live=False, + start=pd.to_datetime(END, utc=True), + end=pd.to_datetime(END, utc=True), + ) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/marketplace/mean_reversion_by_marketcap.py similarity index 100% rename from catalyst/examples/mean_reversion_by_marketcap.py rename to catalyst/examples/marketplace/mean_reversion_by_marketcap.py From 2becc4157adc5ce8336a7aeca77670a9d72abebf Mon Sep 17 00:00:00 2001 From: lenak25 Date: Tue, 20 Feb 2018 14:51:25 +0200 Subject: [PATCH 112/117] MAINT: cosmetics --- catalyst/marketplace/utils/auth_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 4979b6f3..ab3c668d 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -47,11 +47,11 @@ def get_key_secret(pubAddr, wallet='mew'): if wallet == 'mew': print('\nObtaining a key/secret pair to streamline all future ' 'requests with the authentication server.\n' - 'Visit https://www.myetherwallet.com/signmsg.html and sign the' + 'Visit https://www.myetherwallet.com/signmsg.html and sign the ' 'following message:\n{}'.format(nonce)) signature = input('Copy and Paste the "sig" field from ' 'the signature here (without the double quotes, ' - 'only the HEX value:\n') + 'only the HEX value):\n') else: raise MarketplaceWalletNotSupported(wallet=wallet) From ab644bd73239fd85d87b4fb36b5c448c152312b0 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 20 Feb 2018 19:17:03 +0200 Subject: [PATCH 113/117] BUG: modified dual_moving_average example modified long_window to be larger than short_window --- catalyst/examples/dual_moving_average.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 363edba1..00eb01af 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -21,7 +21,7 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages short_window = 2 - long_window = 2 + long_window = 3 # Skip as many bars as long_window to properly compute the average context.i += 1 From 9936b38e09b07a829755be8cac6bc56ec4c2dada Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 21 Feb 2018 08:51:59 -0700 Subject: [PATCH 114/117] DOC: updated Visual C++ instructions for Windows & Python 3 --- docs/source/install.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/source/install.rst b/docs/source/install.rst index 59279683..5c24159f 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -454,12 +454,22 @@ about matplotlib backends, please refer to the Windows Requirements -------------------- -In Windows, you will first need to install the `Microsoft Visual C++ Compiler -for Python 2.7 -`_. This -package contains the compiler and the set of system headers necessary for -producing binary wheels for Python 2.7 packages. If it's not already in your -system, download it and install it before proceeding to the next step. +In Windows, you will first need to install the Microsoft Visual C++ Compiler, +which is different depending on the version of Python that you plan to use: + +* Python 3.5, 3.6: `Visual C++ 2015 Build Tools + `_, + which installs Visual C++ version 14.0. **This is the recommended version** + +* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7 + `_, which + installs version Visual C++ version 9.0 + +This package contains the compiler and the set of system headers necessary for +producing binary wheels for Python packages. If it's not already in your +system, download it and install it before proceeding to the next step. If you +need additional help, or are looking for other versions of Visual C++ for +Windows (only advanced users), follow `this link `_. Once you have the above compiler installed, the easiest and best supported way to install Catalyst in Windows is to use :ref:`Conda `. If you didn't From 7ee875ee7029cade5fada75e99831b4f6d5c05e0 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 21 Feb 2018 13:06:57 -0500 Subject: [PATCH 115/117] BLD: adjusted sample algo --- catalyst/examples/mean_reversion_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 81a3b182..c697a88a 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,8 +37,8 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 40 - context.RSI_OVERBOUGHT = 60 + context.RSI_OVERSOLD = 60 + context.RSI_OVERBOUGHT = 70 context.CANDLE_SIZE = '15T' context.start_time = time.time() From 20f8a75f4a8906641bbb1b198d05cc15148fff02 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 21 Feb 2018 20:42:39 +0200 Subject: [PATCH 116/117] BUG: for issue #237, update positions before checking balances --- catalyst/exchange/exchange.py | 11 ++++++++--- catalyst/exchange/exchange_algorithm.py | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c1b64f7c..3b57256b 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -656,16 +656,21 @@ class Exchange: return df - def _check_low_balance(self, currency, balances, amount): + def _check_low_balance(self, currency, balances, amount, open_orders=None): free = balances[currency]['free'] if currency in balances else 0.0 + if open_orders: + # TODO: make sure that this works + free += sum([order.amount for order in open_orders]) + if free < amount: return free, True else: return free, False - def sync_positions(self, positions, cash=None, check_balances=False): + def sync_positions(self, positions, open_orders=None, cash=None, + check_balances=False): """ Update the portfolio cash and position balances based on the latest ticker prices. @@ -694,7 +699,7 @@ class Exchange: balances=balances, amount=cash, ) - if is_lower: + if is_lower and not open_orders: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 5c9f8771..c827af4a 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -631,23 +631,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if base_currency is None: base_currency = exchange.base_currency - # Don't check the cash if there are open orders. This could - # results in false positives. orders = [] for asset in self.blotter.open_orders: asset_orders = self.blotter.open_orders[asset] if asset_orders: orders += asset_orders - required_cash = self.portfolio.cash if not orders else None - cash, positions_value = exchange.sync_positions( - positions=exchange_positions, - check_balances=check_balances, - cash=required_cash, - ) - total_cash += cash - total_positions_value += positions_value - # Applying modifications to the original positions for position in exchange_positions: tracker.update_position( @@ -657,6 +646,16 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): last_sale_price=position.last_sale_price, ) + required_cash = self.portfolio.cash if not orders else None + cash, positions_value = exchange.sync_positions( + positions=exchange_positions, + open_orders=orders, + check_balances=check_balances, + cash=required_cash, + ) + total_cash += cash + total_positions_value += positions_value + if not check_balances: total_cash = self.portfolio.cash From 4337abd60a4bfa2f8335dba2c8c3bb9173f38cda Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 21 Feb 2018 15:51:28 -0700 Subject: [PATCH 117/117] DOC: linking example_algo to their sources --- docs/source/beginner-tutorial.rst | 158 +----- docs/source/example-algos.rst | 898 +----------------------------- 2 files changed, 19 insertions(+), 1037 deletions(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 41afe3c0..291dd205 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in the ``examples`` directory: `dual_moving_average.py `_. -.. code-block:: python - - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.utils.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - exchange = list(context.exchanges.values())[0] - base_currency = exchange.base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python In order to run the code above, you have to ingest the needed data first: diff --git a/docs/source/example-algos.rst b/docs/source/example-algos.rst index 0136b899..ec5b74a0 100644 --- a/docs/source/example-algos.rst +++ b/docs/source/example-algos.rst @@ -52,35 +52,8 @@ Buy BTC Simple Algorithm Source code: `examples/buy_btc_simple.py `_ -.. code-block:: python - - ''' - Run this example, by executing the following from your terminal: - catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - If you want to run this code using another exchange, make sure that - the asset is available on that exchange. For example, if you were to run - it for exchange Poloniex, you would need to edit the following line: - - context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' - - and specify exchange poloniex as follows: - catalyst ingest-exchange -x poloniex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - To see which assets are available on each exchange, visit: - https://www.enigma.co/catalyst/status - ''' - - from catalyst.api import order, record, symbol - - def initialize(context): - context.asset = symbol('btc_usd') - - def handle_data(context, data): - order(context.asset, 1) - record(btc = data.current(context.asset, 'price')) +.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py + :language: python This simple algorithm does not produce any output nor displays any chart. @@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart. Buy and Hodl Algorithm ~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/buy_and_hodl.py `_ - First ingest the historical pricing data needed to run this algorithm: .. code-block:: bash @@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an earlier date, you'll get an error), and the most recent date you can choose is one day prior to the current date. +Source code: `examples/buy_and_hodl.py `_ -.. code-block:: python - - #!/usr/bin/env python - # - # Copyright 2017 Enigma MPC, Inc. - # Copyright 2015 Quantopian, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - import pandas as pd - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order_target_value, symbol, record, - cancel_order, get_open_orders, ) - - - def initialize(context): - context.ASSET_NAME = 'btc_usd' - context.TARGET_HODL_RATIO = 0.8 - context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO - - context.is_buying = True - context.asset = symbol(context.ASSET_NAME) - - context.i = 0 - - - def handle_data(context, data): - context.i += 1 - - starting_cash = context.portfolio.starting_cash - target_hodl_value = context.TARGET_HODL_RATIO * starting_cash - reserve_value = context.RESERVE_RATIO * starting_cash - - # Cancel any outstanding orders - orders = get_open_orders(context.asset) or [] - for order in orders: - cancel_order(order) - - # Stop buying after passing the reserve threshold - cash = context.portfolio.cash - if cash <= reserve_value: - context.is_buying = False - - # Retrieve current asset price from pricing data - price = data.current(context.asset, 'price') - - # Check if still buying and could (approximately) afford another purchase - if context.is_buying and cash > price: - print('buying') - # Place order to make position in asset equal to target_hodl_value - order_target_value( - context.asset, - target_hodl_value, - limit_price=price * 1.1, - ) - - record( - price=price, - volume=data.current(context.asset, 'volume'), - cash=cash, - starting_cash=context.portfolio.starting_cash, - leverage=context.account.leverage, - ) - - - def analyze(context=None, results=None): - - # Plot the portfolio and asset data. - ax1 = plt.subplot(611) - results[['portfolio_value']].plot(ax=ax1) - ax1.set_ylabel('Portfolio Value (USD)') - - ax2 = plt.subplot(612, sharex=ax1) - ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME)) - results[['price']].plot(ax=ax2) - - trans = results.ix[[t != [] for t in results.transactions]] - buys = trans.ix[ - [t[0]['amount'] > 0 for t in trans.transactions] - ] - ax2.scatter( - buys.index.to_pydatetime(), - results.price[buys.index], - marker='^', - s=100, - c='g', - label='' - ) - - ax3 = plt.subplot(613, sharex=ax1) - results[['leverage', 'alpha', 'beta']].plot(ax=ax3) - ax3.set_ylabel('Leverage ') - - ax4 = plt.subplot(614, sharex=ax1) - results[['starting_cash', 'cash']].plot(ax=ax4) - ax4.set_ylabel('Cash (USD)') - - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]] = results[[ - 'treasury_period_return', - 'algorithm_period_return', - 'benchmark_period_return', - ]] - - ax5 = plt.subplot(615, sharex=ax1) - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]].plot(ax=ax5) - ax5.set_ylabel('Percent Change') - - ax6 = plt.subplot(616, sharex=ax1) - results[['volume']].plot(ax=ax6) - ax6.set_ylabel('Volume (mCoins/5min)') - - plt.legend(loc=3) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=10000, - data_frequency='daily', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace='buy_and_hodl', - base_currency='usd', - start=pd.to_datetime('2015-03-01', utc=True), - end=pd.to_datetime('2017-10-31', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png @@ -278,166 +102,13 @@ one day prior to the current date. Dual Moving Average Crossover ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Source Code: `examples/dual_moving_average.py `_ - This strategy is covered in detail in the last part of `this tutorial `_. -.. code-block:: python +Source Code: `examples/dual_moving_average.py `_ - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - base_currency = context.exchanges.values()[0].base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png @@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of Mean Reversion Algorithm ~~~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/mean_reversion_simple.py `_ - This algorithm is based on a simple momentum strategy. When the cryptoasset goes up quickly, we're going to buy; when it goes down quickly, we're going to sell. Hopefully, we'll ride the waves. @@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type: python mean_reversion_simple.py -.. code-block:: python +Source code: `examples/mean_reversion_simple.py `_ - import os - import tempfile - import time - - import numpy as np - import pandas as pd - import talib - from logbook import Logger - - from catalyst import run_algorithm - from catalyst.api import symbol, record, order_target_percent, get_open_orders - from catalyst.exchange.stats_utils import extract_transactions - # We give a name to the algorithm which Catalyst will use to persist its state. - # In this example, Catalyst will create the `.catalyst/data/live_algos` - # directory. If we stop and start the algorithm, Catalyst will resume its - # state using the files included in the folder. - from catalyst.utils.paths import ensure_directory - - NAMESPACE = 'mean_reversion_simple' - log = Logger(NAMESPACE) - - - # To run an algorithm in Catalyst, you need two functions: initialize and - # handle_data. - - def initialize(context): - # This initialize function sets any data or variables that you'll use in - # your algorithm. For instance, you'll want to define the trading pair (or - # trading pairs) you want to backtest. You'll also want to define any - # parameters or values you're going to use. - - # In our example, we're looking at Neo in USD. - context.neo_eth = symbol('neo_usd') - context.base_price = None - context.current_day = None - - context.RSI_OVERSOLD = 30 - context.RSI_OVERBOUGHT = 80 - context.CANDLE_SIZE = '15T' - - context.start_time = time.time() - - - def handle_data(context, data): - # This handle_data function is where the real work is done. Our data is - # minute-level tick data, and each minute is called a frame. This function - # runs on each frame of the data. - - # We flag the first period of each day. - # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle - # would only execute once. This method works with minute and daily - # frequencies. - today = data.current_dt.floor('1D') - if today != context.current_day: - context.traded_today = False - context.current_day = today - - # We're computing the volume-weighted-average-price of the security - # defined above, in the context.neo_eth variable. For this example, we're - # using three bars on the 15 min bars. - - # The frequency attribute determine the bar size. We use this convention - # for the frequency alias: - # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases - prices = data.history( - context.neo_eth, - fields='close', - bar_count=50, - frequency=context.CANDLE_SIZE - ) - - # Ta-lib calculates various technical indicator based on price and - # volume arrays. - - # In this example, we are comp - rsi = talib.RSI(prices.values, timeperiod=14) - - # We need a variable for the current price of the security to compare to - # the average. Since we are requesting two fields, data.current() - # returns a DataFrame with - current = data.current(context.neo_eth, fields=['close', 'volume']) - price = current['close'] - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - - price_change = (price - context.base_price) / context.base_price - cash = context.portfolio.cash - - # Now that we've collected all current data for this frame, we use - # the record() method to save it. This data will be available as - # a parameter of the analyze() function for further analysis. - record( - price=price, - volume=current['volume'], - price_change=price_change, - rsi=rsi[-1], - cash=cash - ) - - # We are trying to avoid over-trading by limiting our trades to - # one per day. - if context.traded_today: - return - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.neo_eth) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.neo_eth): - return - - # Another powerful built-in feature of the Catalyst backtester is the - # portfolio object. The portfolio object tracks your positions, cash, - # cost basis of specific holdings, and more. In this line, we calculate - # how long or short our position is at this minute. - pos_amount = context.portfolio.positions[context.neo_eth].amount - - if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: - log.info( - '{}: buying - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - # Set a style for limit orders, - limit_price = price * 1.005 - order_target_percent( - context.neo_eth, 1, limit_price=limit_price - ) - context.traded_today = True - - elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: - log.info( - '{}: selling - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - limit_price = price * 0.995 - order_target_percent( - context.neo_eth, 0, limit_price=limit_price - ) - context.traded_today = True - - - def analyze(context=None, perf=None): - end = time.time() - log.info('elapsed time: {}'.format(end - context.start_time)) - - import matplotlib.pyplot as plt - # The base currency of the algo exchange - base_currency = context.exchanges.values()[0].base_currency.upper() - - # Plot the portfolio value over time. - ax1 = plt.subplot(611) - perf.loc[:, 'portfolio_value'].plot(ax=ax1) - ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency)) - - # Plot the price increase or decrease over time. - ax2 = plt.subplot(612, sharex=ax1) - perf.loc[:, 'price'].plot(ax=ax2, label='Price') - - ax2.set_ylabel('{asset}\n({base})'.format( - asset=context.neo_eth.symbol, base=base_currency - )) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - ax4 = plt.subplot(613, sharex=ax1) - perf.loc[:, 'cash'].plot( - ax=ax4, label='Base Currency ({})'.format(base_currency) - ) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - - perf['algorithm'] = perf.loc[:, 'algorithm_period_return'] - - ax5 = plt.subplot(614, sharex=ax1) - perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5) - ax5.set_ylabel('Percent\nChange') - - ax6 = plt.subplot(615, sharex=ax1) - perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI') - ax6.set_ylabel('RSI') - ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod') - ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod') - - if not transaction_df.empty: - ax6.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'rsi'], - marker='^', - s=100, - c='green', - label='' - ) - ax6.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'rsi'], - marker='v', - s=100, - c='red', - label='' - ) - plt.legend(loc=3) - start, end = ax6.get_ylim() - ax6.yaxis.set_ticks(np.arange(0, end, end/5)) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - pass - - - if __name__ == '__main__': - # The execution mode: backtest or live - MODE = 'backtest' - - if MODE == 'backtest': - folder = os.path.join( - tempfile.gettempdir(), 'catalyst', NAMESPACE - ) - ensure_directory(folder) - - timestr = time.strftime('%Y%m%d-%H%M%S') - out = os.path.join(folder, '{}.p'.format(timestr)) - # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000 - run_algorithm( - capital_base=10000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-10-01', utc=True), - end=pd.to_datetime('2017-11-10', utc=True), - output=out - ) - log.info('saved perf stats: {}'.format(out)) - - elif MODE == 'live': - run_algorithm( - capital_base=0.5, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bittrex', - live=True, - algo_namespace=NAMESPACE, - base_currency='usd', - live_graph=False - ) +.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png @@ -763,8 +158,6 @@ strategy. Simple Universe ~~~~~~~~~~~~~~~ -Source code: `examples/simple_universe.py `_ - This example aims to provide an easy way for users to learn how to collect data from any given exchange and select a subset of the available currency pairs for trading. You simply need to specify the exchange and @@ -791,142 +184,10 @@ of the file: catalyst ingest-exchange -x bitfinex -f minute -.. code-block:: bash - - python simple_universe.py - -Credits: This code was originally submitted by `Abner Ayala-Acevedo -`_. Thank you! - -.. code-block:: python - - from datetime import timedelta - - import numpy as np - import pandas as pd - - from catalyst import run_algorithm - from catalyst.exchange.utils.exchange_utils import get_exchange_symbols - from catalyst.api import (symbols, ) - - - def initialize(context): - context.i = -1 # minute counter - context.exchange = context.exchanges.values()[0].name.lower() - context.base_currency = context.exchanges.values()[0].base_currency.lower() - - - def handle_data(context, data): - context.i += 1 - lookback_days = 7 # 7 days - - # current date & time in each iteration formatted into a string - now = data.current_dt - date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ') - lookback_date = now - timedelta(days=lookback_days) - # keep only the date as a string, discard the time - lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] - - one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute' - # update universe everyday at midnight - if not context.i % one_day_in_minutes: - context.universe = universe(context, lookback_date, date) - - # get data every 30 minutes - minutes = 30 - # get lookback_days of history data: that is 'lookback' number of bins - lookback = one_day_in_minutes / minutes * lookback_days - if not context.i % minutes and context.universe: - # we iterate for every pair in the current universe - for coin in context.coins: - pair = str(coin.symbol) - - # Get 30 minute interval OHLCV data. This is the standard data - # required for candlestick or indicators/signals. Return Pandas - # DataFrames. 30T means 30-minute re-sampling of one minute data. - # Adjust it to your desired time interval as needed. - opened = fill(data.history(coin, 'open', - bar_count=lookback, frequency='30T')).values - high = fill(data.history(coin, 'high', - bar_count=lookback, frequency='30T')).values - low = fill(data.history(coin, 'low', - bar_count=lookback, frequency='30T')).values - close = fill(data.history(coin, 'price', - bar_count=lookback, frequency='30T')).values - volume = fill(data.history(coin, 'volume', - bar_count=lookback, frequency='30T')).values - - # close[-1] is the last value in the set, which is the equivalent - # to current price (as in the most recent value) - # displays the minute price for each pair every 30 minutes - print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format( - now=now, - pair=pair, - o=opened[-1], - h=high[-1], - l=low[-1], - c=close[-1], - v=volume[-1], - )) - - # ------------------------------------------------------------- - # --------------- Insert Your Strategy Here ------------------- - # ------------------------------------------------------------- - - - def analyze(context=None, results=None): - pass - - - # Get the universe for a given exchange and a given base_currency market - # Example: Poloniex BTC Market - def universe(context, lookback_date, current_date): - # get all the pairs for the given exchange - json_symbols = get_exchange_symbols(context.exchange) - # convert into a DataFrame for easier processing - df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) - df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1) - df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1) - - # Filter all the pairs to get only the ones for a given base_currency - df = df[df['base_currency'] == context.base_currency] - - # Filter all the pairs to ensure that pair existed in the current date range - df = df[df.start_date < lookback_date] - df = df[df.end_daily >= current_date] - context.coins = symbols(*df.symbol) # convert all the pairs to symbols - - return df.symbol.tolist() - - - # Replace all NA, NAN or infinite values with its nearest value - def fill(series): - if isinstance(series, pd.Series): - return series.replace([np.inf, -np.inf], np.nan).ffill().bfill() - elif isinstance(series, np.ndarray): - return pd.Series(series).replace( - [np.inf, -np.inf], np.nan - ).ffill().bfill().values - else: - return series - - - if __name__ == '__main__': - start_date = pd.to_datetime('2017-11-10', utc=True) - end_date = pd.to_datetime('2017-11-13', utc=True) - - performance = run_algorithm(start=start_date, end=end_date, - capital_base=100.0, # amount of base_currency - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - data_frequency='minute', - base_currency='btc', - live=False, - live_graph=False, - algo_namespace='simple_universe') +Source code: `examples/simple_universe.py `_ +.. literalinclude:: ../../catalyst/examples/simple_universe.py + :language: python .. _portfolio_optimization: @@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used in writting the following article: `Markowitz Portfolio Optimization for Cryptocurrencies `_. -.. code-block:: python +Source code: `examples/simple_universe.py `_ - ''' - You can run this code using the Python interpreter: - - $ python portfolio_optimization.py - ''' - - from __future__ import division - import os - import pytz - import numpy as np - import pandas as pd - from scipy.optimize import minimize - import matplotlib.pyplot as plt - from datetime import datetime - - from catalyst.api import record, symbol, symbols, order_target_percent - from catalyst.utils.run_algo import run_algorithm - - np.set_printoptions(threshold='nan', suppress=True) - - - def initialize(context): - # Portfolio assets list - context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', - 'xmr_usdt') - context.nassets = len(context.assets) - # Set the time window that will be used to compute expected return - # and asset correlations - context.window = 180 - # Set the number of days between each portfolio rebalancing - context.rebalance_period = 30 - context.i = 0 - - - def handle_data(context, data): - # Only rebalance at the beggining of the algorithm execution and - # every multiple of the rebalance period - if context.i == 0 or context.i%context.rebalance_period == 0: - n = context.window - prices = data.history(context.assets, fields='price', - bar_count=n+1, frequency='1d') - pr = np.asmatrix(prices) - t_prices = prices.iloc[1:n+1] - t_val = t_prices.values - tminus_prices = prices.iloc[0:n] - tminus_val = tminus_prices.values - # Compute daily returns (r) - r = np.asmatrix(t_val/tminus_val-1) - # Compute the expected returns of each asset with the average - # daily return for the selected time window - m = np.asmatrix(np.mean(r, axis=0)) - # ### - stds = np.std(r, axis=0) - # Compute excess returns matrix (xr) - xr = r - m - # Matrix algebra to get variance-covariance matrix - cov_m = np.dot(np.transpose(xr),xr)/n - # Compute asset correlation matrix (informative only) - corr_m = cov_m/np.dot(np.transpose(stds),stds) - - # Define portfolio optimization parameters - n_portfolios = 50000 - results_array = np.zeros((3+context.nassets,n_portfolios)) - for p in xrange(n_portfolios): - weights = np.random.random(context.nassets) - weights /= np.sum(weights) - w = np.asmatrix(weights) - p_r = np.sum(np.dot(w,np.transpose(m)))*365 - p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365) - - #store results in results array - results_array[0,p] = p_r - results_array[1,p] = p_std - #store Sharpe Ratio (return / volatility) - risk free rate element - #excluded for simplicity - results_array[2,p] = results_array[0,p] / results_array[1,p] - i = 0 - for iw in weights: - results_array[3+i,p] = weights[i] - i += 1 - - #convert results array to Pandas DataFrame - results_frame = pd.DataFrame(np.transpose(results_array), - columns=['r','stdev','sharpe']+context.assets) - #locate position of portfolio with highest Sharpe Ratio - max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] - #locate positon of portfolio with minimum standard deviation - min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] - - #order optimal weights for each asset - for asset in context.assets: - if data.can_trade(asset): - order_target_percent(asset, max_sharpe_port[asset]) - - #create scatter plot coloured by Sharpe Ratio - plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn') - plt.xlabel('Volatility') - plt.ylabel('Returns') - plt.colorbar() - #plot red star to highlight position of portfolio with highest Sharpe Ratio - plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200) - #plot green star to highlight position of minimum variance portfolio - plt.show() - print(max_sharpe_port) - record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m) - context.i += 1 - - - def analyze(context=None, results=None): - # Form DataFrame with selected data - data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']] - - # Save results in CSV file - filename = os.path.splitext(os.path.basename(__file__))[0] - data.to_csv(filename + '.csv') - - - # Bitcoin data is available from 2015-3-2. Dates vary for other tokens. - start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) - end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) - results = run_algorithm(initialize=initialize, - handle_data=handle_data, - analyze=analyze, - start=start, - end=end, - exchange_name='poloniex', - capital_base=100000, ) +.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py + :language: python .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ. :align: center