DEV: encoding compatible to unix as well

catalyst/__main__.py

@@ -14,6 +14,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
+from catalyst.utils.run_server import run_server
 
 try:
     __IPYTHON__
@@ -505,6 +506,370 @@ def live(ctx,
     return perf
 
 
+@main.command(name='serve')
+@click.option(
+    '-f',
+    '--algofile',
+    default=None,
+    type=click.File('r'),
+    help='The file that contains the algorithm to run.',
+)
+@click.option(
+    '-t',
+    '--algotext',
+    help='The algorithm script to run.',
+)
+@click.option(
+    '-D',
+    '--define',
+    multiple=True,
+    help="Define a name to be bound in the namespace before executing"
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
+)
+@click.option(
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute'}),
+    default='daily',
+    show_default=True,
+    help='The data frequency of the simulation.',
+)
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The starting capital for the simulation.',
+)
+@click.option(
+    '-b',
+    '--bundle',
+    default='poloniex',
+    metavar='BUNDLE-NAME',
+    show_default=True,
+    help='The data bundle to use for the simulation.',
+)
+@click.option(
+    '--bundle-timestamp',
+    type=Timestamp(),
+    default=pd.Timestamp.utcnow(),
+    show_default=False,
+    help='The date to lookup data on or before.\n'
+         '[default: <current-time>]'
+)
+@click.option(
+    '-s',
+    '--start',
+    type=Date(tz='utc', as_timestamp=True),
+    help='The start date of the simulation.',
+)
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='The end date of the simulation.',
+)
+@click.option(
+    '-o',
+    '--output',
+    default='-',
+    metavar='FILENAME',
+    show_default=True,
+    help="The location to write the perf data. If this is '-' the perf"
+         " will be written to stdout.",
+)
+@click.option(
+    '--print-algo/--no-print-algo',
+    is_flag=True,
+    default=False,
+    help='Print the algorithm to stdout.',
+)
+@ipython_only(click.option(
+    '--local-namespace/--no-local-namespace',
+    is_flag=True,
+    default=None,
+    help='Should the algorithm methods be resolved in the local namespace.'
+))
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the targeted exchange.',
+)
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='A label assigned to the algorithm for data storage purposes.'
+)
+@click.option(
+    '-c',
+    '--base-currency',
+    help='The base currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
+@click.pass_context
+def run(ctx,
+        algofile,
+        algotext,
+        define,
+        data_frequency,
+        capital_base,
+        bundle,
+        bundle_timestamp,
+        start,
+        end,
+        output,
+        print_algo,
+        local_namespace,
+        exchange_name,
+        algo_namespace,
+        base_currency):
+    """Run a backtest for the given algorithm on the server.
+    """
+
+    if (algotext is not None) == (algofile is not None):
+        ctx.fail(
+            "must specify exactly one of '-f' / '--algofile' or"
+            " '-t' / '--algotext'",
+        )
+
+    # check that the start and end dates are passed correctly
+    if start is None and end is None:
+        # check both at the same time to avoid the case where a user
+        # does not pass either of these and then passes the first only
+        # to be told they need to pass the second argument also
+        ctx.fail(
+            "must specify dates with '-s' / '--start' and '-e' / '--end'"
+            " in backtest mode",
+        )
+    if start is None:
+        ctx.fail("must specify a start date with '-s' / '--start'"
+                 " in backtest mode")
+    if end is None:
+        ctx.fail("must specify an end date with '-e' / '--end'"
+                 " in backtest mode")
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    if base_currency is None:
+        ctx.fail("must specify a base currency with '-c' in backtest mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    click.echo('Running in backtesting mode.', sys.stdout)
+
+    perf = run_server(
+        initialize=None,
+        handle_data=None,
+        before_trading_start=None,
+        analyze=None,
+        algofile=algofile,
+        algotext=algotext,
+        defines=define,
+        data_frequency=data_frequency,
+        capital_base=capital_base,
+        data=None,
+        bundle=bundle,
+        bundle_timestamp=bundle_timestamp,
+        start=start,
+        end=end,
+        output=output,
+        print_algo=print_algo,
+        local_namespace=local_namespace,
+        environ=os.environ,
+        live=False,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency,
+        analyze_live=None,
+        live_graph=False,
+        simulate_orders=True,
+        auth_aliases=None,
+        stats_output=None,
+    )
+
+    if output == '-':
+        click.echo(str(perf), sys.stdout)
+    elif output != os.devnull:  # make the catalyst magic not write any data
+        perf.to_pickle(output)
+
+    return perf
+
+
+@main.command(name='serve-live')
+@click.option(
+    '-f',
+    '--algofile',
+    default=None,
+    type=click.File('r'),
+    help='The file that contains the algorithm to run.',
+)
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
+@click.option(
+    '-t',
+    '--algotext',
+    help='The algorithm script to run.',
+)
+@click.option(
+    '-D',
+    '--define',
+    multiple=True,
+    help="Define a name to be bound in the namespace before executing"
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
+)
+@click.option(
+    '-o',
+    '--output',
+    default='-',
+    metavar='FILENAME',
+    show_default=True,
+    help="The location to write the perf data. If this is '-' the perf will"
+         " be written to stdout.",
+)
+@click.option(
+    '--print-algo/--no-print-algo',
+    is_flag=True,
+    default=False,
+    help='Print the algorithm to stdout.',
+)
+@ipython_only(click.option(
+    '--local-namespace/--no-local-namespace',
+    is_flag=True,
+    default=None,
+    help='Should the algorithm methods be resolved in the local namespace.'
+))
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the targeted exchange.',
+)
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='A label assigned to the algorithm for data storage purposes.'
+)
+@click.option(
+    '-c',
+    '--base-currency',
+    help='The base currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
+@click.option(
+    '--live-graph/--no-live-graph',
+    is_flag=True,
+    default=False,
+    help='Display live graph.',
+)
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
+@click.pass_context
+def serve_live(ctx,
+               algofile,
+               capital_base,
+               algotext,
+               define,
+               output,
+               print_algo,
+               local_namespace,
+               exchange_name,
+               algo_namespace,
+               base_currency,
+               end,
+               live_graph,
+               auth_aliases,
+               simulate_orders):
+    """Trade live with the given algorithm on the server.
+    """
+    if (algotext is not None) == (algofile is not None):
+        ctx.fail(
+            "must specify exactly one of '-f' / '--algofile' or"
+            " '-t' / '--algotext'",
+        )
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    if algo_namespace is None:
+        ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
+    if base_currency is None:
+        ctx.fail("must specify a base currency '-c' in live execution mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.', sys.stdout)
+
+    else:
+        click.echo('Running in live trading mode.', sys.stdout)
+
+    perf = run_server(
+        initialize=None,
+        handle_data=None,
+        before_trading_start=None,
+        analyze=None,
+            algofile=algofile,
+        algotext=algotext,
+        defines=define,
+        data_frequency=None,
+        capital_base=capital_base,
+        data=None,
+        bundle=None,
+        bundle_timestamp=None,
+        start=None,
+        end=end,
+        output=output,
+        print_algo=print_algo,
+        local_namespace=local_namespace,
+        environ=os.environ,
+        live=True,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency,
+        live_graph=live_graph,
+        analyze_live=None,
+        simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
+        stats_output=None,
+    )
+
+    if output == '-':
+        click.echo(str(perf), sys.stdout)
+    elif output != os.devnull:  # make the catalyst magic not write any data
+        perf.to_pickle(output)
+
+    return perf
+
+
 @main.command(name='ingest-exchange')
 @click.option(
     '-x',
@@ -767,18 +1132,12 @@ def bundles():
 @main.group()
 @click.pass_context
 def marketplace(ctx):
-    """Access the Enigma Data Marketplace to:\n
-    - Register and Publish new datasets (seller-side)\n
-    - Subscribe and Ingest premium datasets (buyer-side)\n
-    """
     pass
 
 
 @marketplace.command()
 @click.pass_context
 def ls(ctx):
-    """List all available datasets.
-    """
     click.echo('Listing of available data sources on the marketplace:',
                sys.stdout)
     marketplace = Marketplace()
@@ -793,8 +1152,6 @@ def ls(ctx):
 )
 @click.pass_context
 def subscribe(ctx, dataset):
-    """Subscribe to an exisiting dataset.
-    """
     if dataset is None:
         ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
                  "List available dataset on the marketplace with "
@@ -833,8 +1190,6 @@ def subscribe(ctx, dataset):
 )
 @click.pass_context
 def ingest(ctx, dataset, data_frequency, start, end):
-    """Ingest a dataset (requires subscription).
-    """
     if dataset is None:
         ctx.fail("must specify a dataset to clean with '--dataset'\n"
                  "List available dataset on the marketplace with "
@@ -852,10 +1207,8 @@ def ingest(ctx, dataset, data_frequency, start, end):
 )
 @click.pass_context
 def clean(ctx, dataset):
-    """Clean/Remove local data for a given dataset.
-    """
     if dataset is None:
-        ctx.fail("must specify a dataset to clean up with '--dataset'\n"
+        ctx.fail("must specify a dataset to ingest with '--dataset'\n"
                  "List available dataset on the marketplace with "
                  "'catalyst marketplace ls'")
     click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
@@ -867,8 +1220,6 @@ def clean(ctx, dataset):
 @marketplace.command()
 @click.pass_context
 def register(ctx):
-    """Register a new dataset.
-    """
     marketplace = Marketplace()
     marketplace.register()
 
@@ -892,8 +1243,6 @@ def register(ctx):
 )
 @click.pass_context
 def publish(ctx, dataset, datadir, watch):
-    """Publish data for a registered dataset.
-    """
     marketplace = Marketplace()
     if dataset is None:
         ctx.fail("must specify a dataset to publish data for "

catalyst/exchange/exchange_bundle.py

@@ -843,6 +843,7 @@ class ExchangeBundle:
         field: str
         data_frequency: str
         algo_end_dt: pd.Timestamp
+        force_auto_ingest: 
 
         Returns
         -------

catalyst/utils/run_server.py

@@ -0,0 +1,103 @@
+#!flask/bin/python
+import base64
+
+import requests
+import pandas as pd
+import json
+
+
+def convert_date(date):
+    """
+    when transferring dates by json,
+    converts it to str
+    :param date:
+    :return: str(date)
+    """
+    if isinstance(date, pd.Timestamp):
+        return date.__str__()
+
+
+def run_server(
+        initialize,
+        handle_data,
+        before_trading_start,
+        analyze,
+        algofile,
+        algotext,
+        defines,
+        data_frequency,
+        capital_base,
+        data,
+        bundle,
+        bundle_timestamp,
+        start,
+        end,
+        output,
+        print_algo,
+        local_namespace,
+        environ,
+        live,
+        exchange,
+        algo_namespace,
+        base_currency,
+        live_graph,
+        analyze_live,
+        simulate_orders,
+        auth_aliases,
+        stats_output,
+        ):
+
+    # address to send
+    url = 'http://sandbox.enigma.co/api/catalyst/serve'
+    # url = 'http://127.0.0.1:5000/api/catalyst/serve'
+
+    # argument preparation - encode the file for transfer
+    if algotext:
+        algotext = base64.b64encode(algotext)
+    else:
+        algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8')
+        algofile = None
+
+    json_file = {'arguments': {
+        'initialize': initialize,
+        'handle_data': handle_data,
+        'before_trading_start': before_trading_start,
+        'analyze': analyze,
+        'algotext': algotext,
+        'defines': defines,
+        'data_frequency': data_frequency,
+        'capital_base': capital_base,
+        'data': data,
+        'bundle': bundle,
+        'bundle_timestamp': bundle_timestamp,
+        'start': start,
+        'end': end,
+        'local_namespace': local_namespace,
+        'environ': None,
+        'analyze_live': analyze_live,
+        'stats_output': stats_output,
+        'algofile': algofile,
+        'output': output,
+        'print_algo': print_algo,
+        'live': live,
+        'exchange': exchange,
+        'algo_namespace': algo_namespace,
+        'base_currency': base_currency,
+        'live_graph': live_graph,
+        'simulate_orders': simulate_orders,
+        'auth_aliases': auth_aliases,
+    }}
+
+    response = requests.post(url,
+                             json=json.dumps(
+                                    json_file,
+                                    default=convert_date
+                                )
+                             )
+
+    if response.status_code == 500:
+        raise Exception("issues with cloud connections, "
+                        "unable to run catalyst on the cloud")
+    received_data = response.json()
+    cloud_log_tail = base64.b64decode(received_data["log"])
+    print(cloud_log_tail)