satellite_leak_detection/notebooks/3a_hyperopt.ipynb

In this notebook I use hyperoptimisation to find the best data filters and the best decision tree parameters.

In [1]:

from path import Path
import arrow
import json
import pytz
from pprint import pprint
from tqdm import tqdm_notebook as tqdm
import re, os, collections, itertools, uuid, logging
import tempfile
# import tables

import zipfile
import urllib

import ee
import pyproj
import numpy as np
import scipy as sp
import pandas as pd
import geopandas as gpd
from matplotlib import pyplot as plt
import seaborn as sns
import shapely


%matplotlib inline
# %precision 4
plt.style.use('fivethirtyeight')
plt.rcParams['figure.figsize'] = (12, 6) # bigger plots

In [301]:

from collections import OrderedDict
import tabulate
from IPython.display import Markdown, display

In [2]:

os.environ['CUDA_VISIBLE_DEVICES']="" # don't need this for skleanr

In [3]:

import keras
import tensorflow
import sklearn
from keras.utils.io_utils import HDF5Matrix
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Reshape, InputLayer, Permute, RepeatVector, Dropout, LocallyConnected1D
from keras.layers.core import Activation
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import UpSampling1D
from keras.layers.convolutional import Convolution1D, MaxPooling1D, MaxPooling2D
from keras.layers.core import Flatten
from keras.optimizers import SGD
from keras.datasets import mnist
from keras.layers import (Input, LocallyConnected1D, ZeroPadding1D, Cropping1D, Embedding, Merge, merge,
    Cropping2D, Convolution1D, Convolution2D, Deconvolution2D, BatchNormalization, UpSampling1D, RepeatVector)
from keras.layers import embeddings, convolutional, activations, normalization, advanced_activations, ZeroPadding2D, UpSampling2D
from keras.models import Model
from keras_tqdm import TQDMCallback, TQDMNotebookCallback

Using TensorFlow backend.

In [4]:

# %load_ext autoreload
# %autoreload 2

In [5]:

# %load_ext autoreload
# %autoreload 2

helper_dir = str(Path('.').abspath())
if helper_dir not in os.sys.path:
    os.sys.path.append(helper_dir)
    
from leak_helpers.earth_engine import display_ee, get_boundary, tifs2np, bands_s2, download_image, bands_s2
from leak_helpers.geometry import diffxy, resample_polygon
from leak_helpers.modelling import ImageDataGenerator, dice_coef_loss
from leak_helpers.visualization import imshow_bands
from leak_helpers.analysis import parse_classification_report, find_best_dummy_classification, calculate_result_class
from leak_helpers.modelling.filters import is_not_cloudy, is_not_center_cloudy, is_image_within, is_leak, filter_split_data

In [6]:

keras.__version__, tensorflow.__version__

Out [6]:

('1.2.2', '1.0.0')

In [302]:

# params

data_path = [
#         Path('../data/scraped_satellite_images/s2-AUTX_v6_COPERNICUS-S2'),
    Path('../data/scraped_satellite_images/l7-AUTX_v2_LANDSAT-LE7_L1T'),
#     Path('../data/scraped_satellite_images/l8-AUTX_v2_LANDSAT-LC8_L1T'),
#     Path('../data/scraped_satellite_images/s1-all_COPERNICUS-S1_GRD'),   # small
][0]

notebook_name = 'leak_detection_CNN'
batch_size=128
random_seed = 1337
test_fraction = 0.4
timespan_before = 60*60*24*2 # we will only take image that where X seconds before the leak (e.g. 1 day)
max_cloud_cover = 1.1 # cloud cover must less than this
thresh = 0.5 # balance precision vs recall
balanced_classes = False 
normalized = True

# derived
seconds_in_a_day = 60*60*24
ts=arrow.utcnow().format('YYYYMMDD-HH-mm-ss')
model_name = '{notebook_name:}__{ts:}'.format(ts=ts,notebook_name=notebook_name)
outdir = Path('../output').joinpath(model_name)
outdir.makedirs_p()
outdir

target_names = ['no leak','leak']

In [303]:

script_metadata = json.load(open(data_path.joinpath('script_metadata.json')))
# test load
import h5py
metadatas = json.load(open(data_path.joinpath('data_metadata.json')))
with h5py.File(data_path.joinpath('data.h5'),'r') as h5f:
    X_raw = h5f['X'][:]
    y_raw = h5f['y'][:]
X_raw.shape, y_raw, metadatas[0].keys()

Out [303]:

((3564, 14, 25, 25),
 array([False, False, False, ..., False, False,  True], dtype=bool),
 dict_keys(['distance', 'name', 'scale', 'leak', 'crs', 'image']))

In [ ]:

Get data

In [ ]:

In [304]:

X_train, y_train, metadata_train, X_val, y_val, metadata_val, X_test, y_test, metadata_test = filter_split_data(
    X_raw,
    y_raw,
    metadatas,
    max_cloud_cover=1.1, 
    timespan_before=timespan_before,
    test_fraction=test_fraction, 
    random_seed=random_seed,
    balanced_classes=balanced_classes,
    normalized=normalized,
    filter_center_cloudy=True,
)   
print(X_train.shape,y_train.shape, len(metadata_train))
print(X_test.shape,y_test.shape, len(metadata_test))
print(X_val.shape,y_val.shape, len(metadata_val))

(926, 14, 25, 25) (926,) 926
(883, 14, 25, 25) (883,) 883
(398, 14, 25, 25) (398,) 398

In [ ]:

model

• batch norm helped
• normalising X helped

Refs:

• conv only net uses strides instead of MaxPooling
• example of heatmap generationg https://github.com/heuritech/convnets-keras/blob/master/convnetskeras/convnets.py

In [305]:

# conv model

def get_model(input_shape, batch_norm=True, activation='relu', border_mode='same', channels = 14*2, pooling = 2):    
    
    model = Sequential()
    # batch_input_shape=(None, X_train.shape[1], pixel_length, pixel_length)
    model.add(InputLayer(input_shape=input_shape, name='input'))
    model.add(Permute((2,3,1)))

    model.add(Convolution2D(channels,3,3,border_mode=border_mode,subsample=(pooling, pooling)))
    if batch_norm: model.add(BatchNormalization())
    model.add(Activation(activation))

    model.add(Convolution2D(channels*3,3,3,border_mode=border_mode,subsample=(3, 3)))
    if batch_norm: model.add(BatchNormalization())
    model.add(Activation(activation))

    model.add(Convolution2D(channels*2*3,3,3,border_mode=border_mode,subsample=(pooling, pooling)))
    if batch_norm: model.add(BatchNormalization())
    model.add(Activation(activation))

    model.add(Convolution2D(channels*2*3*2,1,1,border_mode=border_mode,subsample=(pooling, pooling)))
    if batch_norm: model.add(BatchNormalization())
    model.add(Activation(activation))
    
    model.add(Convolution2D(1,1,1,border_mode=border_mode))
    model.add(Flatten())
    model.add(Activation('sigmoid'))
    
    model.compile(loss='binary_crossentropy',optimizer='nadam', metrics=['accuracy'])
    
    return model

input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3])
# model = get_model(input_shape, border_mode='same' ,batch_norm=True)

Hyperopt

hyperopt nn

In [306]:

# hypropt nn
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.metrics import roc_auc_score
import sys
from sklearn.dummy import DummyClassifier

space = {
        
            # model
            'batch_norm': hp.choice('batch_norm', [False, True]),
            'activation':hp.choice('batch_norm', ['relu','tanh','hard_sigmoid','sigmoid']),
            'channels': 14*2,
            'pooling':2,
            'optimizer':'nadam',
    
            # data
            'max_cloud_cover':hp.uniform('max_cloud_cover', 0.01, 0.3),
            'timespan_before': seconds_in_a_day*1, # hp.uniform('timespan_before', seconds_in_a_day*1, seconds_in_a_day*7),
            'balanced_classes':hp.choice('balanced_classes', [False, True]),
            'normalized': hp.choice('normalized', [False, True]),
            'filter_center_cloudy': hp.choice('filter_center_cloudy', [False, True]),
    
            # std is 5% so max should prob be ~5%
            'channel_shift_range': 0.007, #hp.uniform('channel_shift_range', 0.0, 0.07),
            'rotation_range': 25, #hp.uniform('rotation_range', 0.0, 45),
    
            'batch_size' : batch_size,
            'samples_per_epoch': len(X_raw)*4,
            'nb_epochs' :  40,
        }

# from keras.models import Sequential
# from keras.layers import InputLayer
# from keras.layers.core import Dense, Dropout, Activation
# from keras.optimizers import Adadelta, Adam, rmsprop

# TODO make everything run of the params
# TODO encode params into hyperopt
# TODO run some trials to try differen't data types, could use decision tree, or even null model?
def f_nn(params):   

    # Get data
    X_train, y_train, metadata_train, X_val, y_val, metadata_val, X_test, y_test, metadata_test = filter_split_data(
        X_raw,
        y_raw,
        metadatas,
        max_cloud_cover=params['max_cloud_cover'], 
        timespan_before=params['timespan_before'],
        test_fraction=test_fraction, 
        random_seed=random_seed,
        balanced_classes=params['balanced_classes'],
        normalized=params['normalized'],
        filter_center_cloudy=params['filter_center_cloudy']
    )    

    datagen = ImageDataGenerator(
    #     featurewise_center=True,
    #     featurewise_std_normalization=True,
        channel_shift_range=params['channel_shift_range'],
        rotation_range=params['rotation_range'],
    #     width_shift_range=0.05,
    #     height_shift_range=0.05,
        horizontal_flip=True,
        vertical_flip=True,
       dim_ordering='th',
    #     rescale=0.02,
    #     zoom_range=0.05,
    #     shear_range=0.05,
    )

    # datagen.fit(X_train)

    # Get model
    input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3])
    model = get_model(
        input_shape, 
        border_mode=params['border_mode'],
        batch_norm=params['batch_norm'],
        channels=params['channels'],
        pooling=params['pooling']
        
    )

    if balanced_classes:
        model.compile(loss='binary_crossentropy',optimizer=params['optimizer'], metrics=['accuracy','matthews_correlation'])
    else:
        model.compile(loss=dice_coef_loss,optimizer=params['optimizer'], metrics=['accuracy','matthews_correlation'])

    # pretrain/test
    history = model.fit_generator(
        datagen.flow(X_train, y_train, batch_size=params['batch_size']),
        samples_per_epoch=params['samples_per_epoch'],
        verbose=0, 
        nb_epoch=params['nb_epoch'], 
        validation_data=[X_val,y_val],
        callbacks=[
              TQDMNotebookCallback(),
                keras.callbacks.EarlyStopping(patience=6, monitor='loss'),
                keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=3, verbose=1)
        ]
    )

    # results
    last_metrics = [x[-1] for x in history.history.values()]
    last_metrics = dict(zip(history.history.keys(),last_metrics))

    # accuracy X_train
    model.compile(loss='binary_crossentropy',optimizer='nadam', metrics=['accuracy','mean_squared_error','precision','recall','fbeta_score','fmeasure','matthews_correlation'])
    score = model.evaluate(X_val,y_val, batch_size=batch_size, verbose=False)
    metrics = dict(zip(model.metrics_names,score))

    # score on X_val
    y_pred = model.predict(X_val, verbose=False).T[0]
    report = parse_classification_report(sklearn.metrics.classification_report(y_val>thresh,y_pred>thresh,target_names=target_names))
    loss=-metrics['matthews_correlation']
    
    X_train2 = X_train.reshape((-1,14*24*24))
    X_val2 = X_val.reshape((-1,14*24*24))
    
    clf = DummyClassifier(strategy='uniform')
    clf.fit(X_train2,y_train)
    y_pred = clf.predict(X_val2)
    score_dummy = clf.score(X_val2, y_val)
    matthews_corrcoef_dummy = sklearn.metrics.matthews_corrcoef(y_val>thresh, y_pred>thresh)
    report_dummy = parse_classification_report(sklearn.metrics.classification_report(y_val>thresh,y_pred>thresh,target_names=target_names))
    
    
    
    sys.stdout.flush() 
    return dict(
                # core info
                loss=-loss,
                status=STATUS_OK,
        
                # info
                history=history.history, 
                metrics=dict(
                    last_metrics=last_metrics,
                    report=report.to_dict(),
                    metrics=metrics,
                ),
                dummy_metrics=dict(
                    report_dummy=report_dummy.to_dict(),
                    score_dummy=score_dummy,
                    matthews_corrcoef_dummy=matthews_corrcoef_dummy,
                ),
                
                # data dumps as json
                attachments=dict(
                    model=model.to_json()
                )
               )
            


# trials = Trials()
# best = fmin(f_nn, space, algo=tpe.suggest, max_evals=2, trials=trials, verbose=2)
# print ('best: ')
# print (best)

hyperopt filter

In [307]:

"""
This experiment will look at differen't way of filtering data
and will use decision trees as quick ways of evaluating data.
Mathews cooeffecient is used as loss since it performs well for unbalanced datasets.
"""

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.metrics import roc_auc_score
from hyperopt.mongoexp import MongoTrials
import sys
from sklearn import tree
from sklearn.dummy import DummyClassifier
import time
seconds_in_a_day = 60*60*24

trials = Trials(exp_key='data_filters_%s'%data_path.basename())
# trials = MongoTrials('mongo://localhost:27017/hyperopt_leaks2/jobs', exp_key='data_filters_v2')



space = {

            'max_cloud_cover': hp.uniform('max_cloud_cover', 0.01, 1.0),
            'timespan_before': hp.uniform('timespan_before', seconds_in_a_day*0.5, seconds_in_a_day*7),
            'balanced_classes':hp.choice('balanced_classes', [False, True]),
            'normalized': hp.choice('normalized', [False, True]),
            'filter_center_cloudy': False, #hp.choice('filter_center_cloudy', [False, True]),
    
            # std is 5% so max should prob be ~5%
            'channel_shift_range': hp.uniform('channel_shift_range', 0.0, 0.25),
            'rotation_range': hp.uniform('rotation_range', 0.0, 45),
            'width_shift_range': hp.uniform('width_shift_range', 0.0, 0.5),
            'height_shift_range': hp.uniform('height_shift_range', 0.0, 0.5),
            'horizontal_flip': True, #hp.choice('horizontal_flip', [False, True]),
            'vertical_flip': True, #hp.choice('vertical_flip', [False, True]),
            'rescale': hp.uniform('rescale', 0.0, 0.5),
            'zoom_range': 0 ,# hp.uniform('zoom_range', 0.0, 0.5),
            'shear_range': 0, # hp.uniform('shear_range', 0.0, 0.5),
    
            'max_depth': 50,
            'batch_size' : 2000*10,
            'nb_epochs' :  1,
        }


def f_dt(params):
    t0 = time.time()

    # Get data
    X_train, y_train, metadata_train, X_val, y_val, metadata_val, X_test, y_test, metadata_test = filter_split_data(
        X_raw,
        y_raw,
        metadatas,
        max_cloud_cover=params['max_cloud_cover'], 
        timespan_before=params['timespan_before'],
        test_fraction=test_fraction,
        val_fraction=test_fraction,
        random_seed=random_seed,
        balanced_classes=params['balanced_classes'],
        normalized=params['normalized'],
        filter_center_cloudy=params['filter_center_cloudy']
    )  

    datagen = ImageDataGenerator(
    #     featurewise_center=True,
    #     featurewise_std_normalization=True,
        channel_shift_range=params['channel_shift_range'],
        rotation_range=params['rotation_range'],
        width_shift_range=params['width_shift_range'],
        height_shift_range=params['height_shift_range'],
        horizontal_flip=params['horizontal_flip'],
        vertical_flip=params['vertical_flip'],
        rescale=params['rescale'],
        zoom_range=params['zoom_range'],
        shear_range=params['shear_range'],
        dim_ordering='th',
    )

    # datagen.fit(X_train)
    
    gen = datagen.flow(X_train, y_train, batch_size=params['batch_size']*params['nb_epochs'])
    X_trainb, y_trainb = next(gen)
    X_train2b = X_trainb.reshape((len(X_trainb),-1))
    X_val2 = X_val.reshape((len(X_val),-1))
    X_test2 = X_test.reshape((len(X_test),-1))
    
    t1=time.time()-t0

    # Get model    
    model = tree.DecisionTreeRegressor(max_depth=params['max_depth'])
    model.fit(X_train2b, y_trainb) 
    y_pred = model.predict(X_val2)
    score = model.score(X_val2, y_val)


    # score on X_val
    report = parse_classification_report(sklearn.metrics.classification_report(y_val>thresh,y_pred>thresh,target_names=target_names))
    matthews_corrcoef = sklearn.metrics.matthews_corrcoef(y_val>thresh, y_pred>thresh)
    loss=-matthews_corrcoef
    
    # do a dummy score too
    clf = DummyClassifier(strategy='uniform')
    clf.fit(X_train2b,y_trainb)
    y_pred = clf.predict(X_val2)
    score_dummy = clf.score(X_val2, y_val)
    matthews_corrcoef_dummy = sklearn.metrics.matthews_corrcoef(y_val>thresh, y_pred>thresh)
    report_dummy = parse_classification_report(sklearn.metrics.classification_report(y_val>thresh,y_pred>thresh,target_names=target_names))
    
    
#     sys.stdout.flush()
    print(t1,'loss',loss)
    return dict(loss=-loss,
                status=STATUS_OK,
                run_time=t1,
                metrics=dict(
                    report=report.to_dict(),
                    matthews_corrcoef=matthews_corrcoef,
                ),             
                dummy_metrics=dict(
                    report_dummy=report_dummy.to_dict(),
                    score_dummy=score_dummy,
                    matthews_corrcoef_dummy=matthews_corrcoef_dummy,
                ),
                attachments=dict(                
#                     model=model.__dict__
                )
               )
            

In [ ]:

best = fmin(f_dt, space, algo=tpe.suggest, max_evals=1000, trials=trials, verbose=1)
print ('best: ')
print (best)

1.595038652420044 loss -0.0386642846727
0.7030308246612549 loss 0.0932923927765
1.8368890285491943 loss -0.00368705710051
1.4079639911651611 loss -0.0166153571975
1.2733204364776611 loss 0.0494893096183
1.181145191192627 loss -0.0272881007883
1.0876176357269287 loss -0.0873704056661

/home/isisilon/.virtualenvs/py3syspck/lib/python3.4/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/home/isisilon/.virtualenvs/py3syspck/lib/python3.4/site-packages/sklearn/metrics/classification.py:516: RuntimeWarning: invalid value encountered in double_scalars
  mcc = cov_ytyp / np.sqrt(var_yt * var_yp)

1.743326187133789 loss -0.0
1.926445722579956 loss 0.0193785923943
1.1760334968566895 loss 0.0256270290006
1.0056226253509521 loss 0.0151752911359
0.6968693733215332 loss -0.0
1.6277260780334473 loss 0.0313566404476
1.0868237018585205 loss 0.0327399851106
1.5668323040008545 loss -0.0130328376203
1.2413296699523926 loss 0.0866871379433
1.6522138118743896 loss -0.0115787928458
1.666116714477539 loss 0.032069708152
1.478454351425171 loss 0.0554018715248
1.4123950004577637 loss 0.0432574237166
0.8946154117584229 loss -0.0573102276588
1.3224105834960938 loss -0.0328301167799
0.896176815032959 loss 0.0690553296203
1.2526631355285645 loss -0.0116495714929
0.7282388210296631 loss -0.0490225639152
1.40065336227417 loss -0.0580845778331
1.062377691268921 loss -0.0385831356798
1.9305071830749512 loss -0.00965632680232
1.2048382759094238 loss -0.00383326827693
1.3617136478424072 loss 0.0413901492425
0.6661429405212402 loss 0.0660413176414
1.8069255352020264 loss -0.00166692218296
1.3109245300292969 loss 0.0129863050497
1.1031620502471924 loss 0.0430726496665
1.2229349613189697 loss 0.0108699471106
1.1721446514129639 loss -0.077683279551
1.4071061611175537 loss -0.0438634463307
0.7453796863555908 loss 0.0217796037852
1.3287794589996338 loss 0.00682451169305
1.309830904006958 loss -0.052583048205
0.6199619770050049 loss 0.0999807747763
0.9225940704345703 loss 0.0700152751617
0.6101760864257812 loss 0.0503144528418
1.5531032085418701 loss 0.0435480061659
1.0505895614624023 loss 0.023719131803
0.8544209003448486 loss -0.0194424132609
1.5737848281860352 loss 0.0127326155371
0.9572079181671143 loss 0.0444937556926
1.6960012912750244 loss 0.0044726058412
0.8747713565826416 loss 0.0530595449973
1.0718188285827637 loss 0.0212958855
1.4130747318267822 loss -0.0348723689193
1.6548922061920166 loss 0.0353705673104
0.8295555114746094 loss -0.019279313829
1.3360414505004883 loss -0.00261073502835
1.7084426879882812 loss 0.00219235985541
0.9738762378692627 loss 0.0568051873845
1.614677906036377 loss -0.0107594694354
1.5387177467346191 loss 0.0176764509818
0.6341819763183594 loss 0.00328230714366
0.9057796001434326 loss 0.0618931372168
0.6017038822174072 loss -0.116887940771
1.061206579208374 loss -0.0378005077239
1.5743980407714844 loss 0.0382214483929
1.1713993549346924 loss 0.0690436611898
1.332535743713379 loss -0.0238934099208
0.9691309928894043 loss -0.0186232596099
0.9107766151428223 loss 0.0127178228831
1.2141294479370117 loss 0.00759240268123
0.8369030952453613 loss -0.116529696664
0.992255687713623 loss 0.0122812687697
1.63787841796875 loss 0.0134579230323
0.6554107666015625 loss 0.00119038762326
1.7501065731048584 loss 0.0614732962055
1.2191259860992432 loss -0.0501327966455
0.7595231533050537 loss 0.0114108774431
0.6717150211334229 loss -0.0109562134162
1.1173820495605469 loss -0.0618928733173
1.3781368732452393 loss -0.0112584600853
0.873100996017456 loss 0.0903319761035
0.8814060688018799 loss -0.00137026019844
1.2248499393463135 loss 0.0772080107003
1.048046350479126 loss 0.0369481080435
0.6875677108764648 loss 0.0455947313531
1.265657901763916 loss 0.0334124726781
0.7941527366638184 loss 0.000484192752986
0.6798803806304932 loss -0.0
1.7228796482086182 loss -0.0355747649438
1.1057746410369873 loss 0.0711045157968
1.7806882858276367 loss -8.94488927178e-05
0.9567673206329346 loss -0.0417146318964
1.4447898864746094 loss -0.0357176371229
1.6102213859558105 loss -0.0482595423299
1.7290699481964111 loss -0.0169082056544
0.852022647857666 loss -0.0242807227571
1.2406377792358398 loss 0.0338834593377
0.9319663047790527 loss 0.146705722762
1.2706272602081299 loss -0.0273949329549
0.7375099658966064 loss -0.0222425591861
0.7998201847076416 loss 0.038686711521
1.1476457118988037 loss -0.00120406221701
1.3158257007598877 loss -0.00933390173225
1.1581008434295654 loss 0.0062051897926
0.5744457244873047 loss 0.102184964982
0.6817755699157715 loss -0.0
0.5449984073638916 loss -0.0
0.5889229774475098 loss -0.0
0.6153061389923096 loss -0.0
0.8628952503204346 loss -0.0
1.0558323860168457 loss 0.0322737021256
0.6251144409179688 loss -0.044509847741
1.5085465908050537 loss 0.0447944331866
0.8061904907226562 loss 0.0572947955087
1.4444513320922852 loss -0.00361967060397
1.4304285049438477 loss -0.0465433515149
0.8511807918548584 loss -0.068126616735
0.5867478847503662 loss -0.0
1.125075101852417 loss -0.064826993097
1.0349152088165283 loss 0.0449242270766
0.8231236934661865 loss 0.119058832269
0.833223819732666 loss -0.0407356201158
1.1510579586029053 loss 0.0420082461241
0.934943675994873 loss -0.0439431325156
1.0907318592071533 loss -0.0407359086189
0.8918247222900391 loss -0.0
1.0513832569122314 loss 0.0356479391417
1.1104772090911865 loss 0.00731015994035
1.408228874206543 loss -0.133003640706
0.973811149597168 loss 0.0451101496672
1.3069138526916504 loss -0.0
1.2364673614501953 loss 0.0377409796137
0.851762056350708 loss 0.0598666606811
1.345979928970337 loss 0.00653099537468
1.1836156845092773 loss 0.0387386753699
1.0684335231781006 loss 0.0742648052688
1.2261989116668701 loss -0.0137099976738
1.1661624908447266 loss -0.0329718915356
1.3868944644927979 loss -0.0
1.1146290302276611 loss 0.0635641726164
1.8587803840637207 loss -0.00458912545953
1.4971342086791992 loss -0.045020356031
1.2714331150054932 loss -0.0428223813581
0.7388341426849365 loss 0.224575080296
0.9647023677825928 loss -0.0355700501987
1.2559854984283447 loss -0.112022144028
0.7002313137054443 loss 0.053266897675
0.7441058158874512 loss -0.0
0.7224020957946777 loss -0.0114653193415
0.7976553440093994 loss 0.0086322446841
0.6011712551116943 loss -0.0
0.9683444499969482 loss -0.0520434570825
0.6496193408966064 loss -0.142822452774
0.7613770961761475 loss -0.0
1.7269132137298584 loss 0.0128480534762
0.8062286376953125 loss 0.113423068656
0.9843013286590576 loss -0.0207524220712
0.8339107036590576 loss -0.137102124277
1.1088101863861084 loss -0.0741536011437
0.8982605934143066 loss 0.0687672280911
1.1768219470977783 loss 0.0623487803619
0.8841915130615234 loss -0.0616938884593
0.6374380588531494 loss -0.0
0.9129416942596436 loss 0.13050777928
1.4777328968048096 loss 0.0441109256145
1.1985671520233154 loss 0.0651169352801
1.0956332683563232 loss -0.00792740084713
1.3011178970336914 loss -0.0050815734162
1.220155954360962 loss 0.0577113170327
1.0504302978515625 loss -0.0841245865555
1.272538423538208 loss -0.00238716668334
1.2349607944488525 loss -0.00642535782539
1.2985484600067139 loss -0.00787953227911
1.6275668144226074 loss 0.00126928085048
1.275913953781128 loss -0.012422599875
1.6009936332702637 loss -0.0544613236995
1.346703290939331 loss -0.0106132370652
0.9042720794677734 loss 0.0160205853577
0.8895664215087891 loss 0.0231146547074
1.2004714012145996 loss -0.0878327250163
1.2653095722198486 loss 0.00513448186877
0.8661878108978271 loss -0.0
1.056656837463379 loss 0.0128588627472
0.7781689167022705 loss -0.0846895567655
1.06817626953125 loss 0.0147581293181
1.1168842315673828 loss 0.0165267291918
0.9096884727478027 loss 0.0730450476665
1.8387911319732666 loss 0.0243226147172
1.3280961513519287 loss 0.00252870706323
1.1553597450256348 loss -0.00263481972505
0.7319984436035156 loss -0.0682525031202
1.4143457412719727 loss -0.0460114848826
1.0314042568206787 loss -0.0
0.8726749420166016 loss -0.105613726734
0.9312679767608643 loss -0.0113155812799
0.6728718280792236 loss -0.108179912582
1.58372163772583 loss 0.0883216207531
1.328761100769043 loss -0.0905272707416
1.0624229907989502 loss -0.0205571175969
0.718254566192627 loss 0.00510017737124
1.239013433456421 loss -0.00527981373667
0.5938320159912109 loss -0.0
0.9526882171630859 loss 0.00729927007299
1.1819355487823486 loss -0.0189743095742
0.8125112056732178 loss 0.0776202482096
0.8347675800323486 loss -0.0131490801626
1.4646801948547363 loss 0.0322193898535
1.2850096225738525 loss -0.00145023939453
0.5780060291290283 loss -0.0452981254343
1.2683050632476807 loss 0.0204236884723
0.8873462677001953 loss 0.0649352824413
1.4087839126586914 loss 0.0470264292919
1.0920274257659912 loss -1.60246890532e-17
0.7435271739959717 loss 0.0467686465901
0.6104557514190674 loss -0.103083578431
0.8999857902526855 loss 0.0335190282443
0.5675115585327148 loss -0.11996255412
0.6062302589416504 loss 0.0854981960071
1.385622262954712 loss -0.00708545677544
1.0545649528503418 loss -0.088311387964
0.8331112861633301 loss -0.0742442184566
1.4473271369934082 loss -0.0495677735251
1.6564116477966309 loss -0.0118873146362
0.9508056640625 loss -0.0158855295116
1.214658498764038 loss 0.032832505387
0.6311907768249512 loss 0.125164957183
1.8857269287109375 loss -0.00665124682644
1.6624643802642822 loss -0.00593407477667
0.6244237422943115 loss 0.0564715652604
0.7941701412200928 loss -0.00362484831885
0.5914764404296875 loss 0.0245741136319
1.185908317565918 loss 0.0625306069675
0.9425904750823975 loss -0.0344190237371
0.739734411239624 loss -0.0396124549096
1.7436907291412354 loss 0.0207798838346
0.6109311580657959 loss 0.0624080874038
0.887667179107666 loss -0.0299882685992
0.6273677349090576 loss -0.00716114874039
1.8764030933380127 loss -0.0446068803531
1.4935698509216309 loss -0.0329964727906
1.646672010421753 loss 0.0282081093629
1.359626293182373 loss 0.0827951975822
0.9599535465240479 loss -0.0242216334248
0.8965458869934082 loss -0.0334149074796
0.8898842334747314 loss 0.0544915826349
0.97625732421875 loss -0.0674166794269
1.3371529579162598 loss -0.0190759211223
1.0880393981933594 loss -0.0355079941504
0.6052131652832031 loss 0.179039970747
0.7585365772247314 loss 0.0814378114426
1.2565340995788574 loss 0.0688105690204
1.700885534286499 loss 0.0355831380676
0.836794376373291 loss -0.0277279198035
1.3266935348510742 loss 0.000616297395268
1.0807526111602783 loss 0.025265782303
0.6690149307250977 loss -0.0
0.7723128795623779 loss 0.0740133816681
0.6162757873535156 loss -0.180735758041
0.6254394054412842 loss -0.053593307857
0.6167583465576172 loss 0.0351643377343
0.6260216236114502 loss 0.0114707866935
0.6687088012695312 loss 0.0650521662018
0.8499691486358643 loss -0.103863873375
0.7313566207885742 loss 0.0218912342853
0.800161600112915 loss -0.0465820628584
0.8931105136871338 loss 0.0681799324189
0.7244887351989746 loss -0.0474570115536
0.9013831615447998 loss 0.0748384480339
0.6224260330200195 loss -0.0
0.7572798728942871 loss -0.0444908798733
0.6181795597076416 loss -0.113423623453
0.6235337257385254 loss 0.0742315527404
0.5809857845306396 loss 0.130871839614
0.7870333194732666 loss -0.161476406487
0.8197917938232422 loss -0.113394663875
1.0151422023773193 loss -0.0117070017336
0.8511686325073242 loss -0.0666688374126
0.6355428695678711 loss 0.130732203725
0.6608762741088867 loss -0.0
0.7700026035308838 loss 0.104364570124
0.6694729328155518 loss 0.0685919395489
0.7360856533050537 loss -0.11104392983
0.6703271865844727 loss -0.128346672014
0.8340926170349121 loss -0.00809201691481
0.6265885829925537 loss 0.0304431282774
0.6198718547821045 loss -0.0
0.7517120838165283 loss -0.00963265995672
0.6595849990844727 loss 0.141428329615
0.9543981552124023 loss -0.0403551018643
1.5319733619689941 loss -0.0367552427787
0.7052953243255615 loss -0.168549965616
1.629629135131836 loss 0.00590417602837
1.5468635559082031 loss 0.00731889649218
0.6960842609405518 loss 0.00971328156231
0.708319902420044 loss 0.0579711900497
0.884023904800415 loss -0.0804342274119
0.7745778560638428 loss -0.017954546351
0.9118368625640869 loss -0.0787690062096
0.646277904510498 loss -0.187596198911
1.584261178970337 loss -0.0167338709049
0.8912017345428467 loss 0.110444259899
0.6290709972381592 loss 0.0701652950995
1.5293290615081787 loss -0.0226577075318
1.5704498291015625 loss -0.0574706316299
0.8637986183166504 loss 0.123965321754
0.6768815517425537 loss 0.136438498065
0.7258210182189941 loss 0.0681598176591
0.5948197841644287 loss -0.0991189255567
0.9021413326263428 loss -0.0786844364608
0.9102156162261963 loss -0.0442993911872
0.8312222957611084 loss 0.0451298701299
0.7466747760772705 loss 0.0957689343611
0.6502716541290283 loss 0.123531549007
0.717228889465332 loss -0.169220771297
0.5793960094451904 loss 0.0481078986736
1.1109960079193115 loss 0.000148486093877
0.6365237236022949 loss -0.233118821357
0.8560211658477783 loss -0.0186989398002
0.9205100536346436 loss -0.13897279002
0.8603379726409912 loss -0.0404320087475
0.6256051063537598 loss -0.115162585126
0.6190330982208252 loss 0.0430506910359
1.8858165740966797 loss -0.0148456234053
0.9813737869262695 loss -0.0656532164299
0.7410244941711426 loss 0.0819680012537
1.7096445560455322 loss -0.0309851428862
1.7374324798583984 loss 0.0332334752144
0.656376838684082 loss 0.0792765188848
0.8672158718109131 loss -0.104548750744
0.7355263233184814 loss -0.199718596002
1.0816049575805664 loss 0.0431473791475
0.8540570735931396 loss -0.0195965450417
0.6247105598449707 loss 0.0413701456934
0.8444550037384033 loss -0.0391767729012
1.5278067588806152 loss -0.0304681398375
0.6412947177886963 loss 0.048113657423
1.1438729763031006 loss -0.0883533011295
1.5017602443695068 loss 0.0146599261893
0.9255697727203369 loss 0.0408347947443
0.7621235847473145 loss 0.0941497622381
1.1891093254089355 loss -0.0229785259821
0.5510604381561279 loss -0.0887992379444
0.7080883979797363 loss -0.0559297435205
1.1038293838500977 loss -0.0729324957489
0.56815505027771 loss 0.0678844233302
0.8690965175628662 loss -0.0614308457127
0.8792247772216797 loss -0.0753499751802
0.8266885280609131 loss -0.0128113911453
0.6253194808959961 loss 0.0283325867246
1.7609972953796387 loss 0.00983533330076
0.6522014141082764 loss 0.00593241642211
0.800828218460083 loss -0.039276280561
0.9572291374206543 loss -0.0547891591642
1.378739356994629 loss 0.000668362000551
0.6280074119567871 loss -0.0529442046223
0.7708072662353516 loss 0.0793127961669
0.8982110023498535 loss -0.0862875267559
1.5705759525299072 loss -0.021992514471
0.819199800491333 loss 0.0568108782543
0.6245849132537842 loss -0.0931641541445
1.3084261417388916 loss 0.0531191896859
0.6763179302215576 loss -0.0222979633307
0.8855314254760742 loss 0.0118035705299
0.7748112678527832 loss -0.0857415087317
0.6131620407104492 loss 0.0718491834491
0.6947085857391357 loss 0.142986205436
1.4035253524780273 loss 0.0278423838478
1.3024461269378662 loss -0.0232436337532
2.032801628112793 loss -0.0184197540386
1.4866788387298584 loss -0.0168346544832
1.1261968612670898 loss 0.0177719582828
0.9024298191070557 loss 0.108811246623
0.6977088451385498 loss 0.0766716538893
1.400850534439087 loss 0.073984726055
1.0872952938079834 loss 0.0604425780514
1.2858600616455078 loss -0.0452753068883
0.8975028991699219 loss 0.0123638030995
0.9381847381591797 loss -0.0111462678634
0.9077610969543457 loss -0.0225721650111
1.6980910301208496 loss 0.0236090386501
1.530531644821167 loss -0.0258748978206
1.7171704769134521 loss -0.0633460366171
1.268232822418213 loss -0.0470247966956
0.7107362747192383 loss -0.00555020492776
0.8781700134277344 loss 0.097395621521
0.805680513381958 loss -0.0517612358466
1.4493155479431152 loss -0.0844747324687
0.9556694030761719 loss 0.0147981899731
1.7127974033355713 loss -0.0356169997161
1.9584522247314453 loss -0.0352629781421
0.9015700817108154 loss -0.0924539446912
1.9149584770202637 loss -0.00794953353552
0.9381206035614014 loss 0.0366596089566
1.1401758193969727 loss 0.0990619745995
1.008387565612793 loss -0.00973608309437
1.2541706562042236 loss 0.0164519085609
0.9614682197570801 loss -0.0607014425394
0.7869200706481934 loss -0.108392274195
0.6666150093078613 loss 0.0658401021775
0.9282586574554443 loss -0.0
0.6385796070098877 loss 0.0905976508334
1.022139072418213 loss -0.0631913732539
0.7731952667236328 loss -0.095428402398
0.7420549392700195 loss 0.0644743149451
0.8499624729156494 loss -0.00863662202512
0.945167064666748 loss -0.0915883050779
0.7280013561248779 loss 0.173384657924
0.6662602424621582 loss -0.0838220874549
0.8279883861541748 loss -0.110806669363
0.6429991722106934 loss 0.172613366514
0.62784743309021 loss 0.111369209278
0.6359422206878662 loss 0.0292838933271
0.8831648826599121 loss -0.0743029690614
0.9263384342193604 loss 0.0465728665172
0.7317500114440918 loss 0.206999386669
0.6702392101287842 loss 0.100503781526
0.6894106864929199 loss -0.0469659346822
0.7816047668457031 loss 0.0786055796407
0.8380076885223389 loss 0.0479557022512
0.9627280235290527 loss 0.0512890975952
0.7734911441802979 loss 0.0064619350018
0.642258882522583 loss -0.0774984950799
0.6575782299041748 loss -0.0874386600619
0.6508216857910156 loss 0.122304469149
0.8262345790863037 loss -0.00514048262303
0.8728644847869873 loss 0.0455647283263
0.880244255065918 loss 0.0538965197898
1.0289990901947021 loss -0.0188870935345
0.8954801559448242 loss 0.0105851224805
0.858149528503418 loss 0.0350851617911
0.6639113426208496 loss 0.0835343790618
1.2273271083831787 loss 0.0845653963489
0.8824150562286377 loss -0.0665190105238
0.731224775314331 loss -0.0
0.6813962459564209 loss 0.0559540889527
0.7975661754608154 loss -0.00405993046135
0.701866865158081 loss -0.0276435920325
1.8514676094055176 loss 0.036090434614
0.6405460834503174 loss -0.0543358164785
0.9079174995422363 loss -0.0017160048889
1.80204176902771 loss -0.016489285924
0.9552202224731445 loss 0.00979535101076
0.6773080825805664 loss -0.102069430827
0.9342315196990967 loss 0.0646406088921
0.9697771072387695 loss 0.016618726161
1.611084222793579 loss 0.00517213090188
1.6213562488555908 loss -0.013585704183
1.1365206241607666 loss -0.0100462342483
0.8340551853179932 loss 0.0289923588024
0.6929271221160889 loss -0.0
1.0260512828826904 loss -0.101804226339
1.024073839187622 loss 0.00557444691534
0.7377381324768066 loss -0.0530877773387
0.653372049331665 loss -0.0657081503209
0.8541033267974854 loss 0.0580624255522
1.1920719146728516 loss -0.0431580672192
0.651944637298584 loss 0.170251306152
0.6400363445281982 loss 0.0105809738923
0.6383473873138428 loss -0.0299876326518
0.6522893905639648 loss 0.302127659574
0.6481778621673584 loss 0.0590052943236
0.9067423343658447 loss 0.0027822750468
0.775482177734375 loss -0.0475876221239
0.6312267780303955 loss -0.107009647187
1.321681022644043 loss 0.00639598931788
0.6289224624633789 loss 0.0592852361809
0.5913906097412109 loss 0.180402691064
0.7777912616729736 loss -0.0730753607159
0.752708911895752 loss 0.0627761130112
0.5788469314575195 loss -1.60246890532e-17
1.5205533504486084 loss -0.0500376807359
0.578894853591919 loss 0.0188578571601
0.9674246311187744 loss -0.0200392399453
0.6230981349945068 loss 0.303613707719
0.59116530418396 loss 0.0437521893592
0.582390308380127 loss 0.0457229956859
0.6309168338775635 loss -0.0499501712931
0.6317458152770996 loss -0.0944911182523
0.7944545745849609 loss -0.0274941168884
0.775275468826294 loss 0.0101144347485
1.0592544078826904 loss 0.0158669597815
0.6439845561981201 loss -0.0658506533157
0.562361478805542 loss 0.217327693928
1.253655195236206 loss -0.038904929107
0.7457976341247559 loss 0.0155824498335
0.57781982421875 loss -0.144620305212
0.5554769039154053 loss -0.0361550763031
0.5573074817657471 loss 0.0332844215926
0.588477373123169 loss 0.0421370524277
1.2520477771759033 loss -0.0116694603048
0.7681684494018555 loss 0.002720052225
0.6435463428497314 loss 0.0434756752368
1.4845929145812988 loss -0.0070042843007
0.5805013179779053 loss 0.0744331447696
0.5346319675445557 loss -0.0857492925713
0.6894063949584961 loss 0.00318002335489
0.6249356269836426 loss -0.0575136373397
0.9834654331207275 loss -0.036767126545
0.7947084903717041 loss 0.00618589574132
0.760753870010376 loss 0.122801485828
1.0514411926269531 loss 0.0780425688565
1.5640554428100586 loss -0.0445155860716
0.5823814868927002 loss -0.0498668594716
0.6332778930664062 loss 0.311988770318
0.7470483779907227 loss -0.0760181321946
1.1045465469360352 loss -0.0283446911891
0.9331920146942139 loss 0.0551250478344
0.6408722400665283 loss 0.106425587458
0.5545661449432373 loss 0.108229153499
0.7879197597503662 loss 0.0568956642938
1.0623464584350586 loss 0.0348424650786
0.7667465209960938 loss 0.129287900336
0.7702038288116455 loss -0.0911872197824
0.9906303882598877 loss 0.00214785550583
0.8700385093688965 loss 0.0393050016684
0.7068824768066406 loss -0.0496370285719
0.5875177383422852 loss -0.142137001445
1.2461488246917725 loss -0.0678187475658
0.7462866306304932 loss -0.0279727119432
0.7872388362884521 loss -0.0462938663889
0.6521944999694824 loss -0.0770669989118
1.2595634460449219 loss -0.0112229366084
0.5874342918395996 loss 0.0682230446225
0.6292843818664551 loss 0.0583657677719
1.277355670928955 loss -0.0631339165885
1.148529052734375 loss -0.00296146856885
0.5969994068145752 loss 0.060909634902
0.6432943344116211 loss -0.079549853583
0.7745583057403564 loss 0.185245829028
1.4837536811828613 loss 0.0529171469799
0.8413481712341309 loss -0.0521753038836
1.059098243713379 loss -0.0318456152845
0.6281008720397949 loss -0.106963490396
0.996645450592041 loss 0.0575346284415
1.0245304107666016 loss -0.0496116294367
1.118166446685791 loss -0.023855406116
0.5864231586456299 loss -0.0605227532669
1.0653481483459473 loss -0.0855986149253
0.9454448223114014 loss -0.00674453174082
0.6170730590820312 loss -0.0706381021135
0.6102433204650879 loss 0.132630194016
0.9108474254608154 loss 0.0552245651609
0.6746346950531006 loss -0.210666210154
1.4615600109100342 loss -0.0200369837852
1.0239942073822021 loss -0.00313225787277
1.0438008308410645 loss -0.0421238812825
0.5410184860229492 loss -0.181585677749
0.761336088180542 loss 0.0013059599783
0.8836100101470947 loss -0.0364364430081
0.6854104995727539 loss 0.0504450361794
0.5763771533966064 loss -0.0813073664672
0.6647803783416748 loss -0.00151367539367
0.7726585865020752 loss -0.0357937953105
1.3726677894592285 loss 0.0423002592506
1.5556321144104004 loss 0.0300687482107
0.6134412288665771 loss -0.0353113495418
1.2117252349853516 loss 0.054474312977
0.607586145401001 loss -0.204228737128
0.7913341522216797 loss -0.0460906862107
0.9321341514587402 loss -0.0796652438496
1.2447319030761719 loss 0.0333386164246
1.2216675281524658 loss -0.0306117788978
1.4633495807647705 loss -0.00508977980495
0.7652065753936768 loss 0.00519470685211
0.5909926891326904 loss 0.146457475187
0.557349681854248 loss -0.112151516164
0.6229147911071777 loss 0.136232386642
0.6290717124938965 loss 0.0315834311811
0.7997381687164307 loss 0.058867596215
1.1745498180389404 loss -0.00709220757377
0.6940145492553711 loss -0.0373384068826
0.5802969932556152 loss 0.166692321507
0.6537356376647949 loss -0.101929873451
0.574887752532959 loss -0.0519498260999
0.8989531993865967 loss 0.0548703261169
1.5388939380645752 loss -0.0273620159321
1.0285229682922363 loss 0.0685455926381
1.3521907329559326 loss 0.0243822350642
0.7559089660644531 loss 0.0573214459878
0.8020765781402588 loss 0.0907666273747
0.6422033309936523 loss 0.0152480841033
0.6526174545288086 loss -0.058333513109
0.6239335536956787 loss -0.0111120988971
0.8268945217132568 loss 0.0813569895212
0.6901936531066895 loss 0.128523713932
0.754847526550293 loss 0.0196321768579
0.6329681873321533 loss -0.0386772872244
0.6663472652435303 loss -0.0669418969014
0.8991613388061523 loss 0.0452833412852
0.6060805320739746 loss 0.0832313360027
0.9528865814208984 loss 0.0461894996228
0.8258070945739746 loss 0.0828765163423
0.661505937576294 loss -0.0185927893482
0.7051324844360352 loss -0.0440270997284
0.6298959255218506 loss 0.117490927899
0.6098484992980957 loss -0.118425484673
1.023961067199707 loss 0.0333135506827
0.8071699142456055 loss -0.121333508155
0.8822462558746338 loss -0.164758616987
0.6529858112335205 loss -0.0334423867969
0.8422563076019287 loss -0.126045677661
0.9272317886352539 loss -0.032686022523
0.8018016815185547 loss -0.055335580164
0.6795642375946045 loss -0.0277350098113
0.7768363952636719 loss -0.0210072907951
0.9000840187072754 loss -0.0258652545379
0.6454212665557861 loss -0.0100627524433
0.6493780612945557 loss -0.0777082995277
0.673459529876709 loss 0.0273684210526
1.043868064880371 loss 0.0630231445263
0.7238588333129883 loss -0.121620847596
0.8839395046234131 loss 0.0443472683634
0.6500120162963867 loss -0.0251497727414
0.5864906311035156 loss -0.0354663451066
0.6185610294342041 loss 0.0810892854428
0.9187667369842529 loss 0.0882547343485
0.7178311347961426 loss -0.100572929358
0.9058132171630859 loss -0.0475033192803
0.763094425201416 loss 0.155806081034
0.7611026763916016 loss 0.085688827673
0.6463911533355713 loss -0.13744256927
0.6156811714172363 loss -0.212471698265
1.16823410987854 loss -0.0835472916842
0.7528781890869141 loss 0.0643625417468
0.8762490749359131 loss -0.0427002096023
0.7986154556274414 loss -0.0165597186854
0.9645633697509766 loss -0.086197853396
0.6325094699859619 loss 0.0543245581884
0.6325311660766602 loss -0.00379868588199
0.5586802959442139 loss -0.0482674745967
0.990868091583252 loss -0.0324454108225
0.9625711441040039 loss 0.0324506014469
0.7972569465637207 loss 0.00397957259196
0.8801615238189697 loss 0.00416694577636
0.6399970054626465 loss 0.12770236106
0.749011754989624 loss 0.077330889334
0.6381497383117676 loss 0.129911175653
0.7351274490356445 loss 0.0446656139974
0.5912165641784668 loss -0.0871842563633
1.5234887599945068 loss -0.0449539044901
1.0675709247589111 loss -0.0409221129574
1.0363988876342773 loss 0.0107227796253
0.9509241580963135 loss -0.051573896202
0.746286153793335 loss -0.10721425275
0.980252742767334 loss 0.0196651409706
0.9359958171844482 loss 0.0889831629145
0.6601150035858154 loss 0.0360466564804
0.8689699172973633 loss 0.0103813749558
0.6736986637115479 loss -0.0807734416128
1.1655662059783936 loss 0.0239875689713
0.7353265285491943 loss -0.0506033763699
1.1179301738739014 loss 0.0600905207343
0.6362252235412598 loss -0.11266715931
0.6147654056549072 loss -0.0987313844457
0.6179461479187012 loss -0.0563368285019
0.7706248760223389 loss 0.0940030770434
0.6012580394744873 loss 0.0872093253557
0.937528133392334 loss -0.0521916750479
1.03812837600708 loss -0.0320539164393
0.7304239273071289 loss 0.142857142857
0.7828834056854248 loss -0.151206300238
1.6873955726623535 loss 0.0447644577695
0.7529640197753906 loss 0.097477393325
0.7708570957183838 loss -0.190362444208
1.2396845817565918 loss 0.0417720727562
0.6172645092010498 loss -0.104012823225
1.5226421356201172 loss -0.0270103916739
1.7090611457824707 loss 0.0526959905428
0.5879125595092773 loss -0.155485768403
0.9088435173034668 loss 0.0457210403795
0.64276123046875 loss 0.101993569975
0.9412682056427002 loss 0.0378679996675
0.8087723255157471 loss -0.0231292517007
0.719346284866333 loss 0.0462552766256
1.0338966846466064 loss 0.00975827319355
0.7078244686126709 loss -0.186200499939
1.062319040298462 loss -0.0206078657013
0.7408356666564941 loss -0.0437521893592
1.5988059043884277 loss 0.0117743977157
0.675961971282959 loss 0.0912328360967
0.6257247924804688 loss -0.224679258568
0.8592367172241211 loss -0.0373035978037
0.6293036937713623 loss -0.0579711900497
0.84751296043396 loss -0.00480375223519
0.8877859115600586 loss -0.0464027870135
1.5752992630004883 loss -0.0486208525992
0.6851105690002441 loss 0.0906702160439
0.7404325008392334 loss -0.0123973581096
1.216672658920288 loss -0.00280678134254
0.6571757793426514 loss 0.0227711918929
1.6786916255950928 loss 0.0146783541025
1.0877082347869873 loss -0.0133235690924
0.670750617980957 loss -0.038244005432
0.6494946479797363 loss -0.060888137589
1.068873643875122 loss 0.0771762246221
0.9481017589569092 loss -0.0284364437304
0.8589894771575928 loss -0.00719924569201
1.3119707107543945 loss -0.079728097053
0.8035919666290283 loss -0.0859404840964
0.7692561149597168 loss -0.0
0.7164194583892822 loss -0.0293464486921
1.0704429149627686 loss 0.0272057589027
1.0659725666046143 loss 0.00212219062016
1.0925607681274414 loss -0.0575777640408
0.633458137512207 loss 0.3649002246
0.5341339111328125 loss -0.185312329165
0.6607904434204102 loss 0.374802530691
0.585374116897583 loss -0.246414303882
1.6730096340179443 loss -0.0110726882232
0.6627650260925293 loss 0.306344368452
0.9368484020233154 loss 0.104032008888
1.1048731803894043 loss -0.0525290538176
0.7365634441375732 loss -0.108490319178
1.1061677932739258 loss 0.118395663198
0.6588184833526611 loss 0.0347284611304
1.0130512714385986 loss -0.0374253074585
0.6443796157836914 loss 0.0172589633712
0.7446939945220947 loss -0.107992126233
0.6458868980407715 loss 0.0121640460025
0.6421661376953125 loss -0.077151674981
0.6493241786956787 loss 0.0213691071826
0.6081006526947021 loss -0.031714434561
1.0347607135772705 loss -0.0364068966322
0.919513463973999 loss 0.0451706817648
1.2426159381866455 loss -0.050075116027
0.5953328609466553 loss 0.0220153123503
0.7839562892913818 loss 0.0443690892084
0.7439670562744141 loss -0.0194779638102
1.160048246383667 loss 0.0779752772916
0.7952132225036621 loss -0.0975048816307
0.6398415565490723 loss 0.166428022045
1.0095622539520264 loss 0.0286181903516
0.8327603340148926 loss -0.124013537505
0.5919032096862793 loss -0.151716521227
0.7065796852111816 loss 0.083266012835
0.7780132293701172 loss 0.105616112055
0.6554219722747803 loss 0.10334831938
1.1217820644378662 loss -0.0164419061461
0.7280950546264648 loss 0.0798577553264
0.592090368270874 loss -0.027397312735
0.8999931812286377 loss -0.0442106274075
1.4660577774047852 loss -0.0156702739163
0.6114017963409424 loss -0.133194661082
0.7792131900787354 loss -0.0588887711115
0.6487267017364502 loss -0.0
0.8974058628082275 loss -0.0478347090515
0.986119270324707 loss 0.068199433947
1.400038719177246 loss -0.0249283122311
0.6275384426116943 loss -0.0
1.2837417125701904 loss -0.0267713760679
0.6274905204772949 loss -0.166465773667
0.6300022602081299 loss -0.115470053838
0.8254199028015137 loss -0.0817009225206
1.4049155712127686 loss 0.0506966935195
0.7748146057128906 loss 0.0576964019587
0.6700904369354248 loss 0.0170374810924
1.3036470413208008 loss -0.0274112755504
0.7242529392242432 loss -0.0292177138609
1.0829555988311768 loss 0.0794212798307
0.745380163192749 loss 0.0517087689995
1.0005993843078613 loss 0.161840641137
0.9408903121948242 loss -0.112264028803
1.0498299598693848 loss 0.05859899439
0.7961673736572266 loss -0.109534313572
0.6803662776947021 loss -0.057728176915
0.8751351833343506 loss -0.0630447649664
0.6034634113311768 loss -0.0
1.011850357055664 loss 0.025994340627
1.7557387351989746 loss 0.023221639858
1.338268518447876 loss 0.0990475458312
0.7080097198486328 loss 0.00858927775688
0.6837539672851562 loss -0.0443888015143
0.9141378402709961 loss -0.00246837671067
1.066162347793579 loss -0.132513661202
1.394887924194336 loss 0.0589160760744
1.0494790077209473 loss -0.0857402867116
1.22873854637146 loss 0.0702057417425
1.1752047538757324 loss -0.0269280678504
0.7011222839355469 loss -0.113376632523
1.6502516269683838 loss -0.078303540841
0.6553905010223389 loss -0.0
1.0274648666381836 loss -0.0546268530538
1.136338710784912 loss 0.050202030123
0.8146355152130127 loss -0.100516976244
0.9554874897003174 loss 0.0332608922434
0.7001667022705078 loss -0.093250480824
0.6544435024261475 loss -0.0422637165317
0.7040610313415527 loss 0.064626266662
0.6833798885345459 loss -0.145521375022
0.7051928043365479 loss -0.148522131447
0.7492256164550781 loss 0.0994512269648
0.6805000305175781 loss 0.00312280504909
0.7440974712371826 loss 0.0439740632729
0.6553351879119873 loss 0.0534471969013
0.635404109954834 loss -0.209275398864
1.2562882900238037 loss 0.0768984459845
0.717921257019043 loss -0.118425484673
0.7718939781188965 loss 0.0823356983752
0.6315338611602783 loss -0.0
0.639629602432251 loss -0.0164765540686
0.6248371601104736 loss 0.111472953324
0.6756877899169922 loss -0.0735786241
1.145615577697754 loss -0.0266729176242
0.7499613761901855 loss -0.110096376513
0.6001913547515869 loss 0.272063703362
0.6268424987792969 loss 0.161233878133
0.623772382736206 loss 0.0145973735849
0.769390344619751 loss -0.151214669382
0.7170593738555908 loss 0.0432198439543
0.836777925491333 loss -0.0530831893243
0.680433988571167 loss -0.139686059154
0.5888338088989258 loss -0.0859869362183
0.7348072528839111 loss 0.00848555291628
0.6514322757720947 loss -0.21146027323
0.7759785652160645 loss 0.0344854698225
0.6151797771453857 loss 0.0771852276267
0.6240737438201904 loss 0.00706439190482
0.5938582420349121 loss -0.246753246753
0.6287293434143066 loss -0.0916169659172
0.721463680267334 loss -0.0379671840329
0.6347146034240723 loss 0.0379274579097
0.7941563129425049 loss -0.0693894516546
0.6449282169342041 loss 0.0176455324594
0.7102267742156982 loss -0.0301663738494
0.649193286895752 loss 0.231319028645
0.8076426982879639 loss -0.0515152545245
0.7022056579589844 loss 0.0295927660447
0.6741766929626465 loss -0.00205568194581
0.9006855487823486 loss 0.00976475175785
0.5619826316833496 loss -0.134323635121
1.0238103866577148 loss -0.103447736255
1.0333888530731201 loss 0.0347302730384
0.8595118522644043 loss 0.0261471861472
1.1345360279083252 loss 0.0704120357821
0.6375558376312256 loss -0.000693000693001
0.7497014999389648 loss -0.102987301918
0.6595184803009033 loss 0.160006817004
0.7520427703857422 loss 0.0521331166179
0.7577884197235107 loss -0.0693375245282
0.7334339618682861 loss -0.0171969617716
1.03696870803833 loss 0.0120955675812
0.9062273502349854 loss 0.0415340227693
0.5638659000396729 loss -0.116774841624
0.7682416439056396 loss -0.0
0.8964972496032715 loss -0.0463763541093
0.6407692432403564 loss -0.117030570702
0.6255383491516113 loss 0.23036964956
0.6469650268554688 loss 0.212382500203
0.6516819000244141 loss 0.155808450494
0.576751708984375 loss 0.0373739976015
0.6991660594940186 loss -0.0510809444239
1.0338146686553955 loss -0.0995735067501
0.8403158187866211 loss -0.0779473896258
0.6381821632385254 loss -0.0591777081637
0.6832592487335205 loss -0.0421533610677
0.6138901710510254 loss 0.0452833412852
0.6098330020904541 loss -0.0126533853952
0.9266462326049805 loss -0.0324571947271
0.8146471977233887 loss -0.0927404027752
0.7392101287841797 loss -0.254807692308
1.210402250289917 loss -0.158179259849
1.0808777809143066 loss 0.0449079879068
1.1874070167541504 loss 0.0551468696869
0.8618292808532715 loss 0.112296339872
0.8307287693023682 loss -0.089436817404
0.5953612327575684 loss -0.164319810239
1.3439083099365234 loss -0.0581283091233
0.656125545501709 loss -0.134265939713
0.8931381702423096 loss -0.00543518320687
0.8720066547393799 loss -0.0502374933566
1.0884425640106201 loss -0.0415105160551
0.6721384525299072 loss -0.0404366081933
0.598330020904541 loss -0.100342662341
0.7014987468719482 loss -0.0
0.7393374443054199 loss 0.0214714206574
1.031463384628296 loss 0.0106273863205
0.6237835884094238 loss 0.0721382259167
0.7311820983886719 loss 0.0282524086725
0.8257994651794434 loss -0.00726946449676
0.6031653881072998 loss -0.151804020962
1.0147936344146729 loss -0.0698111713569
0.76639723777771 loss -0.0385987768284
0.6941409111022949 loss -0.0612528648016
0.958491325378418 loss 0.0490778821875
0.5955715179443359 loss -0.0363136519601
0.5636863708496094 loss -0.0920216361679
1.2446448802947998 loss -0.0698890168122
0.5374510288238525 loss -0.0227634376287
0.6879160404205322 loss 0.106395940703
0.7729313373565674 loss 0.143922811122
0.5659761428833008 loss -0.00294408548932
1.3452141284942627 loss -0.0184478884365
0.5716843605041504 loss -0.0564958573541
0.7642364501953125 loss 0.0101917673579
0.7226831912994385 loss -0.0577091475472
0.7755942344665527 loss 0.0398239351288
1.976773977279663 loss -0.0552052826941
0.6990842819213867 loss -0.0979202913764
0.6439402103424072 loss 0.0606908086922
1.5841424465179443 loss 0.0434439835331
0.6753911972045898 loss -0.133543631652
1.0843360424041748 loss 0.0685606663602
0.7684245109558105 loss -0.114707866935
0.9685087203979492 loss 0.0333767234878
0.7372679710388184 loss -0.021868700842
0.706218957901001 loss -0.0732223313627
0.8422489166259766 loss 0.0442632294458
0.6345739364624023 loss 0.0361079545676
0.8797464370727539 loss -0.0162641860342
0.6848366260528564 loss 0.0282934943341
0.7964053153991699 loss -0.0159725220579
0.759530782699585 loss -0.0537440666467
0.6414358615875244 loss -0.228217732294
0.9221184253692627 loss 0.0826271650047
0.6078331470489502 loss -0.0574887051656
0.5720124244689941 loss 0.00560560661664
0.7082974910736084 loss -0.0520307367094
0.6930725574493408 loss 0.0778961947682
0.6068668365478516 loss -0.178276062434
0.8428647518157959 loss -0.0558778585944

In [126]:

len(trials.trials)

Out [126]:

564

In [1]:

from bson import json_util
import json
save_path = Path('../output/hyperopt/{}.json'.format(trials._exp_key))
save_path.dirname().makedirs_p()
json.dump(trials.trials,open(save_path,'w'), default=json_util.default)
save_path

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-1-4a286114c1ae> in <module>()
      1 from bson import json_util
      2 import json
----> 3 save_path = Path('../output/hyperopt/{}.json'.format(trials._exp_key))
      4 save_path.dirname().makedirs_p()
      5 json.dump(trials.trials,open(save_path,'w'), default=json_util.default)

NameError: name 'Path' is not defined

In [4]:

# load
trial_data =json.load(open(save_path))
# trial_data =json.load(open(save_path))
len(trial_data)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-4-18bff864bd4d> in <module>()
      1 # load
----> 2 trial_data =json.load(open(save_path))
      3 # trial_data =json.load(open(save_path))
      4 len(trial_data)

NameError: name 'save_path' is not defined

In [288]:

# convert to df


def flatten(d, parent_key='', sep='_'):
    """Flatten dicts."""
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, sep=sep).items())
        else:
            # also flatten lists with one element
            if isinstance(v,list) and len(v)==1:
                v=v[0]
            items.append((new_key, v))
    return dict(items)
# df = pd.DataFrame([flatten(trial) for trial in trials.trials])
df = pd.DataFrame([flatten(trial) for trial in trial_data])
# df

In [3]:

# lets make a table of major results to record
mean_of=50
metric='result_metrics_report_f1-score_leak'
# metric='result_metrics_matthews_corrcoef'
whitelist = [col for col in df.columns if ('misc_vals' in col) or (metric in col)]+[
    'result_dummy_metrics_report_dummy_f1-score_leak',
    'result_dummy_metrics_report_dummy_support_leak',
    
#     'result_metrics_report_f1-score_leak',
#     'result_metrics_matthews_corrcoef'
]
df50=df.sort_values(metric, ascending=False)[:mean_of][whitelist]

# convert to days
df50['misc_vals_timespan_before']=df50['misc_vals_timespan_before']/seconds_in_a_day

# columns
mean=df50.mean()
std=df50.std()
corr=df50.corr()[metric]
units=['bool', 'frac',
       'frac', 'frac',
       'bool', 'frac', 'deg',
       'days', 'frac',
       '','',
       'int']

# make into df
filter_results = pd.DataFrame(collections.OrderedDict(
    mean=mean,
    std=std,
    corr=corr))
filter_results['units']=units
filter_results=filter_results.round(3)

# print
print('top {mean_of:} scores, \n\n- by metric="{metric:}"\n- for key="{key:}"\n- number of trials={n:}\n'.format(
    mean_of=mean_of,
    key=save_path, 
    metric=metric,
    n=len(trials.trials)
))
import tabulate
markdown=tabulate.tabulate(filter_results,tablefmt='pipe',headers=filter_results.columns)
print(markdown)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-3-b47b94b6e60a> in <module>()
      3 metric='result_metrics_report_f1-score_leak'
      4 # metric='result_metrics_matthews_corrcoef'
----> 5 whitelist = [col for col in df.columns if ('misc_vals' in col) or (metric in col)]+[
      6     'result_dummy_metrics_report_dummy_f1-score_leak',
      7     'result_dummy_metrics_report_dummy_support_leak',

NameError: name 'df' is not defined

In [143]:

df.result_metrics_matthews_corrcoef.hist()

Out [143]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fee8fd0aef0>

• don't normalise
• do balance
• shift off 0.005 ish
• cloud cover of less than 0.7 (no effect)
• rotation eange of ~20 deg

Best filters (for f1 values) for dataset:

• l7_joined

  • mean of best 50 f1 values

      misc_vals_balanced_classes                                     0.980000
      misc_vals_channel_shift_range                                  0.083127
      misc_vals_height_shift_range                                   0.164816
      misc_vals_max_cloud_cover                                      0.109005
      misc_vals_normalized                                           0.000000
      misc_vals_rescale                                              0.329541
      misc_vals_rotation_range                                       9.903196
      misc_vals_timespan_before                                  97939.315502 (1.13 days)
      misc_vals_width_shift_range                                    0.270787
      result_metrics_report_support_avg / total                    414.480000
      result_metrics_matthews_corrcoef                               0.023856
      result_metrics_report_f1-score_avg / total                     0.418200
      result_metrics_report_f1-score_leak                            0.678600
    

  • Correlations (how important each is)

      misc_vals_balanced_classes                      0.623902
      misc_vals_channel_shift_range                  -0.052222
      misc_vals_max_cloud_cover                      -0.021244
      misc_vals_normalized                           -0.196311
      misc_vals_rotation_range                       -0.113902
      misc_vals_timespan_before                       0.084534
    

In [299]:

filters = ['misc_vals_','result_metrics_','result_dummy_metrics_report_dummy_f1-score_leak','result_run_time']
cols = [col for col in df.columns if any([f in col for f in filters])]
df1 = df[cols]
# df1

In [ ]:

In [300]:

# heat map of correlations
import seaborn as sns
corr = df1.corr()
sns.heatmap(corr, 
            xticklabels=corr.columns.values,
            yticklabels=corr.columns.values)
# plt.title('')
# plt.savefig('/tmp/cryptocorr.png')

Out [300]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fee9006d860>

rotation range etc

Open the plot below in a new window

We see rotation range above 25 tends to lose predictive power with 20 being ideal

Cloud cover restrictions seem to do it below 0.6... but I scraped that data to be below 0.3?

Channel shifts>0.01 seems to lose predictive power

In [33]:

df1.columns

Out [33]:

Index(['misc_vals_balanced_classes', 'misc_vals_channel_shift_range',
       'misc_vals_max_cloud_cover', 'misc_vals_normalized',
       'misc_vals_rotation_range', 'misc_vals_timespan_before',
       'misc_vals_filter_center_cloudy',
       'result_dummy_metrics_matthews_corrcoef_dummy', 'result_loss',
       'result_metrics_matthews_corrcoef',
       'result_metrics_report_f1-score_leak',
       'result_metrics_report_precision_leak',
       'result_metrics_report_precision_no leak',
       'result_metrics_report_recall_avg / total',
       'result_metrics_report_recall_leak',
       'result_metrics_report_recall_no leak',
       'result_metrics_report_support_avg / total', 'result_run_time'],
      dtype='object')

In [37]:

# this show the relationship between predictive power and days before
# so to get a good correlation we want 
sns.set()
plt.figure(figsize=(15,15))
df5=df[[
    'result_metrics_matthews_corrcoef',
    'misc_vals_channel_shift_range',
       'misc_vals_max_cloud_cover',        'misc_vals_rotation_range'
]].copy()
df5['n']=df['result_metrics_report_support_avg / total']
df5['f-score']=df1['result_metrics_report_f1-score_leak']
df5['precision*recall_leak']=df['result_metrics_report_recall_leak']*df['result_metrics_report_precision_leak']
df5['precision_leak']=df['result_metrics_report_precision_leak']
df5['misc_vals_timespan_before']=df.misc_vals_timespan_before/seconds_in_a_day
sns.pairplot(df5,size=6)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-37-2278525d9e86> in <module>()
     13 df5['precision_leak']=df['result_metrics_report_precision_leak']
     14 df5['misc_vals_timespan_before']=df.misc_vals_timespan_before/seconds_in_a_day
---> 15 sns.pairplot(df5,size=6)

/home/isisilon/.virtualenvs/py3syspck/lib/python3.4/site-packages/seaborn/linearmodels.py in pairplot(data, hue, hue_order, palette, vars, x_vars, y_vars, kind, diag_kind, markers, size, aspect, dropna, plot_kws, diag_kws, grid_kws)
   1607     if grid.square_grid:
   1608         if diag_kind == "hist":
-> 1609             grid.map_diag(plt.hist, **diag_kws)
   1610         elif diag_kind == "kde":
   1611             diag_kws["legend"] = False

/home/isisilon/.virtualenvs/py3syspck/lib/python3.4/site-packages/seaborn/axisgrid.py in map_diag(self, func, **kwargs)
   1346                 else:
   1347                     func(vals, color=self.palette, histtype="barstacked",
-> 1348                          **kwargs)
   1349             else:
   1350                 for k, label_k in enumerate(self.hue_names):

/usr/local/lib/python3.4/dist-packages/matplotlib/pyplot.py in hist(x, bins, range, normed, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, hold, data, **kwargs)
   2956                       histtype=histtype, align=align, orientation=orientation,
   2957                       rwidth=rwidth, log=log, color=color, label=label,
-> 2958                       stacked=stacked, data=data, **kwargs)
   2959     finally:
   2960         ax.hold(washold)

/usr/local/lib/python3.4/dist-packages/matplotlib/__init__.py in inner(ax, *args, **kwargs)
   1810                     warnings.warn(msg % (label_namer, func.__name__),
   1811                                   RuntimeWarning, stacklevel=2)
-> 1812             return func(ax, *args, **kwargs)
   1813         pre_doc = inner.__doc__
   1814         if pre_doc is None:

/usr/local/lib/python3.4/dist-packages/matplotlib/axes/_axes.py in hist(self, x, bins, range, normed, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, **kwargs)
   6008             # this will automatically overwrite bins,
   6009             # so that each histogram uses the same bins
-> 6010             m, bins = np.histogram(x[i], bins, weights=w[i], **hist_kwargs)
   6011             m = m.astype(float)  # causes problems later if it's an int
   6012             if mlast is None:

/home/isisilon/.virtualenvs/py3syspck/lib/python3.4/site-packages/numpy/lib/function_base.py in histogram(a, bins, range, normed, weights, density)
    664     if mn > mx:
    665         raise ValueError(
--> 666             'max must be larger than min in range parameter.')
    667     if not np.all(np.isfinite([mn, mx])):
    668         raise ValueError(

ValueError: max must be larger than min in range parameter.

<matplotlib.figure.Figure at 0x7f542852dd30>

How many days before?

This show how many days before we can go before we lose predictive power (either good or bad)

If you look at the top right graph, we lose predictive power at greater than 5 days. Before that its exponential increase but I also get a drop of in data.

So: 2 days looks like the sweet spot.

In [ ]:

In [ ]:

# this show the relationship between predictive power and days before
# so to get a good correlation we want 
sns.set()
plt.figure(figsize=(15,15))
df5=df[[
    'result_metrics_matthews_corrcoef'
]].copy()
df5['n']=df['result_metrics_report_support_avg / total']
df5['misc_vals_timespan_before']=df.misc_vals_timespan_before/seconds_in_a_day
sns.pairplot(df5,size=6)

In [ ]:

Hyperopt for randomforest

In [ ]:

In [35]:

"""
This experiment will look at the best random forest params
"""

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.metrics import roc_auc_score
import sys
from sklearn.dummy import DummyClassifier
import sklearn.ensemble
import time
seconds_in_a_day = 60*60*24

trials2 = Trials(exp_key='data_filters_v3_rf_ATX')
# trials = MongoTrials('mongo://localhost:27017/hyperopt_leaks2/jobs', exp_key='data_filters_v2')

def hp_int(label,min,max):
    return hp.choice(label, np.arange(min, max, dtype=int)) 


space = {



            # rf
            'n_estimators': hp_int('n_estimators',2,300),
            'criterion': hp.choice('criterion',['gini','entropy']),
            'max_features': hp_int('max_features',2,300),
            'max_depth': hp_int('max_depth',2,300),
            'min_samples_split': hp_int('min_samples_split',1,10),
            'min_samples_leaf': hp_int('min_samples_leaf',1,10),
            'bootstrap':hp.choice('bootstrap',[False,True]),
    
    
            'min_weight_fraction_leaf': 0, # hp.float('min_weight_fraction_leaf',2,10),
            'max_leaf_nodes': None, # hp.int('max_leaf_nodes',2,300),
            'min_impurity_split': 1e-7, #hp.float('min_impurity_split',1e-12,1e-3),
            
            'oob_score': False, # hp.choice('oob_score',[False,True]),
            'n_jobs':4,
            'random_state':0,
            'verbose':0,
            'warm_start':False,
            'class_weight':None,
        
    
            # data filter
            'max_cloud_cover': 0.3,
            'timespan_before': seconds_in_a_day*3,
            'balanced_classes':True,
            'normalized': False,
            'filter_center_cloudy': True,
    
            # std is 5% so max should prob be ~5%
            'channel_shift_range': 0.07,
            'rotation_range': 25,
    
            'max_depth': 50,
            'batch_size' : 2000*10,
            'nb_epochs' :  1,
        }


def f_rf(params):
    t0 = time.time()

    # Get data
    X_train, y_train, metadata_train, X_val, y_val, metadata_val, X_test, y_test, metadata_test = filter_split_data(
        X_raw,
        y_raw,
        metadatas,
        max_cloud_cover=params['max_cloud_cover'], 
        timespan_before=params['timespan_before'],
        test_fraction=test_fraction, 
        random_seed=random_seed,
        balanced_classes=params['balanced_classes'],
        normalized=params['normalized'],
        filter_center_cloudy=params['filter_center_cloudy']
    )   

    datagen = ImageDataGenerator(
    #     featurewise_center=True,
    #     featurewise_std_normalization=True,
        channel_shift_range=params['channel_shift_range'],
        rotation_range=params['rotation_range'],
    #     width_shift_range=0.05,
    #     height_shift_range=0.05,
        horizontal_flip=True,
        vertical_flip=True,
       dim_ordering='th',
    #     rescale=0.02,
    #     zoom_range=0.05,
    #     shear_range=0.05,
    )

    # datagen.fit(X_train)
    
    gen = datagen.flow(X_train, y_train, batch_size=params['batch_size']*params['nb_epochs'])
    X_trainb, y_trainb = next(gen)
    X_train2b = X_trainb.reshape((-1,14*25*25))
    X_val2 = X_val.reshape((-1,14*25*25))
    X_test2 = X_test.reshape((-1,14*25*25))
    
    t1=time.time()-t0

    # Get model    
    model = sklearn.ensemble.RandomForestClassifier(
        n_estimators=params['n_estimators'],
        criterion=params['criterion'],
        max_features=params['max_features'],
        max_depth=params['max_depth'],
        min_samples_split=params['min_samples_split'], 
        min_samples_leaf=params['min_samples_leaf'],
        max_leaf_nodes=params['max_leaf_nodes'],
        min_impurity_split=params['min_impurity_split'],
        bootstrap=params['bootstrap'],
        oob_score=params['oob_score'],
        n_jobs=params['n_jobs'],
#         random_state=params['random_state'],
        verbose=params['verbose'],
        warm_start=params['warm_start'],
        class_weight=params['class_weight'],
        
    )
    model.fit(X_train2b, y_trainb) 
    y_pred = model.predict(X_val2)
    score = model.score(X_val2, y_val)

    # score on X_val
    report = parse_classification_report(sklearn.metrics.classification_report(y_val>thresh,y_pred>thresh,target_names=target_names))
    matthews_corrcoef = sklearn.metrics.matthews_corrcoef(y_val>thresh, y_pred>thresh)
    loss=-matthews_corrcoef
    
#     sys.stdout.flush()
    print(t1,'loss',loss)
    return dict(loss=loss,
                status=STATUS_OK,
                run_time=t1,
                metrics=dict(
                    report=report.to_dict(),
                    matthews_corrcoef=matthews_corrcoef,
                ),             
                attachments=dict(                
#                     model=model.__dict__
                )
               )
            

In [48]:

best = fmin(f_rf, space, algo=tpe.suggest, max_evals=1000, trials=trials2, verbose=1)
print ('best: ')
print (best)

2.272477149963379 loss -0.0831334234745
2.0736782550811768 loss -0.0924210974528
2.413914918899536 loss -0.078942762829
2.1177151203155518 loss -0.152185012789
2.129666805267334 loss -0.0765051303491
2.2079720497131348 loss -0.0576978278033
2.241548776626587 loss -0.0908495972473
2.179779052734375 loss -0.103385151515
2.162714958190918 loss -0.11771247078
2.3401927947998047 loss -0.0884890730573
2.8878297805786133 loss -0.0783214847039
2.1903300285339355 loss -0.0522391331706
2.349170446395874 loss -0.0539767866521
2.5587830543518066 loss -0.106040281861
2.3124563694000244 loss -0.0747560013207
2.6566598415374756 loss -0.092875417357
2.196125030517578 loss -0.0610194766288
2.644954204559326 loss -0.13901848785
2.333570718765259 loss -0.0455991092667
2.249497652053833 loss -0.0759148602085
2.262049436569214 loss -0.065980782972
2.004255533218384 loss -0.10676754388
2.469154119491577 loss -0.00437870597846
2.393832206726074 loss -0.081317742885
2.2206265926361084 loss -0.0785766360178
2.216156244277954 loss -0.0505477749992
2.2465553283691406 loss -0.113740925451
2.3417046070098877 loss -0.0940001068469
2.612652540206909 loss -0.117114046469
2.2212135791778564 loss -0.0457760601246
2.1960608959198 loss -0.0649733170326
2.1780216693878174 loss -0.0979144673189
2.5605523586273193 loss -0.0686757800803
2.5702219009399414 loss -0.105484461654
2.2017927169799805 loss -0.101090948864
2.198988437652588 loss -0.0634117025284
2.497110366821289 loss -0.131897710042
2.632697105407715 loss -0.12090793662
2.81476092338562 loss -0.0516948720741
2.164499521255493 loss -0.037378704278
2.207096576690674 loss -0.142844902562
2.301442861557007 loss -0.0187473970968
2.168757915496826 loss -0.050767683583
2.581979274749756 loss -0.0802991350031
2.1655192375183105 loss -0.0434177252632
2.264699935913086 loss -0.0919981829868
2.5627617835998535 loss -0.0858664255344
2.2008018493652344 loss -0.0765051303491
2.14874529838562 loss -0.107180265742
2.184567928314209 loss -0.0535391503585
2.2430219650268555 loss -0.0920819515108
2.212372303009033 loss -0.0401036900654
2.2695999145507812 loss -0.0583675898273
2.3197624683380127 loss -0.138453013963
2.4439830780029297 loss -0.053038145883
2.0221474170684814 loss -0.101650736417
1.954366683959961 loss -0.0763788603524
1.9254844188690186 loss -0.090657853112
1.9014248847961426 loss -0.138804658913
1.9059479236602783 loss -0.103385151515
1.8416681289672852 loss -0.0678477941487
1.8779652118682861 loss -0.120082068543
1.9419021606445312 loss -0.0343338366094
1.9368500709533691 loss -0.0607694779249
1.923565149307251 loss -0.0945299586087
1.90217924118042 loss -0.133308692815
2.251796245574951 loss -0.101844986261
1.9694671630859375 loss -0.034663421535
1.879384994506836 loss -0.0945299586087
1.9158179759979248 loss -0.0330036793899
1.997110366821289 loss -0.0908495972473
2.5520095825195312 loss -0.145150835495
1.9399936199188232 loss -0.0896524689473
2.1461238861083984 loss -0.0577825399784
1.891322135925293 loss -0.0576978278033
1.9212026596069336 loss -0.0589586235376
2.071608543395996 loss -0.0946264229947
1.9063045978546143 loss -0.0637288837661
2.0294086933135986 loss -0.090657853112
1.9250898361206055 loss -0.042349275732
1.9384419918060303 loss -0.0615676690749
1.9407880306243896 loss -0.0601662612155
1.9225695133209229 loss -0.038491734636
1.8768441677093506 loss -0.0621714210991
2.135751247406006 loss -0.101090948864
1.9698553085327148 loss -0.00823135783324
1.9567601680755615 loss -0.107776520892
2.147087335586548 loss -0.0990058028914
2.142184019088745 loss -0.131438427922
2.1391549110412598 loss -0.0675782564573
2.105844497680664 loss -0.0984156987341
1.919830322265625 loss -0.0677352277008
2.1734542846679688 loss -0.0642829849834
1.9006235599517822 loss -0.0390167952541
1.9997014999389648 loss -0.0697978803731
1.9252631664276123 loss -0.0808882910299
1.8836431503295898 loss -0.122693767174
1.9430854320526123 loss -0.0307635798384
1.8931622505187988 loss -0.0765051303491
1.917811632156372 loss -0.12499727821
1.8973944187164307 loss -0.0302186275653
1.8989388942718506 loss -0.059334654635
1.8455822467803955 loss -0.0550660792952
1.9154362678527832 loss -0.0725203518877
1.890376329421997 loss -0.160756787017
1.8901684284210205 loss -0.108963595926
1.9257233142852783 loss -0.0914612730523
1.9181849956512451 loss -0.0884890730573
2.136522054672241 loss -0.0884890730573
1.9116249084472656 loss -0.117113809787
2.154127597808838 loss -0.108891098811
1.9067628383636475 loss -0.0879195896906
2.150751829147339 loss -0.0736249228418
2.059560775756836 loss -0.0934504487509
1.9569735527038574 loss -0.0686757800803
1.9151723384857178 loss -0.130260461722
1.9398069381713867 loss -0.0923087985585
1.9037628173828125 loss -0.104400409176
1.8856403827667236 loss -0.0467579069759
1.9639427661895752 loss -0.108282437439
2.0048279762268066 loss -0.0703685321551
2.2502973079681396 loss -0.115207510486
1.8935842514038086 loss -0.137358404247
1.8894720077514648 loss -0.0549410471251
1.9946928024291992 loss -0.128307047722
1.8838300704956055 loss -0.0379430058714
1.9047865867614746 loss -0.0827032342273
1.867363691329956 loss -0.065980782972
2.908477544784546 loss -0.020335762705
1.9859187602996826 loss 0.0247354743726
2.0023863315582275 loss -0.060159376665
1.9454665184020996 loss -0.0615924671148
2.0272600650787354 loss -0.0846829672436
2.1472408771514893 loss -0.0522391331706
2.109426975250244 loss -0.099464477935
2.076936721801758 loss -0.0291363936002
1.8960051536560059 loss -0.0753318751078
1.9494848251342773 loss -0.133895105602
1.926264762878418 loss -0.0797175876814
1.901444673538208 loss -0.081317742885
1.9200818538665771 loss 0.00666507790177
1.918950080871582 loss -0.129676549992
2.2538864612579346 loss -0.0159346962976
1.9304168224334717 loss -0.045710297902
2.1862120628356934 loss -0.119771981413
1.9967834949493408 loss -0.0500456560705
1.964953899383545 loss -0.116509535157
2.1903748512268066 loss -0.0543883674915
2.0945780277252197 loss -0.0533921467724
1.9008405208587646 loss -0.0769181636803
1.9007441997528076 loss -0.0984228537846
2.03547739982605 loss -0.129443363728
1.8992705345153809 loss -0.129676549992
1.9064445495605469 loss -0.0527588507085
1.9087543487548828 loss -0.0557653745944
2.2297580242156982 loss -0.134827545713
2.021090269088745 loss -0.0624209766392
1.9064786434173584 loss -0.0858664255344
1.9093778133392334 loss -0.0132009112551
1.9023089408874512 loss -0.0457760601246
1.906846523284912 loss -0.111240883313
1.9312796592712402 loss -0.0714411015991
1.9411375522613525 loss -0.0577825399784
1.9291369915008545 loss -0.113337985781
1.9406585693359375 loss -0.0319408723268
1.911888599395752 loss -0.0528129861336
2.207719087600708 loss -0.111563763532
1.9166085720062256 loss -0.0566296908362
1.975830316543579 loss -0.0950598929015
1.9226136207580566 loss -0.0494438002923
1.9955275058746338 loss -0.136311298458
2.0204832553863525 loss -0.0654523544061
1.9152593612670898 loss -0.02691523355
2.2059710025787354 loss -0.0759148602085
1.972764253616333 loss -0.045047851217
1.959801435470581 loss -0.104400409176
1.9620394706726074 loss -0.104937885349
1.9734573364257812 loss -0.0852706416016
1.9744393825531006 loss 0.0220544935899
2.031165361404419 loss -0.0927118387897
1.9312865734100342 loss -0.117858238196
1.929194450378418 loss -0.134827545713
1.9032747745513916 loss -0.0329723100498
1.938305139541626 loss -0.100540073557
1.9479811191558838 loss -0.0318618676535
2.008589506149292 loss -0.136828979716
1.9159047603607178 loss -0.0582383492343
1.8974754810333252 loss -0.0390167952541
1.8935036659240723 loss -0.182557198471
2.2246391773223877 loss -0.0797316611004
2.19687557220459 loss -0.0670383440778
1.919950246810913 loss -0.034663421535
2.0170676708221436 loss -0.115638843244
1.89961576461792 loss -0.0792163562663
2.0726208686828613 loss -0.0785766360178
1.919705867767334 loss -0.0488984420113
1.922788381576538 loss -0.0407901616791
1.958472490310669 loss -0.0753318751078
1.9718108177185059 loss -0.038491734636
1.9257545471191406 loss -0.096457881503
1.8958408832550049 loss -0.128128320067
1.9814541339874268 loss -0.0681240207462
1.9440035820007324 loss -0.112144896181
1.874507188796997 loss -0.0689386539477
1.9221022129058838 loss -0.0455991092667
1.9322073459625244 loss -0.060452284031
1.9342303276062012 loss -0.112188472982
1.9454269409179688 loss -0.0634117025284
1.9201056957244873 loss -0.0329723100498
1.9690392017364502 loss -0.0966991564162
1.9372045993804932 loss -0.0875635383479
2.2621545791625977 loss -0.103385151515
2.2119712829589844 loss -0.145160976366
1.9101645946502686 loss -0.050767683583
1.8958659172058105 loss -0.0252770815233
1.8857791423797607 loss -0.0620999761538
1.9361152648925781 loss 0.0115294004597
1.9150688648223877 loss -0.0467579069759
1.9113922119140625 loss -0.113337985781
1.9403338432312012 loss -0.103871896467
1.98020339012146 loss -0.108963595926
1.8617527484893799 loss -0.149094336151
2.2284095287323 loss -0.0197801718498
2.1718361377716064 loss -0.0291359532429
2.258470058441162 loss -0.119574637852
2.1982245445251465 loss -0.108963595926
2.1535863876342773 loss -0.0841468018363
1.9435997009277344 loss -0.0958381413214
1.907564640045166 loss -0.0633479082521
1.9140510559082031 loss -0.0551632158577
1.9043726921081543 loss -0.0499935291897
2.254030466079712 loss -0.0406524037444
2.008908271789551 loss -0.0719776326581
1.95255446434021 loss -0.0979144673189
1.9246985912322998 loss -0.12090793662
1.9108197689056396 loss -0.131384736749
2.0288233757019043 loss -0.103385151515
1.9703264236450195 loss -0.035234543452
1.9840888977050781 loss -0.0197944557418
2.322270631790161 loss -0.0296762789796
1.9008126258850098 loss -0.0970871462163
1.912318468093872 loss -0.110431465139
2.160705804824829 loss -0.0747560013207
1.916093349456787 loss -0.0302186275653
1.897564172744751 loss -0.0990058028914
1.87799072265625 loss -0.107180265742
2.053199291229248 loss -0.106040281861
1.9825963973999023 loss -0.119574637852
1.8810725212097168 loss -0.105802017746
1.9084827899932861 loss -0.0924892048676
1.9849252700805664 loss -0.0368103788305
1.9359517097473145 loss -0.0753318751078
1.9162437915802002 loss -0.101650736417
2.2428641319274902 loss -0.0966991564162
2.1030454635620117 loss -0.115638843244
1.9424123764038086 loss -0.0845935181281
2.0821516513824463 loss -0.129443363728
2.205582618713379 loss -0.0176145929256
1.9402730464935303 loss -0.0703685321551
2.0109660625457764 loss -0.140186567326
1.936178207397461 loss -0.0715298560645
2.03877329826355 loss -0.107180265742
1.911595106124878 loss -0.0966991564162
1.8958818912506104 loss -0.0873580309709
2.201850414276123 loss -0.0769181636803
2.1555745601654053 loss -0.092875417357
1.9671976566314697 loss -0.109578322485
2.234816074371338 loss -0.0861136768759
2.071816921234131 loss -0.0851880207504
1.9329428672790527 loss -0.038491734636
1.977942943572998 loss -0.0445006302187
2.135798931121826 loss -0.0697978803731
2.023602247238159 loss -0.102219590445
1.9720029830932617 loss -0.101090948864
1.8906338214874268 loss -0.0582383492343
1.8680779933929443 loss -0.0648428980937
2.1064252853393555 loss -0.175259331341
1.9557545185089111 loss -0.00823135783324
2.213651180267334 loss -0.100540073557
1.9559099674224854 loss -0.0747560013207
1.944065809249878 loss -0.107776520892
1.9384546279907227 loss -0.0357073349424
1.9307761192321777 loss -0.00165473015587
1.9483838081359863 loss -0.0610194766288
1.896169900894165 loss -0.0488984420113
2.0633668899536133 loss -0.134634358055
2.0191967487335205 loss -0.0919981829868
1.9431431293487549 loss -0.110431465139
1.926288366317749 loss -0.0896530595712
1.984740972518921 loss -0.0197801718498
2.0062997341156006 loss -0.0627565093911
1.895641565322876 loss -0.0960072321874
1.903278112411499 loss -0.0481804840783
1.8935964107513428 loss -0.0549410471251
1.9077117443084717 loss -0.085719376627
1.942826509475708 loss -0.12801829402
2.213685989379883 loss -0.0862580709034
1.9019503593444824 loss -0.0566296908362
1.9988312721252441 loss -0.042349275732
1.9184081554412842 loss -0.0225875383826
1.9577953815460205 loss -0.0189465278566
2.0706441402435303 loss -0.121535696697
1.8973779678344727 loss -0.10987998078
2.193061590194702 loss -0.0287357506825
1.9655392169952393 loss -0.132049548567
2.158751964569092 loss -0.144750492657
1.9524664878845215 loss -0.129676549992
1.9212746620178223 loss -0.102797672466
1.9058544635772705 loss -0.0648428980937
1.8945338726043701 loss -0.0829668607001
1.8670852184295654 loss -0.107263522845
1.9509377479553223 loss 0.0416476125745
2.1542348861694336 loss -0.0461545519182
1.9038922786712646 loss -0.0986127867169
2.217942953109741 loss -0.0621714210991
1.9098715782165527 loss -0.0725203518877
1.9047234058380127 loss -0.114277792572
1.9519076347351074 loss -0.0642829849834
2.232672691345215 loss -0.0927118387897
1.941432237625122 loss -0.0522247154021
1.9372804164886475 loss -0.125291404382
1.9442286491394043 loss -0.055498725006
1.93833327293396 loss -0.0358304118749
1.920067548751831 loss -0.0499935291897
1.902808427810669 loss -0.0785766360178
1.9718055725097656 loss -0.0870830670117
1.9944806098937988 loss 0.01867265903
2.024838924407959 loss -0.177196240387
1.911667823791504 loss -0.128307047722
1.9657883644104004 loss -0.117857285352
1.9650189876556396 loss -0.0791434767152
1.9580447673797607 loss -0.105802017746
2.3080053329467773 loss -0.10880560338
1.9423129558563232 loss -0.124722532752
2.165111541748047 loss -0.0648428980937
1.9804704189300537 loss -0.101650736417
1.8774073123931885 loss -0.0745582620265
2.0549228191375732 loss -0.10933513042
1.9657909870147705 loss -0.0769181636803
1.9374985694885254 loss -0.0895965300934
1.8997125625610352 loss -0.119771981413
1.9880359172821045 loss -0.0587839105193
1.9250879287719727 loss -0.0428857992265
1.9847872257232666 loss 0.0259319797026
1.9815216064453125 loss -0.15176469786
1.885209321975708 loss -0.0423213655761
2.0495846271514893 loss -0.0972625433635
1.9281814098358154 loss -0.0610194766288
1.976227045059204 loss -0.0457760601246
1.9350345134735107 loss -0.060452284031
1.9203529357910156 loss -0.0637288837661
2.2321674823760986 loss -0.135217054768
2.2005929946899414 loss -0.117625391112
2.2187044620513916 loss -0.102219590445
1.9159905910491943 loss -0.0654523544061
2.177461624145508 loss -0.0550660792952
1.9101414680480957 loss -0.0252990589163
1.9243059158325195 loss -0.0392200904583
1.9497718811035156 loss -0.0109760640518
1.948272466659546 loss -0.0824099163152
1.9187142848968506 loss -0.0807822270444
1.8625762462615967 loss -0.059334654635
1.9540917873382568 loss -0.104757384634
1.8972125053405762 loss -0.0961443514271
1.9835329055786133 loss -0.0858664255344
2.5884623527526855 loss -0.0759148602085
1.9186582565307617 loss -0.0979144673189
1.9250435829162598 loss -0.157979674619
1.9216904640197754 loss -0.0703685321551
1.9327452182769775 loss -0.104418431604
1.9403955936431885 loss -0.0838322838844
1.9285871982574463 loss -0.0455991092667
1.9811525344848633 loss -0.0412048213264
2.2429120540618896 loss -0.0610404684162
1.951716423034668 loss -0.0996051609608
2.0094964504241943 loss -0.0307635798384
2.224017858505249 loss -0.0285764144289
2.1986002922058105 loss -0.0362571714083
2.25356388092041 loss -0.153320265401
1.9188570976257324 loss -0.042349275732
1.8817839622497559 loss -0.119219888003
1.933318853378296 loss -0.0774641309732
1.912841558456421 loss -0.0862580709034
1.954021692276001 loss -0.0576978278033
2.015620708465576 loss -0.101460668794
1.9316892623901367 loss -0.0890666432078
1.9657917022705078 loss -0.125871458426
1.908172845840454 loss -0.0148529007107
2.164991855621338 loss -0.103385151515
1.979856252670288 loss -0.0126416694105
2.1778740882873535 loss -0.0824099163152
1.9427530765533447 loss -0.143404783643
2.5551035404205322 loss -0.0818602591107
2.0171585083007812 loss -0.111563763532
2.02069091796875 loss -0.0771028655743
1.9287006855010986 loss -0.092875417357
2.00661039352417 loss -0.034663421535
1.936842679977417 loss -0.125871458426
2.0855844020843506 loss -0.128307047722
1.9202523231506348 loss -0.0818602591107
1.9583485126495361 loss -0.0516948720741
1.9435343742370605 loss -0.0826368878015
2.1918535232543945 loss -0.00877785956727
1.9735386371612549 loss -0.0693542081034
1.9328815937042236 loss -0.108963595926
1.9244298934936523 loss -0.0455991092667
2.1126062870025635 loss -0.0777082525203
2.1559016704559326 loss -0.0258208157727
2.116103410720825 loss -0.0858664255344
2.204299211502075 loss -0.0528129861336
1.9863295555114746 loss -0.0990058028914
1.9361588954925537 loss -0.0714411015991
1.8644506931304932 loss -0.0274662057428
1.967883586883545 loss -0.035234543452
2.1228113174438477 loss -0.0428857992265
1.8586852550506592 loss -0.0912001663337
2.1371073722839355 loss -0.0368103788305
1.9206969738006592 loss -0.133504792335
1.9185163974761963 loss -0.104589015905
1.9055554866790771 loss -0.0733245169438
2.1277997493743896 loss -0.100882254023
2.2032182216644287 loss -0.0324710786346
2.009094715118408 loss -0.0681240207462
1.9270055294036865 loss -0.101650736417
1.9237165451049805 loss -0.0545670823768
1.9313204288482666 loss -0.0489999519538
1.8966560363769531 loss -0.0615924671148
1.90690016746521 loss -0.113740925451
1.9879436492919922 loss -0.0777089298895
1.9135658740997314 loss 0.00274324915757
1.953009843826294 loss -0.0759148602085
1.8932819366455078 loss -0.152185012789
1.8465368747711182 loss -0.0950598929015
2.141949415206909 loss -0.127487854271
2.2162423133850098 loss -0.0703685321551
1.923685073852539 loss -0.0120833593845
1.8998298645019531 loss -0.130252109708
2.039860963821411 loss -0.02691523355
1.953913927078247 loss -0.044605665052
1.954866647720337 loss -0.00279402893985
2.19350266456604 loss -0.184599238564
1.8808281421661377 loss -0.0347895213758
1.9641449451446533 loss -0.085719376627
1.9846813678741455 loss -0.0851880207504
1.9189631938934326 loss -0.0739375273237
2.1662938594818115 loss -0.0829668607001
1.9082891941070557 loss -0.0774641309732
2.20949125289917 loss -0.0560615538904
1.8774309158325195 loss -0.0582383492343
1.991793155670166 loss -0.0670394509301
2.1279053688049316 loss -0.118942685103
1.8845868110656738 loss -0.0654087776056
1.9825890064239502 loss -0.107117842075
1.9065778255462646 loss -0.0170522132427
1.9010422229766846 loss -0.0677352277008
1.9245591163635254 loss -0.132957607018
2.220524787902832 loss -0.0511066876889
1.8842661380767822 loss 0.00882795411419
1.8703839778900146 loss -0.0978346693994
1.9612538814544678 loss -0.0340961162028
2.179927110671997 loss -0.0654087776056
2.109015464782715 loss -0.0721208720098
2.0488908290863037 loss -0.0478475160291
1.9448528289794922 loss -0.0774641309732
1.9026081562042236 loss -0.0852706416016
2.0179781913757324 loss -0.0615924671148
1.9007294178009033 loss -0.0405751724118
1.949885368347168 loss -0.0538405374417
1.9186301231384277 loss -0.132049548567
1.8672761917114258 loss -0.107768450127
1.9501726627349854 loss -0.10776475426
1.8864731788635254 loss -0.108455876758
1.931056261062622 loss -0.0368103788305
1.8590431213378906 loss -0.0115388131193
2.0217020511627197 loss -0.0639142974308
1.947420358657837 loss -0.138453013963
1.9340651035308838 loss -0.0598907286721
1.8602228164672852 loss -0.0335324652823
1.8460638523101807 loss -0.0841032250358
1.9225103855133057 loss -0.135811920839
1.8782484531402588 loss -0.0966991564162
1.9063780307769775 loss -0.115207510486
1.920142412185669 loss -0.0654087776056
1.8873217105865479 loss -0.0721208720098
1.9113550186157227 loss -0.103385151515
2.121018409729004 loss -0.0791434767152
1.9271609783172607 loss -0.125871458426
2.228130340576172 loss -0.0390167952541
1.9099020957946777 loss -0.144659744951
1.9287922382354736 loss -0.14140532996
1.9719154834747314 loss -0.127065804724
1.9241156578063965 loss -0.0440277841537
1.9681377410888672 loss -0.055498725006
1.8798880577087402 loss 0.00496689036363
2.21374249458313 loss -0.0675782564573
2.1108546257019043 loss -0.106040281861
1.8775737285614014 loss -0.119219888003
1.912513256072998 loss -0.149729811755
1.9754579067230225 loss -0.0725203518877
1.9363734722137451 loss -0.0707983124832
1.9536025524139404 loss -0.0384783314586
1.9679887294769287 loss -0.0736249228418
2.0812721252441406 loss -0.0621714210991
1.922593355178833 loss -0.0610404684162
2.5439000129699707 loss -0.134063658197
2.1816906929016113 loss -0.0631804446647
2.215517044067383 loss -0.0629139648498
2.2244374752044678 loss -0.0203373592371
1.9721968173980713 loss -0.020335762705
1.989311695098877 loss -0.0841032250358
1.9392085075378418 loss -0.101650736417
1.974106788635254 loss -0.103982204388
1.9216537475585938 loss -0.12801829402
1.973402738571167 loss 0.0116210645483
1.9388139247894287 loss -0.0780169055256
2.229820966720581 loss -0.132049548567
2.009270191192627 loss -0.138127158172
1.9998407363891602 loss -0.0774641309732
1.9943995475769043 loss -0.0762569644046
1.9008476734161377 loss -0.0920819515108
2.120903491973877 loss 0.00774652631886
1.989898681640625 loss -0.113740925451
1.8869378566741943 loss -0.090657853112
1.9778459072113037 loss -0.122040563661
1.9435186386108398 loss -0.130252109708
1.8525416851043701 loss -0.0501729924353
1.8730037212371826 loss -0.0732755926162
1.9552223682403564 loss -0.0846638713101
2.100559711456299 loss -0.125291404382
1.9310054779052734 loss -0.0432272198633
1.9272971153259277 loss -0.0917504325898
2.1740355491638184 loss -0.123311606276
1.9930121898651123 loss -0.0337575541548
1.8690989017486572 loss -0.0484212447931
1.9542129039764404 loss -0.0923087985585
1.9032058715820312 loss -0.138804658913
2.123246431350708 loss -0.0670383440778
1.9052517414093018 loss -0.0171078215042
1.9012515544891357 loss -0.101460668794
1.9308326244354248 loss -0.0358304118749
1.9165618419647217 loss -0.0445006302187
2.17138671875 loss -0.0879195896906
1.95145583152771 loss -0.0319408723268
1.9685394763946533 loss -0.10776475426
1.9809017181396484 loss -0.104400409176
1.886171579360962 loss -0.134634358055
1.8963291645050049 loss -0.0725203518877
2.5200464725494385 loss -0.171789453037
1.9196479320526123 loss -0.114703402149
1.8822154998779297 loss -0.0472785987005
1.910759449005127 loss -0.0595558231915
1.9118695259094238 loss -0.070945801059
1.9161512851715088 loss -0.0923087985585
2.1222546100616455 loss -0.154997205803
2.282870054244995 loss -0.0160398923931
1.9277703762054443 loss -0.0719776326581
1.9837000370025635 loss -0.0929877833362
1.9141554832458496 loss -0.0829668607001
1.9014463424682617 loss -0.155506178767
1.917360782623291 loss -0.0401036900654
2.1994659900665283 loss -0.0516704211024
1.981156349182129 loss -0.0833246830567
1.8948731422424316 loss -0.128945759453
1.9693033695220947 loss -0.0538405374417
2.0849812030792236 loss -0.0615924671148
2.2397449016571045 loss -0.0539767866521
1.8667850494384766 loss -0.0686757800803
2.0220696926116943 loss -0.0763414023954
1.9221038818359375 loss -0.0775427471314
2.0010218620300293 loss -0.0774641309732
1.932893991470337 loss -0.111563763532
1.9226999282836914 loss -0.070945801059
2.2186150550842285 loss -0.050767683583
2.118023157119751 loss -0.0380041139071
1.8998048305511475 loss -0.119771981413
1.9228711128234863 loss -0.108359136836
1.8987383842468262 loss -0.135811920839
2.2333552837371826 loss -0.0802991350031
1.893449306488037 loss -0.160116892747
1.8976693153381348 loss -0.11395031109
2.0041027069091797 loss -0.0368812305215
1.9702744483947754 loss -0.100540073557
1.9883308410644531 loss -0.0769181636803
1.914872407913208 loss -0.0274662057428
1.8997986316680908 loss -0.0557653745944
1.9005825519561768 loss -0.124722532752
1.9080641269683838 loss -0.113337985781
1.9056317806243896 loss -0.0572032974223
1.9151663780212402 loss -0.0472785987005
1.9005146026611328 loss -0.0627565093911
1.9568562507629395 loss -0.135310689424
1.8865666389465332 loss -0.0763788603524
1.9712512493133545 loss -0.112736243651
1.9189929962158203 loss -0.180301253349
2.0096535682678223 loss -0.117113809787
1.9266440868377686 loss -0.0455991092667
1.9485201835632324 loss -0.0701708865185
1.887988805770874 loss -0.175259331341
1.9064404964447021 loss -0.0654087776056
1.8905978202819824 loss -0.065980782972
1.9435796737670898 loss -0.0774641309732
1.94435453414917 loss -0.102219590445
1.9214532375335693 loss -0.060452284031
2.1895952224731445 loss -0.0683445672708
2.1737232208251953 loss -0.0879195896906
1.9751896858215332 loss -0.16276696151
1.9129300117492676 loss -0.0484212447931
1.9147801399230957 loss -0.0841468018363
1.9469242095947266 loss -0.0627565093911
1.9277422428131104 loss -0.0488984420113
1.9255149364471436 loss -0.0659755251347
2.298863172531128 loss -0.157732972495
1.9230260848999023 loss -0.0742855990646
1.900740385055542 loss -0.113740925451
1.9702301025390625 loss -0.146094193624
1.9034309387207031 loss -0.125871458426
1.8919222354888916 loss -0.065980782972
1.9207649230957031 loss -0.0862580709034
2.1885032653808594 loss -0.142297514241
1.9089634418487549 loss -0.0807822270444
1.8940529823303223 loss -0.0946589434248
2.1237030029296875 loss -0.0527588507085
1.889650583267212 loss -0.00878315601428
1.9303061962127686 loss -0.068333438029
1.8998429775238037 loss -0.0274662057428
1.90924072265625 loss -0.0499935291897
1.936182975769043 loss -0.105205779195
2.214877128601074 loss -0.0654523544061
2.1542181968688965 loss -0.10987998078
2.188228130340576 loss -0.096457881503
1.9608469009399414 loss -0.0516704211024
2.0128626823425293 loss -0.117082084945
1.931058406829834 loss -0.117082084945
1.9310872554779053 loss -0.116612578372
2.0333054065704346 loss -0.0698673007399
1.9133803844451904 loss -0.0197944557418
1.9177203178405762 loss -0.0769181636803
1.9309663772583008 loss -0.070945801059
1.896791934967041 loss -0.0934940255514
1.8783197402954102 loss -0.128307047722
1.9364402294158936 loss -0.132957607018
1.887648582458496 loss -0.114573417068
1.9419867992401123 loss -0.0247354743726
1.9531035423278809 loss -0.0785766360178
1.8920254707336426 loss -0.0934940255514
1.9599134922027588 loss -0.0665041450337
2.1346490383148193 loss -0.138453013963
2.1908118724823 loss -0.0560615538904
2.181903839111328 loss 0.0483573115287
1.9828460216522217 loss -0.0858664255344
1.9887337684631348 loss -0.014825797394
2.187124252319336 loss -0.0358096478098
1.9084248542785645 loss -0.00437870597846
1.8770396709442139 loss -0.0765051303491
2.1109042167663574 loss -0.0870830670117
1.8766603469848633 loss -0.0870830670117
1.8981237411499023 loss -0.0977667544706
1.9123046398162842 loss -0.0697978803731
1.8892617225646973 loss -0.0851880207504
1.9234163761138916 loss -0.0331503719126
1.9326987266540527 loss -0.0879275907053
1.898040771484375 loss -0.122555772184
1.8898491859436035 loss -0.0692336823449
1.9590249061584473 loss -0.105484461654
2.060631036758423 loss -0.0692336823449
2.220353603363037 loss -0.0197801718498
1.9747130870819092 loss -0.184058955381
1.9225451946258545 loss -0.102797672466
2.1493537425994873 loss -0.0797316611004
1.9832799434661865 loss -0.117114046469
2.1966562271118164 loss -0.0538405374417
1.9240188598632812 loss -0.101650736417
1.9851324558258057 loss -0.045047851217
1.9752049446105957 loss -0.0934504487509
1.8904304504394531 loss -0.0742855990646
1.995676040649414 loss -0.0792163562663
2.182711124420166 loss -0.103982204388
2.2595033645629883 loss -0.135805239153
1.942941665649414 loss -0.12319274307
2.200901746749878 loss -0.0774641309732
1.9033658504486084 loss -0.0864705034479
1.8680906295776367 loss -0.0955979781264
1.910517692565918 loss -0.0358096478098
1.9112012386322021 loss -0.038491734636
2.0044524669647217 loss -0.0950598929015
1.945418357849121 loss -0.035234543452
2.188807725906372 loss -0.104418431604
2.0596354007720947 loss -0.0841032250358
1.9592907428741455 loss -0.162663219073
2.5637452602386475 loss -0.0955979781264
1.859910249710083 loss -0.0627565093911
1.9301397800445557 loss -0.092875417357
2.209507703781128 loss -0.0820901514522
1.979600429534912 loss -0.126462865313
1.9127590656280518 loss -0.0484212447931
1.8920214176177979 loss -0.0670383440778
2.0446290969848633 loss -0.112813284284
1.9070568084716797 loss -0.0197801718498
1.9458963871002197 loss -0.0362571714083
1.9579033851623535 loss -0.100213959866
2.1233935356140137 loss -0.0972625433635
1.928635835647583 loss -0.0628469848145
1.9141483306884766 loss -0.0984156987341
1.9274227619171143 loss -0.0137243023206
1.91721773147583 loss -0.0688466684171
1.9588894844055176 loss -0.069551069747
1.906846046447754 loss -0.0929877833362
1.9312717914581299 loss -0.00714100132668
2.1439738273620605 loss -0.122040563661
1.9114792346954346 loss -0.0927118387897
1.9950478076934814 loss -0.059334654635
1.9456124305725098 loss -0.0576978278033
1.9041047096252441 loss -0.0999979633389
1.8824801445007324 loss -0.0126290274086
2.095101833343506 loss -0.0675782564573
2.25677490234375 loss -0.0501297984862
2.2305784225463867 loss -0.0440277841537
1.9883482456207275 loss -0.0495838090463
1.9838688373565674 loss -0.0302186275653
1.8928513526916504 loss -0.0308341800636
1.9109294414520264 loss -0.139743244015
1.901076078414917 loss -0.0428817617292
1.8950722217559814 loss -0.0912001663337
1.8585379123687744 loss -0.0763788603524
1.9116346836090088 loss -0.129596820645
1.8885679244995117 loss -0.100213959866
2.1263959407806396 loss -0.0412048213264
1.9245975017547607 loss -0.104400409176
1.908933162689209 loss -0.14333195396
1.9205083847045898 loss -0.0076857861187
1.9290952682495117 loss -0.00496689036363
2.217933416366577 loss -0.0263668186678
2.0220980644226074 loss -0.0375605745343
2.146307945251465 loss -0.12090793662
1.9362120628356934 loss -0.065911901372
1.936361312866211 loss -0.144945536999
2.0294015407562256 loss -0.111486150256
1.9462153911590576 loss -0.101090948864
1.8787472248077393 loss -0.135310689424
2.176412343978882 loss -0.0914612730523
1.91922926902771 loss -0.018122032655
1.8740031719207764 loss -0.176490407062
1.9028117656707764 loss -0.0368103788305
1.9030230045318604 loss -0.150581585759
1.8886864185333252 loss -0.0873580309709
2.5322911739349365 loss -0.148354255212
2.2620882987976074 loss -0.0950598929015
1.906693696975708 loss -0.105484461654
1.9980545043945312 loss -0.0697978803731
1.9782679080963135 loss -0.055498725006
1.8662574291229248 loss -0.056105130691
2.166640520095825 loss -0.0908495972473
2.0151138305664062 loss -0.130252109708
1.9139604568481445 loss 0.000523138021582
1.8724334239959717 loss 0.0406524037444
1.9618017673492432 loss -0.135811920839
1.9107754230499268 loss -0.14140532996
1.9472877979278564 loss -0.114277792572
1.9288733005523682 loss -0.0934940255514
2.3089981079101562 loss -0.153534213601
1.8743219375610352 loss -0.0730694000407
1.9125115871429443 loss -0.0621714210991
1.9360668659210205 loss -0.0759148602085
1.878854751586914 loss -0.0390167952541
2.059025526046753 loss -0.0132009112551
2.119232177734375 loss -0.0384783314586
2.095733642578125 loss -0.145160976366
1.97562837600708 loss -0.0703685321551
1.8942863941192627 loss -0.0313112782991
2.209096908569336 loss -0.0390167952541
1.986891746520996 loss -0.0808882910299
1.9453644752502441 loss -0.0989394837663
1.9411883354187012 loss -0.0745582620265
2.255173921585083 loss -0.104418431604
1.9067716598510742 loss -0.128559795226
2.184583902359009 loss -0.132049548567
2.1776158809661865 loss -0.109578322485
2.159377098083496 loss -0.123081441725
1.9820759296417236 loss -0.0543883674915
2.178093910217285 loss -0.14852509246
1.891432523727417 loss -0.0571622071753
1.88694167137146 loss -0.0777089298895
1.932133436203003 loss -0.0390167952541
1.8945176601409912 loss -0.14333195396
1.9700920581817627 loss -0.097726148993
1.9506914615631104 loss -0.141746938241
1.8519232273101807 loss -0.103871896467
1.884427547454834 loss -0.0946264229947
2.165431499481201 loss -0.118678183793
1.9122178554534912 loss -0.0835312442692
1.9782629013061523 loss -0.102280056849
1.991415023803711 loss -0.0620101320136
2.121047258377075 loss -0.0902467259954
2.027583122253418 loss -0.0335324652823
1.8798563480377197 loss -0.0557653745944
1.949753999710083 loss -0.0727190298522
1.894061803817749 loss -0.0319842974191
2.0533370971679688 loss -0.0384783314586
1.8800013065338135 loss -0.0649345069288
2.1620988845825195 loss -0.105205779195
1.9774670600891113 loss -0.00612399210195
1.9149410724639893 loss -0.068333438029
2.2421600818634033 loss -0.128559795226
2.1372902393341064 loss -0.0890666432078
1.865915298461914 loss -0.0340765998179
1.8641622066497803 loss -0.0615676690749
2.1246213912963867 loss -0.0865900373488
1.9008829593658447 loss -0.0703860647583
2.1834468841552734 loss -0.0858664255344
2.16695237159729 loss -0.160443506823
1.9172377586364746 loss -0.121492119896
2.0475220680236816 loss -0.0557653745944
2.209203004837036 loss -0.122219873003
2.126762628555298 loss -0.0589586235376
1.8866541385650635 loss -0.104937885349
2.134695529937744 loss -0.148360425709
2.2427966594696045 loss -0.0721208720098
1.9621937274932861 loss -0.118322030892
1.8597986698150635 loss -0.0478475160291
1.912693738937378 loss 0.0115939654806
1.9137616157531738 loss -0.0225189139807
2.180527448654175 loss -0.0884890730573
1.914808750152588 loss 0.0335388080564
1.9202139377593994 loss -0.0577825399784
2.029897451400757 loss -0.060452284031
1.8986592292785645 loss -0.153320265401
1.8930869102478027 loss -0.101090948864
2.0184226036071777 loss 0.0197944557418
1.8945832252502441 loss -0.0516948720741
1.9525401592254639 loss -0.0862580709034
2.2060534954071045 loss -0.0225189139807
1.9507296085357666 loss -0.070945801059
2.002955198287964 loss -0.115948450709
1.9561405181884766 loss -0.081317742885
1.8865718841552734 loss -0.159126255471
2.2532811164855957 loss -0.115948450709
1.8771560192108154 loss -0.050767683583
2.019864082336426 loss -0.0505477749992
2.122347116470337 loss -0.0626375224311
2.032717704772949 loss -0.0120833593845
1.8453588485717773 loss -0.108963595926
1.876387119293213 loss -0.0885654117926
1.8875951766967773 loss -0.0455991092667
1.8863112926483154 loss -0.0648428980937
1.9795289039611816 loss -0.116525862084
1.8863210678100586 loss -0.0714411015991
1.9277422428131104 loss -0.0675782564573
1.8933625221252441 loss -0.0890772574425
1.9100306034088135 loss -0.0718038073954
1.9373962879180908 loss -0.0736249228418
1.927372694015503 loss -0.115381407307
1.852698564529419 loss -0.106605497036
1.8847930431365967 loss -0.12090793662
1.9624767303466797 loss -0.0940340593793
1.8593952655792236 loss -0.112736243651
1.852637529373169 loss -0.0802516509777
1.9757494926452637 loss -0.148623893226
2.1757495403289795 loss -0.0730694000407
1.9027879238128662 loss -0.127487854271
2.1268630027770996 loss -0.00602081130644
1.8953642845153809 loss -0.119771981413
1.9058351516723633 loss -0.115638843244
1.8558070659637451 loss -0.0769181636803
1.885059118270874 loss -0.0395585347883
1.8448805809020996 loss -0.060452284031
1.877781629562378 loss -0.0406524037444
1.9901773929595947 loss -0.121041059743
2.0743188858032227 loss -0.0340961162028
1.8555965423583984 loss -0.0358096478098
1.8756439685821533 loss -0.120788841889
1.8925817012786865 loss -0.117625391112
2.1787397861480713 loss -0.0802991350031
1.9244794845581055 loss -0.0841032250358
2.5569405555725098 loss -0.114277792572
2.1725404262542725 loss -0.0516704211024
1.9036762714385986 loss -0.0407713635208
1.8916471004486084 loss -0.10880560338
1.863675832748413 loss -0.115948450709
1.8976972103118896 loss -0.0533921467724
1.8579752445220947 loss -0.134618469241
1.8916127681732178 loss -0.0902467259954
1.8955557346343994 loss -0.0979144673189
1.921990156173706 loss -0.0670383440778
2.063081741333008 loss -0.115948450709
2.1444132328033447 loss -0.123311606276
2.133897304534912 loss -0.0457760601246
1.8834035396575928 loss -0.0499935291897
1.9205288887023926 loss -0.154997205803
1.9174177646636963 loss -0.112696390573
1.9570033550262451 loss -0.132957607018
1.9284241199493408 loss -0.0419666607379
1.9792187213897705 loss -0.0254147068845
1.970583200454712 loss -0.0445006302187
1.9203853607177734 loss -0.106040281861
2.1914303302764893 loss -0.0718038073954
2.1740264892578125 loss -0.0467143301753
1.919823408126831 loss -0.122555772184
2.1652991771698 loss -0.117114046469
1.903014898300171 loss -0.121492119896
2.0672409534454346 loss -0.110203516252
2.216630697250366 loss -0.0390167952541
1.945488452911377 loss -0.0374106832607
1.9149587154388428 loss -0.148354255212
1.9200406074523926 loss -0.00767349595924
2.1756551265716553 loss -0.0784571070466
1.9760687351226807 loss -0.0461545519182
1.9127483367919922 loss -0.0637288837661
1.9252471923828125 loss -0.0654087776056
2.1900582313537598 loss -0.0814852346285
2.2364442348480225 loss -0.148940455883
1.9871978759765625 loss -0.0798289259856
1.8943121433258057 loss -0.122693767174
1.9571926593780518 loss -0.0329307978835
1.9539451599121094 loss -0.103871896467
2.026322603225708 loss -0.0587839105193
2.281940221786499 loss -0.0984156987341
2.042287588119507 loss -0.0769181636803
1.9403440952301025 loss -0.0511691896873
2.1219136714935303 loss -0.0654087776056
2.184234142303467 loss -0.0966991564162
2.224534749984741 loss -0.0214593655723
1.913994550704956 loss -0.0397344293485
1.937483787536621 loss -0.147239609672
1.9693429470062256 loss -0.11771247078
1.9181501865386963 loss -0.0785766360178
1.8544762134552002 loss -0.107263522845
1.845444917678833 loss -0.0914612730523
2.002821683883667 loss -0.105832695994
2.111935615539551 loss -0.056105130691
2.346569538116455 loss -0.0120833593845
1.8831851482391357 loss -0.0335388080564
1.934401512145996 loss -0.0528129861336
1.8980915546417236 loss -0.128559795226
2.0068023204803467 loss -0.0626375224311
1.9175426959991455 loss -0.085719376627
1.9139907360076904 loss -0.0615924671148
2.01072096824646 loss -0.0349325588233
1.9846551418304443 loss -0.055498725006
1.9855327606201172 loss -0.127974151297
1.9402661323547363 loss -0.176972975147
2.038374423980713 loss -0.118146728826
1.8805246353149414 loss -0.0404222737637
2.0239107608795166 loss -0.184058955381
1.890681505203247 loss -0.0772189157487
1.9313075542449951 loss -0.104937885349
1.881563663482666 loss -0.0774641309732
2.150602102279663 loss -0.0873580309709
1.9293596744537354 loss -0.102219590445
2.086696147918701 loss -0.0550660792952
1.8965351581573486 loss -0.122555772184
1.883742332458496 loss -0.0220544935899
2.2133936882019043 loss -0.110992674275
1.8922266960144043 loss -0.0241791313382
2.0148770809173584 loss -0.103871896467
1.9736952781677246 loss -0.100213959866
1.9761126041412354 loss -0.0439573019772
1.9419691562652588 loss -0.0940340593793
2.2537240982055664 loss -0.0439573019772
best: 
{'min_samples_split': 3, 'max_features': 52, 'criterion': 1, 'bootstrap': 1, 'n_estimators': 274, 'min_samples_leaf': 6}

In [ ]:

trials._exp_key=str(data_path.basename())
len(trials.trials), len(trials2.trials)

In [ ]:

from bson import json_util
import json
save_path = Path('../output/hyperopt/{}.json'.format(trials._exp_key))
save_path.dirname().makedirs_p()
json.dump(trials.trials,open(save_path,'w'), default=json_util.default)
save_path

In [50]:

import collections

def flatten(d, parent_key='', sep='_'):
    """Flatten dicts."""
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, sep=sep).items())
        else:
            # also flatten lists with one element
            if isinstance(v,list) and len(v)==1:
                v=v[0]
            items.append((new_key, v))
    return dict(items)
df = pd.DataFrame([flatten(trial) for trial in trials2.trials])
# df = pd.DataFrame([flatten(trial) for trial in trial_data])
# df

In [51]:

# df.sort_values('result_metrics_matthews_corrcoef')

In [52]:

df2=df[[
#     'book_time',
#  'exp_key',
#  'misc_cmd',
#  'misc_idxs_bootstrap',
#  'misc_idxs_criterion',
#  'misc_idxs_max_features',
#  'misc_idxs_min_samples_leaf',
#  'misc_idxs_min_samples_split',
#  'misc_idxs_n_estimators',
#  'misc_tid',
 'misc_vals_bootstrap',
 'misc_vals_criterion',
 'misc_vals_max_features',
 'misc_vals_min_samples_leaf',
 'misc_vals_min_samples_split',
 'misc_vals_n_estimators',
#  'misc_workdir',
#  'owner',
#  'refresh_time',
#  'result_loss',
 'result_metrics_matthews_corrcoef',
#  'result_metrics_report_f1-score_avg / total',
 'result_metrics_report_f1-score_leak',
#  'result_metrics_report_f1-score_no leak',
#  'result_metrics_report_precision_avg / total',
 'result_metrics_report_precision_leak',
#  'result_metrics_report_precision_no leak',
#  'result_metrics_report_recall_avg / total',
 'result_metrics_report_recall_leak',
#  'result_metrics_report_recall_no leak',
#  'result_metrics_report_support_avg / total',
#  'result_metrics_report_support_leak',
#  'result_metrics_report_support_no leak',
#  'result_run_time',
#  'result_status',
#  'spec',
#  'state',
#  'tid',
#  'version'
   ]].sort_values('result_metrics_matthews_corrcoef',ascending=False)

• critenrion = 'entropy'
• max_features ~ 100-260
• min_samples_leaf=5
• min_samples_split=5?
• n_estimators=~160
• boostrap, yues?

In [53]:

df2

Out [53]:

	misc_vals_bootstrap	misc_vals_criterion	misc_vals_max_features	misc_vals_min_samples_leaf	misc_vals_min_samples_split	misc_vals_n_estimators	result_metrics_matthews_corrcoef	result_metrics_report_f1-score_leak	result_metrics_report_precision_leak	result_metrics_report_recall_leak
482	1	1	52	6	3	274	0.184599	0.60	0.62	0.59
708	0	1	153	7	6	96	0.184059	0.60	0.62	0.59
984	0	1	170	5	6	74	0.184059	0.60	0.62	0.59
230	0	1	140	6	6	132	0.182557	0.59	0.62	0.56
635	1	1	89	7	6	127	0.180301	0.62	0.61	0.63
1	0	1	22	6	1	208	0.179479	0.59	0.62	0.55
369	0	1	142	5	0	207	0.177196	0.58	0.63	0.53
981	1	1	113	7	6	208	0.176973	0.59	0.62	0.57
783	1	1	45	6	7	297	0.176490	0.58	0.63	0.54
318	1	1	248	7	3	99	0.175259	0.60	0.62	0.58
639	0	1	107	7	2	36	0.175259	0.60	0.62	0.58
589	1	1	83	6	3	286	0.171789	0.61	0.61	0.61
647	0	1	54	7	2	294	0.162767	0.57	0.62	0.53
730	0	1	33	7	6	274	0.162663	0.59	0.61	0.56
146	1	1	238	2	6	253	0.160757	0.58	0.61	0.54
851	0	1	49	5	3	136	0.160444	0.59	0.61	0.58
620	0	1	4	6	6	152	0.160117	0.58	0.61	0.55
876	0	1	7	6	6	56	0.159126	0.56	0.62	0.52
409	1	1	190	6	3	174	0.157980	0.60	0.60	0.60
653	0	1	107	7	2	79	0.157733	0.57	0.62	0.53
600	1	1	3	2	6	37	0.155506	0.56	0.62	0.51
595	1	1	131	6	3	102	0.154997	0.60	0.60	0.59
929	1	0	145	6	6	57	0.154997	0.60	0.60	0.59
801	1	1	65	6	7	105	0.153534	0.60	0.60	0.60
420	1	1	141	6	3	167	0.153320	0.58	0.61	0.56
867	1	1	109	5	3	196	0.153320	0.58	0.61	0.56
473	0	1	22	6	3	105	0.152185	0.59	0.61	0.57
45	1	1	99	7	1	58	0.152185	0.59	0.61	0.57
386	0	1	75	3	4	76	0.151765	0.61	0.60	0.62
785	1	1	69	6	7	212	0.150582	0.59	0.60	0.58
...	...	...	...	...	...	...	...	...	...	...
127	1	1	257	8	1	166	0.008231	0.51	0.53	0.49
319	1	1	107	7	3	33	0.008231	0.51	0.53	0.49
771	1	1	229	4	4	270	0.007686	0.51	0.53	0.50
945	1	1	20	6	6	108	0.007673	0.49	0.53	0.46
749	1	0	245	1	1	165	0.007141	0.52	0.53	0.50
842	0	1	90	4	4	99	0.006124	0.54	0.53	0.55
901	1	1	125	1	6	260	0.006021	0.50	0.53	0.47
772	0	1	254	6	0	22	0.004967	0.53	0.53	0.52
694	0	0	157	7	6	150	0.004379	0.51	0.53	0.48
64	1	1	203	0	8	58	0.004379	0.51	0.53	0.48
481	0	1	260	1	4	267	0.002794	0.53	0.53	0.54
324	0	0	248	7	3	37	0.001655	0.52	0.53	0.51
795	1	1	152	6	7	6	-0.000523	0.53	0.53	0.52
471	0	1	255	2	0	270	-0.002743	0.51	0.53	0.50
535	0	1	151	5	6	17	-0.004967	0.50	0.53	0.48
182	0	1	65	3	5	38	-0.006665	0.49	0.53	0.45
562	1	1	206	7	3	19	-0.007747	0.49	0.53	0.46
499	0	1	68	6	5	119	-0.008828	0.50	0.52	0.47
258	1	1	207	6	6	18	-0.011529	0.53	0.52	0.53
861	1	1	193	7	7	30	-0.011594	0.55	0.52	0.57
555	0	1	199	1	6	23	-0.011621	0.49	0.52	0.45
368	1	1	200	2	1	36	-0.018673	0.52	0.52	0.51
869	1	0	84	1	6	70	-0.019794	0.50	0.52	0.48
220	1	1	109	1	6	130	-0.022054	0.49	0.52	0.46
171	0	1	266	4	3	204	-0.024735	0.50	0.52	0.48
385	0	1	58	1	2	0	-0.025932	0.39	0.51	0.32
864	0	1	228	6	1	3	-0.033539	0.49	0.51	0.47
796	1	1	160	6	7	11	-0.040652	0.50	0.51	0.49
353	1	1	67	4	8	4	-0.041648	0.46	0.51	0.41
690	0	1	279	3	6	61	-0.048357	0.49	0.50	0.47

1000 rows × 10 columns

In [55]:

df2[:50].mean()

Out [55]:

misc_vals_bootstrap                       0.540000
misc_vals_criterion                       0.980000
misc_vals_max_features                  102.040000
misc_vals_min_samples_leaf                5.840000
misc_vals_min_samples_split               4.380000
misc_vals_n_estimators                  160.840000
result_metrics_matthews_corrcoef          0.157871
result_metrics_report_f1-score_leak       0.584600
result_metrics_report_precision_leak      0.609400
result_metrics_report_recall_leak         0.564200
dtype: float64

In [5]:

df.iloc[482]

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-246e620b3f38> in <module>()
----> 1 df.iloc[482]

NameError: name 'df' is not defined

In [ ]: