{
    "model_dir": "models/llama-30b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-16 05:02:38",
    "eval_score": -0.1602959858560005,
    "eval_brier_score": 0.1602959858560005,
    "eval_average_probability": 0.7153305772141936,
    "eval_accuracy": 0.756,
    "eval_probabilities": [
        0.3482837292998797,
        0.3783023810703453,
        0.34406304589137116,
        0.8852480913936146,
        0.6681274406585637,
        0.8500176736518327,
        0.9665866666127438,
        0.9561804019659782,
        0.9807953440967611,
        0.18831737576956908,
        0.12242655374602998,
        0.26740214401059,
        0.880038145406822,
        0.8052538621579634,
        0.630639587350845,
        0.41109749132686335,
        0.7413337254007217,
        0.2992694352813535,
        0.7696056344381557,
        0.7766334084528915,
        0.4873436125382436,
        0.928503848645545,
        0.7351840743750679,
        0.46734931174079575,
        0.6339005977667397,
        0.8526291600948045,
        0.1786043450571611,
        0.9616284482809195,
        0.9558227226997268,
        0.9587585737980441,
        0.9373760104304291,
        0.5801834064721548,
        0.8272531994010429,
        0.2116238092572872,
        0.5309095468354046,
        0.1025723132430989,
        0.9701110552723634,
        0.8371009811263762,
        0.9288307120313521,
        0.9711763996832323,
        0.458705323302918,
        0.7135146586741994,
        0.9848092663104608,
        0.9554936668613504,
        0.8332116279714664,
        0.7905046089943973,
        0.5567507026317117,
        0.9489541711820604,
        0.9301455895002383,
        0.8464996496459468,
        0.9004826625560879,
        0.7208559153171017,
        0.8464612353312494,
        0.8276526131450594,
        0.1558288262483752,
        0.3485243981224758,
        0.15980296462573462,
        0.9292745760731886,
        0.35744574422626707,
        0.6061029618631597,
        0.9960385629983888,
        0.9476978022125558,
        0.819740932950565,
        0.03587666663690403,
        0.9474383609471961,
        0.26036954975411064,
        0.9931442695621624,
        0.9787663017476513,
        0.7507425430179797,
        0.929139074510726,
        0.7405343680590921,
        0.9714817344365474,
        0.7955279806225356,
        0.935970239360868,
        0.7780281847431593,
        0.2799703823957431,
        0.41536723726778124,
        0.5932816007185956,
        0.03365123627982236,
        0.9897061390004118,
        0.2718470845931489,
        0.8796234049326078,
        0.9886268302338611,
        0.9044284134833441,
        0.16580912752730415,
        0.9328960723769664,
        0.2637685013683311,
        0.8206042090874732,
        0.8189278915043761,
        0.4755349058449232,
        0.9978040186460166,
        0.24198121837387887,
        0.9985129492608306,
        0.7912216436133522,
        0.7816732935667001,
        0.3317669805868085,
        0.5178242160424771,
        0.8731469539158524,
        0.6564760492388417,
        0.8825078218550447,
        0.6075638524454983,
        0.8136129248500024,
        0.9360148252585867,
        0.8908431013443432,
        0.8832936705948122,
        0.9950257549249668,
        0.9959231927115826,
        0.9930887520382322,
        0.9978908096666151,
        0.998194903400527,
        0.9961161856307522,
        0.9741935529545909,
        0.7100506504257222,
        0.7927435434525484,
        0.8312118033505079,
        0.7000629037890058,
        0.9754502172485313,
        0.9801819251450693,
        0.958578314944936,
        0.9865325093613755,
        0.27783587542761545,
        0.9849419778044982,
        0.5762200411740057,
        0.9705428164121723,
        0.8924753510396473,
        0.982846068175998,
        0.7816658909217868,
        0.7589410802941363,
        0.475017592700799,
        0.4773647537322373,
        0.9902569349084885,
        0.9670223956719262,
        0.9482286655748237,
        0.8841851699490643,
        0.9896666357786182,
        0.4371720388629631,
        0.47788360733262997,
        0.7118399828990372,
        0.5700085763400243,
        0.9185192446759636,
        0.9874131753602005,
        0.8953480638605148,
        0.9952433723547541,
        0.4036071017733496,
        0.9568859685239114,
        0.43855371327296644,
        0.4657743711989162,
        0.7819312284791321,
        0.7723597540920757,
        0.8472502415588077,
        0.5640077518066451,
        0.4672561172313241,
        0.47434407898650666,
        0.9818992848163407,
        0.9553553066899774,
        0.993775220271461,
        0.42785605321182063,
        0.4580973848196294,
        0.5058844286196256,
        0.680684575403971,
        0.20624146838811502,
        0.9877147298162767,
        0.9851662886030403,
        0.6891276366772126,
        0.9147106636489923,
        0.7797938954621293,
        0.4621184340959983,
        0.28652817447531975,
        0.7485447919135981,
        0.16663489203198345,
        0.16563512671012043,
        0.9373133237491007,
        0.9000801450433846,
        0.8267582453356539,
        0.918764118315305,
        0.9122092102045225,
        0.9219828806912368,
        0.8339107655574973,
        0.9566984387372508,
        0.699205569995832,
        0.5642406447350026,
        0.7321117026351822,
        0.07041075891173643,
        0.44118307606845525,
        0.9935080414575415,
        0.5838243075073661,
        0.07602892239871589,
        0.6842252515450816,
        0.13643477972850354,
        0.9533623374560461,
        0.5102010660137288,
        0.9624148080079036,
        0.5827833659593701,
        0.3937173457214009,
        0.4183655532041177,
        0.8155786242128972,
        0.9666890639219299,
        0.9136034824504835,
        0.8026972935256482,
        0.6915380179537132,
        0.5438234708420076,
        0.41813126006040396,
        0.6145922944533997,
        0.9128724636282429,
        0.9878613434354494,
        0.9985944810122943,
        0.9968112985741862,
        0.927573280614265,
        0.6777335256933413,
        0.398975676164364,
        0.6604935281265021,
        0.41376350453866095,
        0.503725895822542,
        0.6873639730796464,
        0.8948202362158899,
        0.8262095425832101,
        0.9461241721203385,
        0.9682656850252713,
        0.9961109192292251,
        0.9963729706829286,
        0.9986378738499114,
        0.6691563448315321,
        0.9065326901742689,
        0.9348192157833585,
        0.8996075085337669,
        0.981544617966517,
        0.7929050544667985,
        0.7966620412293048,
        0.2541401240388722,
        0.9636662885855094,
        0.8562910633123442,
        0.3168834775747087,
        0.42141123849693585,
        0.16654552393771627,
        0.9410682448051426,
        0.9580714923462634,
        0.9517394456331092,
        0.8976951103933799,
        0.9393489526621494,
        0.9851142497136983,
        0.9992470124892179,
        0.9990014831824566,
        0.9988728397120907,
        0.8926704694625638,
        0.9664791557182622,
        0.6768583560357694,
        0.9097465361027998,
        0.9837506808003315,
        0.9063756792759121,
        0.7571433561443484,
        0.6021767215901802,
        0.7897829266670828,
        0.35205442770073936,
        0.5180170751437194,
        0.4184453186267005,
        0.8439254538265742,
        0.5218118757444076,
        0.9453575720164186,
        0.6206628035807287,
        0.7989924702478353,
        0.9677665123487394,
        0.9247984070052898,
        0.6847115433661146,
        0.7144194384106587,
        0.8997856910963092,
        0.47772495746196975,
        0.8578525246164402,
        0.9133032353125251,
        0.9162668729807256,
        0.927711367167412,
        0.9906704527602238,
        0.28605681332993027,
        0.9069367998969399,
        0.3815427990142765,
        0.5372943912341306,
        0.4823988378646796,
        0.8054457952380122,
        0.7919206349372578,
        0.3737484340012249,
        0.08366829016073585,
        0.10267466289943385,
        0.08500515739896708,
        0.963806778687632,
        0.9235918434139467,
        0.9027713553777498,
        0.9982842238304499,
        0.8548779173814132,
        0.9781676215102154,
        0.892052549744723,
        0.9896081054272707,
        0.6261460929221344,
        0.9899996275547724,
        0.9684557216445062,
        0.516341474464463,
        0.995282974874602,
        0.975666711848419,
        0.9927104232697546,
        0.9906089362426832,
        0.9777618121094027,
        0.9693751774129954,
        0.9918600961975593,
        0.0377481437300933,
        0.9976095630248185,
        0.505780326433546,
        0.8898887773452625,
        0.6357283364072068,
        0.40875969216319424,
        0.6663470170747248,
        0.7899569234638015,
        0.31029487449349646,
        0.3069318096634421,
        0.6852885312600903,
        0.9973518791349223,
        0.04427084237341456,
        0.8859488371361542,
        0.9687756022596398,
        0.9750742228357443,
        0.8130214260846955,
        0.9708704196605836,
        0.9893659127500364,
        0.7605260376798695,
        0.3580599928885142,
        0.23546160488616225,
        0.1855033715574513,
        0.857124172891642,
        0.9267283150066444,
        0.43210684804592764,
        0.9518728918864202,
        0.8040334345041391,
        0.8571381716662386,
        0.11263091262236591,
        0.7374689658357767,
        0.5280373228086105,
        0.8295297948904461,
        0.7214054262771518,
        0.703110447333888,
        0.9879274373276764,
        0.934247404557961,
        0.908032109433639,
        0.28911812046922303,
        0.9139501182899412,
        0.12223000944383344,
        0.4085585732757162,
        0.3520942848094857,
        0.23281864646673922,
        0.6675647204043917,
        0.735537513265502,
        0.5210359240281881,
        0.9741745556726085,
        0.9120914120840438,
        0.9316589482145893,
        0.049531263247419924,
        0.08681656534706828,
        0.029420399201542692,
        0.9866142447271007,
        0.9107147642562972,
        0.7755309917352194,
        0.9909432861714187,
        0.997914978817138,
        0.3787970043529012,
        0.9089822212808241,
        0.0866431078472829,
        0.9172063914781335,
        0.9900191308021457,
        0.7112877936181379,
        0.7982271987774979,
        0.6375864699883405,
        0.4588315649782437,
        0.6244544342047518,
        0.7474920564962062,
        0.9925794002858057,
        0.2938496124386301,
        0.8860945024893111,
        0.8874132590206638,
        0.8145132053081475,
        0.701640819489057,
        0.9551181720348829,
        0.7750723572440175,
        0.34404258909281815,
        0.43751986803906306,
        0.8712432824093016,
        0.7323986242682926,
        0.9836424308689056,
        0.9671515600270906,
        0.9952925956521708,
        0.9829821907596669,
        0.9849750737446379,
        0.7381354285604097,
        0.7673242214899141,
        0.9791028277692181,
        0.8153915289934034,
        0.7828098418353894,
        0.6116846843821868,
        0.8746914063679969,
        0.6676950196975383,
        0.9211319917911136,
        0.843351852107736,
        0.5927711419194429,
        0.39282366799203094,
        0.9854056644103267,
        0.974432951332648,
        0.18146905870865593,
        0.9956913088644569,
        0.9895649272909052,
        0.979412396602162,
        0.9980977121070569,
        0.9980948428289234,
        0.9978630034026087,
        0.8754219072045079,
        0.7237186945380356,
        0.8012164539613912,
        0.1064429673955648,
        0.8624930912553788,
        0.22007552055825905,
        0.9614415321025308,
        0.285787790586001,
        0.821475432850232,
        0.6219046050024402,
        0.8584198658089781,
        0.6850818110160728,
        0.10761860671322909,
        0.06165651125371614,
        0.2049824627686472,
        0.9712588060295666,
        0.9749626609346688,
        0.943403386097038,
        0.9118743971490628,
        0.9568620614207388,
        0.16246123250466543,
        0.9312831030143861,
        0.9446830787728298,
        0.9257449759943558,
        0.8050504460798991,
        0.917578632289532,
        0.9443167239631106,
        0.28464660707901446,
        0.46673259601461337,
        0.8014598821997417,
        0.5147390035742745,
        0.37415778572404645,
        0.942917733264327,
        0.7240810317847948,
        0.6183443639613802,
        0.35696945792135715,
        0.6557271590179223,
        0.45794998353655353,
        0.6391857390666649,
        0.8409977752111826,
        0.9976680381691225,
        0.9995023421354746,
        0.4271278170441859,
        0.9643744929176656,
        0.8678590947851383,
        0.6282732017882603,
        0.4946485418478618,
        0.5775089482493851,
        0.7818754503331515,
        0.8603114895446637,
        0.13315727246869313,
        0.6321722579375965,
        0.8806676062116489,
        0.9592025750291225,
        0.5178934190471527,
        0.4362419755471012,
        0.7674834347222856,
        0.8965901558651117,
        0.9044666718236636,
        0.8502662114797137,
        0.9521185285701248,
        0.9760859425760409,
        0.9957880891502178,
        0.060517663534970234,
        0.1584316059948152,
        0.09766770200071839,
        0.9960378424259719,
        0.7490796269126034,
        0.9983802590459592,
        0.7010124540221994,
        0.21540509642592554,
        0.883273084643376,
        0.8458353935083158,
        0.9921991966075455,
        0.9565989555209453,
        0.6861781895221926,
        0.23124415482061655,
        0.2789927721773747,
        0.974519244782859,
        0.965410852385001,
        0.9766730723857964,
        0.7369681125218612,
        0.9679781553514502,
        0.7106550298400295,
        0.750640018088822,
        0.6305999534336778,
        0.9197171242112517,
        0.9345018357441529,
        0.9482761089777015,
        0.9253211516795594,
        0.4566901952947894,
        0.6046445626218383,
        0.1883754148419593,
        0.8580602496365893,
        0.9271839955181358,
        0.18745653755059288,
        0.9853308689673338,
        0.4362687916638279,
        0.991692617773852,
        0.7322590464953957,
        0.8124020488509772,
        0.741404506251846,
        0.5548726623030186,
        0.8307258638487562,
        0.5408931179542475,
        0.2570031218776686,
        0.4121389944191537,
        0.8776081653273926,
        0.9628764230300787,
        0.6122037934628579,
        0.35562809518633476,
        0.9635674042740044,
        0.9795898091045022,
        0.9780240256716629,
        0.23114900688028447,
        0.5112820719755945,
        0.8392862084148448,
        0.9991709355587056,
        0.9994758403630871,
        0.999530622448215,
        0.9883949174410971,
        0.9954918818169072,
        0.9988087361806876,
        0.9172298321684752,
        0.512215258979031,
        0.9063306342488172,
        0.9845954694132131,
        0.9480394462288949,
        0.9632393396629543,
        0.9467055817669626,
        0.970532073661207,
        0.9911634858977529,
        0.9973815644115188,
        0.9977348868375615,
        0.9960273560522643,
        0.9322927364068022,
        0.8918734212298634,
        0.8904655515842633,
        0.21435258602311014,
        0.9790947227553685,
        0.9739347380678925,
        0.526866420177302,
        0.841239520848795,
        0.18309783386210932,
        0.5622534888457946,
        0.7900044380331482,
        0.5746581576271784,
        0.6397493816022859,
        0.8109994451592443,
        0.796815857785305,
        0.9403537244529927,
        0.9492766617412843,
        0.9833465280075316,
        0.9544636556171896,
        0.9021959447617447,
        0.9148972115428702,
        0.9910671479889225,
        0.9899890022839585,
        0.9930623104068543,
        0.4969016866985283,
        0.9172211049574555,
        0.7783653613433845,
        0.8841484421134813,
        0.8916847672929814,
        0.9789772449022675,
        0.9990727653948631,
        0.9949320899660549,
        0.9898901056788,
        0.24978144084448406,
        0.4133948642066546,
        0.6972243142993134,
        0.9731488979825651,
        0.834597765003432,
        0.6018544156331703,
        0.982285222602242,
        0.9560370115179697,
        0.9932556074457511,
        0.7583974946458854,
        0.23596131694064193,
        0.2197768935469634,
        0.9562244143266814,
        0.7958346721137389,
        0.9081527153549205,
        0.9945878193175554,
        0.9951273511594135,
        0.945946208493047,
        0.6871990774081982,
        0.5824570471422151,
        0.7118946822519394,
        0.4759519993255025,
        0.9230557143905683,
        0.8074626439458131,
        0.09810225405019363,
        0.10134026057957979,
        0.782808725070329,
        0.40320019630960735,
        0.32779011449426765,
        0.5222271816255813,
        0.9952365450873182,
        0.9961426807840965,
        0.985471359008213,
        0.6149514756440467,
        0.42474502388249014,
        0.8226944749450196,
        0.7685998020808963,
        0.9140494543596499,
        0.7817881678223233,
        0.6161427841445997,
        0.3837997373943493,
        0.6002983047856918,
        0.38454472278404417,
        0.4985733279590814,
        0.2422422850252357,
        0.5446234436315016,
        0.42753417588064074,
        0.5161790766257691,
        0.9609282119792704,
        0.8639248387466316,
        0.12785604504681003,
        0.983496325748218,
        0.8272292654356762,
        0.9977938252669634,
        0.7553393436044342,
        0.4126629811565957,
        0.7281638369921986,
        0.9206849554983553,
        0.9905974783430489,
        0.3014905928785945,
        0.9997128734792569,
        0.9991569649319902,
        0.9992783000270095,
        0.9802326182302509,
        0.9937580660375622,
        0.9935135932069823,
        0.9016142391749976,
        0.9819307083170147,
        0.9333983594085686,
        0.728604626438466,
        0.6713452511421085,
        0.4672459737821737,
        0.8278760551292996,
        0.45741424858898866,
        0.5006177678253302,
        0.6610538936153391,
        0.5340768871342858,
        0.4158468013925058,
        0.9804250762260913,
        0.985007555209467,
        0.9939921328609622,
        0.221119938856445,
        0.18899783646431587,
        0.17853401111787828,
        0.8108773930647952,
        0.6660307689297544,
        0.2017453005202136,
        0.3521495842095583,
        0.7173561662285606,
        0.14597703518952176,
        0.9898170176193182,
        0.9361496658587538,
        0.9664149371528107,
        0.9946006070448458,
        0.9799789949514319,
        0.9936367800759488,
        0.4381788539490673,
        0.07613396328078038,
        0.1854667531922786,
        0.9910419685720075,
        0.9840435622329676,
        0.9973076202051944,
        0.7444396347122462,
        0.055166600623406496,
        0.24547194504678435,
        0.9248807746474513,
        0.8857111491683518,
        0.8648557138887764,
        0.9964391919726295,
        0.999082034732223,
        0.9728338116049408,
        0.14387617296238087,
        0.31708583866237117,
        0.09513128934107463,
        0.1098194879167499,
        0.11453909392731654,
        0.10718072077787433,
        0.6830309198340987,
        0.37179905441509076,
        0.6817178953326635,
        0.7889507323475526,
        0.7186031884974836,
        0.9688451491058756,
        0.7851481101629425,
        0.8284649324206429,
        0.4479530890573225,
        0.0924434720821522,
        0.17646518175237505,
        0.28010082669048764,
        0.5168660449390295,
        0.8834294318064987,
        0.9485971759060231,
        0.35482351194951356,
        0.12165056530807905,
        0.8442152387946288,
        0.9962755331008303,
        0.990110965377304,
        0.9968792103899246,
        0.9301925197224078,
        0.8975385239732893,
        0.9086177377643376,
        0.3905846693832626,
        0.4967158124272359,
        0.8360636206812255,
        0.48947458468343485,
        0.8285822320062416,
        0.5478548354015778,
        0.6138542401190111,
        0.8110672986075866,
        0.5488907488735744,
        0.6090397172382643,
        0.7559246811859499,
        0.3256539818282496,
        0.9970160977439657,
        0.9958518459651241,
        0.9928567009807621,
        0.2661115861123205,
        0.771429319613574,
        0.18659085755352856,
        0.9114752756359458,
        0.8213147923072197,
        0.9939077003138168,
        0.7287378679031513,
        0.5933865152465665,
        0.4741298600881667,
        0.9029202360633348,
        0.9581313474472939,
        0.9212961780259411,
        0.10183784164260279,
        0.29301017333950874,
        0.0636904089680359,
        0.677679432658763,
        0.9267657734019976,
        0.9930394016121549
    ],
    "eval_rms_calibration_error": 0.30141389487139975
}