{
    "model_dir": "models/llama-13b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-14 00:15:02",
    "eval_score": -0.1498359961936773,
    "eval_brier_score": 0.1498359961936773,
    "eval_average_probability": 0.7262082472191324,
    "eval_accuracy": 0.7706666666666667,
    "eval_probabilities": [
        0.5321437057296083,
        0.7523073621895225,
        0.5416386957418207,
        0.9964154385322935,
        0.9286889144346034,
        0.9787842744339728,
        0.9794758650641628,
        0.9623648328045664,
        0.9660353996876058,
        0.9874545548264746,
        0.9940746870584459,
        0.998683482926758,
        0.6066972486340345,
        0.6339868018357984,
        0.6967710813467866,
        0.481137174404888,
        0.7947025846081094,
        0.5205534584308296,
        0.5861641980227741,
        0.8620001057647008,
        0.8900558816848283,
        0.9608653938332872,
        0.9705074599942657,
        0.9974138266644968,
        0.6225217623841957,
        0.9225283236610567,
        0.29624852265355583,
        0.7401834468786094,
        0.8151589983294492,
        0.8762922474475872,
        0.9820562260988562,
        0.20888189858621664,
        0.9914185100636583,
        0.9638586710420166,
        0.6963085532405354,
        0.2920148570542163,
        0.7891646429867054,
        0.8438699688597507,
        0.7772870965375203,
        0.7991586565475399,
        0.5343978348426586,
        0.2337972745936518,
        0.7415182055403986,
        0.4898791834336534,
        0.406957784932964,
        0.39128740627020075,
        0.2175147065310838,
        0.9593441663908088,
        0.15987131252799902,
        0.6603277666293442,
        0.941139371380797,
        0.18465856312885254,
        0.5499478805043478,
        0.3826684776288768,
        0.5184429474665382,
        0.16242123023145882,
        0.04583491428471169,
        0.9953378324550288,
        0.8253002356921246,
        0.9051212762158957,
        0.9960233727645017,
        0.9893220626522674,
        0.777722594899702,
        0.53246377400244,
        0.9894092191335541,
        0.9945145597235021,
        0.9093793979020149,
        0.6383444776641216,
        0.27876567169833794,
        0.9164587934780217,
        0.3034306881894985,
        0.4338509472616884,
        0.902474045905746,
        0.9780226871433538,
        0.8227589949941531,
        0.39897834576870267,
        0.8618198885774251,
        0.6555622653592019,
        0.195979427421571,
        0.9886075638528768,
        0.32431709294158517,
        0.9803317017592226,
        0.9662077545237737,
        0.9952332179331477,
        0.6625215869948327,
        0.9277729292274218,
        0.7176122556054381,
        0.3755041125206511,
        0.3720588492798575,
        0.3901358935264609,
        0.996213445396134,
        0.5512919383964797,
        0.9957388670630262,
        0.1888542811968974,
        0.6287602572877318,
        0.7277833450109558,
        0.9474799473412744,
        0.8103694959835831,
        0.9518730897815092,
        0.9970874633029091,
        0.11164855951491648,
        0.984592306117654,
        0.5881386771939707,
        0.3929462131864776,
        0.3243409667176705,
        0.6986037062983769,
        0.7936832711538645,
        0.7281319627093199,
        0.8929978628535974,
        0.8465417669078463,
        0.891510769201278,
        0.994429689684301,
        0.9767661539654054,
        0.8064282725114025,
        0.6528502678845984,
        0.9505143719025557,
        0.47737102565812545,
        0.84566582811388,
        0.7404483825218381,
        0.6779383997449111,
        0.9678625332294739,
        0.9960811957092193,
        0.8842026932851998,
        0.9927386934922611,
        0.998316083127517,
        0.9942465264913065,
        0.8201126968970655,
        0.16601786921686665,
        0.7765434750022189,
        0.8271594578586767,
        0.9675008032512663,
        0.7577986925168054,
        0.9055797300450935,
        0.9260839505676403,
        0.9738612490950824,
        0.2867298595862837,
        0.742479545645562,
        0.28532110700309415,
        0.09891153909767926,
        0.7452594133774461,
        0.3034119243161138,
        0.6177649488392023,
        0.9950113530463509,
        0.9937327016288163,
        0.8658784850898685,
        0.8476319230744451,
        0.7161919296779101,
        0.8546210495273698,
        0.9715694875465697,
        0.6396076794542163,
        0.023048840211654542,
        0.00538893358072889,
        0.9076001969093693,
        0.8520333700073334,
        0.6709238816059915,
        0.9688002237562289,
        0.3941715343424579,
        0.5614642720100157,
        0.37778378303116045,
        0.5492824960875163,
        0.4393779813482811,
        0.8468234177369729,
        0.6357168627053837,
        0.45583579211255326,
        0.9317240661014583,
        0.8069996072667336,
        0.28845942678546577,
        0.29919023276493156,
        0.5526150200100461,
        0.2295426344321424,
        0.9790501817987727,
        0.7980381545910162,
        0.9415096266559801,
        0.8702359285521526,
        0.24367005902464353,
        0.42768927675757884,
        0.6167577261941768,
        0.8485808948169674,
        0.9849556457198487,
        0.7119803956646722,
        0.8749830114637789,
        0.7216045509083838,
        0.5062923895566623,
        0.9516194756256406,
        0.9577774941295494,
        0.9731410201169121,
        0.447886292338019,
        0.7496261658154897,
        0.40079437551173497,
        0.38127594842061885,
        0.9635095553314663,
        0.9940914646780837,
        0.5985163581031284,
        0.4910974830732844,
        0.5172380729899199,
        0.6461010814700975,
        0.8432085585365385,
        0.7729673861293682,
        0.9596947425306334,
        0.9655236311326321,
        0.8570534209821261,
        0.7065147319211331,
        0.9628825773319067,
        0.9715640148842902,
        0.9798061473201368,
        0.9879214129801631,
        0.9606146844724539,
        0.9881102958169454,
        0.4334709951135701,
        0.48744283744366435,
        0.45773443723202345,
        0.10841690222471927,
        0.32557813451323914,
        0.6714869798958963,
        0.7291257137366067,
        0.31612827182173586,
        0.9687119969659561,
        0.3593192166949496,
        0.9508341546962947,
        0.9586966954530433,
        0.9896761221967985,
        0.9917940040613996,
        0.8718624191237813,
        0.9174969530306738,
        0.8832812818489443,
        0.8963178868696985,
        0.9912279153053094,
        0.9682889822546598,
        0.2339115627097633,
        0.3963258855970886,
        0.8463329308872877,
        0.21104776786742308,
        0.7126408056358581,
        0.11297945228808859,
        0.8942698865667542,
        0.9732243971883917,
        0.8636029257375031,
        0.9936372311413965,
        0.9815003944220255,
        0.9658368726454256,
        0.9995357170473664,
        0.9997768379355896,
        0.9957374856667149,
        0.016810138659267615,
        0.625744079550866,
        0.41649911470161444,
        0.9481137088156273,
        0.9973748383630704,
        0.9856314253374556,
        0.8683564112012547,
        0.13587292232457782,
        0.48287810554135346,
        0.9697897556140078,
        0.29718553164971584,
        0.6564333773308374,
        0.9621622145002665,
        0.9200143413602964,
        0.9058070734788324,
        0.09500699717436334,
        0.7967833177488765,
        0.9168183020835509,
        0.9898260248161947,
        0.9866923025428642,
        0.9778250739476811,
        0.38494780717884985,
        0.9694554112433617,
        0.8669249254361029,
        0.985602852304723,
        0.9801755725600301,
        0.9451065445787897,
        0.9985191763862198,
        0.626511380742125,
        0.9989997050961694,
        0.7132328171752407,
        0.5803617005069888,
        0.4050910031408489,
        0.9743283994705809,
        0.9955204040025749,
        0.9847230565962176,
        0.23946705939340718,
        0.2504595429933524,
        0.32331275733832965,
        0.7781319321408697,
        0.6603465726287439,
        0.7376846155197625,
        0.9943965342561184,
        0.9546931365361399,
        0.9957236802982894,
        0.9977175734434807,
        0.9985807823293543,
        0.7515264127530763,
        0.9659956755245168,
        0.9745518480700819,
        0.89709750075,
        0.9951115002544422,
        0.9750828179238081,
        0.9911220896292745,
        0.8539024647122831,
        0.9602605023481634,
        0.6779960936986112,
        0.9870908818744203,
        0.07569753727576126,
        0.9908876170202917,
        0.7519834060471163,
        0.5114350261448876,
        0.29128089250421907,
        0.8439033235845357,
        0.11835061960640826,
        0.05142425554640246,
        0.4851485675765131,
        0.25571695232900615,
        0.36699037628691045,
        0.9429203711619776,
        0.08797856172733964,
        0.8923756831976206,
        0.9206148816291977,
        0.9472473859611692,
        0.9099002322055597,
        0.9898406209397985,
        0.9937410590428625,
        0.9599516809182959,
        0.35492801429857107,
        0.4454408892104093,
        0.19555350480711955,
        0.9678616774619412,
        0.9854832490213918,
        0.8759801125669525,
        0.7714229801494253,
        0.017255562562099896,
        0.4703334960496476,
        0.9565109781338792,
        0.981750435809722,
        0.9775479875088297,
        0.8000892462748606,
        0.8505259775857414,
        0.8948057512598694,
        0.987303213680041,
        0.9934076715746827,
        0.8391112110418082,
        0.9828639102950764,
        0.9846710071135923,
        0.9875577781920457,
        0.06725026925077299,
        0.3885377961837856,
        0.11270028613871352,
        0.9809428289643898,
        0.9669615905288483,
        0.9940606290955467,
        0.988534098948984,
        0.5137326598745545,
        0.8846914676235581,
        0.38269751526055124,
        0.6210908911258916,
        0.6398987755482596,
        0.5407275451342715,
        0.751764107315217,
        0.5223639993256783,
        0.9716861571791324,
        0.8726272838704159,
        0.2238181159542863,
        0.4217106289397944,
        0.41599457426648023,
        0.575432139486906,
        0.995595527714981,
        0.9566774041659716,
        0.9716515454592538,
        0.9763288401463325,
        0.7050887989598534,
        0.8619893491539881,
        0.7608844288027707,
        0.8122585598995326,
        0.5536000683558873,
        0.948123332345878,
        0.9416075549779391,
        0.9293513291881285,
        0.7398940481536427,
        0.7924856783815974,
        0.8113061526058998,
        0.6013952348834256,
        0.691708487558749,
        0.9248179578707516,
        0.8312816968813779,
        0.9358454337943473,
        0.9317098935350827,
        0.9430046593931884,
        0.8162560791099516,
        0.954576636697253,
        0.439308732967973,
        0.8962769335599134,
        0.940401822583704,
        0.31568974425714913,
        0.43030011202180807,
        0.26310010004970297,
        0.9167883316822287,
        0.9515803281624097,
        0.8360233157636554,
        0.9015373643337079,
        0.8136710737486825,
        0.7392216910538758,
        0.8776621303223235,
        0.9669783409278728,
        0.9429268067171138,
        0.8572498904694127,
        0.723024969415799,
        0.9224449168370622,
        0.9970892006609751,
        0.9973569906150177,
        0.9832262809223327,
        0.49194628922234585,
        0.6372032578913698,
        0.6135692629979467,
        0.8618122008861239,
        0.28737215644816066,
        0.8569275432609433,
        0.9517954562992481,
        0.6806874441628525,
        0.9068167334646873,
        0.7758646987765705,
        0.7635660685534643,
        0.40113575398901524,
        0.6112440398802479,
        0.6587465335982683,
        0.7955952713735832,
        0.5634560016097039,
        0.9529549938984965,
        0.5437452669972838,
        0.9696063304224071,
        0.9184674167850452,
        0.9320927176193902,
        0.9354970609911056,
        0.9915934879833751,
        0.876761183037267,
        0.9900225765635357,
        0.9545802197760636,
        0.9790160887802662,
        0.9306778505582937,
        0.985018902823512,
        0.9760834033797646,
        0.7205792162753969,
        0.8594802901929696,
        0.5242229661368457,
        0.44649469368766936,
        0.5120810910710759,
        0.7950652562494949,
        0.9392617045897996,
        0.9907991733633991,
        0.9837407896785503,
        0.9946223494193724,
        0.9968529072228426,
        0.9940168895595957,
        0.9959014305421839,
        0.9901511990929318,
        0.3549938887645508,
        0.7282867616027753,
        0.5257927855880565,
        0.46975195689321547,
        0.8787559095691746,
        0.8076193530345719,
        0.801760526915581,
        0.10502970743930287,
        0.1525899328115804,
        0.4394337727645906,
        0.9052592981441459,
        0.9793161827793283,
        0.9800944185671205,
        0.9582144485850829,
        0.9359365286175415,
        0.990322535040572,
        0.7576182823012233,
        0.2740037249550037,
        0.414812212690019,
        0.36189685850258596,
        0.5515611487083081,
        0.7518334381529295,
        0.9622009204096525,
        0.8736581970619268,
        0.9953474750623695,
        0.9413614243894218,
        0.6425353675507897,
        0.9904226257938854,
        0.9909197291003409,
        0.9993525879530054,
        0.9955634442710826,
        0.9240378744752276,
        0.7035042085256847,
        0.7925865701623052,
        0.6946924133629835,
        0.6109461165384681,
        0.6730250823493366,
        0.9117306951477845,
        0.9572976139379301,
        0.7258138347320341,
        0.8891775131915631,
        0.997593270308027,
        0.8972847822320656,
        0.4029255534625007,
        0.09800457929188197,
        0.4535783956238163,
        0.4103399039188098,
        0.440651597594101,
        0.36278306000694344,
        0.8198331193758167,
        0.8305093459110063,
        0.7833912403638319,
        0.9376142788558895,
        0.8757203217254313,
        0.9394309641309907,
        0.24621630012120155,
        0.2329248229877242,
        0.16386859093047357,
        0.2402527822019943,
        0.8911374859553858,
        0.4970947596942347,
        0.4922039278032683,
        0.321644993540934,
        0.9706274172247725,
        0.9212738033432863,
        0.9044671341444614,
        0.9334156108715012,
        0.8072865575269,
        0.695519150000823,
        0.7639137454868329,
        0.16544583672961174,
        0.7277043534933016,
        0.6791081963033953,
        0.9974102640865764,
        0.9982258306271593,
        0.9987791938344731,
        0.9962194934154089,
        0.9393603915616203,
        0.9381509302729449,
        0.8102610892870675,
        0.7040834219809597,
        0.7964514278273164,
        0.9156293317016251,
        0.9601596094085282,
        0.9162437767558725,
        0.1314831303380887,
        0.9918757092044086,
        0.9844952745824859,
        0.1865179344210453,
        0.7750158088965149,
        0.2895394215656055,
        0.9793374185705449,
        0.9893853241263466,
        0.787854829394387,
        0.9120938986218953,
        0.8334039501068973,
        0.9201885948447117,
        0.9569010607569196,
        0.947715447927881,
        0.16626989170332676,
        0.9228081266231472,
        0.3003014743063662,
        0.8979566089990949,
        0.23732899078328018,
        0.9027083525427378,
        0.0436543764326391,
        0.25657572074637497,
        0.4839725576286698,
        0.6781985878927624,
        0.9985645555634688,
        0.8988814825277909,
        0.9774243233798147,
        0.3098017013984377,
        0.3668530852438893,
        0.7686100341981551,
        0.9870172802715474,
        0.963553908248085,
        0.9834965331263684,
        0.5630285142522454,
        0.8321303711049843,
        0.654963975827348,
        0.9980636407312687,
        0.9977994828168923,
        0.9990784305187614,
        0.9876084257857589,
        0.9880549138268854,
        0.9875427444220128,
        0.9800431400571367,
        0.951890617405156,
        0.9977588706524987,
        0.6662611448788041,
        0.7211497891575512,
        0.7577257424127956,
        0.468618506571523,
        0.30817828402073766,
        0.501314153544726,
        0.9315554678970348,
        0.8067255417337292,
        0.8652874675518349,
        0.9952231677138761,
        0.9905804799168594,
        0.9968792094440576,
        0.4063551143669025,
        0.4183668817411378,
        0.30673193229186063,
        0.683170503087152,
        0.976279539500537,
        0.9915454590058567,
        0.9893398710827954,
        0.20840625410422134,
        0.7522087036330882,
        0.6639065360644159,
        0.651487441047674,
        0.31049535807230283,
        0.8717689022727245,
        0.6608783642250562,
        0.510837617674006,
        0.8910265528913042,
        0.7157498718393817,
        0.9604436025327114,
        0.6094070361815838,
        0.9281394777829985,
        0.9648827332810419,
        0.14233109824233492,
        0.2474942468008552,
        0.07269726523813358,
        0.2056561313377807,
        0.2874256918675092,
        0.696163122092173,
        0.36098277277698504,
        0.3529657980392793,
        0.519858440735209,
        0.9772222053939466,
        0.9555958907200673,
        0.3110236412025751,
        0.9765244817951015,
        0.9785487048609468,
        0.9865569234161923,
        0.42168055922550884,
        0.09337233927465326,
        0.5282677124817843,
        0.8305172140794185,
        0.9927577963354649,
        0.916036506545004,
        0.9994840297845874,
        0.9990368805680914,
        0.9973496054864804,
        0.14480820764204297,
        0.3107893288641862,
        0.3225797927871926,
        0.9313284103962411,
        0.9911681824770397,
        0.930125516571835,
        0.7568167450888392,
        0.0751427861877052,
        0.11870544628800594,
        0.321331833802722,
        0.39953828706873595,
        0.5086638901043136,
        0.551196054230811,
        0.39295376853995845,
        0.5825392584055363,
        0.9519409780982964,
        0.8801219326495143,
        0.9580701823823895,
        0.9596615340404726,
        0.8344892917975639,
        0.4470904111532461,
        0.9460954885183918,
        0.9705938153012319,
        0.7001957334711773,
        0.7058727587691398,
        0.9152584990294215,
        0.9487284444974211,
        0.5727625733345967,
        0.17162266972018972,
        0.08563897734566425,
        0.5361720112632444,
        0.12676441100249985,
        0.15391776859901907,
        0.4931384600265347,
        0.6652788855318079,
        0.9259653300322171,
        0.5366045567645233,
        0.6177283654410551,
        0.646267970893549,
        0.8309311008663018,
        0.8274970709880519,
        0.9366343764101264,
        0.8658343747710204,
        0.8551584534887833,
        0.9787688640346135,
        0.993710762091379,
        0.9596979856335146,
        0.9357697602070487,
        0.6901383584699947,
        0.7367146677997757,
        0.583256341896293,
        0.07953772461243483,
        0.36667047181023504,
        0.4773358474825552,
        0.62142047696734,
        0.6489321100715572,
        0.6313306062727888,
        0.9390236025360768,
        0.9663987029926713,
        0.653385587921173,
        0.7018635280103412,
        0.35002119616626204,
        0.830319094379947,
        0.2930320033432952,
        0.24383037559298504,
        0.33878668906777143,
        0.9585674749652207,
        0.9338671403453905,
        0.6042152222986289,
        0.9980021118423141,
        0.5356783872981559,
        0.9762191288104468,
        0.5816611217939401,
        0.5705520746936923,
        0.9255292818355814,
        0.43565171588171225,
        0.33311285138822916,
        0.6874243918541141,
        0.6138098486136232,
        0.8403034956127288,
        0.8339779800771578,
        0.828568087094341,
        0.5985609983766822,
        0.7825118751061773,
        0.7856019931355573,
        0.8945556439518909,
        0.921384120487391,
        0.7492548613321668,
        0.7881101289968927,
        0.9339227596651893,
        0.7451249405810754,
        0.860431401107631,
        0.75011569296917,
        0.964014123723442,
        0.9906402139363188,
        0.9857397583752495,
        0.18754546572219363,
        0.9912290411987539,
        0.9298603326230516,
        0.9122439222409425,
        0.9663955389186705,
        0.9632241850355402,
        0.9473479319636873,
        0.9167449261741369,
        0.9288688430210013,
        0.991091274666395,
        0.9226729900647876,
        0.9591421355322669,
        0.7395582105352451,
        0.9477415508668295,
        0.9325250582689044
    ],
    "eval_rms_calibration_error": 0.3022047826091495
}