{
    "model_dir": "models/llama-7b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-14 00:01:10",
    "eval_score": -0.1741718582977794,
    "eval_brier_score": 0.1741718582977794,
    "eval_average_probability": 0.6932057476608959,
    "eval_accuracy": 0.736,
    "eval_probabilities": [
        0.7002755494508828,
        0.5651353570053743,
        0.4680493024798681,
        0.8670978878996684,
        0.11231297314582613,
        0.6666897697145636,
        0.9669580538487665,
        0.9354411208828165,
        0.9686260159426312,
        0.8979633340913015,
        0.926314811127332,
        0.8927195713519366,
        0.708897090739126,
        0.28895958292537766,
        0.2775257355270395,
        0.9239324248305024,
        0.9649277432846113,
        0.1971218210155258,
        0.2473766479202403,
        0.8760372118198565,
        0.8251655745489265,
        0.9741964927235814,
        0.9802196067515676,
        0.9872397255780967,
        0.8016114070580832,
        0.7101000813889908,
        0.3822777848532373,
        0.11803232908003924,
        0.06100269197449833,
        0.11464975462142146,
        0.9844155345481266,
        0.12497043962241236,
        0.9316604350502944,
        0.9687551461386713,
        0.7758442727641239,
        0.2549401881173178,
        0.4790931883622605,
        0.5424199075799441,
        0.5235697459567571,
        0.7061069120639023,
        0.671604821930911,
        0.269926594862498,
        0.28152345554022606,
        0.19846307812871844,
        0.15020234121715315,
        0.7119139053899275,
        0.423160910302295,
        0.6672090205745005,
        0.36569792664818157,
        0.8413041145754476,
        0.981291926618221,
        0.32589555354532906,
        0.5743608350720004,
        0.6396467107449486,
        0.26698804752666055,
        0.37536646639566273,
        0.14826641529745233,
        0.9270836853152662,
        0.9380304190514328,
        0.7048508860831431,
        0.9950147646045686,
        0.9715399378700159,
        0.5242978596350476,
        0.10768610006515988,
        0.2435679054628328,
        0.9231883499785795,
        0.8824851544685592,
        0.7376607253160014,
        0.36894710667398367,
        0.9359293914040822,
        0.9021545863104021,
        0.2695264990492969,
        0.8010109751131737,
        0.9831094501032513,
        0.7885850593407764,
        0.5379063229445218,
        0.2784370547457639,
        0.1851295730732908,
        0.3742387051188156,
        0.9907947875540621,
        0.3830821080853374,
        0.7757514349022266,
        0.8475304292801313,
        0.9799319361290018,
        0.8958892745330038,
        0.4203421948681591,
        0.9201283471256158,
        0.6200959510712934,
        0.5823133655773062,
        0.7724862021632855,
        0.9900772926944792,
        0.5772789442364427,
        0.9480787078574834,
        0.8115204600034027,
        0.3249089450256021,
        0.934107010313393,
        0.9798139022965601,
        0.9608920028130085,
        0.9631143590939183,
        0.9757284400505281,
        0.41289960389279123,
        0.9158084049121868,
        0.8177840146998419,
        0.47813534971569227,
        0.7499251580170021,
        0.9890814921392369,
        0.9814647613589638,
        0.9873363044756481,
        0.6274443102370129,
        0.6692570786992342,
        0.6251808885486166,
        0.9960912041389594,
        0.9828716919868716,
        0.9565304404155536,
        0.9481730408105211,
        0.8245253421425809,
        0.5307421770583756,
        0.7727802282339173,
        0.8994879353681587,
        0.6855398814979691,
        0.964179625618933,
        0.9837042750860981,
        0.9758358379444547,
        0.9639375466510218,
        0.9835816425081693,
        0.9947349457030615,
        0.8738845458011693,
        0.2630248544282775,
        0.49042432114829226,
        0.23052943817402452,
        0.4638782418984954,
        0.5256454470277114,
        0.32701546281607846,
        0.40584670654883637,
        0.6600070039465946,
        0.4059611166053672,
        0.8999091311262386,
        0.4907318586918762,
        0.9045033950466531,
        0.8132720083749092,
        0.14115233973709168,
        0.6308499723682789,
        0.9949858151235799,
        0.9851290708647377,
        0.27236301801046026,
        0.5391701290329051,
        0.34070922055283265,
        0.976655548779613,
        0.31761222157735525,
        0.24284035690073935,
        0.7006965237437965,
        0.22779563141206852,
        0.955582328322826,
        0.5360102076220458,
        0.473484344365357,
        0.4960750662561044,
        0.23343402476376496,
        0.21485871114614385,
        0.24449946971138525,
        0.14590114331903795,
        0.5663009739357744,
        0.4495837284107455,
        0.7669793038500461,
        0.7984148545182737,
        0.9054549224367789,
        0.8872216686599599,
        0.7119744557713882,
        0.5516966777303671,
        0.33645443094214544,
        0.36323196473896907,
        0.9307917513325731,
        0.7354884359961364,
        0.7185388506704067,
        0.8852538932511926,
        0.6112007852478141,
        0.7924700479346962,
        0.4755693185847448,
        0.8062011866332339,
        0.98789250993949,
        0.7221719324819712,
        0.8740073678631499,
        0.3564320779113565,
        0.9228494524514022,
        0.9853245493312502,
        0.9694501434117074,
        0.9973090818783165,
        0.7950221610332707,
        0.7297340940961335,
        0.5993051267583294,
        0.3882520768014076,
        0.8152297611819531,
        0.9954839203131746,
        0.962824857331114,
        0.9409321486024967,
        0.9742785777639986,
        0.17991514123080576,
        0.9887518404372874,
        0.37100741582562563,
        0.38996632999548203,
        0.9776454643489758,
        0.39722223026085557,
        0.6571422004149754,
        0.9866412903013709,
        0.9441939019199626,
        0.9321188571092405,
        0.9414167286110949,
        0.874809388239011,
        0.986096333046412,
        0.9198432172202342,
        0.15248356297993085,
        0.3397671661055153,
        0.4943206657048409,
        0.6197606962352504,
        0.5950551054240445,
        0.10209490958071754,
        0.09992020920219156,
        0.9811136654654927,
        0.3834507584919793,
        0.9939614888053515,
        0.9855960587672424,
        0.9988310648649493,
        0.9943414908312453,
        0.7725574837314465,
        0.9183802068283202,
        0.843332376759955,
        0.8837351545915085,
        0.9738230199110262,
        0.8304410361024156,
        0.9502786794361738,
        0.8180163694275192,
        0.855701557363995,
        0.2726210865088722,
        0.4528790457865923,
        0.3746290118044097,
        0.9787920708274624,
        0.9914212329003339,
        0.9758540585662221,
        0.9446087509465361,
        0.952400627202449,
        0.9687910529362938,
        0.9991198634850662,
        0.9990004228825908,
        0.9874655071550109,
        0.6634190408157515,
        0.8828617615707209,
        0.07712779632822993,
        0.9858879491300443,
        0.9802408364233254,
        0.9383288803223644,
        0.9593516246848454,
        0.15507335874395659,
        0.18131362347088567,
        0.8105325058247551,
        0.8623911070159063,
        0.5891507086792241,
        0.743639647480704,
        0.9164162161160366,
        0.5878341987632298,
        0.9120584896530685,
        0.4102137239343736,
        0.8648555461875848,
        0.9666963097920533,
        0.9697938702911227,
        0.8925256387036903,
        0.9400090691280725,
        0.5697477445502784,
        0.8615583710379091,
        0.9090163620448073,
        0.958486582405349,
        0.7836312987355163,
        0.9913490311438872,
        0.08158883786023396,
        0.9960505834800041,
        0.5970650850203174,
        0.7048075310439548,
        0.29306185317149847,
        0.9725705617409717,
        0.9568374007504292,
        0.9709761448211442,
        0.5306453483594386,
        0.3394466570649324,
        0.080526592817419,
        0.18691719472600293,
        0.3201264413379875,
        0.5029361453327197,
        0.894874783835395,
        0.655263707778754,
        0.7816062780922957,
        0.9249626532749841,
        0.9933810619091374,
        0.37730620018255967,
        0.8847932426292465,
        0.928277179738713,
        0.6330465291947999,
        0.7651397366231711,
        0.766929034442933,
        0.9188006666779501,
        0.8721904573851609,
        0.7926001251276923,
        0.6640368473875409,
        0.6420468905901718,
        0.10238745726949568,
        0.8564723966473264,
        0.918114756532904,
        0.9353536170759031,
        0.6156505420920637,
        0.6751299796788123,
        0.09073461329025144,
        0.38922052191066325,
        0.5653846198572566,
        0.5872501425487964,
        0.9873192692297256,
        0.9824529404732584,
        0.23667266743515558,
        0.9528196739484505,
        0.9841763414441514,
        0.9899193617271483,
        0.9355990248813355,
        0.8744585984792195,
        0.967424132093656,
        0.8991692070845686,
        0.9003262038887997,
        0.8747054311959866,
        0.7897042692969528,
        0.5649449553771927,
        0.9742097315936163,
        0.11517233752182054,
        0.8177242325951533,
        0.17101770107368897,
        0.5903528038456566,
        0.4747706702005993,
        0.9742317835181789,
        0.9432188887046726,
        0.5596207157909736,
        0.3223652581899798,
        0.410662181447798,
        0.9822752407714925,
        0.9915047195191908,
        0.45143191160755675,
        0.9698047987363938,
        0.9006484196967643,
        0.9536644756364887,
        0.1775277984172361,
        0.11774370904338628,
        0.08402290715310583,
        0.43839497007285205,
        0.39872590938352337,
        0.6041074579870681,
        0.8683110568935907,
        0.13646145286982184,
        0.6088753863010999,
        0.8702448664378746,
        0.9122024432066929,
        0.7677322551099092,
        0.29500690796263235,
        0.8216871137834425,
        0.6125676222686145,
        0.6725263755474673,
        0.9465978285275267,
        0.7537094935366262,
        0.4200439912707071,
        0.2442305670463205,
        0.5336613963196297,
        0.9914256663044564,
        0.9477292492196631,
        0.8283782500472869,
        0.8759022106985251,
        0.7095750818744571,
        0.4367031618722645,
        0.34975168658425726,
        0.39734885284854926,
        0.08884599645301856,
        0.7743275870161959,
        0.8478328468924804,
        0.8712847253539925,
        0.7777558040394185,
        0.8394875427950248,
        0.5986738946929829,
        0.8935676662693073,
        0.5920792299520841,
        0.9685086929752538,
        0.0456914221695226,
        0.38351240241099727,
        0.0343443971426793,
        0.9106287749215339,
        0.9179779226266412,
        0.8891024368462049,
        0.10424186457437667,
        0.17336905167153338,
        0.31499260827956826,
        0.2252118681003894,
        0.8392253218013473,
        0.32168379695921034,
        0.7360482696786922,
        0.9116615205016295,
        0.9057718553842538,
        0.9553044006118915,
        0.3530276495699104,
        0.4312396000047023,
        0.524429336512451,
        0.9378620282962313,
        0.9798210529697503,
        0.7009378699244545,
        0.4089715810812261,
        0.650098811551059,
        0.9960916498916954,
        0.9947557323816146,
        0.9961762339500897,
        0.15207340233974942,
        0.1449865754950932,
        0.13310803552695039,
        0.9722460960231261,
        0.9031749941323011,
        0.8783398191294471,
        0.9962273648203385,
        0.7388203665851438,
        0.997671909440874,
        0.7252639160635567,
        0.80585979225472,
        0.8372120281164042,
        0.6030929935964386,
        0.5365050979223966,
        0.5631857983972779,
        0.9085702249245923,
        0.9531032378626904,
        0.9739915590990671,
        0.9971519557294345,
        0.9181700045104967,
        0.837344959912748,
        0.5277766970035652,
        0.9640221315980354,
        0.6478132080854023,
        0.9933979379290582,
        0.9090110110243014,
        0.981247115013658,
        0.903518757595072,
        0.8368268391872778,
        0.9728788025529648,
        0.10388288278634944,
        0.31627116262573113,
        0.7367376538874542,
        0.1420877553038713,
        0.33191777895745683,
        0.8048335177104791,
        0.8785252397850193,
        0.9343737634150964,
        0.9379471508110364,
        0.9986928918202125,
        0.9846492319020933,
        0.9922703047608921,
        0.9875250693595635,
        0.9815798777187168,
        0.10526081870398178,
        0.5554629801827529,
        0.6778408096565823,
        0.39778878568245946,
        0.9343247813120069,
        0.9306019252699684,
        0.8246061272269684,
        0.12271521284959704,
        0.4303424220405855,
        0.970469158187509,
        0.84830332449681,
        0.9927557404550011,
        0.8773450248886764,
        0.9474081491425098,
        0.9571204242022856,
        0.9635341128516403,
        0.5063458991794764,
        0.8645546039593306,
        0.2078999518375957,
        0.7481047941906903,
        0.8196267172082016,
        0.6272175536067777,
        0.9826138243849516,
        0.8088372894590319,
        0.5993553665998047,
        0.8984506270865305,
        0.8351332395683834,
        0.95291954966145,
        0.9328125369554351,
        0.995487440531523,
        0.9863512395559256,
        0.08473606162672664,
        0.5059795270953321,
        0.24030024266162323,
        0.5728236919447849,
        0.7241605375834663,
        0.5623271456423898,
        0.8882247734416945,
        0.7721740058583951,
        0.7154566469091695,
        0.33483930339600115,
        0.9926327571523298,
        0.8854131039068044,
        0.9267493724780185,
        0.4182292563650386,
        0.4683442196417129,
        0.1865125650876887,
        0.33944074703935,
        0.6986204355291861,
        0.9880470611919054,
        0.9561097491844963,
        0.3561481456752228,
        0.9581380938423786,
        0.3398599286818024,
        0.9647450932814664,
        0.8430468782517605,
        0.6919449309561331,
        0.7515011532204174,
        0.9057351518805,
        0.5500003160687899,
        0.5625594511386015,
        0.7864508479372989,
        0.19872815037386915,
        0.9201032822569623,
        0.9053144702062446,
        0.9549325980183268,
        0.8778437112298924,
        0.867613081984981,
        0.835580946368038,
        0.7726963964035655,
        0.34846135722392346,
        0.9045770395909797,
        0.9333589363402628,
        0.9985625696356355,
        0.9958457522708404,
        0.9941302135713567,
        0.9919820813362217,
        0.9413902843030728,
        0.9798933971238526,
        0.669437372880883,
        0.7473203715234638,
        0.5452328367931984,
        0.973569764010786,
        0.9715127899677694,
        0.9679788117626464,
        0.3005173730290883,
        0.9810654447439534,
        0.9886869906654449,
        0.7365689143973592,
        0.8871055352668838,
        0.771162090835281,
        0.9939892363831946,
        0.9867833712658552,
        0.7353827489290764,
        0.08953187330427059,
        0.8734440248850177,
        0.8823767512996115,
        0.25419195891635626,
        0.8662859836515463,
        0.49103322758956985,
        0.7299985687427313,
        0.8120184271728307,
        0.6514588890259572,
        0.7325832647182334,
        0.9580449609477661,
        0.496465883592515,
        0.1328100139699625,
        0.4509974743407165,
        0.4815826256899581,
        0.9815450257531444,
        0.9800836656527278,
        0.941235483705549,
        0.8525241989762684,
        0.7877274383528682,
        0.9572924124289837,
        0.871894045607493,
        0.9014157966238134,
        0.7883641581680029,
        0.9570212666842681,
        0.8941761679582062,
        0.8703453258598455,
        0.8592628868446185,
        0.9937862097155299,
        0.9803585493359762,
        0.9428987896512021,
        0.9889961743013022,
        0.9908801884754461,
        0.9823210667667697,
        0.9788506290803441,
        0.991356830219052,
        0.7152683754456076,
        0.9654710615135186,
        0.9239382136019831,
        0.19140053110870375,
        0.6253956886620529,
        0.8330144615036557,
        0.8286909833390095,
        0.5207907614187908,
        0.6114387738960633,
        0.993346601212787,
        0.9912305368097732,
        0.933269501760659,
        0.6813679057092138,
        0.8174629522221105,
        0.8385102081863772,
        0.5477899959696633,
        0.6989037395775299,
        0.9621028572316744,
        0.9106717349680188,
        0.09471691575053337,
        0.39688660899127803,
        0.4622296259051981,
        0.6617392280590757,
        0.42687078107088106,
        0.8594391015942691,
        0.9183581005109295,
        0.8453584339748206,
        0.9936710462101732,
        0.8887324959180073,
        0.9782621081443565,
        0.8953911988846551,
        0.9763399212899977,
        0.9754039242527179,
        0.10087519455129693,
        0.1081087212398951,
        0.10011594843651735,
        0.14611610593832589,
        0.5628124018305815,
        0.5631804191373113,
        0.42645611146796913,
        0.23500867614798102,
        0.23571338819943016,
        0.8845101192749911,
        0.9020895659528465,
        0.2276014710599622,
        0.44207334457296077,
        0.7876755844564447,
        0.6567363025516675,
        0.38732467142223004,
        0.15113792235301315,
        0.4383458941615897,
        0.4825475661664893,
        0.6102542858731218,
        0.3828264429027116,
        0.9944583213309401,
        0.9972408062728458,
        0.9803741818125064,
        0.4039693494672231,
        0.6012052598786986,
        0.6396926552658082,
        0.9899139847027254,
        0.9986120048676957,
        0.9270744165241682,
        0.9069797989145247,
        0.333432225734915,
        0.2612800975432744,
        0.8172141100981887,
        0.9254627504654427,
        0.9221547674353656,
        0.3741609719985145,
        0.3759924050711085,
        0.6596875392384286,
        0.6396847676523643,
        0.7105477676017732,
        0.882778256379049,
        0.5260955063423622,
        0.42732817590100375,
        0.37410272463617195,
        0.7846442601014568,
        0.8455326363991313,
        0.728641462814231,
        0.5614812099482875,
        0.8823395198372121,
        0.8495276309851183,
        0.2526573870711499,
        0.06616525660037825,
        0.10997444576733462,
        0.5028967223283564,
        0.16561799546807662,
        0.12390394270693936,
        0.8687315916969323,
        0.4157126059075126,
        0.6316708552091043,
        0.1867503829674693,
        0.6512820260037848,
        0.7102254487748092,
        0.6688869377074751,
        0.5748240237508266,
        0.7609378787257037,
        0.7423095014292587,
        0.7239746510991593,
        0.9230787936231245,
        0.9590222569629047,
        0.9383154590588417,
        0.6999759355482444,
        0.5525480758257334,
        0.5607396069154549,
        0.7498764261674574,
        0.07610787621933246,
        0.22210968013535973,
        0.5781867014044819,
        0.6424221030855957,
        0.8283599283496068,
        0.09673061394368263,
        0.619493179450269,
        0.14547057130504082,
        0.16911845578005702,
        0.7459117835676546,
        0.405781272319102,
        0.8251441839333107,
        0.21375112760147108,
        0.13329581019854353,
        0.23351000731983945,
        0.9899189798939454,
        0.9601380533201916,
        0.4180264512226718,
        0.9840645738073058,
        0.7042976955525521,
        0.9474082536642912,
        0.43715465480314075,
        0.6868350200551696,
        0.6387225323487689,
        0.3329394044949287,
        0.5089025454020072,
        0.3429611760507493,
        0.9399625564790963,
        0.9852167702547573,
        0.9339039756953184,
        0.2032016898871535,
        0.886670487331828,
        0.860987812642676,
        0.9758153773657788,
        0.8015185772740205,
        0.6037448423266123,
        0.82890591485178,
        0.9450377921238888,
        0.9636904924756896,
        0.9733289044819671,
        0.9624667820448829,
        0.9746014727767054,
        0.967355024290868,
        0.9885672637282957,
        0.6302367091370091,
        0.6641432347881604,
        0.9774858752130937,
        0.8858296172792236,
        0.47749249540257893,
        0.8886741999224331,
        0.8316687478991125,
        0.9226020178556821,
        0.9465689473981289,
        0.8856051711969872,
        0.9619180131736448,
        0.8304588966126767,
        0.9352312129834387,
        0.706813123207101,
        0.9359366517637588,
        0.8020830054683445
    ],
    "eval_rms_calibration_error": 0.29396100252265095
}