GENIES/results/evaluations/zero_shot/openllama-3b-code/counterfactual_python.json

{
    "model_dir": "models/openllama-3b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-20 01:18:03",
    "eval_loss": 0.6855528354644775,
    "eval_score": -0.24693697690963745,
    "eval_brier_score": 0.24693696352363126,
    "eval_average_probability": 0.5038591234286626,
    "eval_accuracy": 0.728,
    "eval_probabilities": [
        0.500970184803009,
        0.496784508228302,
        0.4993571937084198,
        0.508660614490509,
        0.5061156153678894,
        0.506000816822052,
        0.5286304354667664,
        0.5103843808174133,
        0.5212336182594299,
        0.5048821568489075,
        0.5041775703430176,
        0.5087447762489319,
        0.49785590171813965,
        0.4521206319332123,
        0.5034769773483276,
        0.5115383267402649,
        0.5145734548568726,
        0.5012235641479492,
        0.49889329075813293,
        0.49549323320388794,
        0.5133496522903442,
        0.5124505758285522,
        0.5264015197753906,
        0.5234212875366211,
        0.5066211819648743,
        0.5017524361610413,
        0.4893896281719208,
        0.46124789118766785,
        0.46204832196235657,
        0.454241544008255,
        0.501081109046936,
        0.5001449584960938,
        0.5005605220794678,
        0.5077939033508301,
        0.502457320690155,
        0.4987269341945648,
        0.4988250136375427,
        0.49324730038642883,
        0.5010948181152344,
        0.5041933059692383,
        0.5007741451263428,
        0.48477575182914734,
        0.5033477544784546,
        0.5098629593849182,
        0.5077076554298401,
        0.5021180510520935,
        0.498973548412323,
        0.5013256669044495,
        0.519022524356842,
        0.4990164637565613,
        0.4990164637565613,
        0.5015813112258911,
        0.504256010055542,
        0.5202166438102722,
        0.5064262747764587,
        0.5027457475662231,
        0.5078521370887756,
        0.5049152374267578,
        0.5097820162773132,
        0.5087863206863403,
        0.5114139914512634,
        0.5044566988945007,
        0.5071080327033997,
        0.4963286817073822,
        0.5214171409606934,
        0.548811137676239,
        0.51263028383255,
        0.5064045786857605,
        0.49581968784332275,
        0.509410560131073,
        0.5168331861495972,
        0.5190591812133789,
        0.5062248110771179,
        0.5122695565223694,
        0.503420889377594,
        0.5025742650032043,
        0.5049248337745667,
        0.49799010157585144,
        0.5037423372268677,
        0.5073911547660828,
        0.5002367496490479,
        0.4981295168399811,
        0.5052854418754578,
        0.5183582901954651,
        0.5015555024147034,
        0.505914032459259,
        0.5014377236366272,
        0.5005828142166138,
        0.4996398687362671,
        0.4905194044113159,
        0.5090922713279724,
        0.503400444984436,
        0.5089356899261475,
        0.5347216129302979,
        0.5071797370910645,
        0.5098351240158081,
        0.5192470550537109,
        0.5124058127403259,
        0.5301664471626282,
        0.5049180388450623,
        0.5000271201133728,
        0.5060870051383972,
        0.40586674213409424,
        0.3902398347854614,
        0.396027535200119,
        0.5040051937103271,
        0.5045312643051147,
        0.5051714181900024,
        0.49538180232048035,
        0.5023332238197327,
        0.4960085153579712,
        0.5138449668884277,
        0.5170817375183105,
        0.5052311420440674,
        0.49007853865623474,
        0.5129286050796509,
        0.4920649528503418,
        0.5437439680099487,
        0.5525939464569092,
        0.5656300187110901,
        0.5098072290420532,
        0.5161752700805664,
        0.5113860368728638,
        0.5042080879211426,
        0.5130418539047241,
        0.50495845079422,
        0.49109625816345215,
        0.49998730421066284,
        0.4964646100997925,
        0.5047580599784851,
        0.4997142553329468,
        0.49690696597099304,
        0.5067181587219238,
        0.5070297122001648,
        0.507431149482727,
        0.5038367509841919,
        0.5036360025405884,
        0.5027250051498413,
        0.5036431550979614,
        0.4874284565448761,
        0.4796009361743927,
        0.4925854802131653,
        0.5072410106658936,
        0.5016095042228699,
        0.5050998330116272,
        0.49349620938301086,
        0.5050386190414429,
        0.5215150117874146,
        0.4852117896080017,
        0.5025849342346191,
        0.46939849853515625,
        0.46491891145706177,
        0.4623912274837494,
        0.5203028917312622,
        0.5154476761817932,
        0.5087627172470093,
        0.33128300309181213,
        0.33021414279937744,
        0.3293665647506714,
        0.5001717805862427,
        0.5177879333496094,
        0.5078604817390442,
        0.5,
        0.5093836784362793,
        0.5190008878707886,
        0.5046488046646118,
        0.48422926664352417,
        0.5007224678993225,
        0.4938701391220093,
        0.4961037039756775,
        0.5048656463623047,
        0.448853462934494,
        0.4570222795009613,
        0.46112313866615295,
        0.5828206539154053,
        0.5616961717605591,
        0.5838040709495544,
        0.5027809739112854,
        0.5100746750831604,
        0.5003323554992676,
        0.4987685978412628,
        0.5013771653175354,
        0.5017814636230469,
        0.5207206010818481,
        0.5122572183609009,
        0.5144466757774353,
        0.5,
        0.5047221183776855,
        0.4934835433959961,
        0.4935219883918762,
        0.5053851008415222,
        0.4976706802845001,
        0.5037175416946411,
        0.4997114837169647,
        0.5041770935058594,
        0.5,
        0.5024816989898682,
        0.5150716304779053,
        0.5010956525802612,
        0.5097493529319763,
        0.5008370876312256,
        0.5075068473815918,
        0.5118759870529175,
        0.5036650896072388,
        0.5517098307609558,
        0.5836011171340942,
        0.5529770255088806,
        0.5031465291976929,
        0.5023642182350159,
        0.49590614438056946,
        0.4999067783355713,
        0.5035690665245056,
        0.5042980313301086,
        0.49049142003059387,
        0.4934176504611969,
        0.49174463748931885,
        0.5070598721504211,
        0.49055078625679016,
        0.5067522525787354,
        0.5018609762191772,
        0.4929507374763489,
        0.5013871788978577,
        0.5175962448120117,
        0.5141819715499878,
        0.5476434826850891,
        0.506237268447876,
        0.49921056628227234,
        0.5078132152557373,
        0.6311295628547668,
        0.5946775078773499,
        0.5744703412055969,
        0.4743514657020569,
        0.48009759187698364,
        0.42851966619491577,
        0.5257964730262756,
        0.5309323668479919,
        0.527654767036438,
        0.5159448981285095,
        0.5167407989501953,
        0.5155971050262451,
        0.5106343626976013,
        0.5177637934684753,
        0.5102661848068237,
        0.4899745285511017,
        0.4881518483161926,
        0.4934544265270233,
        0.5006555318832397,
        0.5130288600921631,
        0.5168496966362,
        0.5071462988853455,
        0.49434131383895874,
        0.5040183663368225,
        0.5025233626365662,
        0.5027341246604919,
        0.5010457634925842,
        0.5101775527000427,
        0.5070446133613586,
        0.49665430188179016,
        0.49976685643196106,
        0.5003578662872314,
        0.5027175545692444,
        0.5576273202896118,
        0.5491969585418701,
        0.5356360673904419,
        0.5038136839866638,
        0.5128395557403564,
        0.5370712280273438,
        0.5241684317588806,
        0.5179513096809387,
        0.5182470083236694,
        0.5085433721542358,
        0.49784931540489197,
        0.5103546380996704,
        0.5096749663352966,
        0.5099910497665405,
        0.5024098753929138,
        0.5110453367233276,
        0.504889190196991,
        0.5053549408912659,
        0.5070963501930237,
        0.49518024921417236,
        0.5000243782997131,
        0.5048550367355347,
        0.49485912919044495,
        0.5007480382919312,
        0.5228056311607361,
        0.5114395022392273,
        0.5327605605125427,
        0.5053304433822632,
        0.49930843710899353,
        0.5042390823364258,
        0.49612173438072205,
        0.5249044299125671,
        0.5194700360298157,
        0.5219619870185852,
        0.5223132371902466,
        0.5307206511497498,
        0.41836243867874146,
        0.4202955365180969,
        0.39941516518592834,
        0.49150732159614563,
        0.49253740906715393,
        0.5039065480232239,
        0.49892014265060425,
        0.5252069234848022,
        0.45216476917266846,
        0.5034113526344299,
        0.49833524227142334,
        0.5002152323722839,
        0.5264812707901001,
        0.5243950486183167,
        0.5036559700965881,
        0.5099906325340271,
        0.4797499179840088,
        0.5341811776161194,
        0.5018944144248962,
        0.5067447423934937,
        0.5042115449905396,
        0.494479238986969,
        0.4977363646030426,
        0.5009199976921082,
        0.482907235622406,
        0.4729105532169342,
        0.4720969498157501,
        0.5081229209899902,
        0.5044323205947876,
        0.5052971839904785,
        0.5007666349411011,
        0.4960619807243347,
        0.4990233778953552,
        0.5049806237220764,
        0.5068820118904114,
        0.5090386271476746,
        0.5320157408714294,
        0.5296431183815002,
        0.5352203845977783,
        0.5060250759124756,
        0.5166700482368469,
        0.5063849091529846,
        0.5548340678215027,
        0.5346148014068604,
        0.5109890699386597,
        0.49924710392951965,
        0.4998376965522766,
        0.5004709362983704,
        0.5039472579956055,
        0.5008442997932434,
        0.5034943222999573,
        0.5010762810707092,
        0.5023916363716125,
        0.501835286617279,
        0.49970775842666626,
        0.4969094395637512,
        0.4959605038166046,
        0.504686713218689,
        0.505339503288269,
        0.509051501750946,
        0.5067998170852661,
        0.4989430606365204,
        0.49872955679893494,
        0.501207709312439,
        0.49854519963264465,
        0.5178167223930359,
        0.5050942897796631,
        0.5033279061317444,
        0.5062288641929626,
        0.5140684843063354,
        0.47636762261390686,
        0.5094546675682068,
        0.5325468182563782,
        0.547585129737854,
        0.5497810244560242,
        0.40581661462783813,
        0.403932124376297,
        0.4048600196838379,
        0.5201290249824524,
        0.5267062187194824,
        0.517704427242279,
        0.5017743706703186,
        0.49744364619255066,
        0.4982297122478485,
        0.49418580532073975,
        0.4596405327320099,
        0.48596006631851196,
        0.5303112864494324,
        0.5333977937698364,
        0.5388766527175903,
        0.5035620331764221,
        0.5149791836738586,
        0.5052111148834229,
        0.5289404392242432,
        0.4888772666454315,
        0.5412939786911011,
        0.5054585337638855,
        0.5060225129127502,
        0.5042941570281982,
        0.5023972988128662,
        0.5001347064971924,
        0.5012677907943726,
        0.5068709850311279,
        0.5120370388031006,
        0.5060581564903259,
        0.33261001110076904,
        0.33802205324172974,
        0.3375372886657715,
        0.5052134990692139,
        0.5086450576782227,
        0.5078516006469727,
        0.49745187163352966,
        0.4970923364162445,
        0.49634671211242676,
        0.4425778090953827,
        0.6197919845581055,
        0.5599571466445923,
        0.5332528352737427,
        0.5411068201065063,
        0.5393255949020386,
        0.5003900527954102,
        0.499696284532547,
        0.5012677907943726,
        0.5027869939804077,
        0.4996708631515503,
        0.49762555956840515,
        0.5288078784942627,
        0.5415676236152649,
        0.5233530402183533,
        0.5020250678062439,
        0.5300424098968506,
        0.5085127949714661,
        0.5045857429504395,
        0.5155760645866394,
        0.521147608757019,
        0.5099625587463379,
        0.5075632929801941,
        0.5085800886154175,
        0.5154402256011963,
        0.518467903137207,
        0.5226883292198181,
        0.4993947744369507,
        0.5026452541351318,
        0.5023595094680786,
        0.5091685056686401,
        0.5098196268081665,
        0.5136571526527405,
        0.5122909545898438,
        0.5075740814208984,
        0.5061596632003784,
        0.500450074672699,
        0.5142565369606018,
        0.49366411566734314,
        0.512259840965271,
        0.5188214778900146,
        0.5081765651702881,
        0.5041402578353882,
        0.5052698254585266,
        0.5191062092781067,
        0.5140888690948486,
        0.5152820944786072,
        0.5120208263397217,
        0.5058736801147461,
        0.5053455829620361,
        0.5191574692726135,
        0.5171671509742737,
        0.5207257270812988,
        0.5226534008979797,
        0.5038933753967285,
        0.5029594898223877,
        0.5090439915657043,
        0.5459175109863281,
        0.5289027690887451,
        0.5369828343391418,
        0.5001278519630432,
        0.4970058500766754,
        0.4957161843776703,
        0.5115951299667358,
        0.5000411868095398,
        0.5152435302734375,
        0.5092509984970093,
        0.5552324652671814,
        0.5027036070823669,
        0.5035876631736755,
        0.508826494216919,
        0.5114865899085999,
        0.5028659105300903,
        0.5046254396438599,
        0.49894556403160095,
        0.5075969696044922,
        0.5114575028419495,
        0.5085527300834656,
        0.5037980675697327,
        0.5132295489311218,
        0.5083207488059998,
        0.5008257627487183,
        0.5037099123001099,
        0.5086794495582581,
        0.4973137378692627,
        0.4903889000415802,
        0.4944702982902527,
        0.49109455943107605,
        0.4932193458080292,
        0.5021556615829468,
        0.5011597275733948,
        0.5022556185722351,
        0.4848055839538574,
        0.5104755759239197,
        0.5005772709846497,
        0.5059554576873779,
        0.4995576739311218,
        0.5004389882087708,
        0.5014299154281616,
        0.5006479620933533,
        0.5028347373008728,
        0.5022112131118774,
        0.5040447115898132,
        0.5017208456993103,
        0.5070478320121765,
        0.5087504386901855,
        0.5027660131454468,
        0.49456942081451416,
        0.5170398354530334,
        0.5137752294540405,
        0.5164532661437988,
        0.4834638237953186,
        0.504025399684906,
        0.5005398988723755,
        0.5027192831039429,
        0.5054307579994202,
        0.510046660900116,
        0.5119743943214417,
        0.5101907849311829,
        0.5027155876159668,
        0.5250675082206726,
        0.5225296020507812,
        0.5467936992645264,
        0.516041100025177,
        0.5172780156135559,
        0.5240437388420105,
        0.49153992533683777,
        0.5091395378112793,
        0.5040318965911865,
        0.414498507976532,
        0.41352754831314087,
        0.41513651609420776,
        0.5060999393463135,
        0.5067991614341736,
        0.49642741680145264,
        0.503525972366333,
        0.5063744783401489,
        0.5076482892036438,
        0.5028959512710571,
        0.49801650643348694,
        0.49513015151023865,
        0.5163694620132446,
        0.4949152171611786,
        0.5164241790771484,
        0.5003659725189209,
        0.4967953860759735,
        0.49427565932273865,
        0.4841659963130951,
        0.47644293308258057,
        0.5296847224235535,
        0.5118234157562256,
        0.508397102355957,
        0.5068962574005127,
        0.42363548278808594,
        0.42098814249038696,
        0.4293871819972992,
        0.5251568555831909,
        0.5373177528381348,
        0.4988855719566345,
        0.5038472414016724,
        0.5037625432014465,
        0.5041975975036621,
        0.50287926197052,
        0.5122319459915161,
        0.5147387385368347,
        0.5135214924812317,
        0.5254594087600708,
        0.5241959095001221,
        0.5320197939872742,
        0.5121166110038757,
        0.5084090828895569,
        0.46966052055358887,
        0.4742954671382904,
        0.47359737753868103,
        0.49520549178123474,
        0.5028493404388428,
        0.507783830165863,
        0.5028325319290161,
        0.510682225227356,
        0.5030914545059204,
        0.5134630799293518,
        0.5193805694580078,
        0.5118701457977295,
        0.5008203387260437,
        0.4996098577976227,
        0.503538966178894,
        0.43794652819633484,
        0.48573562502861023,
        0.47868600487709045,
        0.5231475234031677,
        0.5111391544342041,
        0.5289663672447205,
        0.5216875076293945,
        0.5208057165145874,
        0.5060639977455139,
        0.6046751141548157,
        0.5737674236297607,
        0.5468388795852661,
        0.4989253282546997,
        0.49847427010536194,
        0.5021525025367737,
        0.5080226063728333,
        0.49869924783706665,
        0.504740297794342,
        0.4977407157421112,
        0.5181952714920044,
        0.5074754357337952,
        0.49563077092170715,
        0.5002098083496094,
        0.5044012665748596,
        0.5082769393920898,
        0.4946777820587158,
        0.4985077977180481,
        0.4964359700679779,
        0.4868772029876709,
        0.49366918206214905,
        0.5252262949943542,
        0.5172072052955627,
        0.5162110924720764,
        0.4916239380836487,
        0.49885934591293335,
        0.49619024991989136,
        0.5066179633140564,
        0.5154577493667603,
        0.5017552971839905,
        0.511522650718689,
        0.5183762907981873,
        0.5111926794052124,
        0.4426725208759308,
        0.4643474221229553,
        0.4684525430202484,
        0.5103098154067993,
        0.5271591544151306,
        0.5097290277481079,
        0.5027443170547485,
        0.5016157031059265,
        0.4981469511985779,
        0.49970147013664246,
        0.5000057816505432,
        0.4948943555355072,
        0.49948248267173767,
        0.5009827613830566,
        0.5012971758842468,
        0.5224245190620422,
        0.5172944664955139,
        0.5157141089439392,
        0.5039563179016113,
        0.5004876852035522,
        0.5005677938461304,
        0.5227108597755432,
        0.5341619253158569,
        0.589228093624115,
        0.5058031678199768,
        0.5029232501983643,
        0.5046074986457825,
        0.5075929164886475,
        0.4395717978477478,
        0.4440706670284271,
        0.4546566307544708,
        0.4766254425048828,
        0.4417601227760315,
        0.5015750527381897,
        0.4999995529651642,
        0.5037434697151184,
        0.5121504068374634,
        0.5226823687553406,
        0.5175746083259583,
        0.5216269493103027,
        0.5172232389450073,
        0.5134932398796082,
        0.5054219961166382,
        0.50579833984375,
        0.4999583661556244,
        0.4860891103744507,
        0.4948967695236206,
        0.5073456168174744,
        0.529381513595581,
        0.5255016684532166,
        0.5228716731071472,
        0.5045745968818665,
        0.5043326020240784,
        0.5007140636444092,
        0.49113765358924866,
        0.4825814962387085,
        0.49868398904800415,
        0.49936285614967346,
        0.5061454176902771,
        0.49580180644989014,
        0.49800002574920654,
        0.502526581287384,
        0.5038126111030579,
        0.4979727864265442,
        0.4976326525211334,
        0.4962560832500458,
        0.5055243372917175,
        0.5135884284973145,
        0.47404375672340393,
        0.5043215155601501,
        0.505220353603363,
        0.501786470413208,
        0.5057412385940552,
        0.4991275668144226,
        0.5181820392608643,
        0.5008467435836792,
        0.49934470653533936,
        0.5008792877197266,
        0.5037215948104858,
        0.5097768902778625,
        0.5143536925315857,
        0.5061233639717102,
        0.5393557548522949,
        0.5315257906913757,
        0.5063165426254272,
        0.5007197856903076,
        0.5044089555740356,
        0.5103486180305481,
        0.504534125328064,
        0.5415477156639099,
        0.5332672595977783,
        0.5339085459709167,
        0.5342319011688232,
        0.5125057101249695,
        0.5156344771385193,
        0.5030832886695862,
        0.4900912046432495,
        0.5046457052230835,
        0.5007675886154175,
        0.5052964091300964,
        0.5066726207733154,
        0.5016164779663086,
        0.5138458609580994,
        0.5078047513961792,
        0.4994020164012909,
        0.5128543972969055,
        0.5042791962623596,
        0.5140072703361511,
        0.5075104832649231,
        0.5186010003089905,
        0.5223562121391296
    ],
    "eval_runtime": 55.5691,
    "eval_samples_per_second": 13.497,
    "eval_steps_per_second": 0.432,
    "eval_rms_calibration_error": 0.0002823842366536655
}