GENIES/results/evaluations/zero_shot/openllama-3b-code_low_quality/code.json

{
    "model_dir": "models/openllama-3b",
    "distribution_id": "code",
    "timestamp": "2023-10-20 01:45:17",
    "eval_loss": 0.6734113097190857,
    "eval_score": -0.24034792184829712,
    "eval_brier_score": 0.24034791807741335,
    "eval_average_probability": 0.5099883956909179,
    "eval_accuracy": 0.852,
    "eval_probabilities": [
        0.5422244668006897,
        0.5702081322669983,
        0.5897537469863892,
        0.5196388959884644,
        0.5171331167221069,
        0.5112442374229431,
        0.511946439743042,
        0.5050100684165955,
        0.5077146291732788,
        0.5085302591323853,
        0.5167051553726196,
        0.5184034705162048,
        0.5091130137443542,
        0.511625349521637,
        0.5223994851112366,
        0.5045440196990967,
        0.4930412769317627,
        0.501183271408081,
        0.5048266053199768,
        0.502834677696228,
        0.49954357743263245,
        0.5045831799507141,
        0.5090726017951965,
        0.5112781524658203,
        0.48149144649505615,
        0.48447859287261963,
        0.48557060956954956,
        0.5046032667160034,
        0.5038072466850281,
        0.5069986581802368,
        0.50338214635849,
        0.5133063793182373,
        0.4978688955307007,
        0.5163767337799072,
        0.4999377131462097,
        0.49697813391685486,
        0.5081990361213684,
        0.5162377953529358,
        0.5062076449394226,
        0.5309246778488159,
        0.5294354557991028,
        0.5269574522972107,
        0.5115239024162292,
        0.5092887878417969,
        0.5053049921989441,
        0.5053457021713257,
        0.5156709551811218,
        0.5137386918067932,
        0.5208756923675537,
        0.5218968391418457,
        0.4521472454071045,
        0.54551762342453,
        0.5549445152282715,
        0.522935152053833,
        0.5048756003379822,
        0.5009864568710327,
        0.5042321681976318,
        0.522911787033081,
        0.4992101490497589,
        0.5164030194282532,
        0.5198647379875183,
        0.518846869468689,
        0.522488534450531,
        0.5009836554527283,
        0.5073531866073608,
        0.50246661901474,
        0.5047900676727295,
        0.4915965795516968,
        0.49543333053588867,
        0.5054842233657837,
        0.5042405724525452,
        0.5063852667808533,
        0.5030543208122253,
        0.5080687999725342,
        0.5076045989990234,
        0.5447825789451599,
        0.5159820318222046,
        0.5337051749229431,
        0.5336003303527832,
        0.5171427726745605,
        0.517551600933075,
        0.5045181512832642,
        0.49856966733932495,
        0.5059130787849426,
        0.5094031691551208,
        0.5111196041107178,
        0.5002939701080322,
        0.5072001814842224,
        0.5061689019203186,
        0.5037775635719299,
        0.5057733654975891,
        0.5020975470542908,
        0.49648842215538025,
        0.5016038417816162,
        0.5004729628562927,
        0.5030299425125122,
        0.4989413321018219,
        0.4892643988132477,
        0.4933824837207794,
        0.5072891712188721,
        0.5051856637001038,
        0.4862854778766632,
        0.5165996551513672,
        0.5287705659866333,
        0.5072972178459167,
        0.5225600004196167,
        0.5257136225700378,
        0.5311734676361084,
        0.5038524866104126,
        0.5099451541900635,
        0.5010844469070435,
        0.5187233090400696,
        0.5344878435134888,
        0.5257521867752075,
        0.5228607058525085,
        0.5182521939277649,
        0.5122039318084717,
        0.5337194800376892,
        0.5089041590690613,
        0.5089266896247864,
        0.5063369274139404,
        0.5104950666427612,
        0.5158017873764038,
        0.5193319916725159,
        0.5244753956794739,
        0.5129930973052979,
        0.5367105007171631,
        0.5328570008277893,
        0.5328800678253174,
        0.5084843635559082,
        0.526677668094635,
        0.5048330426216125,
        0.498958021402359,
        0.5048369765281677,
        0.5067878365516663,
        0.5179623961448669,
        0.5106804370880127,
        0.5070980191230774,
        0.4982161819934845,
        0.5034466981887817,
        0.5005354881286621,
        0.5031301975250244,
        0.50094074010849,
        0.5014841556549072,
        0.5055330395698547,
        0.497612327337265,
        0.49823352694511414,
        0.503166913986206,
        0.5050889849662781,
        0.5065737962722778,
        0.5143272876739502,
        0.5111739039421082,
        0.50710529088974,
        0.5019229054450989,
        0.5023345351219177,
        0.5022093653678894,
        0.502101719379425,
        0.5067555904388428,
        0.514532208442688,
        0.5092668533325195,
        0.5013504028320312,
        0.5037813186645508,
        0.4989570379257202,
        0.48995596170425415,
        0.4932006597518921,
        0.525719165802002,
        0.5147213935852051,
        0.5213214755058289,
        0.5035498738288879,
        0.5021922588348389,
        0.5006192922592163,
        0.5120872855186462,
        0.5073980093002319,
        0.506177544593811,
        0.5140325427055359,
        0.5096889138221741,
        0.5344274640083313,
        0.5060514211654663,
        0.5,
        0.5127777457237244,
        0.516960084438324,
        0.518375039100647,
        0.5205522179603577,
        0.5365955829620361,
        0.5523321628570557,
        0.5265539288520813,
        0.5188300013542175,
        0.524042010307312,
        0.5162591934204102,
        0.5125643610954285,
        0.5171934366226196,
        0.5134482383728027,
        0.506524384021759,
        0.5065891742706299,
        0.5101574659347534,
        0.5081825852394104,
        0.5062384009361267,
        0.5070027112960815,
        0.5226550102233887,
        0.5096282958984375,
        0.5328139662742615,
        0.5045304894447327,
        0.5011062622070312,
        0.5011856555938721,
        0.4961327016353607,
        0.5152153372764587,
        0.5078116655349731,
        0.49610769748687744,
        0.5040565133094788,
        0.5027555227279663,
        0.4963926672935486,
        0.5091403126716614,
        0.5076393485069275,
        0.5012883543968201,
        0.4989052712917328,
        0.499929279088974,
        0.4976796805858612,
        0.48631930351257324,
        0.509963870048523,
        0.49870455265045166,
        0.5017575025558472,
        0.5039205551147461,
        0.49712371826171875,
        0.5105989575386047,
        0.5076045989990234,
        0.4997764825820923,
        0.49987688660621643,
        0.5016835927963257,
        0.5055238008499146,
        0.5090301632881165,
        0.51304030418396,
        0.5214385390281677,
        0.5233283638954163,
        0.5369470715522766,
        0.5091187953948975,
        0.5108232498168945,
        0.4997406601905823,
        0.5020187497138977,
        0.5019068121910095,
        0.5011363625526428,
        0.5091995000839233,
        0.5092717409133911,
        0.5061067342758179,
        0.5145857334136963,
        0.5052123069763184,
        0.524703323841095,
        0.4786945879459381,
        0.5033524632453918,
        0.5019180178642273,
        0.5147377848625183,
        0.5078659653663635,
        0.5561439394950867,
        0.5022138953208923,
        0.5027583837509155,
        0.4994491934776306,
        0.511159360408783,
        0.5158342719078064,
        0.5234062075614929,
        0.5158433318138123,
        0.5075637102127075,
        0.5038635730743408,
        0.5098400115966797,
        0.5346475839614868,
        0.5222823619842529,
        0.516936182975769,
        0.5040733218193054,
        0.505962073802948,
        0.5017675161361694,
        0.49683061242103577,
        0.4960806965827942,
        0.5181149840354919,
        0.5274290442466736,
        0.5096858739852905,
        0.5229678153991699,
        0.5265181660652161,
        0.5259150266647339,
        0.5313199758529663,
        0.5308035016059875,
        0.5211430191993713,
        0.5024867653846741,
        0.5104284882545471,
        0.5013209581375122,
        0.5328329205513,
        0.5210949182510376,
        0.5129390358924866,
        0.5151540040969849,
        0.5246942639350891,
        0.5203620791435242,
        0.5197563767433167,
        0.5069754123687744,
        0.513199508190155,
        0.5092555284500122,
        0.5043804049491882,
        0.5027359127998352,
        0.5059687495231628,
        0.5004253387451172,
        0.50426185131073,
        0.5028682947158813,
        0.49438056349754333,
        0.503026008605957,
        0.5128796696662903,
        0.5129856467247009,
        0.5225658416748047,
        0.5156459808349609,
        0.5290369391441345,
        0.5274497866630554,
        0.5006155371665955,
        0.5015630722045898,
        0.5041406154632568,
        0.5086390376091003,
        0.502719521522522,
        0.49968254566192627,
        0.49691885709762573,
        0.496632844209671,
        0.4991168975830078,
        0.5018452405929565,
        0.5029025077819824,
        0.5028390288352966,
        0.5298980474472046,
        0.520592212677002,
        0.5224959254264832,
        0.5044342875480652,
        0.5007914900779724,
        0.504147469997406,
        0.517231822013855,
        0.524175226688385,
        0.5104496479034424,
        0.510511040687561,
        0.5011006593704224,
        0.5109196901321411,
        0.5156968235969543,
        0.5136783719062805,
        0.5091547966003418,
        0.513139009475708,
        0.5139148831367493,
        0.5157771706581116,
        0.5176898241043091,
        0.49904051423072815,
        0.5110146999359131,
        0.5149508118629456,
        0.5184158682823181,
        0.5285097360610962,
        0.5128417015075684,
        0.5097334980964661,
        0.5115482211112976,
        0.5284339785575867,
        0.5118167400360107,
        0.5210898518562317,
        0.5083521604537964,
        0.5109272599220276,
        0.5050947666168213,
        0.49014222621917725,
        0.47184863686561584,
        0.486232191324234,
        0.5179099440574646,
        0.5035367608070374,
        0.5046292543411255,
        0.5020134449005127,
        0.5413309335708618,
        0.5145843625068665,
        0.5040649175643921,
        0.5033746957778931,
        0.5028993487358093,
        0.515024721622467,
        0.5156537294387817,
        0.5176782608032227,
        0.509166419506073,
        0.5186745524406433,
        0.5283344984054565,
        0.4978286027908325,
        0.5022167563438416,
        0.5030627846717834,
        0.5162106156349182,
        0.5081959366798401,
        0.5097419619560242,
        0.5180526375770569,
        0.5178495049476624,
        0.5075857043266296,
        0.5101503729820251,
        0.5201207995414734,
        0.5150333046913147,
        0.5328444242477417,
        0.5110933780670166,
        0.5260260105133057,
        0.5135388970375061,
        0.5071825385093689,
        0.5123045444488525,
        0.5281783938407898,
        0.5255563855171204,
        0.5231887102127075,
        0.5381652116775513,
        0.5413077473640442,
        0.5393857359886169,
        0.5002676844596863,
        0.506951630115509,
        0.5021503567695618,
        0.5126178860664368,
        0.5024851560592651,
        0.4969557821750641,
        0.5106510519981384,
        0.5152631402015686,
        0.5044511556625366,
        0.5033252239227295,
        0.5002672672271729,
        0.49958857893943787,
        0.5024677515029907,
        0.5040038824081421,
        0.5051151514053345,
        0.5139780044555664,
        0.5158061981201172,
        0.5097646713256836,
        0.5049800872802734,
        0.48508232831954956,
        0.5120894908905029,
        0.5077346563339233,
        0.4935062825679779,
        0.5098990797996521,
        0.5038893222808838,
        0.49861276149749756,
        0.5036178231239319,
        0.5134252905845642,
        0.5155611038208008,
        0.5140738487243652,
        0.4995171129703522,
        0.4948500990867615,
        0.5009986162185669,
        0.5040332674980164,
        0.502763569355011,
        0.505271315574646,
        0.49887406826019287,
        0.5155588984489441,
        0.5199329257011414,
        0.5057901740074158,
        0.5061373114585876,
        0.507921040058136,
        0.5100623369216919,
        0.5143282413482666,
        0.5091789364814758,
        0.5039224028587341,
        0.4986761212348938,
        0.4901842474937439,
        0.5144708752632141,
        0.5152061581611633,
        0.5194900631904602,
        0.5040911436080933,
        0.5008752346038818,
        0.5047605037689209,
        0.4882727265357971,
        0.5023645758628845,
        0.4986720681190491,
        0.495639830827713,
        0.49091199040412903,
        0.4883432686328888,
        0.5189281702041626,
        0.500710129737854,
        0.5099128484725952,
        0.49988794326782227,
        0.5066614151000977,
        0.501319944858551,
        0.5047677755355835,
        0.5091196298599243,
        0.5048549175262451,
        0.5283441543579102,
        0.5315538048744202,
        0.5298905968666077,
        0.4752649664878845,
        0.48300760984420776,
        0.48614415526390076,
        0.5019829273223877,
        0.5057280659675598,
        0.5064323544502258,
        0.5048173069953918,
        0.5018240213394165,
        0.49666884541511536,
        0.4735313951969147,
        0.5022531747817993,
        0.5046836733818054,
        0.5028074383735657,
        0.49736708402633667,
        0.49632588028907776,
        0.5085153579711914,
        0.5136122703552246,
        0.4857151210308075,
        0.4946988523006439,
        0.5210273265838623,
        0.5165740251541138,
        0.5040848851203918,
        0.5002362132072449,
        0.5037636160850525,
        0.5158263444900513,
        0.5092717409133911,
        0.5269027352333069,
        0.5593379735946655,
        0.5223743915557861,
        0.45358937978744507,
        0.500726580619812,
        0.5033595561981201,
        0.5241034626960754,
        0.5018541812896729,
        0.5069624185562134,
        0.5168576836585999,
        0.4973188042640686,
        0.5030398964881897,
        0.5001656413078308,
        0.4864903390407562,
        0.49112337827682495,
        0.49388381838798523,
        0.498783677816391,
        0.4995008707046509,
        0.5028586387634277,
        0.5241028666496277,
        0.5300202965736389,
        0.5309483408927917,
        0.5041775107383728,
        0.5049067139625549,
        0.5133087038993835,
        0.537175178527832,
        0.526378870010376,
        0.5152541995048523,
        0.5120044946670532,
        0.5138468742370605,
        0.5123676657676697,
        0.5237870216369629,
        0.5312932729721069,
        0.5279895663261414,
        0.5011212825775146,
        0.517037034034729,
        0.5181301236152649,
        0.5155410170555115,
        0.5115770101547241,
        0.5269577503204346,
        0.5238876342773438,
        0.5142295360565186,
        0.512324869632721,
        0.4997820556163788,
        0.5070739388465881,
        0.5050877332687378,
        0.5172950029373169,
        0.5122784972190857,
        0.5076084733009338,
        0.5174896717071533,
        0.5268682241439819,
        0.5284265875816345,
        0.516572117805481,
        0.5181859135627747,
        0.5225483179092407,
        0.5169415473937988,
        0.5129841566085815,
        0.5192045569419861,
        0.5050333738327026,
        0.5037442445755005,
        0.5035388469696045,
        0.5250753164291382,
        0.5164393186569214,
        0.5084264278411865,
        0.5010970830917358,
        0.4988960921764374,
        0.5064193606376648,
        0.4927840828895569,
        0.5053014159202576,
        0.5014286041259766,
        0.415587455034256,
        0.42928850650787354,
        0.4535161554813385,
        0.4962415397167206,
        0.5,
        0.5024387240409851,
        0.5206809043884277,
        0.5176339745521545,
        0.5178564786911011,
        0.5037612915039062,
        0.5045397281646729,
        0.5032064318656921,
        0.5126780867576599,
        0.5088513493537903,
        0.5247228145599365,
        0.5245518088340759,
        0.5108722448348999,
        0.5110131502151489,
        0.5551489591598511,
        0.5565901398658752,
        0.5793383121490479,
        0.5110635757446289,
        0.5002498626708984,
        0.5125951170921326,
        0.5494127869606018,
        0.546820342540741,
        0.5482358932495117,
        0.5267900824546814,
        0.5260858535766602,
        0.5282099843025208,
        0.5001460909843445,
        0.4999200999736786,
        0.5026781558990479,
        0.5081444978713989,
        0.5096195340156555,
        0.5103094577789307,
        0.5005435943603516,
        0.5069341063499451,
        0.5085194706916809,
        0.5083454847335815,
        0.5203776955604553,
        0.5067756175994873,
        0.504410445690155,
        0.5064358711242676,
        0.5047038793563843,
        0.5011685490608215,
        0.5046114325523376,
        0.5026635527610779,
        0.5488613247871399,
        0.5201014280319214,
        0.5048767924308777,
        0.49831289052963257,
        0.5058740377426147,
        0.5056147575378418,
        0.520018994808197,
        0.5208775401115417,
        0.5208579897880554,
        0.508791983127594,
        0.5037039518356323,
        0.5131917595863342,
        0.5486952066421509,
        0.5352980494499207,
        0.5246399641036987,
        0.5277990102767944,
        0.5220047235488892,
        0.5215880274772644,
        0.507652997970581,
        0.5095207691192627,
        0.5065920948982239,
        0.52098548412323,
        0.511240541934967,
        0.5142531394958496,
        0.5009468793869019,
        0.5221679210662842,
        0.5273546576499939,
        0.5073772668838501,
        0.4947069585323334,
        0.5033441185951233,
        0.5005367398262024,
        0.4999784231185913,
        0.5002249479293823,
        0.5291810631752014,
        0.5157052874565125,
        0.5015344023704529,
        0.5334077477455139,
        0.5036166310310364,
        0.47573038935661316,
        0.5197317600250244,
        0.5207000374794006,
        0.5081007480621338,
        0.5175943374633789,
        0.5179082155227661,
        0.5218060612678528,
        0.5206034779548645,
        0.5086657404899597,
        0.5155102610588074,
        0.5324530005455017,
        0.5122565031051636,
        0.5234271883964539,
        0.5297174453735352,
        0.5266913771629333,
        0.5187420845031738,
        0.5051543116569519,
        0.5082353949546814,
        0.5193212032318115,
        0.49998050928115845,
        0.4883177578449249,
        0.4983581304550171,
        0.5212194323539734,
        0.5159685611724854,
        0.5206034183502197,
        0.40001535415649414,
        0.47296029329299927,
        0.40106651186943054,
        0.5108518600463867,
        0.5020033121109009,
        0.5106759071350098,
        0.51277756690979,
        0.5155114531517029,
        0.5174870491027832,
        0.5,
        0.5,
        0.5078127980232239,
        0.5168669819831848,
        0.5322387218475342,
        0.5025379657745361,
        0.5075793266296387,
        0.5093427300453186,
        0.5070285797119141,
        0.5193621516227722,
        0.5282872915267944,
        0.5153216123580933,
        0.5000935196876526,
        0.5040934085845947,
        0.5084739327430725,
        0.5216040015220642,
        0.5159774422645569,
        0.5229853987693787,
        0.4951478838920593,
        0.5046601295471191,
        0.5020456910133362,
        0.5011123418807983,
        0.5101583003997803,
        0.5107006430625916,
        0.5021799206733704,
        0.49916404485702515,
        0.49706581234931946,
        0.5338659286499023,
        0.5131047368049622,
        0.5335462093353271,
        0.5112351775169373,
        0.5159450769424438,
        0.502342939376831,
        0.4952423572540283,
        0.5060675144195557,
        0.5006026029586792,
        0.5131306648254395,
        0.5071924328804016,
        0.5083837509155273,
        0.5147340893745422,
        0.5224024057388306,
        0.5248688459396362,
        0.5073816180229187,
        0.5019468069076538,
        0.517042338848114,
        0.5160278081893921,
        0.516767144203186,
        0.5128143429756165,
        0.5055994987487793,
        0.5162386298179626,
        0.5123273134231567,
        0.5084894299507141,
        0.527879536151886,
        0.5131903886795044,
        0.4999699890613556,
        0.5021787285804749,
        0.5019017457962036,
        0.5124260187149048,
        0.49997439980506897,
        0.49353447556495667,
        0.47750070691108704,
        0.47777655720710754,
        0.47922492027282715,
        0.5154041051864624,
        0.5051892399787903,
        0.5096495151519775,
        0.5096337795257568,
        0.5159083008766174,
        0.5117741823196411
    ],
    "eval_runtime": 54.6528,
    "eval_samples_per_second": 13.723,
    "eval_steps_per_second": 0.439,
    "eval_rms_calibration_error": 0.0008263103961944629
}