GENIES/results/evaluations/zero_shot/openllama-3b-code_easy/code_hard.json

{
    "model_dir": "models/openllama-3b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-20 01:20:54",
    "eval_loss": 0.6877474188804626,
    "eval_score": -0.24786555767059326,
    "eval_brier_score": 0.24786556860993772,
    "eval_average_probability": 0.5027546538511912,
    "eval_accuracy": 0.7186666666666667,
    "eval_probabilities": [
        0.5009541511535645,
        0.4991297423839569,
        0.4983240067958832,
        0.5087032914161682,
        0.49204862117767334,
        0.49780744314193726,
        0.5127522945404053,
        0.537738025188446,
        0.5129897594451904,
        0.4946988523006439,
        0.5210273265838623,
        0.5165740251541138,
        0.5326241254806519,
        0.5320203900337219,
        0.538191020488739,
        0.5105686187744141,
        0.49722591042518616,
        0.500602662563324,
        0.5088046789169312,
        0.5222324132919312,
        0.5222846865653992,
        0.5215458869934082,
        0.5273125171661377,
        0.5239026546478271,
        0.5180680155754089,
        0.5171548128128052,
        0.5182163715362549,
        0.4816742539405823,
        0.5175365805625916,
        0.5185783505439758,
        0.5064588785171509,
        0.5059816241264343,
        0.5055780410766602,
        0.5049747228622437,
        0.49865880608558655,
        0.4988449215888977,
        0.4645281732082367,
        0.4676106572151184,
        0.46278098225593567,
        0.5043057203292847,
        0.4943769574165344,
        0.5004122853279114,
        0.5273630619049072,
        0.5257828235626221,
        0.5250824093818665,
        0.502755880355835,
        0.5008992552757263,
        0.5022150278091431,
        0.497560054063797,
        0.4999421536922455,
        0.49749240279197693,
        0.49787765741348267,
        0.5040478706359863,
        0.5004732012748718,
        0.4814645051956177,
        0.48159298300743103,
        0.4803681969642639,
        0.5009512901306152,
        0.5041300058364868,
        0.5024775266647339,
        0.49850431084632874,
        0.4990447163581848,
        0.49794888496398926,
        0.5074923038482666,
        0.5016791820526123,
        0.5057981610298157,
        0.5112265944480896,
        0.5133332014083862,
        0.5052410960197449,
        0.49512067437171936,
        0.5103120803833008,
        0.5034862160682678,
        0.5173513293266296,
        0.5241010189056396,
        0.5216107368469238,
        0.518986701965332,
        0.5227066874504089,
        0.5179375410079956,
        0.5005595088005066,
        0.5019857883453369,
        0.5024380683898926,
        0.509884238243103,
        0.4989665448665619,
        0.504818320274353,
        0.5274378061294556,
        0.5390399694442749,
        0.5003756880760193,
        0.49900031089782715,
        0.5013948082923889,
        0.5027642846107483,
        0.5306578278541565,
        0.53801029920578,
        0.539126992225647,
        0.4130994379520416,
        0.41323211789131165,
        0.413695365190506,
        0.5053542256355286,
        0.5053405165672302,
        0.510122537612915,
        0.5124260187149048,
        0.49997439980506897,
        0.49353447556495667,
        0.4579092264175415,
        0.45309770107269287,
        0.457597017288208,
        0.4992327094078064,
        0.500151515007019,
        0.49872878193855286,
        0.5009836554527283,
        0.5073531866073608,
        0.50246661901474,
        0.499522864818573,
        0.5053948760032654,
        0.5025132894515991,
        0.498482882976532,
        0.5021127462387085,
        0.4961938261985779,
        0.5284340977668762,
        0.50554358959198,
        0.5309867262840271,
        0.5147180557250977,
        0.5139132738113403,
        0.515818178653717,
        0.5150627493858337,
        0.5176246762275696,
        0.5144700407981873,
        0.5147340893745422,
        0.5224024057388306,
        0.5248688459396362,
        0.5149508118629456,
        0.5184158682823181,
        0.5285097360610962,
        0.506483793258667,
        0.5043370723724365,
        0.5037827491760254,
        0.49520227313041687,
        0.4938935339450836,
        0.4985451400279999,
        0.4107288420200348,
        0.4104059040546417,
        0.41457319259643555,
        0.5061694979667664,
        0.50319904088974,
        0.5005083680152893,
        0.48186200857162476,
        0.5035342574119568,
        0.5004662871360779,
        0.48729023337364197,
        0.49482864141464233,
        0.49021971225738525,
        0.4230315387248993,
        0.41954752802848816,
        0.4286539852619171,
        0.47959592938423157,
        0.4741314649581909,
        0.47467702627182007,
        0.5192230939865112,
        0.49911385774612427,
        0.5170983076095581,
        0.5246425867080688,
        0.5104341506958008,
        0.5108966827392578,
        0.5005435943603516,
        0.5069341063499451,
        0.5085194706916809,
        0.5048575401306152,
        0.4997410178184509,
        0.49683666229248047,
        0.508003294467926,
        0.5046340823173523,
        0.5024315714836121,
        0.4978286027908325,
        0.5022167563438416,
        0.5030627846717834,
        0.4257031977176666,
        0.4121972322463989,
        0.42864999175071716,
        0.5178281664848328,
        0.5096238255500793,
        0.49875542521476746,
        0.5147284269332886,
        0.5107414126396179,
        0.5120668411254883,
        0.49172186851501465,
        0.4980837106704712,
        0.5003341436386108,
        0.4987697899341583,
        0.5017468929290771,
        0.5005198121070862,
        0.5160881280899048,
        0.5064770579338074,
        0.5091634392738342,
        0.501404345035553,
        0.5005557537078857,
        0.4995492994785309,
        0.5156750082969666,
        0.5013359785079956,
        0.5037763118743896,
        0.37886151671409607,
        0.36857813596725464,
        0.3738362193107605,
        0.5422244668006897,
        0.5702081322669983,
        0.5897537469863892,
        0.4989883601665497,
        0.5039159655570984,
        0.5019713044166565,
        0.5147560238838196,
        0.5157040357589722,
        0.5080110430717468,
        0.5076059699058533,
        0.49975988268852234,
        0.515874981880188,
        0.4981813430786133,
        0.5011470317840576,
        0.5004943013191223,
        0.5075818300247192,
        0.5054045915603638,
        0.50248783826828,
        0.5244343280792236,
        0.5255465507507324,
        0.5256333351135254,
        0.5008946657180786,
        0.5051144361495972,
        0.5134977698326111,
        0.5037509799003601,
        0.5007672309875488,
        0.5059165358543396,
        0.499546617269516,
        0.4942046105861664,
        0.4979848265647888,
        0.5107523798942566,
        0.5023475289344788,
        0.5036648511886597,
        0.5099871158599854,
        0.5070180892944336,
        0.5080859661102295,
        0.4867812693119049,
        0.4800567030906677,
        0.48355263471603394,
        0.501395583152771,
        0.5036728978157043,
        0.5023757219314575,
        0.527726411819458,
        0.5322823524475098,
        0.5392914414405823,
        0.5001704692840576,
        0.5018320679664612,
        0.5045567154884338,
        0.5272818803787231,
        0.5267221331596375,
        0.522538423538208,
        0.49949419498443604,
        0.5005879402160645,
        0.5038205981254578,
        0.5007840394973755,
        0.507729172706604,
        0.5102759599685669,
        0.502423882484436,
        0.5064696669578552,
        0.5019562840461731,
        0.5233514904975891,
        0.5232648253440857,
        0.5319072604179382,
        0.5009849071502686,
        0.5034413933753967,
        0.49993541836738586,
        0.5297547578811646,
        0.5304645299911499,
        0.5332890748977661,
        0.5135123133659363,
        0.5201650261878967,
        0.5182808041572571,
        0.5025556683540344,
        0.5095629692077637,
        0.5061773061752319,
        0.484458327293396,
        0.5061435103416443,
        0.5007451176643372,
        0.5125772953033447,
        0.5143836736679077,
        0.5111652612686157,
        0.5188631415367126,
        0.5230873823165894,
        0.5215491056442261,
        0.528350830078125,
        0.5358057618141174,
        0.522598922252655,
        0.4979894161224365,
        0.49755680561065674,
        0.4980669915676117,
        0.4999699890613556,
        0.5021787285804749,
        0.5019017457962036,
        0.5020527839660645,
        0.4953957200050354,
        0.5045327544212341,
        0.5062636137008667,
        0.49943286180496216,
        0.5024356245994568,
        0.5258026719093323,
        0.5214945077896118,
        0.5143948793411255,
        0.5107553005218506,
        0.5204503536224365,
        0.5153393149375916,
        0.5233345627784729,
        0.5317865014076233,
        0.5349575877189636,
        0.5015318393707275,
        0.5038819313049316,
        0.5002246499061584,
        0.5242816805839539,
        0.5016770958900452,
        0.5029898881912231,
        0.5158433318138123,
        0.5075637102127075,
        0.5038635730743408,
        0.4365612864494324,
        0.4955875873565674,
        0.5003886818885803,
        0.5037612915039062,
        0.5045397281646729,
        0.5032064318656921,
        0.5046544075012207,
        0.5081546902656555,
        0.506134569644928,
        0.5346421003341675,
        0.5159380435943604,
        0.5262466669082642,
        0.5019809603691101,
        0.5080664753913879,
        0.5009700059890747,
        0.49661320447921753,
        0.5048591494560242,
        0.4994228184223175,
        0.5004201531410217,
        0.5004626512527466,
        0.5004665851593018,
        0.501192569732666,
        0.5028157234191895,
        0.5030494332313538,
        0.5004416108131409,
        0.5107994079589844,
        0.5085605382919312,
        0.4587014615535736,
        0.45772749185562134,
        0.4584256708621979,
        0.5571109056472778,
        0.5273408889770508,
        0.527249276638031,
        0.5180985331535339,
        0.5105831027030945,
        0.504063069820404,
        0.5015424489974976,
        0.5016186833381653,
        0.5038890838623047,
        0.5246437788009644,
        0.5165879130363464,
        0.5226777195930481,
        0.5034899115562439,
        0.49908989667892456,
        0.5071726441383362,
        0.502647876739502,
        0.4989399015903473,
        0.5035512447357178,
        0.5127728581428528,
        0.5077216029167175,
        0.5082169771194458,
        0.5193491578102112,
        0.4990854561328888,
        0.4990854561328888,
        0.4908117353916168,
        0.5026227831840515,
        0.4980575144290924,
        0.5041861534118652,
        0.5024716258049011,
        0.5029170513153076,
        0.4921966791152954,
        0.49953576922416687,
        0.5005469918251038,
        0.4507313072681427,
        0.46941184997558594,
        0.4645062983036041,
        0.4951478838920593,
        0.5046601295471191,
        0.5020456910133362,
        0.4997764825820923,
        0.49987688660621643,
        0.5016835927963257,
        0.5057981610298157,
        0.5063574314117432,
        0.519572913646698,
        0.5044342875480652,
        0.5007914900779724,
        0.504147469997406,
        0.5051930546760559,
        0.505500078201294,
        0.49819451570510864,
        0.5145785212516785,
        0.5204763412475586,
        0.5201772451400757,
        0.5367105007171631,
        0.5328570008277893,
        0.5328800678253174,
        0.529439389705658,
        0.5339477062225342,
        0.5322490334510803,
        0.518363356590271,
        0.5167111158370972,
        0.5258252620697021,
        0.5061048269271851,
        0.5106028318405151,
        0.5106737017631531,
        0.5381652116775513,
        0.5413077473640442,
        0.5393857359886169,
        0.4660997688770294,
        0.4182272255420685,
        0.45618513226509094,
        0.5268338322639465,
        0.5262309312820435,
        0.5247222185134888,
        0.4882727265357971,
        0.5023645758628845,
        0.4986720681190491,
        0.42487382888793945,
        0.42970842123031616,
        0.4242590069770813,
        0.5217719078063965,
        0.5224560499191284,
        0.5361471772193909,
        0.5203791260719299,
        0.5143694281578064,
        0.5148425102233887,
        0.505043625831604,
        0.5138986110687256,
        0.5416344404220581,
        0.459038108587265,
        0.46910107135772705,
        0.4678616225719452,
        0.5347440838813782,
        0.5320515036582947,
        0.5359301567077637,
        0.5001460909843445,
        0.4999200999736786,
        0.5026781558990479,
        0.4968556761741638,
        0.5028875470161438,
        0.5076910853385925,
        0.5158545970916748,
        0.5019733905792236,
        0.49886375665664673,
        0.5047823786735535,
        0.5056302547454834,
        0.5027822256088257,
        0.5092668533325195,
        0.5013504028320312,
        0.5037813186645508,
        0.5069150328636169,
        0.5066774487495422,
        0.5105799436569214,
        0.5007753372192383,
        0.5042945146560669,
        0.5063353776931763,
        0.5139327049255371,
        0.5321759581565857,
        0.5229036808013916,
        0.49655574560165405,
        0.4956997036933899,
        0.49840429425239563,
        0.5031301975250244,
        0.50094074010849,
        0.5014841556549072,
        0.46834608912467957,
        0.4612736999988556,
        0.4592016041278839,
        0.49960604310035706,
        0.5014097094535828,
        0.5008105039596558,
        0.49892541766166687,
        0.497993528842926,
        0.5023998022079468,
        0.49590054154396057,
        0.4897083044052124,
        0.49919137358665466,
        0.5505585670471191,
        0.5312770009040833,
        0.5408290028572083,
        0.503551185131073,
        0.5023664832115173,
        0.50800621509552,
        0.49993959069252014,
        0.49993959069252014,
        0.4999386966228485,
        0.5162038207054138,
        0.5220046043395996,
        0.5230036377906799,
        0.49507734179496765,
        0.494942307472229,
        0.4916694760322571,
        0.49950525164604187,
        0.5078713893890381,
        0.5082727074623108,
        0.4962415397167206,
        0.5,
        0.5024387240409851,
        0.39882469177246094,
        0.4009437561035156,
        0.40178948640823364,
        0.4820476770401001,
        0.47829508781433105,
        0.47408002614974976,
        0.4909660518169403,
        0.4974615275859833,
        0.4956236183643341,
        0.5238876342773438,
        0.5142295360565186,
        0.512324869632721,
        0.5047900676727295,
        0.4915965795516968,
        0.49543333053588867,
        0.4848581850528717,
        0.4848581850528717,
        0.4873354732990265,
        0.5037482976913452,
        0.5028479695320129,
        0.49165451526641846,
        0.5119800567626953,
        0.5181000232696533,
        0.5111053586006165,
        0.4936477839946747,
        0.4972204864025116,
        0.5018909573554993,
        0.5,
        0.5028125643730164,
        0.504949152469635,
        0.5071046352386475,
        0.5094790458679199,
        0.509076714515686,
        0.5264836549758911,
        0.5239436030387878,
        0.5120601058006287,
        0.5225600004196167,
        0.5257136225700378,
        0.5311734676361084,
        0.49918535351753235,
        0.5001807808876038,
        0.5031403303146362,
        0.4855404496192932,
        0.507258951663971,
        0.49724483489990234,
        0.5,
        0.5136678814888,
        0.5091027617454529,
        0.4983045756816864,
        0.497130811214447,
        0.4936565160751343,
        0.5098475217819214,
        0.5071225166320801,
        0.5050577521324158,
        0.5229678153991699,
        0.5265181660652161,
        0.5259150266647339,
        0.5057901740074158,
        0.5061373114585876,
        0.507921040058136,
        0.4937571883201599,
        0.4902033805847168,
        0.55033940076828,
        0.5232491493225098,
        0.531272828578949,
        0.522799551486969,
        0.501308798789978,
        0.5111252665519714,
        0.516834020614624,
        0.5025580525398254,
        0.5028321743011475,
        0.5016791224479675,
        0.5116210579872131,
        0.5112370848655701,
        0.5029230117797852,
        0.5229653716087341,
        0.5095295906066895,
        0.51495361328125,
        0.5020527839660645,
        0.5015379190444946,
        0.5015674829483032,
        0.482572078704834,
        0.48323363065719604,
        0.4817308187484741,
        0.5031677484512329,
        0.5068406462669373,
        0.5013048052787781,
        0.5000906586647034,
        0.5064936280250549,
        0.5075532793998718,
        0.5281783938407898,
        0.5255563855171204,
        0.5231887102127075,
        0.5028682947158813,
        0.49438056349754333,
        0.503026008605957,
        0.5054842233657837,
        0.5042405724525452,
        0.5063852667808533,
        0.5072040557861328,
        0.5006740093231201,
        0.503508448600769,
        0.5047541260719299,
        0.5162956714630127,
        0.5036306381225586,
        0.49859312176704407,
        0.49943405389785767,
        0.5004180073738098,
        0.502506673336029,
        0.502631425857544,
        0.5022781491279602,
        0.5017004013061523,
        0.5046606659889221,
        0.5034868717193604,
        0.500732421875,
        0.4993216097354889,
        0.5012392401695251,
        0.526041567325592,
        0.5280998349189758,
        0.5208909511566162,
        0.4735313951969147,
        0.5022531747817993,
        0.5046836733818054,
        0.5034245848655701,
        0.5038003325462341,
        0.5081883072853088,
        0.5022950768470764,
        0.5011173486709595,
        0.5049695372581482,
        0.498090535402298,
        0.5038131475448608,
        0.5084542632102966,
        0.49966883659362793,
        0.4991714656352997,
        0.4992195665836334,
        0.497189998626709,
        0.5304344296455383,
        0.5147782564163208,
        0.4982161819934845,
        0.5034466981887817,
        0.5005354881286621,
        0.49964529275894165,
        0.4953494369983673,
        0.4873664975166321,
        0.5059751868247986,
        0.5076417922973633,
        0.5062074661254883,
        0.4335992932319641,
        0.43009260296821594,
        0.43447551131248474,
        0.5442053079605103,
        0.5542563796043396,
        0.5477290749549866,
        0.510228157043457,
        0.5035006999969482,
        0.5023659467697144,
        0.5197411775588989,
        0.5173681378364563,
        0.5146010518074036,
        0.5027673840522766,
        0.49884191155433655,
        0.502178966999054,
        0.5041917562484741,
        0.502571165561676,
        0.503524661064148,
        0.5050333738327026,
        0.5037442445755005,
        0.5035388469696045,
        0.499571830034256,
        0.5013175010681152,
        0.5023362040519714,
        0.5000914335250854,
        0.5011729598045349,
        0.5011788010597229,
        0.5191941857337952,
        0.5197611451148987,
        0.5191386342048645,
        0.4999289810657501,
        0.5038092732429504,
        0.5040368437767029,
        0.5236762166023254,
        0.5101822018623352,
        0.5058136582374573,
        0.5094401240348816,
        0.5053980946540833,
        0.5058819651603699,
        0.4947691559791565,
        0.4953126609325409,
        0.4969939589500427,
        0.5340363383293152,
        0.5448473691940308,
        0.5440835356712341,
        0.5091491341590881,
        0.49445435404777527,
        0.4987610876560211,
        0.5212538838386536,
        0.5199126601219177,
        0.5200256705284119,
        0.5023741722106934,
        0.5029817819595337,
        0.503196120262146,
        0.5189456939697266,
        0.5004532933235168,
        0.519112229347229,
        0.5039882063865662,
        0.5127840638160706,
        0.5045323967933655,
        0.5000935196876526,
        0.5040934085845947,
        0.5084739327430725,
        0.5046032667160034,
        0.5038072466850281,
        0.5069986581802368,
        0.5188038349151611,
        0.5276033282279968,
        0.5202265381813049,
        0.505096435546875,
        0.5096435546875,
        0.49902263283729553,
        0.3428906798362732,
        0.3589918613433838,
        0.3601532578468323,
        0.5052120685577393,
        0.49895596504211426,
        0.4957111179828644,
        0.5006155371665955,
        0.5015630722045898,
        0.5041406154632568,
        0.5005367398262024,
        0.4999784231185913,
        0.5002249479293823,
        0.4723547697067261,
        0.47603368759155273,
        0.4988456964492798,
        0.49953755736351013,
        0.5,
        0.5002236366271973,
        0.5037704706192017,
        0.5022208094596863,
        0.5012670755386353,
        0.47026702761650085,
        0.47416388988494873,
        0.4698916971683502
    ],
    "eval_runtime": 69.9282,
    "eval_samples_per_second": 10.725,
    "eval_steps_per_second": 0.343,
    "eval_rms_calibration_error": 0.0005820668041037758
}