GENIES/results/evaluations/cra/llama-30b-code_easy/code_hard.json

{
    "model_dir": "models/llama-30b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-16 06:35:51",
    "eval_score": -0.15580325066797343,
    "eval_brier_score": 0.15580325066797343,
    "eval_average_probability": 0.6732792099521543,
    "eval_accuracy": 0.7586666666666667,
    "eval_probabilities": [
        0.9783086449643994,
        0.8065401123243088,
        0.6005784814786941,
        0.6696448637387,
        0.35877608830014746,
        0.31315699042160855,
        0.8716183187312617,
        0.973705056617621,
        0.7807697016516715,
        0.5431231384394613,
        0.7170036647455255,
        0.28237562666530125,
        0.7124886429382478,
        0.8005518846709531,
        0.7734583967854423,
        0.8749092146953341,
        0.8010167008492142,
        0.8733646169158225,
        0.3248417053983569,
        0.6505565242848844,
        0.29873259754385056,
        0.9142396580805898,
        0.9209631256468935,
        0.9451741612658289,
        0.6948347696558208,
        0.8705035844244414,
        0.8695776927321265,
        0.8783103125298111,
        0.8547014659994902,
        0.889521499132625,
        0.5244751370932018,
        0.8379158361934427,
        0.35092646447414744,
        0.806963151135724,
        0.6503092366624645,
        0.8529384322255992,
        0.8696657564125843,
        0.8881574496587009,
        0.7582802317483981,
        0.7640842327072507,
        0.5560230079462054,
        0.6737695900437948,
        0.9731946583017536,
        0.9726785615675908,
        0.9706435578523462,
        0.7114810037167868,
        0.32650281023368133,
        0.44337589792286236,
        0.7631464043026154,
        0.8672660686561943,
        0.8724467797221751,
        0.6929316134589285,
        0.07129503591825148,
        0.7162362553478177,
        0.7709987715399537,
        0.6072911520068893,
        0.5839202235824303,
        0.6282904174860073,
        0.7161445358515709,
        0.5201196086348915,
        0.49461006000198404,
        0.32540905473992404,
        0.43349089216970843,
        0.8273311409950153,
        0.4992458614898929,
        0.4645601559627621,
        0.573197625017167,
        0.564006407754619,
        0.7519541642446198,
        0.8739938663047051,
        0.9011406185781334,
        0.826050608311055,
        0.8875947018512024,
        0.933331616957226,
        0.9739643906873813,
        0.6301267047162475,
        0.6020477464619527,
        0.9000416434287274,
        0.31062085858558564,
        0.3227406106367685,
        0.46872903327015236,
        0.7111452778444176,
        0.321531176195339,
        0.5023974687034527,
        0.8746923547195593,
        0.8783199902491987,
        0.4819356283595426,
        0.4028834338351952,
        0.4180357577495166,
        0.44983195648266977,
        0.8986234509766626,
        0.8507150883380038,
        0.8955924074858538,
        0.7487229943687176,
        0.7379688100121442,
        0.7739998549842549,
        0.8756292115309563,
        0.4991583305119178,
        0.7502434832172977,
        0.9412504569986573,
        0.9050467354261953,
        0.8672664740817995,
        0.07962256387415512,
        0.1365009816175457,
        0.1377266287632819,
        0.3382154546821293,
        0.8914485684788261,
        0.37848426040902766,
        0.7659792311752718,
        0.46194040646723455,
        0.5559568165623973,
        0.553447682169967,
        0.7713320268474673,
        0.589430297529574,
        0.9086177556124435,
        0.7761888522954109,
        0.6767614140131955,
        0.9137234574645308,
        0.8147167434608221,
        0.9066180400364403,
        0.691551561484359,
        0.7284511897959464,
        0.877142212632704,
        0.6991247522598355,
        0.9104795738833606,
        0.9859680089485946,
        0.7056155373241917,
        0.4321761783422736,
        0.6412632589069843,
        0.9020036990563516,
        0.8342198235527618,
        0.9301092862528806,
        0.7082320618985001,
        0.5870963762704027,
        0.7022759077755348,
        0.9495637100211817,
        0.8617986360430026,
        0.895488829805811,
        0.534407860326705,
        0.3398816229616317,
        0.37065190550351057,
        0.8871877739674914,
        0.814555245933349,
        0.46712376026615765,
        0.5588337106821394,
        0.5326093877398018,
        0.3856360618348187,
        0.6826503753382294,
        0.8449921891942513,
        0.8036164351146,
        0.9403950341416263,
        0.9396576694466758,
        0.927771033999074,
        0.20758000189920656,
        0.12924104858482408,
        0.11372720547427036,
        0.9290304296652022,
        0.8325467473982648,
        0.8739379011630372,
        0.6486485403080432,
        0.532620407680488,
        0.3178534385918756,
        0.802578963666013,
        0.7880969821336938,
        0.7405134895775555,
        0.20655217765188005,
        0.779977698929195,
        0.6348703398827676,
        0.7128582432873866,
        0.5784424569082803,
        0.6138537063368625,
        0.7016057898184536,
        0.386370254800414,
        0.6940625935598047,
        0.9507506554799104,
        0.6721523909641843,
        0.4027234575674067,
        0.8157627707209951,
        0.7728078433842055,
        0.48869021958195746,
        0.944959014963899,
        0.9361463337869196,
        0.9520953384212647,
        0.7987697095954613,
        0.11488343512682642,
        0.6516830455737233,
        0.7131884594114366,
        0.8302593044201758,
        0.7660841781622023,
        0.8519867424038698,
        0.8150557166113268,
        0.3589225002018839,
        0.5806813553026687,
        0.4778290984100359,
        0.37208678343983603,
        0.36758036053160636,
        0.7898862124352046,
        0.8056388358413415,
        0.9122854659714016,
        0.9418492482028834,
        0.864891271099801,
        0.8609651612185952,
        0.8642163071832614,
        0.8938150523075765,
        0.723934922397754,
        0.5783610033387212,
        0.39908838343887043,
        0.2682069945842297,
        0.6371845277500783,
        0.2287265388612999,
        0.4978769408946577,
        0.5585792020751623,
        0.4800771463449103,
        0.5922490140326433,
        0.3000246380340743,
        0.24519643168324134,
        0.7600093614826094,
        0.8332117327579415,
        0.7614081997712911,
        0.6511354229378166,
        0.612847054712498,
        0.7563355829319197,
        0.35633255076229253,
        0.4398774805936126,
        0.584824859618789,
        0.8863255670492068,
        0.3630806687568716,
        0.8943594435983486,
        0.8124705618402519,
        0.6195511715741608,
        0.8059030531949144,
        0.8708510318873918,
        0.8269297033023619,
        0.915459319139361,
        0.4492649375305864,
        0.4413308729032965,
        0.43357168777906124,
        0.7288495396331404,
        0.6797770603909636,
        0.6298161860550154,
        0.9202127147593832,
        0.48642248302811125,
        0.34027044124055883,
        0.7276790102339967,
        0.7433495834536874,
        0.8516545290365465,
        0.4963782713192715,
        0.7649719210222456,
        0.6224572276418341,
        0.7570342965828185,
        0.8911968345469341,
        0.8656960497771269,
        0.6583922336556747,
        0.6011078826505843,
        0.7851561478295063,
        0.3342843154688265,
        0.19067962758071522,
        0.3608172173716811,
        0.4699041238347111,
        0.5178753424744289,
        0.329930163673425,
        0.9600713390406441,
        0.8892529217660861,
        0.9320683826078122,
        0.7023892799569843,
        0.4782561198858476,
        0.624740737161278,
        0.8988574825911017,
        0.7996963700168465,
        0.8697409468848869,
        0.5422831756103469,
        0.13172804963060325,
        0.2599791195396408,
        0.6207051605464372,
        0.7821706098484011,
        0.6114814428331062,
        0.8500471975174337,
        0.9201258009462476,
        0.8113601226363424,
        0.4574252818639868,
        0.6002474257937188,
        0.25399212898125423,
        0.898626567223721,
        0.7254749883570288,
        0.6888051259072812,
        0.6000284420154485,
        0.9281577136734783,
        0.889475218065638,
        0.6206742489413929,
        0.6142822942157489,
        0.8741512303673714,
        0.6174316639680756,
        0.4867202013168831,
        0.7868528249548398,
        0.27623803423256477,
        0.356204373711371,
        0.4816000358623813,
        0.4303166414303171,
        0.6726286662317487,
        0.711716802683988,
        0.9050884882365999,
        0.842269829557459,
        0.6357004534130846,
        0.3890952350323045,
        0.5743579964257552,
        0.38075432077411453,
        0.8270972444360691,
        0.8138653442930646,
        0.7724496673043851,
        0.5764329751481025,
        0.7992318257050393,
        0.3176539771776965,
        0.9421433525424396,
        0.7259453229792191,
        0.5468380391531886,
        0.6447774385385456,
        0.2904020444014386,
        0.17265851093816015,
        0.9159535426180536,
        0.6622926176633385,
        0.9195114057350904,
        0.7892796868196338,
        0.6649566931155895,
        0.6400375102416684,
        0.4968576587428566,
        0.40619839703967453,
        0.4865294292288823,
        0.8606219884250366,
        0.9149463319435249,
        0.9736078448571417,
        0.08054348988338667,
        0.4436450723415543,
        0.12218612741095125,
        0.8066259691037094,
        0.416922363847844,
        0.37418148025143483,
        0.3379738654687886,
        0.20689700839357084,
        0.5193718567978667,
        0.2964646229420982,
        0.7708512988868221,
        0.5839136235353032,
        0.4394257517603338,
        0.8410095328984075,
        0.7583785089066173,
        0.5170204405057881,
        0.39755459008656746,
        0.4214390506215069,
        0.8216828759645438,
        0.782594918703909,
        0.9262381576208758,
        0.9326215862890674,
        0.8813063669733125,
        0.9136473647436416,
        0.6215184612945247,
        0.7495768417017099,
        0.8212431561828949,
        0.8284560504075698,
        0.8088533703483448,
        0.8525067455583075,
        0.5324141518565162,
        0.7919955147783135,
        0.5265895963975671,
        0.92485056905606,
        0.7529403963109625,
        0.6805820402018938,
        0.745691092825592,
        0.8605608505908088,
        0.8808554458003983,
        0.8328591625417425,
        0.784143081784175,
        0.9095170508551752,
        0.4776640475022188,
        0.29061543006878,
        0.7028284408534892,
        0.4332502560087085,
        0.9713964454093694,
        0.9420817066960077,
        0.8241433677219656,
        0.7644289148620846,
        0.7625135946152239,
        0.6543110227121038,
        0.7047494712529042,
        0.49541951732976275,
        0.6571302554845121,
        0.9236058838982814,
        0.6561083100075604,
        0.70916040512806,
        0.8528948877756879,
        0.5653489709847398,
        0.3709408592773479,
        0.42443722604239525,
        0.8567859480799088,
        0.42361087388936014,
        0.5695007115503027,
        0.515202538387782,
        0.6647132229245213,
        0.6990054831101877,
        0.7217024872109186,
        0.8824495224874784,
        0.8528104190031944,
        0.8445629582891588,
        0.8236181791010533,
        0.9369412042184068,
        0.952337230186059,
        0.8645643933769822,
        0.9532265375693949,
        0.9277641229955977,
        0.8070396211993601,
        0.6479105434325695,
        0.7965641218971454,
        0.8908726193679872,
        0.6840848908606703,
        0.8609815395746846,
        0.7511694874379878,
        0.7589741337374292,
        0.8134062962455959,
        0.6439200916336455,
        0.8361447927688748,
        0.5214658348261407,
        0.9303974890203743,
        0.9216885632561215,
        0.918820762469104,
        0.22278938638990406,
        0.8650147285592369,
        0.18107906917321903,
        0.9650364358160594,
        0.8940835548014272,
        0.9517420523879898,
        0.4719807285866428,
        0.5064748717492609,
        0.2590674053748102,
        0.9399025069164956,
        0.9502643173650688,
        0.794273105665277,
        0.6077272858745455,
        0.7407058032306278,
        0.8695960625652212,
        0.5468602284727577,
        0.2369265283410653,
        0.26758246229007243,
        0.872843719441738,
        0.8764170439786343,
        0.8778334572433432,
        0.45501965859394755,
        0.6296050470786432,
        0.7007948514418245,
        0.4042989457538393,
        0.5879965418007606,
        0.12899922326886581,
        0.9565902421998711,
        0.8986605196912436,
        0.8234275077957562,
        0.28529145410189327,
        0.7810518287803397,
        0.866100331033682,
        0.8938371082940663,
        0.7200396112713043,
        0.6983340274222969,
        0.9159253455446482,
        0.9652124307014053,
        0.9537241682097853,
        0.5531456742130472,
        0.29367429068471496,
        0.26459758508860953,
        0.5505715055935366,
        0.8717404218640887,
        0.6190321581070273,
        0.8498820710684013,
        0.7566381493366173,
        0.6688987125897996,
        0.8646175747027607,
        0.5599636159836514,
        0.7392264799333139,
        0.47589385957551,
        0.28193882180904906,
        0.211949899179129,
        0.7809970247325516,
        0.7049634847253319,
        0.8609014388433069,
        0.3517422781058914,
        0.7679493445590684,
        0.3895116785404128,
        0.3947930279040064,
        0.3666740599320727,
        0.522193996343402,
        0.8748038001820341,
        0.8660503847661978,
        0.7834727245230121,
        0.6575122323601092,
        0.18072656561488365,
        0.8113606615170242,
        0.584561371592812,
        0.6123615175257024,
        0.5903162728199582,
        0.9151657696352485,
        0.7687445340522971,
        0.9150152841994282,
        0.4906838473313315,
        0.6370009927342569,
        0.6081438273895481,
        0.660659924331648,
        0.4313087866721592,
        0.8890856574820803,
        0.9491440621547524,
        0.7886798436912048,
        0.47724409940290835,
        0.7240038853524522,
        0.7229244939931445,
        0.7078012080533582,
        0.7709928207168842,
        0.7506148763192857,
        0.7715104984129362,
        0.717274797222008,
        0.8954355702847406,
        0.8332898066725706,
        0.787556379371607,
        0.9233647412935395,
        0.8960048745165837,
        0.9155688901039563,
        0.37224123390808084,
        0.5752733398291624,
        0.804953879486349,
        0.6920140691648042,
        0.6954206955820289,
        0.7080284849846199,
        0.7139250669079992,
        0.8094083911758508,
        0.7140678562850009,
        0.68186301243427,
        0.4985877457434762,
        0.7030798071354485,
        0.6755861158746503,
        0.8507491369330444,
        0.8248167497172767,
        0.7762519416935909,
        0.35559469109143027,
        0.8882595478895791,
        0.8437037764047122,
        0.7015974697333822,
        0.9214097962807807,
        0.8859547955237078,
        0.9585988935189351,
        0.9220284715533427,
        0.9183268807714793,
        0.643449582011442,
        0.5881996806914902,
        0.8352628314170072,
        0.7339943707826814,
        0.8880373150833842,
        0.4548023606379804,
        0.5355328155632284,
        0.6572153708181315,
        0.9689875916491988,
        0.9820256937326199,
        0.35063433139132616,
        0.5324341893520311,
        0.7367449955966422,
        0.41234716848259634,
        0.4399330087591692,
        0.6692848667231759,
        0.8713826106953041,
        0.7209312216159386,
        0.8180461275829534,
        0.27686868558232913,
        0.2299444247841539,
        0.24662070228310784,
        0.3661658032822961,
        0.34531904330538843,
        0.8128901449189426,
        0.9848466397530939,
        0.9539509411106927,
        0.7379346648885894,
        0.23242594417543427,
        0.6706734423730644,
        0.5571305762090498,
        0.4002741451809453,
        0.6779560115943783,
        0.5423765722668793,
        0.9315496976914616,
        0.9057046490164453,
        0.7000019840986667,
        0.9134052341972934,
        0.9438324657057269,
        0.9008661462465466,
        0.3591381729902102,
        0.467906337135206,
        0.3850362649372241,
        0.05382859468395429,
        0.1275406249863854,
        0.0998416967065159,
        0.8790099628700242,
        0.9206966182716463,
        0.612748580609181,
        0.3462173554619723,
        0.5740395657926842,
        0.43332214713195716,
        0.7859346259730166,
        0.8167658273295322,
        0.8525412152427121,
        0.5583422781216283,
        0.5044603887459713,
        0.7968644378346168,
        0.6985415917387132,
        0.9268136189669743,
        0.6548973473834874,
        0.833883105823195,
        0.8469029217858673,
        0.9530336075737937,
        0.3754269823324381,
        0.8360231787910244,
        0.16141205524910918,
        0.20389489221235302,
        0.17839541479557208,
        0.22541023759517653,
        0.6234433795714249,
        0.7900219344428552,
        0.831551043509271,
        0.9095737516605515,
        0.4715557506481918,
        0.8415045782498464,
        0.587675504406905,
        0.3462068797005118,
        0.8193639895568424,
        0.8145294084823912,
        0.8041005027622685,
        0.7889287434061645,
        0.9450516946013523,
        0.6263406970428832,
        0.3331391663391804,
        0.7679708588087762,
        0.48132147654228474,
        0.8889305399974071,
        0.6850610001943674,
        0.7448026911933475,
        0.8357919537011587,
        0.3533121322825017,
        0.7173114190366154,
        0.968456915414706,
        0.44708586291317465,
        0.4803119544303082,
        0.3550623082283282,
        0.6355131013468657,
        0.8951553639663963,
        0.9527254975786172,
        0.6521156970375686,
        0.6388781790276488,
        0.27196894143041905,
        0.6385889052786293,
        0.8380180185153481,
        0.6445433940754159,
        0.3824846859919533,
        0.5990698277768381,
        0.2635510349632605,
        0.8250869853411901,
        0.6447068061529805,
        0.5833772188856685,
        0.9188655528218861,
        0.9422034301611807,
        0.858038051468897,
        0.9295578349126042,
        0.8106464263297686,
        0.649722033868127,
        0.9122562366586496,
        0.8927389870809368,
        0.7965133305654143,
        0.7585085504887502,
        0.48635323898635874,
        0.46367271214579475,
        0.78020129873537,
        0.4736241030722944,
        0.5778953850439443,
        0.6392435739116646,
        0.5674629438929956,
        0.6845868356812712,
        0.5891807075789867,
        0.7943297359601065,
        0.3845903967699644,
        0.7351803478081572,
        0.7631356745360113,
        0.6284168212296563,
        0.860079656903918,
        0.8442732895300393,
        0.8970975655609605,
        0.5790515055322925,
        0.4024004218968191,
        0.45599633753737456,
        0.46036248223133436,
        0.4346954074328725,
        0.7402961314926091,
        0.922874202587653,
        0.8042747772411147,
        0.808036239709925,
        0.9207572017865688,
        0.9283218392264093,
        0.9415711795216841,
        0.8653344163975011,
        0.9308167508777776,
        0.6327899173684415,
        0.8344553950405738,
        0.7809785249410705,
        0.7775571890171735,
        0.9840961619620906,
        0.9882821053254641,
        0.9652204362444531,
        0.799904346656917,
        0.685599192774115,
        0.292215126718251,
        0.7726680845560863,
        0.6507870396065405,
        0.8848318474224122,
        0.9086405173479568,
        0.8811341848264639,
        0.942696514514134,
        0.7854636828477534,
        0.7850642324675439,
        0.8486898878419159,
        0.876780983513546,
        0.8636123489494941,
        0.5614474297499941,
        0.5763702106544171,
        0.6498153347101969,
        0.9454631370277372,
        0.7011247965462175,
        0.601045278774432,
        0.28132847292096014,
        0.702953666712943,
        0.9202595355133817,
        0.9075665407153515,
        0.9572278484740029,
        0.8344380910013401,
        0.8674456581315382,
        0.8440690744507622,
        0.679176568212672,
        0.8434682039822802,
        0.6682507506254903,
        0.43249779189386106,
        0.3433288167496896,
        0.06886999945483396,
        0.050955732969861334,
        0.5423845634187424,
        0.23167784920730997,
        0.6382688689690394,
        0.4140483226440149,
        0.7009081255791455,
        0.7242964064505599,
        0.8619574526187472,
        0.764861652190895,
        0.911603989836965,
        0.6380614070881299
    ],
    "eval_rms_calibration_error": 0.2728741808305065
}