GENIES/results/evaluations/mms/llama-13b-code_easy/code_hard.json

{
    "model_dir": "models/llama-13b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-14 00:14:19",
    "eval_score": -0.2687591076449651,
    "eval_brier_score": 0.2687591076449651,
    "eval_average_probability": 0.5543734601758279,
    "eval_accuracy": 0.5826666666666667,
    "eval_probabilities": [
        0.9105675880809965,
        0.403453270909397,
        0.5974196593538798,
        0.6564895596076106,
        0.5709600720033197,
        0.5588274762787034,
        0.14831777281289482,
        0.21167766899582574,
        0.3069952464697365,
        0.3967473811248237,
        0.21244149283901526,
        0.6384548560962149,
        0.7639592208645936,
        0.21461446785917976,
        0.1985245767695965,
        0.2691726482968043,
        0.5163987933262775,
        0.3707520268131992,
        0.5935186863761236,
        0.4787389438483499,
        0.4243414569570706,
        0.9126292301643979,
        0.6843630964320819,
        0.8734087419911101,
        0.4159638101501282,
        0.42923321090664696,
        0.4508787881986967,
        0.5785111443397315,
        0.5574879437654169,
        0.4654405386119197,
        0.8011496072134996,
        0.9067250295155755,
        0.8239108793262745,
        0.8883293354579028,
        0.4123350331703499,
        0.38817836192159094,
        0.6257686157598864,
        0.6567964092711048,
        0.7618571372225345,
        0.6626946678872353,
        0.7221473487602118,
        0.7735571589064829,
        0.2327010745546847,
        0.20245878740481293,
        0.11604568191155806,
        0.9897941514190322,
        0.6048178275472808,
        0.9689456972322051,
        0.9907947784576525,
        0.9417941596250983,
        0.9537818682276517,
        0.11542702822018924,
        0.3018510311024099,
        0.5154221850369789,
        0.0575888008996727,
        0.06159659708999543,
        0.0686961945030552,
        0.8756031499210044,
        0.9577924385867751,
        0.9552061880006746,
        0.356206531438134,
        0.28708173631738243,
        0.3788979347490419,
        0.9768082786954057,
        0.860721643121633,
        0.9876483916457612,
        0.27547493275128965,
        0.7204246793165201,
        0.660393178014726,
        0.752308432840992,
        0.7559348868167949,
        0.5400562764567747,
        0.6375347217156877,
        0.4326287979587672,
        0.749654636394934,
        0.7900645512058281,
        0.7623691030975457,
        0.56320493589556,
        0.3464996139968072,
        0.7555286616578663,
        0.7087395770667695,
        0.9040471843387868,
        0.8851706766974923,
        0.866426363081412,
        0.2396105176424088,
        0.33889595573681686,
        0.3616180660217134,
        0.7659269529955774,
        0.7307283938387332,
        0.3957683944955625,
        0.6393027269943071,
        0.6610401371157822,
        0.5856402059329786,
        0.7083518053285622,
        0.7244555847171513,
        0.6792259256672347,
        0.6505785355884378,
        0.25368565122541054,
        0.28011336978951384,
        0.7613283908498705,
        0.9404134379723846,
        0.45168894749426763,
        0.9604262917961496,
        0.944186393576939,
        0.9760049938436788,
        0.36201801068479245,
        0.900083649888127,
        0.2589693208888424,
        0.4587562699176421,
        0.9695125869063443,
        0.9353635623641146,
        0.12765518024431977,
        0.12711686543499007,
        0.137449524998134,
        0.6420807904369906,
        0.5997351216356874,
        0.558230615881555,
        0.287788737292465,
        0.4399288724468484,
        0.29987873338243687,
        0.5934193481676348,
        0.7265807251908925,
        0.46193022971240166,
        0.48101975253991663,
        0.4780111344736029,
        0.9259759879889082,
        0.07741943051402496,
        0.09048720573726643,
        0.10684011915612035,
        0.4638552111371988,
        0.666780488398333,
        0.6507465417153292,
        0.8461803506822424,
        0.7107184913256774,
        0.9701305635788899,
        0.7047336880553039,
        0.6682340971393206,
        0.7007721831005134,
        0.20290609441971452,
        0.20334419993697572,
        0.2122401566698161,
        0.5392638800327374,
        0.6964495014761543,
        0.7198989991485221,
        0.96018675089204,
        0.5729614904476366,
        0.8544296307260342,
        0.07169097417211667,
        0.1290832511139327,
        0.13063824443699024,
        0.689501572540493,
        0.7196487725200237,
        0.9117573466495258,
        0.7835425615587415,
        0.7755352709157788,
        0.7710354842502595,
        0.2172642673344419,
        0.9091567725914947,
        0.6465791949062492,
        0.24860853377786754,
        0.06466110386574309,
        0.06167721952731997,
        0.6083788353582349,
        0.96820126864082,
        0.7088452949656614,
        0.6689478157329249,
        0.802652147913973,
        0.4210125553757043,
        0.10602072430157718,
        0.7723838520269649,
        0.7885873161190328,
        0.8499223454541366,
        0.334510884216181,
        0.6227232332115075,
        0.1652944104971809,
        0.1481247488679428,
        0.1557793433517507,
        0.6166252698099626,
        0.5130889688771357,
        0.8257109171194184,
        0.6304352934031588,
        0.6701305939700016,
        0.536262248365486,
        0.8012018827490606,
        0.23348864603381436,
        0.6090180271599279,
        0.5921404050280447,
        0.9382372610429383,
        0.8348181186067818,
        0.945945304121811,
        0.826786568175175,
        0.5060282326117199,
        0.9143380677998294,
        0.4421076861205471,
        0.12924038453920583,
        0.6451262143418651,
        0.9041699610159685,
        0.9678902831469115,
        0.22362255964645328,
        0.26429916381320084,
        0.4202797007042366,
        0.2504979251556264,
        0.1499867218224069,
        0.19086137737353123,
        0.37179458829978035,
        0.8118726483299201,
        0.8192723764827915,
        0.39068642543215204,
        0.42450572341797677,
        0.46068258745987983,
        0.24099416322509534,
        0.14297558512528882,
        0.19268357262160607,
        0.2057186307200456,
        0.179443787460824,
        0.06716248195522512,
        0.2388013441438337,
        0.39428637599648986,
        0.4866413260557808,
        0.7210699047993987,
        0.1399291132193251,
        0.33853426961370525,
        0.4839033677342467,
        0.8327943894074064,
        0.0793985922279351,
        0.9839492578046248,
        0.4988075244684439,
        0.9223747852353215,
        0.7284423919856745,
        0.4327055888924167,
        0.4268171497572546,
        0.9072219935215514,
        0.944249313352429,
        0.9140497280707235,
        0.4114680210358847,
        0.5261179570862597,
        0.43470361824314213,
        0.8037891454005293,
        0.800621641300162,
        0.25006846333307037,
        0.8143084663373846,
        0.7585477650308257,
        0.791281550604279,
        0.4079214440410906,
        0.43248990870190696,
        0.6407069702155308,
        0.02448341378591515,
        0.6144623955409412,
        0.955564151428709,
        0.32503220884816286,
        0.3191967765425511,
        0.22267445894002344,
        0.7865018440338053,
        0.7683099256620903,
        0.8311146476115556,
        0.4646760133942889,
        0.38322864746277885,
        0.3057726857981501,
        0.2354572524775915,
        0.2122058023900798,
        0.2144857751919509,
        0.6508610244547822,
        0.4169972364780982,
        0.4048111072694127,
        0.7476919722503967,
        0.9373685730434635,
        0.5958991973705751,
        0.4257692869814119,
        0.4076175904187336,
        0.5992651284453171,
        0.7222333424711986,
        0.798568332238347,
        0.8469968210622942,
        0.6174500170861649,
        0.7054699426674345,
        0.8162848215923141,
        0.917635052060281,
        0.9218308704835434,
        0.9159576136839865,
        0.3309540457019979,
        0.3657719164001678,
        0.3054275796620064,
        0.5235711992945106,
        0.2153320958390238,
        0.160751349756325,
        0.12826503698530908,
        0.6946491896303456,
        0.27969177627056035,
        0.9475151381637266,
        0.9373148041300592,
        0.9414205577887778,
        0.7299786781148221,
        0.5047262610949741,
        0.826824338899974,
        0.6264009814603625,
        0.6066679944662,
        0.6754673517135658,
        0.737048335458076,
        0.7522068344524627,
        0.9180943625247855,
        0.667807636806891,
        0.8800993027353498,
        0.8589048924989469,
        0.6001686866548673,
        0.5777092532464256,
        0.6854487486994552,
        0.21724653535744454,
        0.2518156858696573,
        0.6039829479813201,
        0.6736589097935086,
        0.6378291484918703,
        0.7465996412005215,
        0.3074498975640392,
        0.7793754627413642,
        0.4826785211791521,
        0.6044281461784009,
        0.10818847320887774,
        0.5658261352265285,
        0.3974862503435258,
        0.22609812532039086,
        0.3060746910915859,
        0.5632290781385644,
        0.7861127637952063,
        0.6247965212790187,
        0.8975195377061981,
        0.2299111106020637,
        0.18257144577697548,
        0.6173313836141097,
        0.7896720781566425,
        0.9908476212020357,
        0.6856621345560614,
        0.9414979746295195,
        0.5650148645752735,
        0.871326323997005,
        0.48663845145439805,
        0.6051520469622508,
        0.5623372665039224,
        0.5735013298543858,
        0.14778817943196987,
        0.0986841663884135,
        0.8272278725388775,
        0.8262366309494231,
        0.8735711679070831,
        0.8395529494300282,
        0.28192731051240577,
        0.18721187228338299,
        0.17845524214576192,
        0.1995191467963576,
        0.7466476517643593,
        0.5629615957645665,
        0.3311559393901189,
        0.29097845333552796,
        0.37029475435821035,
        0.27455615397993266,
        0.6942528659439379,
        0.6780754906765425,
        0.6888043209083279,
        0.43947079614355444,
        0.4347593918070741,
        0.568294304772923,
        0.7431318673999612,
        0.7977402180040404,
        0.774324208959432,
        0.8832342945689559,
        0.20249828424861605,
        0.31937708779194074,
        0.9197390592098829,
        0.9002907389872222,
        0.9097624958510248,
        0.06358693176198504,
        0.33335564311085875,
        0.83069310213049,
        0.6377366866265124,
        0.08271315002000657,
        0.3877499692714877,
        0.4547165078803055,
        0.14228231012072134,
        0.09565518990582489,
        0.6939676485369067,
        0.7840251495670539,
        0.6974066321206195,
        0.5227494847481446,
        0.46056619503462204,
        0.43684372943162897,
        0.8570492163751703,
        0.39485821273921307,
        0.8163234530403499,
        0.843199292688176,
        0.3397595410256431,
        0.8812489381533704,
        0.8730961835636027,
        0.839980906095251,
        0.8161960373891566,
        0.38269651086640755,
        0.7329787954361738,
        0.23763478627021994,
        0.48514142810877353,
        0.27213176494290664,
        0.6636065446053163,
        0.5456218735366011,
        0.441302393090608,
        0.8492507810934522,
        0.6220045722395509,
        0.43101903807413744,
        0.6930076897567458,
        0.9526004372801861,
        0.9535323970863779,
        0.745045679492492,
        0.29684828044647266,
        0.11145528933137239,
        0.291470341972834,
        0.8689493124800972,
        0.8980211403923971,
        0.8936367207513232,
        0.45204046983130214,
        0.46788053370612875,
        0.08897901922728102,
        0.7032370148469107,
        0.9777790899685306,
        0.22652128647594483,
        0.5800998562366738,
        0.32888539776872766,
        0.4745400707503539,
        0.7397070105644392,
        0.7917914158848022,
        0.9814342786407959,
        0.7751867090270564,
        0.8015759309249972,
        0.7876726575212669,
        0.3190824852234725,
        0.26308052205831206,
        0.2916486698754501,
        0.6468614092713741,
        0.5084241855775209,
        0.9043345293267719,
        0.6373571661935726,
        0.6333895477449617,
        0.42196717358420854,
        0.44239667527471366,
        0.4884512496644926,
        0.48311639544284213,
        0.554313562054618,
        0.6233788908103217,
        0.6334997509483108,
        0.8558081500034835,
        0.9674919410455204,
        0.4153295087805741,
        0.2292974015534567,
        0.33618124696105,
        0.7041040432927708,
        0.7947950619326195,
        0.759881728872531,
        0.6653209688699911,
        0.4535688426691662,
        0.5481615996621523,
        0.45635501170951254,
        0.5075273237018734,
        0.3401719410427633,
        0.8017569676409964,
        0.5979762660539081,
        0.2962209794940004,
        0.5201888866543825,
        0.9089126726051242,
        0.9937448973564474,
        0.7782723568880451,
        0.48100807655494526,
        0.47041967484749103,
        0.5359935715598839,
        0.7527509382373826,
        0.985709618467886,
        0.5431149710572553,
        0.5245738803441151,
        0.07557293165841728,
        0.07356206222798445,
        0.39657067419993663,
        0.6333763724277494,
        0.9962443752231976,
        0.5559369109107558,
        0.7579546474766875,
        0.922713817171945,
        0.6541542944955032,
        0.6854953673187475,
        0.9871901830985554,
        0.43343939313258983,
        0.6001418234267519,
        0.6722194367149108,
        0.23372434383532548,
        0.2230982434221908,
        0.24362896550663035,
        0.2956461761687907,
        0.32656042859931905,
        0.9291430547271073,
        0.4600635211552848,
        0.4831324773591076,
        0.8911167677125236,
        0.42006020979377584,
        0.6506017172531682,
        0.9051626658725884,
        0.5250932151419208,
        0.5664798304332319,
        0.5059651769960605,
        0.7980893640133084,
        0.7149686559879528,
        0.7835862526605718,
        0.95138921094132,
        0.5036996642419962,
        0.7508321642699095,
        0.34314379213630936,
        0.4121180434828805,
        0.5057399768336064,
        0.8162764276177858,
        0.7874971809440107,
        0.6968683550815664,
        0.7192213090912971,
        0.6414528951462002,
        0.552590294876178,
        0.9294828346978442,
        0.9319850571184528,
        0.926451679052518,
        0.9087566483995374,
        0.15977807906231578,
        0.08505290962839972,
        0.8720855399506353,
        0.9174133488364945,
        0.9617470785216952,
        0.10574249447280337,
        0.10500105054943742,
        0.1567165856010838,
        0.17682194519960787,
        0.5057794589053932,
        0.3437141549792314,
        0.929516096813786,
        0.23779623367026603,
        0.6858202618099505,
        0.06459584177267574,
        0.0548473738711345,
        0.05424925021743507,
        0.524560685942652,
        0.5821969052218517,
        0.5931312796834813,
        0.43861517482779544,
        0.630112429395009,
        0.5888546462724085,
        0.5797597176761513,
        0.42764946015435445,
        0.8002404827156857,
        0.3463084822752507,
        0.19710457241018975,
        0.6893807440768419,
        0.576795596249438,
        0.6455810751923139,
        0.931559425752479,
        0.5931777228054143,
        0.9397203570240404,
        0.9052781828103249,
        0.14912054946104725,
        0.07582926832153532,
        0.1073161951888365,
        0.9614574894869182,
        0.5293755520180135,
        0.228906179509908,
        0.39063951912503314,
        0.35913507249781723,
        0.3934666749132081,
        0.36914482151899186,
        0.3826137366366321,
        0.4356033357926049,
        0.3031471941841291,
        0.4491076797975417,
        0.36020430624074706,
        0.7562352272966579,
        0.6462691830553762,
        0.6220867744620384,
        0.4566160129451178,
        0.4218123310338646,
        0.3571456771857875,
        0.5344497654866696,
        0.6648700907199244,
        0.21937866530541053,
        0.695383968684185,
        0.8914395952174925,
        0.7275882407815639,
        0.16656854072959948,
        0.9094961342804389,
        0.25303455003932906,
        0.2942897071627842,
        0.5576960008595004,
        0.688748437864819,
        0.0341617122100113,
        0.03364193786272174,
        0.04063351091501237,
        0.1421223771446007,
        0.72985367203197,
        0.3395813722579861,
        0.9121750727367371,
        0.8595749028172303,
        0.5572138463940859,
        0.7605864020490625,
        0.8228123655817403,
        0.8873806952218386,
        0.8141187137145589,
        0.6547830102627914,
        0.6649285169214185,
        0.8649335548068247,
        0.07367004024819179,
        0.7822763986433828,
        0.8289434256739445,
        0.8789454498952465,
        0.7645013359064624,
        0.6166075077451507,
        0.7312579137088988,
        0.7485421133537898,
        0.2486535105787189,
        0.2563231690550928,
        0.2188057910025467,
        0.32362987732691795,
        0.2860045499771401,
        0.3200344641935712,
        0.8171621298379241,
        0.6291350931583551,
        0.833828109582043,
        0.9652630353165527,
        0.3088039827726933,
        0.27784988199161187,
        0.5433116952190645,
        0.7800773735295113,
        0.4698527804914965,
        0.9927072932044333,
        0.5413983147025466,
        0.9587371179418378,
        0.2704616463587065,
        0.47474331019309535,
        0.5914643371473058,
        0.0591425693375642,
        0.05508187711557681,
        0.06174621255871415,
        0.1775613059107365,
        0.6061449659481833,
        0.5427572588126849,
        0.9134659250503357,
        0.5707688540618305,
        0.6999229029823224,
        0.5963776799552722,
        0.6745541871129916,
        0.9470252288283549,
        0.5595951837892335,
        0.7823992614766775,
        0.45810942273097327,
        0.8090323847093002,
        0.9069279321291777,
        0.9016514060755129,
        0.7743290324047757,
        0.6832759251185432,
        0.6083311548895585,
        0.2043606227350766,
        0.2435172228001573,
        0.18479699126417692,
        0.4165475139552549,
        0.7854006526694226,
        0.5215576246711232,
        0.5553638640227401,
        0.9576105497871116,
        0.7767648426019531,
        0.47777237808291684,
        0.3884326019071114,
        0.44399324333460494,
        0.8956744882594664,
        0.6081197635608102,
        0.9179440985142551,
        0.6937420708695213,
        0.7813564172226439,
        0.7584196683586583,
        0.24101576234130376,
        0.6892556200026129,
        0.5415515615316785,
        0.6847601293905683,
        0.280281929809891,
        0.06758379331759365,
        0.566528787082826,
        0.9230871014769146,
        0.7810409628820754,
        0.5530292871500246,
        0.5684650990032007,
        0.6887763800741119,
        0.5493355260622442,
        0.5068306875584364,
        0.8892404003403054,
        0.5027596756988204,
        0.45112463766994176,
        0.33970140801745796,
        0.5984043799170609,
        0.5792890753350232,
        0.5638201250151593,
        0.7435633959297416,
        0.7422433292326864,
        0.7303102849872332,
        0.3858618832807458,
        0.33668791710962576,
        0.45159305159164626,
        0.06312424566014468,
        0.06098280929031095,
        0.06381554215637886,
        0.3861219245354858,
        0.4869175439821515,
        0.9716242934228988,
        0.06919596409639289,
        0.05389159690208777,
        0.7986077968954417,
        0.7661119288094362,
        0.4413190290489534,
        0.7224720053171806,
        0.6852043592731505,
        0.7767143158817639,
        0.47488537014765686,
        0.8193730531922986,
        0.4921819571509573,
        0.8723759823102376,
        0.8191779019042175,
        0.34204704637769134,
        0.7460719875825524,
        0.27882524194215297,
        0.13459705228344943,
        0.2472663303741217,
        0.38173279062088505,
        0.38604305249737403,
        0.3706602685999052,
        0.18241072834067645,
        0.22858504911042066,
        0.20835525045179804,
        0.6186742907271132,
        0.0830259656922081,
        0.07885186609954799,
        0.6709617644208055,
        0.8349046945433423,
        0.9962142450681891,
        0.5290928266035396,
        0.3951165678736127,
        0.6683141540209057,
        0.2877371926379042,
        0.3892855484061203,
        0.3017859178249612,
        0.28502509391026165,
        0.7534391251174971,
        0.35899690697482495,
        0.33991435537565906,
        0.3167368517067983,
        0.3529349183032133,
        0.14611461420062638,
        0.14357199153824746,
        0.15409419630274274
    ],
    "eval_rms_calibration_error": 0.28058138235630087
}