GENIES/results/evaluations/mms/llama-30b-code_easy/code_hard.json

{
    "model_dir": "models/llama-30b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-14 01:51:41",
    "eval_score": -0.22717749328532136,
    "eval_brier_score": 0.22717749328532136,
    "eval_average_probability": 0.6409135627137225,
    "eval_accuracy": 0.676,
    "eval_probabilities": [
        0.9835081852099258,
        0.6986180634633641,
        0.9300249258694078,
        0.48131605411370854,
        0.17982065892206056,
        0.6038228342610791,
        0.9278527721900005,
        0.45783877089464287,
        0.5476273562279252,
        0.2375091094687074,
        0.4924182141909049,
        0.411306451612405,
        0.9455327731025689,
        0.562514017486568,
        0.4924790781682796,
        0.6555820251292291,
        0.8146411431477896,
        0.4372084943047643,
        0.20919375401077933,
        0.275013604473204,
        0.21028192140344704,
        0.9614246281800249,
        0.868400758798248,
        0.9923740310568309,
        0.739844787277093,
        0.5641933762856811,
        0.9485487934299571,
        0.8046812860260092,
        0.8391422583403643,
        0.8156685080436199,
        0.8943445139681608,
        0.9515260137392488,
        0.7994811414105738,
        0.9602211103184668,
        0.8854876356075548,
        0.40336892887861653,
        0.34160120093890955,
        0.12238285415213346,
        0.8314712905820781,
        0.6324969134032183,
        0.8569183894409885,
        0.9302044297784178,
        0.794844605348401,
        0.7465570702976981,
        0.5251567732264452,
        0.9956518453845361,
        0.5095659124253422,
        0.8789502570168136,
        0.9895987235977549,
        0.9911150594392419,
        0.9925695689263894,
        0.00899813070085509,
        0.8534343783482434,
        0.9277832618160631,
        0.02544706802423735,
        0.029100762785042145,
        0.16512604086028324,
        0.2777865469572679,
        0.9727807232815994,
        0.9515455750377572,
        0.05414932118455229,
        0.17477466790057145,
        0.04993644644809725,
        0.9940731197989925,
        0.9921741662782659,
        0.9996774592772136,
        0.07270143323472633,
        0.3677698775941485,
        0.2966330510337249,
        0.5147086506994107,
        0.9270628526135315,
        0.6795226197516513,
        0.9024953220516346,
        0.9119873835685152,
        0.8261138095142819,
        0.9907216835343606,
        0.9670912110920067,
        0.988518261080743,
        0.21370432386398905,
        0.9356079415125593,
        0.9588183539661862,
        0.9564861135968683,
        0.9898313254184596,
        0.9056670516911144,
        0.8287976407356454,
        0.870224829147354,
        0.7275604191873839,
        0.9561505371176322,
        0.9838872569475159,
        0.9473904670791481,
        0.655336068775369,
        0.7205662513689017,
        0.7199689054143716,
        0.7598987993761377,
        0.7374443065243392,
        0.7050272423423232,
        0.8975521715118095,
        0.9285281761475519,
        0.2505746547852409,
        0.9733041135720043,
        0.9924548720429464,
        0.9623462428632968,
        0.9385534029783938,
        0.8771304273375254,
        0.971484067040744,
        0.7754452055426642,
        0.996503239449481,
        0.06792418235725739,
        0.4640005751051138,
        0.988729107603018,
        0.62205232100779,
        0.16263790180658533,
        0.059277496375349736,
        0.3301702602858831,
        0.7630411422641169,
        0.760235281445923,
        0.6482175414160746,
        0.26658759286086453,
        0.5839189325811563,
        0.3177645265298707,
        0.20624032131251735,
        0.9375211267127731,
        0.6558968800675877,
        0.8927283178656875,
        0.8111270411006201,
        0.9759494279186771,
        0.9507503867531479,
        0.9646737182397565,
        0.9830625523278952,
        0.7478295205284289,
        0.34095330961537135,
        0.11551249323455026,
        0.9807243477053205,
        0.9407357406389815,
        0.9805998790592859,
        0.8637921261820335,
        0.9784409028033569,
        0.8632069718820081,
        0.07712256487284841,
        0.06382607653484917,
        0.08054326618312158,
        0.13030241912426288,
        0.9699506798599216,
        0.7926291975251393,
        0.945755473309262,
        0.9542454453516436,
        0.7955036356419437,
        0.32996584479322444,
        0.26695738134091884,
        0.3459373847213074,
        0.2936338914992971,
        0.6464483649389879,
        0.799584522041149,
        0.4263433635284153,
        0.6600382063217023,
        0.8628467017823789,
        0.4678662723903109,
        0.9255009595745204,
        0.9331213664333021,
        0.2757423021445301,
        0.05754658727689743,
        0.2545872774202284,
        0.8590232460442097,
        0.7568925371104948,
        0.8997008423755336,
        0.9525693786340538,
        0.9880121397081133,
        0.6979168894931917,
        0.8954707263405436,
        0.6354454390506121,
        0.8664347299353825,
        0.6517206634301334,
        0.34276009320821454,
        0.6426875666567515,
        0.6925592065699223,
        0.17109214596839273,
        0.5892167116313347,
        0.8722617163067932,
        0.3536920824297549,
        0.9290790162517496,
        0.9360652571746253,
        0.6175060534957868,
        0.2776823438443736,
        0.8377580017505517,
        0.4377761842408864,
        0.9092991144224573,
        0.05314402168364374,
        0.9798959717912232,
        0.8618562637467031,
        0.9837348476798354,
        0.8066769629425463,
        0.919985086946524,
        0.7605495062355057,
        0.13348053241117508,
        0.19290617135198962,
        0.8743801829137952,
        0.9566096009155919,
        0.9753416428492389,
        0.938939001472714,
        0.7467654233006796,
        0.8906924980570312,
        0.3309480425966561,
        0.019787070443168853,
        0.05996669804300803,
        0.12770852859938342,
        0.8494472471325586,
        0.20851593543595093,
        0.3159655274787586,
        0.2394676607023739,
        0.2707799946881446,
        0.172325931561836,
        0.16174259099557828,
        0.20879886004109505,
        0.6193211939765575,
        0.9533259212903947,
        0.7748220539438183,
        0.7479598560539328,
        0.557159990586628,
        0.6825127975336736,
        0.9419139754865704,
        0.6679376599712172,
        0.9118895354732431,
        0.9385415227921826,
        0.9827034235190931,
        0.9736194197913502,
        0.9919297952510573,
        0.5763651897973588,
        0.9599052318137992,
        0.7092815494256088,
        0.05141364972893048,
        0.7471962981335237,
        0.4691658990870221,
        0.9971413772400688,
        0.39234769379739876,
        0.6259237106873301,
        0.6878316023991902,
        0.6132496824532596,
        0.8898908093442356,
        0.9356425321562751,
        0.728136830185351,
        0.8019093877330445,
        0.9177868579364782,
        0.9767362800415256,
        0.6710383128375332,
        0.5859430055807839,
        0.6132335358508958,
        0.6842496708534042,
        0.9886208694141247,
        0.9514026710162135,
        0.7773724622621474,
        0.9840079084930066,
        0.811399419840726,
        0.8031389405545633,
        0.4703813975733363,
        0.7715841034856201,
        0.22633185627918084,
        0.37423280842517115,
        0.46107640663435284,
        0.23054313463309595,
        0.6545072145710752,
        0.5646024028085068,
        0.8996056722859156,
        0.20921570197964146,
        0.8567202824984154,
        0.05318057424261391,
        0.8147274745424,
        0.8682425000776062,
        0.8568690312646248,
        0.9403314489260559,
        0.9680252857969163,
        0.9918551992744075,
        0.9831320521872874,
        0.9089954509385855,
        0.7956987723020568,
        0.9684650375599931,
        0.9825904026849227,
        0.6067309037751456,
        0.6266331830520853,
        0.9051870293929328,
        0.8943989338215683,
        0.6774320564858937,
        0.9053108527958821,
        0.9438756572485267,
        0.8680304818856631,
        0.7931383075590159,
        0.8215863625088363,
        0.9586392563250816,
        0.8753625904266918,
        0.9894130790149254,
        0.9819916843115748,
        0.9385394082967708,
        0.25236955225825475,
        0.10990306770550828,
        0.970345052967231,
        0.4890285993017019,
        0.16078386702690783,
        0.6219437992613126,
        0.41685845197184546,
        0.9482517720346069,
        0.5468646356696997,
        0.7248398173677724,
        0.9846252543153604,
        0.9486692070859479,
        0.2782502514641142,
        0.05382063004614723,
        0.8918320378750839,
        0.27104999511190614,
        0.299339261106234,
        0.9606195733744635,
        0.9104169449186396,
        0.9431989693415364,
        0.910732794562169,
        0.6889934977264324,
        0.9085458130348679,
        0.7580932560298297,
        0.9936102265726722,
        0.8893291357522537,
        0.8018580840616328,
        0.603635497611096,
        0.7733800778090479,
        0.7491490439117462,
        0.044531008093115276,
        0.6449316389978068,
        0.3452407946326643,
        0.9695628795788303,
        0.5339471260455412,
        0.10330853551793484,
        0.8348115216980894,
        0.48488756546226686,
        0.9818472033548132,
        0.4886881214633968,
        0.9174422229405287,
        0.5058338455816671,
        0.9160378600626344,
        0.11814034684437713,
        0.3173513936466156,
        0.8776822669630866,
        0.7617812234208662,
        0.5176637188858181,
        0.09574922191136515,
        0.3350667381687027,
        0.6443085751798314,
        0.966468144275767,
        0.7673018952544087,
        0.9131188938175364,
        0.36462475693511154,
        0.09857949977894409,
        0.0674747765727951,
        0.6396272131849301,
        0.8967394810987394,
        0.4511184021170682,
        0.05690028225529431,
        0.2778665758229,
        0.025663064155162063,
        0.05542823737033986,
        0.22004492823475635,
        0.05733016991121387,
        0.8703091969251924,
        0.28236314566626614,
        0.3247203302065985,
        0.16650192331450286,
        0.7692087362336215,
        0.3239128243868422,
        0.8155550995615141,
        0.2791257529546586,
        0.6739172198619923,
        0.5806251925022796,
        0.5521282859041554,
        0.5820505692312291,
        0.8398801812812661,
        0.6225988139786319,
        0.2217803643705465,
        0.08693346719546391,
        0.5943810622028813,
        0.7355290144428898,
        0.6766299282400401,
        0.6019814965990412,
        0.8777035510341218,
        0.18118530559743373,
        0.5932739919841216,
        0.845767507399041,
        0.6588619193227446,
        0.9401553992012296,
        0.3771042396376798,
        0.9746158863646076,
        0.3385771657327178,
        0.1975912484920448,
        0.941981119141104,
        0.5925690748433257,
        0.9801309314318409,
        0.9934310511132746,
        0.9956551239535266,
        0.9649359719277477,
        0.9638174359258085,
        0.03985804272388471,
        0.8937165884589218,
        0.8861697833248763,
        0.8278633520931356,
        0.923702802201426,
        0.7939293709094973,
        0.9213430251270038,
        0.9325437193759043,
        0.7746228345963276,
        0.41971379000041553,
        0.9775096583551411,
        0.9985018619429783,
        0.9082201677477549,
        0.9435517762437237,
        0.150994498474335,
        0.2856555631311877,
        0.8879401683943484,
        0.366927914002388,
        0.3589908066957671,
        0.1065548155372954,
        0.15000672672815696,
        0.2771711030576278,
        0.5434313805953244,
        0.40078109414414187,
        0.9593956999165472,
        0.1335232574088967,
        0.28907550375832036,
        0.8648125005991624,
        0.8084044510291182,
        0.988161398543422,
        0.20021356498474963,
        0.9555188326474996,
        0.638779994804332,
        0.9176570275122159,
        0.8996364107442707,
        0.6959774224209111,
        0.4987426420674815,
        0.13952366564102414,
        0.7541193995770512,
        0.9469668087552191,
        0.9016780689576283,
        0.5973265281987898,
        0.864191317705022,
        0.2615099650791075,
        0.052308619361153164,
        0.050118541874962674,
        0.25093855341685317,
        0.26904169218180196,
        0.033248692858112494,
        0.3237184997973204,
        0.42762336317153393,
        0.9918824128237336,
        0.9155098633329232,
        0.8634395341374013,
        0.1753451243235168,
        0.9919014492957922,
        0.7595173603049888,
        0.7794731289035108,
        0.9087683545772274,
        0.4925355793360996,
        0.9615283742501927,
        0.6070432466771711,
        0.9302040897825176,
        0.848977527182518,
        0.9752779175790707,
        0.6252558258547883,
        0.8765726709880693,
        0.8053250549600108,
        0.12403834908740896,
        0.9853889812702934,
        0.8340763565176946,
        0.49146209139788943,
        0.8108214733816459,
        0.6551027816422306,
        0.812985829131796,
        0.9832487484628075,
        0.21445042949740034,
        0.09404631805184882,
        0.007171225690277616,
        0.007150487769626124,
        0.6949268449376198,
        0.904353199613758,
        0.9901393136410881,
        0.4987873728409849,
        0.9616640691584644,
        0.9000187583992869,
        0.0051558098471576885,
        0.9863669501234094,
        0.9848986083493968,
        0.381988562318882,
        0.7856504181013549,
        0.6983890378605468,
        0.7241384567642368,
        0.8478127183239651,
        0.6537641562905697,
        0.860834525095869,
        0.5692122393332978,
        0.9656269931446991,
        0.875260663812059,
        0.6136994100352461,
        0.6914307678965713,
        0.2988759355361209,
        0.743766009383909,
        0.8340962858305173,
        0.9320385646529383,
        0.8944594400217107,
        0.8345195615532919,
        0.45061015141189087,
        0.9919201756798491,
        0.9809601874601451,
        0.9098578787414773,
        0.9477733538562026,
        0.26062667852842947,
        0.48011468844718197,
        0.1968594035017759,
        0.27295523362045704,
        0.857073275068054,
        0.7489920706575834,
        0.2106757287645053,
        0.9101079667619809,
        0.30035893524753443,
        0.30365906826623523,
        0.6902778034178785,
        0.9396693701624519,
        0.9424805777587889,
        0.9876153841605779,
        0.08639212463891877,
        0.15832234339292628,
        0.4051682508830816,
        0.7374571497054697,
        0.7901331764995837,
        0.1039757840004202,
        0.019402143952970352,
        0.030974836784043558,
        0.7980635183820621,
        0.9484419426713934,
        0.8668294696703094,
        0.9922574860785023,
        0.4646806829020532,
        0.2407438628126265,
        0.032508660655059196,
        0.09530814705171381,
        0.13698142952884204,
        0.3753366131874433,
        0.9255971882858364,
        0.23803237795693521,
        0.9644439160523159,
        0.6307343761446758,
        0.5535202194672508,
        0.28920828565148543,
        0.2986466361387181,
        0.138824749823645,
        0.514310153005076,
        0.7694048387896536,
        0.9796594392379118,
        0.954781273688241,
        0.987403761158442,
        0.9655985578171912,
        0.024859237978263615,
        0.7719236976355504,
        0.5520916308218677,
        0.3029632275165782,
        0.07362149080541647,
        0.06762805008842043,
        0.9974819722046919,
        0.9868180477217242,
        0.9815964044925085,
        0.9471699045340936,
        0.9722898604194878,
        0.9616868439261737,
        0.15725248288658833,
        0.5058714073178092,
        0.1411932393204276,
        0.08419764924452366,
        0.8303406491187401,
        0.8580651489074467,
        0.8740363655779407,
        0.41925321165408314,
        0.7258737029444394,
        0.49796040291805005,
        0.14504919493776858,
        0.25007574449739756,
        0.9024659788671971,
        0.9379975394129128,
        0.6869480773332385,
        0.9162979662264774,
        0.9481743616636417,
        0.907285193099024,
        0.06622364926482771,
        0.918230416306122,
        0.280871938160424,
        0.8729846594430124,
        0.6516038030348085,
        0.22707670963374582,
        0.009254662530666424,
        0.05016739395159395,
        0.05407463484992568,
        0.35346749186820925,
        0.18865374480870067,
        0.9042383841794069,
        0.9654341152326062,
        0.3916693358753772,
        0.8957706625827334,
        0.9491157537349689,
        0.855432744863176,
        0.7791694435450723,
        0.07772567638930827,
        0.25940949120185486,
        0.2922866668863192,
        0.7653313089783985,
        0.1421416196914309,
        0.9850516718567702,
        0.984453929617616,
        0.9563168098644562,
        0.628183009976627,
        0.9618713919093379,
        0.6952540766085237,
        0.5460095303736158,
        0.501790670725064,
        0.5039322325315215,
        0.5779288625406337,
        0.26530594079505376,
        0.3406936591059475,
        0.669867990132749,
        0.5530041711716165,
        0.7174966133230881,
        0.9039335271270615,
        0.8486105388234222,
        0.8833246234928922,
        0.07190990736571301,
        0.7647711022921001,
        0.9479278036336073,
        0.8361376906691479,
        0.9839203460388686,
        0.27023298541292773,
        0.991611326389611,
        0.7827428343553702,
        0.9721262441591256,
        0.8816970470991209,
        0.24945854000650514,
        0.05496034870099009,
        0.051391689722774636,
        0.2920145120579411,
        0.7922717532694676,
        0.418812106535278,
        0.9936186208721965,
        0.9411296230395164,
        0.8731765436041479,
        0.7138513475576858,
        0.935476404080117,
        0.9996146360299086,
        0.4091595548637631,
        0.5042239103772268,
        0.5799475414523846,
        0.8306831745963826,
        0.9367879448914485,
        0.5550591174943879,
        0.8986787347499061,
        0.8146858390839019,
        0.9262337296214099,
        0.8777493406921654,
        0.9744338736009422,
        0.9803876495386623,
        0.7261686418931645,
        0.6858660890963764,
        0.9493142592589427,
        0.9264138599509915,
        0.9898884965282635,
        0.9909630588430465,
        0.9499649446037995,
        0.9016250523467447,
        0.9831881484682431,
        0.44963913705640346,
        0.6685902975970837,
        0.9123805044032618,
        0.46238462599375596,
        0.8864557820987153,
        0.6006542841058031,
        0.6788906843160899,
        0.8863598302311181,
        0.29536497294488373,
        0.4856225150279372,
        0.0524476803133348,
        0.07422324069279777,
        0.15453626727860337,
        0.9736725459249059,
        0.74030904397483,
        0.983117745892074,
        0.8911759874392406,
        0.96900052812051,
        0.8825187608773213,
        0.9199167742960846,
        0.9941860118173734,
        0.06780966800105004,
        0.1984312060767854,
        0.5663137598304415,
        0.5355222741786854,
        0.8800970454438831,
        0.8334098779318103,
        0.9542043479846816,
        0.9517282673936777,
        0.9524370998538809,
        0.30357107404366523,
        0.17271353547622148,
        0.7827863446569319,
        0.3398002721782775,
        0.2363649258395686,
        0.2538013181634825,
        0.3552618558298548,
        0.2252181250981422,
        0.9745425402912373,
        0.7380377114051341,
        0.016858831599528057,
        0.5938793094467039,
        0.7938409428793769,
        0.9608989007021314,
        0.8103773772394538,
        0.9479059136058373,
        0.9673318783305594,
        0.9609804464026984,
        0.9523812404002062,
        0.6059719987649775,
        0.9815243370472273,
        0.6752880906853789,
        0.39631709984478947,
        0.8145144779355126,
        0.885942525669571,
        0.7809933310546083,
        0.8030599950435039,
        0.5727277693299524,
        0.8904451445973959,
        0.8723490200160082,
        0.05078915135216622,
        0.29376880739715433,
        0.31236145959361583,
        0.266579743087177,
        0.20648049050889627,
        0.04390966223587789,
        0.7397796011263955,
        0.9820209866718375,
        0.9833014372513762,
        0.35788730524609,
        0.6571653508054549,
        0.7903510571146355,
        0.31857643200640046,
        0.3571613566182236,
        0.22922787118758636,
        0.5350826214212103,
        0.44424130276128443,
        0.6120442505280447,
        0.40457166040236026,
        0.09272435180425165,
        0.18252153659196024,
        0.668046671417622,
        0.6300608835220686,
        0.6996260246396441
    ],
    "eval_rms_calibration_error": 0.29652087163465013
}