GENIES/results/evaluations/mms/llama-7b-code_easy/code_hard.json

{
    "model_dir": "models/llama-7b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-14 00:06:13",
    "eval_score": -0.27119261092729446,
    "eval_brier_score": 0.27119261092729446,
    "eval_average_probability": 0.5719051013486147,
    "eval_accuracy": 0.608,
    "eval_probabilities": [
        0.8357455174262569,
        0.836089264067936,
        0.6276084382513512,
        0.93107972524585,
        0.7994910956357882,
        0.6586566055995844,
        0.2458204768756772,
        0.8375288671978727,
        0.4095007351637321,
        0.6511122414112798,
        0.44472237707385676,
        0.974015687127958,
        0.7620438407919738,
        0.20339936051832808,
        0.5294973459344936,
        0.5720659084629155,
        0.8600120362282934,
        0.17342028933301912,
        0.7167575269019426,
        0.7399721676104644,
        0.7024477722302273,
        0.8561833120515334,
        0.6150542798079021,
        0.9064931756075635,
        0.849859482592017,
        0.6738252825473473,
        0.5992693119679243,
        0.0723249497293389,
        0.7865967803789563,
        0.6326775357302594,
        0.5826896798426975,
        0.6012316689349655,
        0.4889293742173796,
        0.5156496556950082,
        0.15306727822340727,
        0.5425942762578918,
        0.4297608171261077,
        0.11142673268170045,
        0.38159082552711776,
        0.4267333582613423,
        0.5450843126375005,
        0.9398429909871452,
        0.6382553931815587,
        0.5343736779634911,
        0.34830106578815706,
        0.9965753600731438,
        0.22886167299774662,
        0.37752902345612716,
        0.9907826839515522,
        0.35797164412307136,
        0.7965873867256011,
        0.27614784245165863,
        0.8153178677287526,
        0.19079183611943162,
        0.13177274034776498,
        0.030565454194152464,
        0.033896548641692115,
        0.14858446771159117,
        0.8588829309811313,
        0.8789125696415703,
        0.34827809576019875,
        0.7315410107596814,
        0.7245684288223015,
        0.9976941652699775,
        0.9816073355379272,
        0.9831448493187777,
        0.1701757828847601,
        0.8067388715196161,
        0.16316453761435712,
        0.2099434435329638,
        0.32037859858880147,
        0.602089239141967,
        0.6862396755975833,
        0.4837089340038554,
        0.9176194634448656,
        0.876939547507505,
        0.13987945033716542,
        0.4493641781050885,
        0.4236603282704873,
        0.056016839248035244,
        0.8868218286594577,
        0.8555247870844341,
        0.9963529538792648,
        0.44451579802320074,
        0.3635103023006342,
        0.44961104444811784,
        0.7946138347375296,
        0.4322168902403445,
        0.6759370544954004,
        0.40635293221572677,
        0.9080224273538602,
        0.9321452328362936,
        0.8779163334772788,
        0.018028207555206954,
        0.10124951649656078,
        0.07513502165628723,
        0.3571416961499956,
        0.9427143753617283,
        0.7913867928673873,
        0.9867027409092824,
        0.7423816334839765,
        0.13909450522530697,
        0.9358791767560958,
        0.4606456667734037,
        0.798321483643171,
        0.05905246139779596,
        0.8872103979468128,
        0.2879680269702827,
        0.06935121356167884,
        0.8352621494706713,
        0.9589115281459404,
        0.6739639647165181,
        0.29790020189190336,
        0.7164191703441796,
        0.1935101942812908,
        0.192600817870768,
        0.16545610222098373,
        0.5115788914192619,
        0.3567164346376971,
        0.3372131177361226,
        0.2185636696846276,
        0.42897886998130563,
        0.496444838315653,
        0.963938682057028,
        0.6444846651578581,
        0.8313167623952611,
        0.6454231947219631,
        0.6438741839539119,
        0.9232124925367275,
        0.5152990462976755,
        0.18809569785217506,
        0.7180019562754691,
        0.9593745352386811,
        0.9289749888587002,
        0.941501921912239,
        0.9489511805634663,
        0.9391852089353662,
        0.7865944182907167,
        0.0420306114425929,
        0.18430651012070387,
        0.2450607613789354,
        0.06817090797580938,
        0.7540525894992721,
        0.8174888553153719,
        0.9190870284986794,
        0.9600616314419439,
        0.85898675481624,
        0.6362074359190693,
        0.7093921996037456,
        0.6301684335301865,
        0.7213297635313733,
        0.7799174784439016,
        0.966710404575456,
        0.3335043958890678,
        0.797869050422612,
        0.3666199012443285,
        0.512064736365244,
        0.2704147571514509,
        0.08100717896550624,
        0.10871482793899068,
        0.06849226067359604,
        0.018541692673919807,
        0.2269564266792482,
        0.5351312343702677,
        0.9415563381444202,
        0.3937103095923816,
        0.8299943892551815,
        0.744209940112963,
        0.8799123879352718,
        0.9780330595210273,
        0.08818371449776907,
        0.7641114670686378,
        0.7403271344205794,
        0.4594376801276595,
        0.5412376580353639,
        0.15596546688714874,
        0.16432713139963295,
        0.9118155706522162,
        0.6900418755782468,
        0.422534964746619,
        0.9772716019049701,
        0.9551430906394826,
        0.8609908713684883,
        0.7551355701996214,
        0.10209757215991545,
        0.17852920917205917,
        0.9408779459481024,
        0.8744386840936663,
        0.9075555346419729,
        0.11687313804473298,
        0.9864755835216406,
        0.1278153070554897,
        0.9892564918411488,
        0.2097285257677689,
        0.19179470453146372,
        0.30580501541998995,
        0.3667225089635723,
        0.7896838265509989,
        0.6737763980845944,
        0.0783582395848833,
        0.1898793710863499,
        0.36765599331129173,
        0.3576531298245958,
        0.12600964795740877,
        0.29317561112158763,
        0.5973886580231351,
        0.26362997664177723,
        0.4024892533485155,
        0.12135603271681072,
        0.4328614512984077,
        0.07289919170936188,
        0.0770087632713834,
        0.07999769981652316,
        0.5026321976004839,
        0.1609650989709116,
        0.8043332346151173,
        0.25657746951230165,
        0.8168672594300874,
        0.6311019780857248,
        0.6721963287033521,
        0.6550181204293856,
        0.8276640743550903,
        0.44858931686953996,
        0.3427207321043899,
        0.47697788449328066,
        0.9862696635804239,
        0.7492941942596714,
        0.8249180298502305,
        0.18336053771148353,
        0.5364347346748173,
        0.21917833614276883,
        0.9873853854673709,
        0.9204216336160957,
        0.9843165584031467,
        0.22315893444420876,
        0.28965826849304166,
        0.23134715317467314,
        0.6786861374176156,
        0.8879018198924156,
        0.8301208011992395,
        0.939892632877674,
        0.9917446067273462,
        0.9357159543824817,
        0.820005043350852,
        0.39811951198826845,
        0.41392268272706245,
        0.3078910543353446,
        0.7182931162051891,
        0.25809303107603854,
        0.9098054809892897,
        0.9926222610526364,
        0.6880295552930658,
        0.8998017587462939,
        0.8589253804981444,
        0.8486448577797802,
        0.49026275079250525,
        0.11180350235525849,
        0.4055864242565292,
        0.4486836593982849,
        0.14261477336274775,
        0.1272670567056522,
        0.46192128976803537,
        0.4139395051652811,
        0.5466470818089044,
        0.9531189153896122,
        0.37971160313334673,
        0.45994352596702165,
        0.9412619191719267,
        0.9152009018708194,
        0.9537940987523656,
        0.7540608616559554,
        0.9670481714285432,
        0.8351566737151246,
        0.8646858282406097,
        0.4303982463822209,
        0.2419286681852474,
        0.8554036501987936,
        0.6509335228664941,
        0.8921251222980096,
        0.8756307238206879,
        0.811881491856821,
        0.832702726505638,
        0.8065627756327384,
        0.8073397277938458,
        0.8659085817917885,
        0.6725311909697542,
        0.6170786569492954,
        0.5838513982985085,
        0.9686610099426536,
        0.799583287874813,
        0.9779174737445262,
        0.4959318582089956,
        0.39949542310291664,
        0.8554806071790235,
        0.646330460009171,
        0.22655020449270957,
        0.7987107995243355,
        0.46507511471578655,
        0.4469913773927586,
        0.5124752219814912,
        0.8701568389873587,
        0.6990359308469821,
        0.7719558447977237,
        0.7597370166970101,
        0.18757191856964373,
        0.6733183847264225,
        0.06697045850914263,
        0.333808558187127,
        0.5703816278686177,
        0.9862022746384314,
        0.42540852902069826,
        0.814466844053512,
        0.3532176710516324,
        0.2476966430351546,
        0.8062702149926461,
        0.9100408312286102,
        0.28860393651000205,
        0.8308502175219631,
        0.7311577742233922,
        0.3831644615074582,
        0.908387677566529,
        0.18964781531053088,
        0.6257370950729899,
        0.5405718520552099,
        0.8586059747978784,
        0.051919873286526474,
        0.04559387565085726,
        0.5194014072881106,
        0.6499064363103082,
        0.9946542519247331,
        0.06178583845632801,
        0.6216932942214975,
        0.06977333438016675,
        0.9345305795284358,
        0.37541601783704537,
        0.882313808103292,
        0.6120147896786352,
        0.2419390753400066,
        0.06113538024302834,
        0.08379334485054157,
        0.48601514848433497,
        0.2560006482585854,
        0.5164520685740321,
        0.7559211453663723,
        0.7902567388086358,
        0.0500888857031806,
        0.2105208687159284,
        0.047069501123579105,
        0.747233882422504,
        0.8815786712624843,
        0.6662477508876652,
        0.4037526657479955,
        0.3679413029081833,
        0.597172851913712,
        0.06788300094413213,
        0.058671661541517595,
        0.2550217743477106,
        0.7389714382465442,
        0.9239838662389472,
        0.8670551817864703,
        0.6052028782739308,
        0.6496399870425487,
        0.6858674318432338,
        0.880016435633601,
        0.011008551363291152,
        0.055672588829093095,
        0.8249578528186712,
        0.9608095524285913,
        0.9604822582847696,
        0.4018899272380976,
        0.9572054576715208,
        0.9801151849369264,
        0.3721117590385651,
        0.8256430821614243,
        0.7469748676418402,
        0.2659373460516451,
        0.792475222444214,
        0.1752502396665544,
        0.6294369103119182,
        0.289583275372884,
        0.06914596150545871,
        0.8323901434390619,
        0.9480685723539943,
        0.9328388592403,
        0.952053679837976,
        0.054124564652869966,
        0.8498407837090928,
        0.6708337921240388,
        0.9734734813282165,
        0.9575103954024227,
        0.9720464241279574,
        0.9563904001738127,
        0.9748699224131689,
        0.03920638531323073,
        0.6149923577933964,
        0.6574168376384056,
        0.8312186256898109,
        0.6874642392270387,
        0.8342865807798004,
        0.7807175766525211,
        0.6735661010159906,
        0.9314114677484335,
        0.289936392098761,
        0.5680246115898685,
        0.9132106102184583,
        0.96619318629184,
        0.9868723166634081,
        0.8642299880627909,
        0.7609678637875951,
        0.5885219986690537,
        0.6673785822324938,
        0.8637297483813596,
        0.8878871301538904,
        0.5888801967905842,
        0.7511129648467394,
        0.5948991625749499,
        0.7834955468930228,
        0.5294778552475042,
        0.9039884355558162,
        0.08190503468402599,
        0.7442376940834216,
        0.9219332644157273,
        0.8522606564609911,
        0.9848092271381963,
        0.9091264605391192,
        0.9399849898041593,
        0.608270684450136,
        0.5842304413939204,
        0.5859822402712267,
        0.28550000716137114,
        0.19290404265301542,
        0.5606416083033088,
        0.799643092943632,
        0.22023088370507438,
        0.723817294664869,
        0.87882718422213,
        0.6033511763544357,
        0.81133560799616,
        0.03587567819994508,
        0.0334526004951996,
        0.035751959947917245,
        0.18961530001568527,
        0.6252122767259469,
        0.17195228886805236,
        0.5999657523915308,
        0.24037594905291815,
        0.2600723109031001,
        0.18563608413199206,
        0.12789282960735004,
        0.8816202451485984,
        0.24023082520375655,
        0.7561605219345944,
        0.31190194218872125,
        0.7821848920189417,
        0.5012063955543594,
        0.9630590800092316,
        0.9543764040413355,
        0.9454563673429884,
        0.9739900240761188,
        0.43330131684923523,
        0.48935701933085696,
        0.8990106711247702,
        0.5405885944101441,
        0.8929032942579508,
        0.9567331261096762,
        0.9366332277305832,
        0.9167589340664294,
        0.7183926178822023,
        0.23281017295501502,
        0.6074321293246652,
        0.14973449856472867,
        0.6161221323299623,
        0.47768365592460615,
        0.14737345027866577,
        0.17790371180158637,
        0.40690718280222826,
        0.9880704496758644,
        0.5342902817105115,
        0.6875864429109769,
        0.8527980743208604,
        0.9828748533968906,
        0.11718787629088179,
        0.9936079439870571,
        0.6701643796129999,
        0.6997926758697522,
        0.5900529355152493,
        0.44242252537019156,
        0.1067540811647395,
        0.41262232153808787,
        0.8691876842761692,
        0.1478342940694093,
        0.8864572696934945,
        0.6829730758075717,
        0.3838473427802027,
        0.5120000967109766,
        0.3161876692902489,
        0.7872994292705445,
        0.9621314066730013,
        0.22710183130377748,
        0.22664212840362818,
        0.518254368398482,
        0.08547673866176442,
        0.45077886424263236,
        0.9167459071233582,
        0.27567612778170014,
        0.4364971625381987,
        0.28866490494420227,
        0.5268283036437094,
        0.23554901280909324,
        0.6656583851312663,
        0.667370885485506,
        0.6554280644491407,
        0.14802031021162676,
        0.8467579178703932,
        0.5705576451734723,
        0.12216201310517327,
        0.5614797612376535,
        0.6940260765560378,
        0.5231039542564608,
        0.9776007641531468,
        0.30075260276530913,
        0.9271908847254954,
        0.4834853609628794,
        0.43739130309090857,
        0.622052238779679,
        0.25467145816064873,
        0.060718383856829594,
        0.061326301014366424,
        0.8673762278316212,
        0.4179326526537328,
        0.6944472599221919,
        0.962757309554056,
        0.4366781945940041,
        0.20891924863107525,
        0.03670563196220866,
        0.16300094886227984,
        0.03920250075289888,
        0.45978563537114975,
        0.09412387237321362,
        0.12363570110964925,
        0.7298202115508282,
        0.7308245934740356,
        0.40869674957427704,
        0.4498594379098577,
        0.08942275686762813,
        0.6338952608166836,
        0.5301880993620152,
        0.306914712776403,
        0.59389199640804,
        0.7371747269883973,
        0.9575875205968376,
        0.721850248313272,
        0.854110107015266,
        0.8334608198957469,
        0.787133533640972,
        0.9130379639618613,
        0.9392622447084666,
        0.6943520188418402,
        0.9715801066760689,
        0.8387990876269253,
        0.810412293648805,
        0.8489296638965538,
        0.9131946844760553,
        0.8733409720332277,
        0.1371657137749983,
        0.5017151399811811,
        0.19542873619694778,
        0.4653265600218742,
        0.3535120591263693,
        0.3270575779122289,
        0.8267216536824517,
        0.9244703654711226,
        0.6771286871308552,
        0.13809511090690285,
        0.3100834798790597,
        0.758484254766692,
        0.4604307186784697,
        0.22337046095021207,
        0.10709621028017058,
        0.9576174008010124,
        0.9124981559724843,
        0.8702677087652102,
        0.5724621862654579,
        0.9774517833666334,
        0.7371783145469323,
        0.5133050262157914,
        0.5629053309934007,
        0.2302798568122283,
        0.01908139655828378,
        0.07938505125926065,
        0.02187380983075371,
        0.7061410201281275,
        0.9381124549482204,
        0.8508762445151457,
        0.905256056477385,
        0.8576658953649142,
        0.8786941219644391,
        0.7710326891578997,
        0.787310743666191,
        0.8444626260571592,
        0.8408237515166981,
        0.16535374749484189,
        0.5281737713609395,
        0.5914435380005681,
        0.11422656422911637,
        0.9674318551482394,
        0.42573474620458573,
        0.9337570261431034,
        0.9706868551606519,
        0.2993216592569965,
        0.7966317521211218,
        0.8026521566062564,
        0.629768423232497,
        0.5840736155504833,
        0.28076340308550074,
        0.16469934117125085,
        0.6283197761114531,
        0.13589523974824524,
        0.7237791132239655,
        0.7527502444929692,
        0.9339193848052761,
        0.5731056945073423,
        0.7571973017501843,
        0.9560703942264726,
        0.5321658090488,
        0.8361009600429077,
        0.6461900212561026,
        0.994838706880102,
        0.7921021430127398,
        0.9635553850114839,
        0.09935252142421173,
        0.6275086978539701,
        0.6734451235628185,
        0.12523799193741036,
        0.13028677270554367,
        0.4342443705434175,
        0.475794339276838,
        0.5575658511248194,
        0.6136398751591132,
        0.7027897873034181,
        0.28368018021505403,
        0.7877995223096127,
        0.6074666843298215,
        0.5659122258035032,
        0.989507581610056,
        0.46625128733933674,
        0.5914463676793735,
        0.0727674746227076,
        0.20586439193482864,
        0.7667975453031944,
        0.5701452400075419,
        0.9393418233648503,
        0.7851231264206545,
        0.5549733554094304,
        0.267198888703583,
        0.6290776854872071,
        0.5959981872531172,
        0.41698392573055326,
        0.46140558900026674,
        0.22686222962441405,
        0.7580407055620967,
        0.9080069822003203,
        0.8960508471269587,
        0.3685439894176966,
        0.6895913878372524,
        0.6588512270492041,
        0.6593726110102597,
        0.8255967477958092,
        0.8221398646101132,
        0.21452701793571227,
        0.08936124390633449,
        0.3674088663044339,
        0.42606168024616553,
        0.1563476077155319,
        0.24065592282801085,
        0.4592464393126218,
        0.7485316773533501,
        0.4356816847635504,
        0.8631460827902773,
        0.9763182109832295,
        0.9123445523601448,
        0.7948958709131826,
        0.4713308653118785,
        0.8171976761702442,
        0.5216142548551737,
        0.5994795884484252,
        0.9282132056868871,
        0.07376515397338734,
        0.04625262683834852,
        0.7985941422296972,
        0.6875683000709932,
        0.6853129945238007,
        0.7389907863959149,
        0.7882064578902439,
        0.7638382748438256,
        0.41697423705252445,
        0.5043485734026273,
        0.6602300657169501,
        0.6871932831022852,
        0.22988375020684007,
        0.04372477198717717,
        0.04181422188605266,
        0.5021252001228247,
        0.553563048759563,
        0.945552572167599,
        0.2854125716028657,
        0.03114799308299243,
        0.20179193013914354,
        0.8352103762008266,
        0.3712434620535186,
        0.6087491186631837,
        0.7536992509733071,
        0.8009381228716327,
        0.9200337014684253,
        0.33905313963399775,
        0.17809414752668362,
        0.7414832323778597,
        0.7365163511932525,
        0.73905697209392,
        0.25028771451678455,
        0.33301469208318124,
        0.8286102818253441,
        0.8025673666907615,
        0.2396886541069313,
        0.611552336752346,
        0.1739150542459439,
        0.13845528227909487,
        0.6292019370356522,
        0.6219936769941173,
        0.735430076488474,
        0.18326721384234276,
        0.03527316135895975,
        0.9650458183669987,
        0.8695315060868737,
        0.9981694854419321,
        0.8931575500140283,
        0.981430589390468,
        0.8426734519044865,
        0.16918751860368741,
        0.5602651317777001,
        0.6502214880102595,
        0.5322616854256683,
        0.698530465448613,
        0.49592434017986475,
        0.4048189136322845,
        0.113628497506155,
        0.44446066110178484,
        0.30634426481159654,
        0.32697889058415686,
        0.08329019804840176
    ],
    "eval_rms_calibration_error": 0.2849196377739891
}