GENIES/results/evaluations/cra/llama-7b-code_easy/code_hard.json

{
    "model_dir": "models/llama-7b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-16 03:12:41",
    "eval_score": -0.20125630779949452,
    "eval_brier_score": 0.20125630779949452,
    "eval_average_probability": 0.648183155400546,
    "eval_accuracy": 0.708,
    "eval_probabilities": [
        0.7609667459214036,
        0.24194455234859572,
        0.16989501326017875,
        0.6808680523273165,
        0.5158028995119683,
        0.5935491722035372,
        0.8333000946744872,
        0.9321447057822994,
        0.590392489000511,
        0.6164653403929407,
        0.9877460266582603,
        0.8889401391101079,
        0.7936457962067766,
        0.6607248241094703,
        0.34857412901854146,
        0.507523673304505,
        0.8538608198989938,
        0.6742771711804998,
        0.967693512890976,
        0.4315889498654635,
        0.617178132568255,
        0.978507416172474,
        0.9608497851686639,
        0.9819239299909316,
        0.8914661637148508,
        0.9108290767080788,
        0.7509061809258418,
        0.5997850790930059,
        0.5831026851842016,
        0.7108852701645691,
        0.7864586214227081,
        0.5793081161192384,
        0.29486595919132674,
        0.9302038659796957,
        0.9347306974268811,
        0.8279555758497175,
        0.569082791520314,
        0.6498959276836734,
        0.7035608567518259,
        0.5537759958157024,
        0.5397438856573775,
        0.6633516574054055,
        0.9819119533473911,
        0.9201328977431604,
        0.9611005172848464,
        0.6346789625646169,
        0.1193072795940503,
        0.8435900350338318,
        0.9847478264158657,
        0.9528617245033676,
        0.9549029254561586,
        0.16380366312716488,
        0.9607913358291592,
        0.9687359499449505,
        0.8553235972516258,
        0.8985739131386813,
        0.8677954471976786,
        0.20013719958697757,
        0.7777920817273327,
        0.03954346302547727,
        0.2992214388978221,
        0.12576852803439814,
        0.08309012364525781,
        0.46249265982257814,
        0.31664387805508853,
        0.46165639023588534,
        0.1394395994699055,
        0.5619274798107191,
        0.09829580913463681,
        0.9202931950856695,
        0.7715694026714914,
        0.8469967569714103,
        0.41881681622915923,
        0.9940577061121822,
        0.7598824935786876,
        0.5152609410771741,
        0.28257703544173096,
        0.7033030907775293,
        0.46982301708903895,
        0.32441246266491763,
        0.5035206142998729,
        0.34974802661644183,
        0.7253290053371826,
        0.7665049560960827,
        0.5210362390038262,
        0.7164078720790722,
        0.6575091844171002,
        0.831713141042286,
        0.24029274540323786,
        0.19959702574377516,
        0.9633939488222572,
        0.888472941431049,
        0.7604024794832427,
        0.7145608666781613,
        0.6175552629549493,
        0.6910717513619242,
        0.6791065813940579,
        0.9707897852985067,
        0.08234914072224536,
        0.602254507787441,
        0.9401744169720796,
        0.9814449808696227,
        0.2781012869755798,
        0.1633779626199949,
        0.20910294705031535,
        0.8039682037609619,
        0.9482841692616957,
        0.8865587961022355,
        0.8672193355484263,
        0.41607226440852507,
        0.4193719623147755,
        0.6219922178072879,
        0.5005758043121732,
        0.3932127604189417,
        0.5056250788832765,
        0.22802839439082692,
        0.3470158485320731,
        0.8039515178427841,
        0.5385855480717147,
        0.7390781072868884,
        0.6937432001952248,
        0.40508235719804886,
        0.0939045482914902,
        0.2800587875290519,
        0.20581384540073022,
        0.49496969230613497,
        0.9317685126459074,
        0.7371109468236867,
        0.4456207586641525,
        0.014660130910106995,
        0.009030169493304733,
        0.007543369498045911,
        0.12828583981738742,
        0.24951026191710335,
        0.2973174124856288,
        0.931045773334582,
        0.7898386507243049,
        0.9101661678985117,
        0.38538602789455156,
        0.6372395785118159,
        0.546627374215434,
        0.5148160229404838,
        0.29275058065008663,
        0.40879048957434116,
        0.7631974951322635,
        0.8918756137301116,
        0.9730214456099529,
        0.7268532965707548,
        0.7322878982188369,
        0.2783943289596998,
        0.5694693134625521,
        0.5560439705341146,
        0.7482823677992181,
        0.06831923018711265,
        0.09827732077478864,
        0.058481938324490876,
        0.6235215327101167,
        0.35545365857592454,
        0.6629339268922926,
        0.17875463759556937,
        0.2226118581843345,
        0.1639162746030952,
        0.9305779603654294,
        0.7507686065267613,
        0.42124353375988566,
        0.6128259248283074,
        0.967303495004081,
        0.9536914958128208,
        0.3184077247302475,
        0.19320564180808175,
        0.0884213063977757,
        0.7916030249191432,
        0.8363673348141071,
        0.4865036058711045,
        0.6730613409207776,
        0.6605189482679351,
        0.7147376628202918,
        0.7249489122068027,
        0.1784743167898628,
        0.28428375605750356,
        0.7969234441262387,
        0.9736941692713974,
        0.9673689835845887,
        0.692096978449282,
        0.8666816075477526,
        0.7288646506187964,
        0.059378652176816545,
        0.43944544017748344,
        0.9008717116192994,
        0.7484583124177199,
        0.9850107655144261,
        0.9452802209910864,
        0.8836758900540755,
        0.19499724582692296,
        0.30673017752260034,
        0.7292970219971682,
        0.6534929407278874,
        0.8231400485557355,
        0.8027214224005622,
        0.46324525988825244,
        0.3297470599743637,
        0.8792097188596829,
        0.7986247698420591,
        0.6178625330556652,
        0.5267028626345921,
        0.7309466767861472,
        0.865369069167656,
        0.7112039184225369,
        0.43594313039202337,
        0.4387341496874574,
        0.2955895487604397,
        0.23540695441772502,
        0.7404494218299164,
        0.4808001373404163,
        0.818282670055976,
        0.4326230535852861,
        0.8537502429991082,
        0.6796554181373482,
        0.5297698376919813,
        0.862074948381015,
        0.8563892056830845,
        0.8488322830779219,
        0.8307792656909504,
        0.8138849081571563,
        0.7510907217278906,
        0.28526816366423907,
        0.34935540023468414,
        0.8861550405193324,
        0.6502127306247196,
        0.8143905670545153,
        0.7294011661145456,
        0.5641175011513239,
        0.908251375906229,
        0.6984628401691622,
        0.3188734579958145,
        0.3147648205818502,
        0.43176325151745765,
        0.6847906100233434,
        0.6186772638803079,
        0.6658780983951396,
        0.9511396915192469,
        0.49584120935059384,
        0.7196597849908719,
        0.9251017284691517,
        0.9539282761049529,
        0.9563970863325733,
        0.5236543728999206,
        0.40658247912502987,
        0.16639526100049773,
        0.897402027589502,
        0.9406515524026489,
        0.9597206687059753,
        0.2505834516496459,
        0.17020744752686512,
        0.1579983569213522,
        0.24608531653069465,
        0.26142150520616403,
        0.4440380640789504,
        0.21258055468993456,
        0.24256040028956238,
        0.21316953787388704,
        0.9320667004544779,
        0.9455688311173912,
        0.9658206147815231,
        0.8858643088898835,
        0.7751391818960249,
        0.9310869882897195,
        0.9551461582670577,
        0.9664037254556886,
        0.9840501679554685,
        0.7553464983539765,
        0.9148266440058818,
        0.9355473499564902,
        0.09027876471502641,
        0.9907768594768144,
        0.49031082038607104,
        0.8891243273551682,
        0.8975870970935338,
        0.9164640000603773,
        0.2869020264632077,
        0.24902627254692605,
        0.8119752519407429,
        0.9915927467674462,
        0.9863038507007076,
        0.9894838964539503,
        0.9546312093858798,
        0.8311812817751251,
        0.5358308555710635,
        0.2961736207229586,
        0.29328359733588605,
        0.590917565576298,
        0.1118282275471413,
        0.42761245298717054,
        0.5290792178271961,
        0.32634741438731096,
        0.18315186706290854,
        0.3674461232226651,
        0.5956793154072992,
        0.7251964954130976,
        0.38858964362253035,
        0.9571563299973784,
        0.9180983344912163,
        0.7386410228460811,
        0.9216389556669887,
        0.2636565392824686,
        0.9540874323640658,
        0.8635895527552317,
        0.8505905017752684,
        0.914605796256452,
        0.09215441221392284,
        0.05513602411628151,
        0.3970233311510961,
        0.4287934924780758,
        0.10406400476629415,
        0.6791285847201185,
        0.8682624804378329,
        0.7611749751188055,
        0.19001467807172653,
        0.9599965157546717,
        0.9472770913499281,
        0.9391427810608381,
        0.4213680477543462,
        0.12838813689996975,
        0.24384938242825713,
        0.8818502720981959,
        0.7724834632017414,
        0.6590705561930286,
        0.9063375220933666,
        0.8103522810065785,
        0.9963711759084751,
        0.43302338927033124,
        0.5244673654679066,
        0.2188426669232022,
        0.9377826391353495,
        0.4950935126597608,
        0.6353870055000642,
        0.2759763172289674,
        0.6630497386411577,
        0.27434356407257615,
        0.8520414560107543,
        0.8516950336002809,
        0.9464261452573287,
        0.6194120939849221,
        0.9924000264991591,
        0.5312550387287618,
        0.7238378781934056,
        0.7458098584608361,
        0.49965042809101246,
        0.9544687153018986,
        0.9043996323350576,
        0.9142108060848305,
        0.8270625522283674,
        0.9001989619084348,
        0.8929212550123763,
        0.3829947285320914,
        0.29295968287709645,
        0.30525851292520084,
        0.986554109603778,
        0.9805819230958528,
        0.9768784064487211,
        0.6548491021571848,
        0.16398717337974153,
        0.7206286835165392,
        0.6389080905785555,
        0.6634952294943118,
        0.8514978441955029,
        0.8874448094141256,
        0.7674464100380528,
        0.8304580087163871,
        0.9487072540850043,
        0.6555385934560632,
        0.667644522576461,
        0.8805161663041147,
        0.9118029382992082,
        0.8276299539054474,
        0.7750064966106075,
        0.6601138839963663,
        0.9086746996588421,
        0.7004534830010027,
        0.5880964177840097,
        0.6748997049762215,
        0.29732776101923314,
        0.24993892965894024,
        0.3594598292870754,
        0.875615567579338,
        0.7714058381110508,
        0.8696790492851896,
        0.2071653452296952,
        0.5584294081694877,
        0.760671918999266,
        0.5684713721965705,
        0.5373477334628749,
        0.9853414651534345,
        0.609740654129889,
        0.43128145137152774,
        0.6689353828118159,
        0.5338165244817306,
        0.2655453079723536,
        0.7087089123532034,
        0.2899249111087195,
        0.46649355030204187,
        0.7610878630686742,
        0.9638209910026986,
        0.9520573272511698,
        0.9667735491625505,
        0.9817331818041285,
        0.977749335242767,
        0.9479922606170526,
        0.7297398290356949,
        0.573814274075013,
        0.9619706692787794,
        0.18878827182913757,
        0.33441909244344337,
        0.2192769230472981,
        0.8797099920442887,
        0.9005981588806679,
        0.8849818315015396,
        0.6466934855880071,
        0.7590845447594073,
        0.6638981334659884,
        0.7966848343590206,
        0.9104800042399755,
        0.9320864809192747,
        0.449632934330973,
        0.710528658041782,
        0.4180805723769827,
        0.08427234594400049,
        0.21362619983881126,
        0.08774439411699828,
        0.8133614345932204,
        0.6470579187705826,
        0.8551484824610682,
        0.7320846742840753,
        0.6773560548004304,
        0.3263779897118192,
        0.8557199275745507,
        0.9086253616968889,
        0.8068755860559056,
        0.7499967522084698,
        0.7396209213716158,
        0.7441597843784691,
        0.8424865150403492,
        0.81362822418554,
        0.893496963859568,
        0.7826611938322507,
        0.5862189110592171,
        0.7328828373485266,
        0.16340871090677744,
        0.3134638982759463,
        0.4233369078215574,
        0.8470158724284126,
        0.6086357673494929,
        0.47031894547589337,
        0.9167580027477278,
        0.22633692178229972,
        0.789374671870706,
        0.7924090858452475,
        0.15909352844430844,
        0.4466048884723334,
        0.5511933375447367,
        0.9151479102592575,
        0.9494367138999459,
        0.07833216537250998,
        0.7723880226731231,
        0.282028541328857,
        0.7440388026516802,
        0.9863944385078699,
        0.9097944227545942,
        0.23883229781669163,
        0.31840548012987646,
        0.35803852541352543,
        0.7414951637967776,
        0.538705833950583,
        0.8101365746555095,
        0.6896162233092394,
        0.6612636024150667,
        0.4601794792221855,
        0.6393009588309408,
        0.7284832852392172,
        0.9508737522628326,
        0.925327338197487,
        0.7223899238071699,
        0.46507665277520627,
        0.1071981141748759,
        0.08377559123287781,
        0.08183349278320882,
        0.6263073721500532,
        0.8054052626059492,
        0.7504965414992475,
        0.18909262318423872,
        0.4560824284392062,
        0.9841299854659489,
        0.08824717302067894,
        0.3500524907234668,
        0.4334778148700124,
        0.6462295225924456,
        0.26162828436457697,
        0.8182989381424602,
        0.9434080704805526,
        0.8938917010529,
        0.9124668001816777,
        0.7298565029308324,
        0.2988916010689575,
        0.9522201156457071,
        0.32097891899341285,
        0.8709464132770129,
        0.08545911063796108,
        0.6627496029486368,
        0.6597953698198948,
        0.6546878254674875,
        0.3420124403298237,
        0.547816891454386,
        0.3608460036840394,
        0.17911180004055471,
        0.6015615174727154,
        0.5215230654382337,
        0.8039893707141091,
        0.685547563867229,
        0.6800362529655424,
        0.936226585076551,
        0.6549560672456967,
        0.6946867096485387,
        0.8791439483253212,
        0.8630860540972782,
        0.7796908556534672,
        0.935083196884654,
        0.9111818546873786,
        0.9896297210873896,
        0.9335125616394844,
        0.8619522249519567,
        0.988573714626054,
        0.7277776749545575,
        0.46664673450151595,
        0.34038126474367303,
        0.21049515123976328,
        0.11762927774301304,
        0.2556928286761375,
        0.8683053877675305,
        0.5349920701237614,
        0.7156377704320693,
        0.8965847444134097,
        0.8068629687961348,
        0.9753885958403357,
        0.8435945596820319,
        0.8652906834903649,
        0.7281143798218713,
        0.29290861090635156,
        0.882070805735071,
        0.9681347912778316,
        0.6557753744564101,
        0.7108343933797094,
        0.6809877033048755,
        0.5771220621619626,
        0.8721384815837734,
        0.9669073709081634,
        0.9199820636270692,
        0.9305083904561979,
        0.8715325198548064,
        0.964027720125954,
        0.5115015581750488,
        0.5616880546863268,
        0.812964319620895,
        0.8778290006829228,
        0.9253056716285967,
        0.14906892188718135,
        0.1425915508340354,
        0.13853497409757787,
        0.9543689701372816,
        0.9174581726639295,
        0.9549705196700131,
        0.992794444980678,
        0.964433879216108,
        0.6276623155334775,
        0.014691331716927492,
        0.49168586790660956,
        0.6827896817174637,
        0.9388973374809811,
        0.7781075214403275,
        0.8634346235720473,
        0.9758382685994564,
        0.9758042192860235,
        0.40457903727447925,
        0.9695599628979688,
        0.9952104664553767,
        0.9895357868143428,
        0.6369061321531773,
        0.28821459839981456,
        0.7085254069550441,
        0.5149992612542845,
        0.44782419806067525,
        0.6181805010883544,
        0.7818570651789657,
        0.8780317256997854,
        0.9109656994187622,
        0.6899554249092175,
        0.5237774178796315,
        0.9717219127937988,
        0.9424009770952657,
        0.9287105341521141,
        0.8971412423968876,
        0.7509656857259683,
        0.6831613403857524,
        0.13299645753409853,
        0.6868365455404232,
        0.7120546078051168,
        0.9910052387920201,
        0.9549517964435146,
        0.8664366373013512,
        0.9776229695418658,
        0.6678744812942468,
        0.7183476464811684,
        0.35369382579025566,
        0.06310795111971679,
        0.02894891349250206,
        0.04609836274419943,
        0.9926526784673266,
        0.9882102673281089,
        0.9879812092043503,
        0.9431750982246204,
        0.9335479755458984,
        0.8299649586281128,
        0.7837374046301608,
        0.8426380001888937,
        0.2813123175655066,
        0.7489991893467622,
        0.9311309205684482,
        0.947412139282352,
        0.525051752244098,
        0.21049926714089484,
        0.6639334160819961,
        0.8835388213656741,
        0.983553432247489,
        0.7585458337225387,
        0.9319671226176798,
        0.9206347656141423,
        0.9348594115528938,
        0.6622314007222595,
        0.8949403977927949,
        0.8461825786057738,
        0.9005994379845901,
        0.6729120809155403,
        0.9545137334397801,
        0.14644230226756866,
        0.9691098502462763,
        0.9026523620990681,
        0.261835537314141,
        0.702522360967702,
        0.08870365615088815,
        0.9278088899518523,
        0.8477638511538935,
        0.7662255868178899,
        0.905019473790464,
        0.9030717394875726,
        0.8297788293249304,
        0.6534523885346651,
        0.5932177859848278,
        0.550527823550965,
        0.9084244306176261,
        0.9661111015453898,
        0.9128807112344137,
        0.6977589981775658,
        0.555755694113991,
        0.29951314073135044,
        0.9391557370036057,
        0.8657612111411328,
        0.9427077042007725,
        0.6304338365715358,
        0.5836826635352753,
        0.9585703046532212,
        0.7399123103859757,
        0.5802190596458235,
        0.5500108623181733,
        0.3836049303392427,
        0.7684791192704544,
        0.544627806565091,
        0.19175554759426086,
        0.0454396381013635,
        0.15015358469446982,
        0.1614982989066253,
        0.37144788046404775,
        0.2355058963007669,
        0.9918860726578104,
        0.9691744584337569,
        0.9780580742331236,
        0.6602642958730388,
        0.7884305026767802,
        0.8827899678232829,
        0.9412337683814423,
        0.18840506843678417,
        0.1714606504375133,
        0.4578419699312799,
        0.6157854345892305,
        0.45118026489778396,
        0.9622401695834794,
        0.9391643365829404,
        0.9414343409900524,
        0.468448969153889,
        0.6782470000928683,
        0.6697706504628764,
        0.972669631377518,
        0.9732890256383601,
        0.9547963520695597,
        0.9577997994812919,
        0.9424617977392408,
        0.9251556903166929,
        0.95225520143593,
        0.45654653059578437,
        0.8837690737987257,
        0.8514067778594121,
        0.634863393709372,
        0.9374947906534981,
        0.9313202224705054,
        0.8575062633244128,
        0.9361801684154094,
        0.6947400729713021,
        0.9050100570374717,
        0.7274922392764744,
        0.48134326859642157,
        0.3629198602340617,
        0.8418487545131663,
        0.5844210927665507,
        0.8195084637997195,
        0.9263586142991528,
        0.7136141360774206,
        0.6783900013165143,
        0.2809769577616779,
        0.5940845352338837,
        0.8628402575249661,
        0.8394963090175116,
        0.9217352135826163,
        0.9747159762862757,
        0.9754590675419706,
        0.9248437744885374,
        0.9535446710011016,
        0.9920299316726252,
        0.12405955357792399,
        0.23735558544762492,
        0.3153689959560394,
        0.31237929878685755,
        0.363803295488637,
        0.7923736933087444,
        0.24957713172656404,
        0.1462553471359904,
        0.15025839397530039,
        0.9357939006473937,
        0.9325916054198435,
        0.9491781845437208,
        0.8530288469555186,
        0.7659364222205822,
        0.5537717923658096
    ],
    "eval_rms_calibration_error": 0.28920527449290784
}