GENIES/results/evaluations/cra/llama-13b-code_easy/code_hard.json

{
    "model_dir": "models/llama-13b",
    "distribution_id": "code_hard",
    "timestamp": "2023-10-16 03:11:03",
    "eval_score": -0.2051598713615981,
    "eval_brier_score": 0.2051598713615981,
    "eval_average_probability": 0.6337841224158034,
    "eval_accuracy": 0.6893333333333334,
    "eval_probabilities": [
        0.8111203547123992,
        0.42876153800514166,
        0.35629003095287326,
        0.5021091712394413,
        0.23910183756455508,
        0.2574705141653944,
        0.547867813934518,
        0.8757544858838489,
        0.28762085764445877,
        0.16946609872563914,
        0.16294566373366617,
        0.16241756198020138,
        0.8569096555600244,
        0.8789925425090209,
        0.8775461245050838,
        0.8284087946990792,
        0.7666050512115135,
        0.7300328380848868,
        0.7914068427459103,
        0.5748542540697583,
        0.5300348339736685,
        0.9213828286661256,
        0.7955033312667606,
        0.8473450900798266,
        0.97679163016992,
        0.9668827382806526,
        0.9663005645357525,
        0.5613125605725862,
        0.8494126208285071,
        0.8185330480536493,
        0.23341527195609912,
        0.7075664284685127,
        0.321122414253448,
        0.5343772021860596,
        0.43222186987824307,
        0.4059319939487054,
        0.8210401820059411,
        0.8219184215143012,
        0.8266947740599668,
        0.32627492882117914,
        0.3017628130530903,
        0.34786988047703504,
        0.9529870466504294,
        0.9379625714426537,
        0.9668399153385198,
        0.8886028456932102,
        0.37915232221910866,
        0.37539397375456235,
        0.8987199507141698,
        0.984103569232523,
        0.972101377678243,
        0.2220711027951222,
        0.3717363824309401,
        0.6874218454464953,
        0.48168594364401796,
        0.44948376013334457,
        0.5487190863302978,
        0.2130734060346301,
        0.6097884209637612,
        0.8987533198761994,
        0.6280418148157871,
        0.6616756559742043,
        0.5523096618483055,
        0.8893255451093267,
        0.7548271795718713,
        0.9851074363948574,
        0.47135066684383453,
        0.32506422530083834,
        0.5426514515257079,
        0.8684079069143301,
        0.8909525506443712,
        0.880428512248448,
        0.33604123397838215,
        0.9831053332589322,
        0.9465141045515892,
        0.07799513902069943,
        0.43151585179464813,
        0.28211432585523905,
        0.06218802796041101,
        0.17760501048137656,
        0.09420646702010527,
        0.7330511445613664,
        0.44346281842073454,
        0.28940511072273994,
        0.7323880075560266,
        0.7690474367041145,
        0.2542852701860673,
        0.703986092885759,
        0.6855972855810957,
        0.4257514475914736,
        0.9822021416531225,
        0.92000100951829,
        0.9157351592918227,
        0.058581791489242446,
        0.06032490036859464,
        0.0636928741369346,
        0.7428284256769665,
        0.28600300181015115,
        0.4700103185528312,
        0.9354283480627368,
        0.8974432303838243,
        0.7581287660494131,
        0.14479595530250666,
        0.12435876914477077,
        0.15419509391148134,
        0.5991978992094242,
        0.5854738533793952,
        0.8294121015879393,
        0.5328844093472022,
        0.5364795175690311,
        0.6471857373217742,
        0.5546238041634266,
        0.5187473991841722,
        0.6325261220254945,
        0.8884970958177859,
        0.7300101929715246,
        0.841039296088527,
        0.7517264507627354,
        0.7401311068290185,
        0.7294352470142906,
        0.7011803386574448,
        0.9394174433033369,
        0.795623835835823,
        0.826944215775091,
        0.38835129297466886,
        0.7974205076801856,
        0.9141173087655848,
        0.8192155137376664,
        0.8799321944888349,
        0.9391358453209678,
        0.7671674278216075,
        0.9335157976990791,
        0.26205589161513226,
        0.176048180933695,
        0.909442351871489,
        0.9901522711821196,
        0.9352919420604001,
        0.9880873918225128,
        0.6006476341468413,
        0.5779964094119718,
        0.6553548608010347,
        0.6463473014908985,
        0.8557553323944747,
        0.9459606079074736,
        0.4361461769739773,
        0.7311885490834564,
        0.4542433372639969,
        0.5965598557988452,
        0.8291015685936834,
        0.7760759784530878,
        0.2503809135764354,
        0.2596556873057609,
        0.23305628872771977,
        0.20198096656416387,
        0.17244415201112295,
        0.20682774447656474,
        0.8945229626918346,
        0.8757844503482278,
        0.9157423300153003,
        0.4991516360773615,
        0.4348760322281906,
        0.34277525124619296,
        0.7693306164939925,
        0.8698816395565712,
        0.9467566022690008,
        0.8874653456393203,
        0.9042942386335459,
        0.9402335799346166,
        0.17962464979226947,
        0.14530890678547437,
        0.1785418687456074,
        0.7750815560898545,
        0.31118269053960235,
        0.35372754439813786,
        0.9095932225445739,
        0.5153589322606328,
        0.6454166044555762,
        0.406406257585993,
        0.3068004692212333,
        0.35786995826659784,
        0.8884608868572186,
        0.9806413635258723,
        0.9271044241860446,
        0.8451381813170494,
        0.5675837390714953,
        0.5792302713693511,
        0.5515792474100583,
        0.48619667532143085,
        0.9197559656148028,
        0.9215278648178156,
        0.9832578681221297,
        0.4681194586075689,
        0.438415028196654,
        0.6717694199971331,
        0.2229369103776978,
        0.5178759514893421,
        0.3966550479317031,
        0.5928255669767548,
        0.8368757655697712,
        0.8631083543552549,
        0.7375028447001647,
        0.8372678095258975,
        0.7487388932347897,
        0.9413180735188967,
        0.6926145615047724,
        0.7809938121587534,
        0.8039941664999641,
        0.3011213041977667,
        0.4056555588472057,
        0.23700422806420596,
        0.4948997685945172,
        0.5031415281132476,
        0.5660657268765314,
        0.717088100378473,
        0.5775528867667545,
        0.5816016820306453,
        0.3543126541053079,
        0.6393846130038758,
        0.8508214356750878,
        0.8651334698152502,
        0.8315522277134558,
        0.7181850984047548,
        0.0936333265919651,
        0.1334551320220036,
        0.252076786191571,
        0.9575113906896677,
        0.9246969879238254,
        0.35419467813784683,
        0.4546374376191091,
        0.3815205394148913,
        0.4955020788280529,
        0.7772228689714014,
        0.7848334897715719,
        0.544538291704932,
        0.3349916568884557,
        0.3020004275926698,
        0.2884289772482117,
        0.8472416822172157,
        0.7068807041797214,
        0.6584037009336637,
        0.9464743146879678,
        0.9184332225196178,
        0.7984343174058872,
        0.6258015337946421,
        0.587005051425573,
        0.7987386461893662,
        0.389885454006514,
        0.6388985656843977,
        0.7919837857299207,
        0.7403569801700616,
        0.7558940155967333,
        0.9080834134937544,
        0.32633407024644456,
        0.5817305767479064,
        0.34460983871651,
        0.9016465885078341,
        0.7638466510224146,
        0.9138429601403913,
        0.39279780752910676,
        0.4632158743700654,
        0.3633689602993956,
        0.8187385079068026,
        0.8061077884366995,
        0.9252397837598069,
        0.6568736201945633,
        0.8694416560222629,
        0.8221913470427815,
        0.7315055190214201,
        0.6922589349222186,
        0.7463691459283359,
        0.8391057270993628,
        0.35302880385483715,
        0.3919079414948035,
        0.9039952525018651,
        0.9164320555526168,
        0.9135071228236814,
        0.6001960443193833,
        0.8490521997813544,
        0.7316772951340937,
        0.6795767659118406,
        0.6532925605324879,
        0.4266647460712135,
        0.9941157737154157,
        0.9588462727315433,
        0.9384234329215033,
        0.9147651366409055,
        0.799736360741409,
        0.9272707390949606,
        0.1287684018775881,
        0.06978363251261521,
        0.08439763531274182,
        0.49030660551525773,
        0.40330238376411087,
        0.8521381739344775,
        0.7749921446274171,
        0.8005526339440598,
        0.7315761070300946,
        0.8547034154947135,
        0.8365323105532967,
        0.8567523345650027,
        0.9818792045962711,
        0.9169625904507985,
        0.9367292590606855,
        0.647387962308453,
        0.5400793482199103,
        0.8130898071493037,
        0.9887582645597679,
        0.9650964330462412,
        0.9456126339231093,
        0.6125792392486504,
        0.6313812369820365,
        0.22007599755463136,
        0.9050885832517037,
        0.7131484856125548,
        0.5174012306146203,
        0.5760057667469499,
        0.6830456865175174,
        0.6200379406796316,
        0.913095582134415,
        0.36893254539205605,
        0.8583274991960708,
        0.447291379622058,
        0.9537583451712823,
        0.8258298867907817,
        0.9647318921240906,
        0.9236038084176635,
        0.8932186859601301,
        0.7331913885319397,
        0.7821556321156482,
        0.9918486848435855,
        0.16357862613115737,
        0.3551497083796004,
        0.2614863453729813,
        0.6014976497735249,
        0.2579723361963163,
        0.3837393541021169,
        0.5454011045411018,
        0.2930282782623831,
        0.6744280697177292,
        0.23100088412171366,
        0.8165677065583503,
        0.10283984113073237,
        0.5107620826530797,
        0.7763666066882556,
        0.4512437688908395,
        0.06383197049277764,
        0.05692251920026581,
        0.052620834119669224,
        0.9818495300536676,
        0.8975295090070974,
        0.9135100598582543,
        0.9705174090335861,
        0.9665613098505111,
        0.9717948060243784,
        0.47422204540375745,
        0.5055883949906669,
        0.623105001604833,
        0.9816222173135417,
        0.964173594051677,
        0.9771287456902995,
        0.6708310771128783,
        0.591213129718314,
        0.46535362263603497,
        0.9210928802252172,
        0.2566409528171969,
        0.7225383310936296,
        0.551436407079336,
        0.549939606056484,
        0.49338857114364376,
        0.48445643704682906,
        0.6511640190585453,
        0.3950530361973381,
        0.15148650349862514,
        0.8128001076047093,
        0.4309614629348906,
        0.603456148607875,
        0.8856923091856791,
        0.8485848768662009,
        0.9191413014332953,
        0.7960682974992166,
        0.8008152004668639,
        0.065512478527358,
        0.19196873723843913,
        0.12463063964214871,
        0.8570119141263707,
        0.8707539153122887,
        0.8616528635989043,
        0.6902499743211516,
        0.29931280207817035,
        0.1012577508378939,
        0.5545874922597768,
        0.5964263729588039,
        0.9769992656458879,
        0.8873627904918736,
        0.6100288201923298,
        0.858013199913948,
        0.6277031745055945,
        0.6333205806679751,
        0.5064192620279395,
        0.9647733059191386,
        0.9733544104429437,
        0.9962405143610584,
        0.9191964753103707,
        0.9626638015052914,
        0.9680740054780432,
        0.8956346293782312,
        0.9628338374534098,
        0.9430101401310467,
        0.9507002092528937,
        0.9090512618597076,
        0.8170016042117199,
        0.4455387691800247,
        0.42084559508277974,
        0.40964440501601673,
        0.5722669626692953,
        0.7182973304406997,
        0.7765944012603313,
        0.3126332421012875,
        0.48316667544066394,
        0.23826692172632935,
        0.9469301739436264,
        0.869313534278969,
        0.9051599719010576,
        0.23690685695043373,
        0.6431487285480696,
        0.528961421674959,
        0.6999507632978276,
        0.3894193172909006,
        0.6385367358424471,
        0.7817629706816283,
        0.7308223358139292,
        0.5635378812156713,
        0.9479073971012276,
        0.9468689808367137,
        0.8990418893363415,
        0.6732929604540706,
        0.6653733626037752,
        0.8667326343243751,
        0.114345159218243,
        0.11892875115158008,
        0.12429718521045818,
        0.9562638755764612,
        0.9649181874454325,
        0.954362396477598,
        0.3714132470562538,
        0.5559082319823843,
        0.49728530900299506,
        0.6689899870529674,
        0.2366702061392278,
        0.9048329931475759,
        0.7039583732601306,
        0.5575995695110998,
        0.2653176573947842,
        0.5982958995731567,
        0.5742872916023205,
        0.8651461413771985,
        0.8888758494605907,
        0.7640750995112445,
        0.84082989251038,
        0.8061540735328068,
        0.8219871643754408,
        0.584698514918495,
        0.429088181985404,
        0.8106860448043928,
        0.8023349502208018,
        0.7659664027252905,
        0.8029029664668398,
        0.8083218314906914,
        0.4821413454676216,
        0.4274390908943394,
        0.3681214402979902,
        0.5872818398467256,
        0.4555316929769888,
        0.669358247497442,
        0.11660437983527869,
        0.09052189146495226,
        0.25923372657701316,
        0.5868291060237829,
        0.15374921276603015,
        0.5465875900470253,
        0.607889293242365,
        0.8353995398615706,
        0.7806737159251285,
        0.30378918008202327,
        0.2756760101218896,
        0.27541251129314076,
        0.7818989250947409,
        0.576280365430434,
        0.696162652784875,
        0.9760260566952037,
        0.681774257044772,
        0.9695215623889778,
        0.7152134283875882,
        0.7841180739654443,
        0.3448793106395892,
        0.9865662020881342,
        0.932118862456996,
        0.951019483202426,
        0.3810266753612255,
        0.3315662557629474,
        0.4528280168358752,
        0.6568146730897844,
        0.7902460631813343,
        0.7570686441302198,
        0.8452440791422483,
        0.930057074308778,
        0.7441556359323627,
        0.6723744462348178,
        0.6706963716475318,
        0.6960341527394469,
        0.7432691734178588,
        0.4697008541881045,
        0.5002727283853821,
        0.7117790479701718,
        0.6729791065537283,
        0.5303344983392283,
        0.7771430546073458,
        0.880504228027637,
        0.9093716359142631,
        0.9118318383762332,
        0.21638102171461104,
        0.4363509502832808,
        0.540384037004236,
        0.4348345076536889,
        0.6010994580476097,
        0.19659989608072595,
        0.19840931804143158,
        0.1786126858302819,
        0.5583741869054093,
        0.7619902765879951,
        0.6906997420037422,
        0.21657310223381887,
        0.2930293283706029,
        0.5075301177686801,
        0.54651347041486,
        0.6668084765121703,
        0.3421673180104743,
        0.682194319103757,
        0.7702577261942108,
        0.6684650126340228,
        0.9505321051617512,
        0.9023939511809953,
        0.9084722441812167,
        0.969603239538609,
        0.9668265323672837,
        0.9440856155646021,
        0.33198383017918975,
        0.33082071353382414,
        0.6438746482561077,
        0.627560618522732,
        0.5657988360238446,
        0.6712568377236048,
        0.4004190959752734,
        0.9013040077788164,
        0.9210163380242252,
        0.4190996105077809,
        0.3659079432353123,
        0.48850707682756983,
        0.6058190530144536,
        0.2102348708864571,
        0.7276826367924996,
        0.8675680287997812,
        0.768285938306158,
        0.8355245012670869,
        0.46659229728722773,
        0.49299800234825,
        0.49149414040778294,
        0.18363180957218683,
        0.46980649271701924,
        0.8532546569780387,
        0.9709287889353009,
        0.7928708650971552,
        0.7508783690766334,
        0.7432322543406495,
        0.6074158728547866,
        0.6526112968940714,
        0.22263790287728058,
        0.36827080427233794,
        0.290377106599297,
        0.8740603641182447,
        0.7620073517331234,
        0.7384950554748292,
        0.9313533656034174,
        0.9627649676821073,
        0.9418950698076397,
        0.19567965087801176,
        0.20182311130591557,
        0.7806003674345142,
        0.11118758749975133,
        0.0911572093803624,
        0.11108815206300247,
        0.6245681020127153,
        0.8217680049468917,
        0.9110329367906879,
        0.10885700417257071,
        0.8826929439496467,
        0.5473926058956812,
        0.9787563399638654,
        0.9907386670119791,
        0.9904680585030661,
        0.7105783064274084,
        0.8931911605445703,
        0.18164628835609936,
        0.2838283803902697,
        0.8292293768386254,
        0.6608957281569782,
        0.7628332501429815,
        0.7313773099310374,
        0.9852312912582478,
        0.3187974231991365,
        0.9177469819071467,
        0.18304159808650688,
        0.16143930493780478,
        0.15268446656531354,
        0.1515975193631992,
        0.8969148332967698,
        0.9132792278198899,
        0.9189708275419766,
        0.7718398037010231,
        0.8775182912327575,
        0.9737867759027329,
        0.8336258994026875,
        0.5238265601809481,
        0.8694018099424243,
        0.8236619841231845,
        0.8823669058520817,
        0.856589519111967,
        0.8240130843863583,
        0.4992302053235235,
        0.81725679660242,
        0.7190017107460196,
        0.03848066561193064,
        0.3701075893840164,
        0.608138163775285,
        0.7930891632131797,
        0.7681227983755697,
        0.17892549068126085,
        0.30272716353297846,
        0.9339008799965586,
        0.8625245113382445,
        0.62494481035884,
        0.47847382071528965,
        0.7047043387745963,
        0.7694524162056012,
        0.769426854214343,
        0.26413629436399944,
        0.23500750427188077,
        0.5540671665244299,
        0.21544284186409868,
        0.25844192634545254,
        0.13574524999022472,
        0.6400756501964764,
        0.2917568195773208,
        0.5999036767394701,
        0.21148750594343563,
        0.14337545211468794,
        0.1788909892106001,
        0.9775499950829397,
        0.9905032485387402,
        0.9790336705416633,
        0.5486386580496876,
        0.5373781372032558,
        0.28811350732170665,
        0.8450558983392537,
        0.8479174439424066,
        0.8649835685549788,
        0.2901484124588315,
        0.8267119610153716,
        0.8014151431391,
        0.757061030525759,
        0.7366784317599782,
        0.5743947838549677,
        0.8652625483005842,
        0.9082417099190984,
        0.6156215203416511,
        0.4386194140233719,
        0.3331544639776817,
        0.6696765908992519,
        0.3044146755751628,
        0.6118057362353625,
        0.5040382716797788,
        0.978352503966534,
        0.9659621463810395,
        0.9740645606937126,
        0.9749904103402554,
        0.9703925982419292,
        0.986136103732717,
        0.8650174072374538,
        0.501179114599123,
        0.9083793336725745,
        0.8573855070830172,
        0.7991231848134337,
        0.7902637073160206,
        0.9114039209442095,
        0.9056626786573809,
        0.9231248049039908,
        0.731223584918972,
        0.8341251587357289,
        0.5092263692055865,
        0.9230231814790003,
        0.9133802101697016,
        0.9316372329244516,
        0.892236942268664,
        0.9216427656753776,
        0.8235445919135951,
        0.1825683909208826,
        0.0877403613092983,
        0.1497651425890717,
        0.8042862333825108,
        0.7421513603293064,
        0.94077316330415,
        0.5642677358825612,
        0.6062246953054354,
        0.6439589982806925,
        0.592208831627488,
        0.9551303004429356,
        0.8699354136597595,
        0.7374312020979437,
        0.3575173873888605,
        0.7576849629361034,
        0.9379070687321349,
        0.9360197461159471,
        0.9490525527024206,
        0.2414614064385773,
        0.40483556942709886,
        0.45027463891354635,
        0.18988088747580234,
        0.5525603243191808,
        0.4883871746150834,
        0.6958351764583728,
        0.37146689772314057,
        0.292425530355787,
        0.26720147127693306,
        0.34721891896324697,
        0.794303664602091,
        0.7823608343857091,
        0.7856817742259691,
        0.596960150438365,
        0.3040526235295127,
        0.5532427100792244,
        0.9212159329654979,
        0.121224868050833,
        0.13639575950860913,
        0.23134555422071645,
        0.9590036355978845,
        0.9673028688577914,
        0.9795264588473932,
        0.6876459933742551,
        0.8598830863899688,
        0.6745425978171942
    ],
    "eval_rms_calibration_error": 0.28268629289852854
}