GENIES/results/evaluations/cra/llama-7b-code/counterfactual_python.json

{
    "model_dir": "models/llama-7b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-16 02:39:45",
    "eval_score": -0.22413425940205714,
    "eval_brier_score": 0.22413425940205714,
    "eval_average_probability": 0.5909104955933082,
    "eval_accuracy": 0.6386666666666667,
    "eval_probabilities": [
        0.32091029398879994,
        0.26502119767170096,
        0.4880332935757866,
        0.18918028717689966,
        0.36443103340033167,
        0.16584661958670577,
        0.9338027821469045,
        0.9341575982577945,
        0.9390181875860292,
        0.5924806023117403,
        0.6641987530755336,
        0.43507002260843924,
        0.6851536084292659,
        0.876853926251999,
        0.48740524202998836,
        0.6362804337569562,
        0.7284779529826688,
        0.777637056936951,
        0.1368032609832388,
        0.2271450412052427,
        0.16158351359285666,
        0.7831041991843808,
        0.7521037761281348,
        0.6115384762474477,
        0.3009586281913479,
        0.7431379633137007,
        0.7368015700555193,
        0.847532299249242,
        0.8687331829060132,
        0.8475430088581016,
        0.7920292361641028,
        0.5888648187996052,
        0.6911953321397883,
        0.771486155979221,
        0.1388128146558757,
        0.7068663704523095,
        0.6034014850149961,
        0.7201934646539876,
        0.7423370486626502,
        0.8432442027139422,
        0.095870191672912,
        0.8021209995749773,
        0.8078245200333327,
        0.819615746849307,
        0.7298099009438794,
        0.5028294548180352,
        0.8777870696368187,
        0.3405897710150222,
        0.587560304881147,
        0.6300510473303697,
        0.3284109010367653,
        0.4901468524462411,
        0.3203485949075622,
        0.6193070225539823,
        0.2046423999118336,
        0.27487094630453635,
        0.16438049995362014,
        0.7334809077144308,
        0.3928434484400797,
        0.49800833110498327,
        0.8609019688790117,
        0.3432099606314762,
        0.2757045008256879,
        0.11302613364310576,
        0.24823944314631463,
        0.4529777573517415,
        0.5723074753228824,
        0.32481376242822524,
        0.1550149266571087,
        0.36151248319397294,
        0.533963677433386,
        0.8782817899629024,
        0.7816310740812971,
        0.9154242594420006,
        0.6497319521176879,
        0.22382533265934043,
        0.7885012656266479,
        0.6248500637713331,
        0.09398143422950478,
        0.7536227639404508,
        0.23373871005952618,
        0.7038673598024405,
        0.8313216333656072,
        0.3055616743521512,
        0.6782955431525574,
        0.8769133283702333,
        0.45875699210968324,
        0.4073339837073172,
        0.2875914298317699,
        0.42214397286589506,
        0.41516440062634513,
        0.07813325163867157,
        0.3052861830009699,
        0.8124423354889786,
        0.254549191364936,
        0.9024386409412213,
        0.7242268011156624,
        0.9324576093619107,
        0.8131970871068932,
        0.18576770232760342,
        0.33642258359014465,
        0.3755908157030591,
        0.7554707182367898,
        0.5673086270786516,
        0.6084943572318269,
        0.9423970987985761,
        0.9319807269635545,
        0.9313609410521742,
        0.9529227005757248,
        0.9584522046748117,
        0.9528669399423572,
        0.5338863546507568,
        0.39789144253653147,
        0.5048223579091901,
        0.319464528470488,
        0.2014134645720373,
        0.6278819619607701,
        0.5139220111132463,
        0.3642153442205288,
        0.36062955311902045,
        0.29324211467597633,
        0.8614279947650281,
        0.7264107376109443,
        0.846687315008847,
        0.9063138940341966,
        0.739276343190144,
        0.5927087557833477,
        0.7673291912306743,
        0.43738519109728335,
        0.5027983567962112,
        0.7687388838937607,
        0.8427676299429929,
        0.22542969864670676,
        0.27337812153525265,
        0.3791777162586631,
        0.6607394154280024,
        0.31915409431783925,
        0.7235911631127974,
        0.468105959883434,
        0.15643983027281994,
        0.7316230149940863,
        0.8695242755811095,
        0.5068783054442902,
        0.6649393494510195,
        0.3378973205311935,
        0.3187483328467138,
        0.4691580897526729,
        0.6415460805781941,
        0.6177821046437135,
        0.7129733703068333,
        0.6317831159671862,
        0.5130631588559634,
        0.7174472985403759,
        0.6500948305434671,
        0.73530942778322,
        0.82660611846085,
        0.8633458869137812,
        0.8527215126907598,
        0.843140843133588,
        0.5007178619657695,
        0.6950036845035346,
        0.7213344622212244,
        0.7200628845089467,
        0.3150380198323297,
        0.4465513982029954,
        0.3564776450100354,
        0.239054967093144,
        0.21947816249940938,
        0.1285251032494282,
        0.1144420952286318,
        0.6634341642122166,
        0.8816767040303114,
        0.8112955812475516,
        0.8373738610554462,
        0.40495206612623874,
        0.5106617084784574,
        0.6666739539296906,
        0.8461535154554995,
        0.23123353163829619,
        0.8310107723573095,
        0.6481105079724875,
        0.6494155583091366,
        0.6899471456599887,
        0.5051582902597105,
        0.22083495329911673,
        0.2374893863939742,
        0.3164171287255507,
        0.7021742805541163,
        0.43813361814495155,
        0.45193929195066024,
        0.376265274853292,
        0.392184891819747,
        0.8065655603599605,
        0.8209865668520442,
        0.8355312951351921,
        0.4586799371344955,
        0.177237337555862,
        0.7130597813128486,
        0.4193562512330392,
        0.23220643683857156,
        0.589009529569958,
        0.89075060733974,
        0.8856674066940782,
        0.5811055209069974,
        0.49059068163513453,
        0.483301005843234,
        0.4336877417826779,
        0.5892076157145452,
        0.674177743762684,
        0.1661317999859404,
        0.5974671397825215,
        0.536913492608415,
        0.548246382222112,
        0.5367534693255503,
        0.3862941143775006,
        0.38078288215118117,
        0.2925645830259199,
        0.6445666227437612,
        0.9318861195081893,
        0.5633346059273063,
        0.6927159302491602,
        0.7481372419169181,
        0.4409633653977314,
        0.5039458574479146,
        0.4952146301263756,
        0.7802979547802729,
        0.6334532534080527,
        0.7802230815676089,
        0.7408702904086788,
        0.7604652640677205,
        0.7546205509983355,
        0.7234708831978456,
        0.6290012711330958,
        0.5787630126506339,
        0.9476502127658326,
        0.9505938170264376,
        0.9510344500838283,
        0.8986632895993714,
        0.885453756846255,
        0.8765625002516255,
        0.7005002285030372,
        0.7058471987231674,
        0.8981256267332867,
        0.4802406907589932,
        0.8764597026744833,
        0.42383972991390895,
        0.30064838932088,
        0.36487281643352854,
        0.7664467966651504,
        0.1947574413279145,
        0.37767180990664895,
        0.2267872163838851,
        0.7236565012737306,
        0.5647573036874598,
        0.58953910604106,
        0.6945059217523915,
        0.4267732272074082,
        0.6411915949504431,
        0.4926254536254566,
        0.5054037813401988,
        0.5725980599817121,
        0.8493679839170433,
        0.889548535262129,
        0.7321604266199855,
        0.6269354322289151,
        0.3743155652982404,
        0.4031975169617057,
        0.6659728424708569,
        0.7362569581639542,
        0.6918786873088779,
        0.551469184103083,
        0.6743335365226988,
        0.7802890876953928,
        0.29118488232360934,
        0.6550150065897887,
        0.7652527293322429,
        0.31039015480753657,
        0.24593810420477902,
        0.3111636170083655,
        0.6833692088771441,
        0.7038131792370409,
        0.7325030152255272,
        0.5345996761866411,
        0.5936378463655764,
        0.7042781548942257,
        0.6664339963799728,
        0.41296505083617197,
        0.17405546725934695,
        0.6670779321813848,
        0.7703104689685508,
        0.6898255631619508,
        0.40112434356198184,
        0.5111983860330507,
        0.8784752238872638,
        0.8323287952072315,
        0.8801214137756092,
        0.8997690979545725,
        0.9148259817056322,
        0.8924051500761789,
        0.8377442416201741,
        0.6206338659482357,
        0.5304471319654265,
        0.527195697383809,
        0.40287790169241466,
        0.41623321470666164,
        0.5404735551791577,
        0.2714649406730493,
        0.26686846532310926,
        0.15774717388205906,
        0.21024805285084414,
        0.2072932939729391,
        0.5941953269893827,
        0.6888585550698344,
        0.36104134161284407,
        0.19697056293769732,
        0.34672793810557057,
        0.6568813142917376,
        0.35756075371249385,
        0.3248455803593089,
        0.4943775030134569,
        0.7140072700434797,
        0.17717427754922177,
        0.16334081003925274,
        0.12533975088684965,
        0.2703481486827798,
        0.5781037183338706,
        0.3391977126645864,
        0.9003497256590228,
        0.8532281132365357,
        0.8710131205543146,
        0.1471479632211231,
        0.32420075013700295,
        0.09572013577190819,
        0.7630032392029888,
        0.7661522178117972,
        0.8505956535421605,
        0.5370407748909536,
        0.4651844941675035,
        0.39504781283865814,
        0.6042691735339768,
        0.26018303609509774,
        0.803873838320855,
        0.5768393019082403,
        0.510326217865612,
        0.5469321881263451,
        0.4151508090174416,
        0.6360276482336631,
        0.2718030149939846,
        0.37070390453787216,
        0.35624593127729076,
        0.5967248115073254,
        0.46587659396848097,
        0.5324779086951588,
        0.25149167847161663,
        0.7903293193178962,
        0.7156851006767896,
        0.616676442463606,
        0.4729187035628283,
        0.8169238484329452,
        0.7872146169064458,
        0.3534075027239477,
        0.3196214785348942,
        0.6389821442679755,
        0.7844535124027973,
        0.8796949250909625,
        0.7415834634069758,
        0.4282780404042946,
        0.8206799908315418,
        0.46885018455614763,
        0.26112262854667,
        0.30319832512143596,
        0.3161227882552988,
        0.7249343644559829,
        0.7620049245187046,
        0.6501323597831747,
        0.9488565131399547,
        0.9483234064507811,
        0.9474484512811794,
        0.25986393082058934,
        0.13663781558215626,
        0.850702870557567,
        0.3067066544754532,
        0.38045374897619766,
        0.14894900102364428,
        0.7827450609148777,
        0.7652243832157679,
        0.8120714660380773,
        0.4591245746746737,
        0.6260215610711084,
        0.37270478306723787,
        0.5118506273543588,
        0.6645234395176958,
        0.5834689477752124,
        0.3216144578738287,
        0.4497086013653244,
        0.33823341555581665,
        0.20822678430279196,
        0.2431594219990576,
        0.12681230176057023,
        0.30606787955891473,
        0.39084344260473697,
        0.2461577451254363,
        0.9631889345311782,
        0.9538965625395865,
        0.9586726433759856,
        0.6970962058185728,
        0.7600978157223042,
        0.8050714416299197,
        0.8414432908118937,
        0.8503128414607056,
        0.8419840227328452,
        0.1820189499631431,
        0.8150885676415314,
        0.5189596287939885,
        0.8140623410092749,
        0.8215764554628564,
        0.7594022956099089,
        0.2774486422540653,
        0.8858133386115147,
        0.6394026630034653,
        0.8227674923869481,
        0.7417985827865848,
        0.5010262322223387,
        0.8968788535554533,
        0.9044980173044525,
        0.7114252641598336,
        0.8791330751184132,
        0.8979776288406556,
        0.4756676976471544,
        0.36197670895190975,
        0.6918694878965373,
        0.8556630587262112,
        0.67816523547217,
        0.8578005179708503,
        0.7457276290250081,
        0.9003313675375072,
        0.9030541119227006,
        0.928320413893282,
        0.5569907433340487,
        0.5571126975097805,
        0.8907859891559105,
        0.5364547948792282,
        0.2530734396354511,
        0.4027101721042283,
        0.7060701412812651,
        0.8754312820606771,
        0.5085412813699135,
        0.8241963943386541,
        0.908240520151147,
        0.854611338550339,
        0.8296200233933355,
        0.6613086098890602,
        0.23891612664956718,
        0.24800370908037428,
        0.4865648294265966,
        0.27037495960788616,
        0.9071287238925846,
        0.9122466673080081,
        0.824866564935449,
        0.6680257284426732,
        0.6237201278001069,
        0.45424201686063226,
        0.27149626536526283,
        0.4445438980385079,
        0.4140505431793443,
        0.12117754574145068,
        0.11317488468837311,
        0.10295632269111636,
        0.927562644404431,
        0.8635750146346797,
        0.8911770015976948,
        0.6476929936110976,
        0.5457157826934367,
        0.3912492542637135,
        0.4767625180363308,
        0.5321687073069521,
        0.5552057343688849,
        0.6023260137159421,
        0.4144712204187938,
        0.11729453528384337,
        0.745281315087765,
        0.4332714919507364,
        0.5861070053943285,
        0.6177709597440295,
        0.44209926220063533,
        0.6248775351389966,
        0.3339919127209172,
        0.4076419149846937,
        0.41188687778554045,
        0.8585738993913106,
        0.8385079123018064,
        0.26017856223803254,
        0.6748661034614768,
        0.6895874222341818,
        0.20957397623312424,
        0.6666759311345466,
        0.6602906769777019,
        0.6940872271925588,
        0.775047688635369,
        0.8566038575922628,
        0.7369877865137626,
        0.8353055254230526,
        0.8066899452109969,
        0.6575303814893616,
        0.6794217163270271,
        0.7881584945948441,
        0.48149786570521247,
        0.29546533990277235,
        0.21386195467157187,
        0.2634131853891951,
        0.6374612197334066,
        0.36132200170130135,
        0.4886087349230935,
        0.26046705915705193,
        0.20395926706241152,
        0.6890058791475875,
        0.8383917383667244,
        0.636705642000689,
        0.7190892923490031,
        0.9414273178707949,
        0.950771671775181,
        0.9376591354973113,
        0.7699365070589005,
        0.3009441091214626,
        0.40798922928234893,
        0.7740395929465266,
        0.575395310061117,
        0.8321407160985197,
        0.49206383909749724,
        0.2406593864307463,
        0.7485228642881958,
        0.9256750046497906,
        0.9200465975510393,
        0.9506719834757859,
        0.7203828840508993,
        0.7639759865687981,
        0.7819621185521518,
        0.752488039273889,
        0.7004090140589997,
        0.1861076186004813,
        0.7944831904430564,
        0.5979068309199551,
        0.734346490865512,
        0.43210544248816557,
        0.5872254930961659,
        0.6601999999248457,
        0.1567668750561434,
        0.3957554991455603,
        0.6081321298635626,
        0.39017678701264635,
        0.6018974145932352,
        0.2804011561399825,
        0.6109967063282606,
        0.582532161382669,
        0.6644412606554984,
        0.43291236643099285,
        0.3988632973807349,
        0.6963595566493584,
        0.8433087881408646,
        0.8901940075387353,
        0.8388750566223766,
        0.6830937613087961,
        0.7497499547907143,
        0.8836076403485483,
        0.7921344184669956,
        0.765492095823406,
        0.8309393206302111,
        0.19742536388835996,
        0.23744564004326463,
        0.5002434241901449,
        0.4779008518108412,
        0.6407423009985465,
        0.5124673321303223,
        0.8506258121419767,
        0.8870898589679616,
        0.9131615196319631,
        0.7864361299034757,
        0.6553359055655432,
        0.7181302602193311,
        0.36220648495650365,
        0.8891769502995376,
        0.7546708590820327,
        0.7365074638054134,
        0.7515057221872439,
        0.8891514314285573,
        0.5057128361758749,
        0.4169218033728396,
        0.25647031155776534,
        0.37057790796973156,
        0.2481523494034746,
        0.2001927164225152,
        0.7062460149431473,
        0.2573396791541417,
        0.7346908698597185,
        0.7170775881324083,
        0.5818881523074008,
        0.654735630314109,
        0.6616014673960398,
        0.4309394882659323,
        0.46988270915498304,
        0.49738014460946217,
        0.2091827109700125,
        0.9262417266610533,
        0.294947840322181,
        0.22935885818703977,
        0.14560444302567044,
        0.954868950186542,
        0.9598773692772238,
        0.9542769832684085,
        0.5799291999626075,
        0.32762943878658213,
        0.7602644294527139,
        0.1474905874152983,
        0.18513039200682746,
        0.40951286610392035,
        0.7141601121322649,
        0.48761265744780136,
        0.5600583494264276,
        0.3662499048771091,
        0.43440681760825806,
        0.6853887157361196,
        0.6490593649592925,
        0.5603086987617849,
        0.5145351466185205,
        0.5508006506659249,
        0.4948919538919182,
        0.7737066965565248,
        0.9412482832300022,
        0.8870098955504979,
        0.9323027545463533,
        0.641875305457739,
        0.4876055270769539,
        0.6172656199975164,
        0.5491832227259689,
        0.506313220326643,
        0.4108847747142432,
        0.5431706347441081,
        0.6202922071286356,
        0.7697838515738877,
        0.9378816512237256,
        0.9361390152215325,
        0.9433224833449059,
        0.6842879079269482,
        0.7635437457895643,
        0.8482574530023139,
        0.7977514648019902,
        0.7664576959161535,
        0.9022812430423826,
        0.7705141108489416,
        0.8505160598248579,
        0.7737445900666405,
        0.22817824083927263,
        0.17856470919109746,
        0.12269433355742675,
        0.8419365431102434,
        0.8672970584327342,
        0.9221913515934066,
        0.9327494600265757,
        0.2514659209140111,
        0.22711672118674406,
        0.6946389807276264,
        0.8527515967979904,
        0.8252022255221736,
        0.47502502354124104,
        0.7677297178211169,
        0.5041583659294675,
        0.5243812225609266,
        0.8138308291196646,
        0.881743691731764,
        0.8225614102702936,
        0.35931572445248994,
        0.7692364364700377,
        0.39737824008147904,
        0.3590038054595821,
        0.3746680134333951,
        0.7213926279675502,
        0.7636997414939809,
        0.8164676940053246,
        0.3313118043419649,
        0.40821150847360493,
        0.807025479467641,
        0.28310123122524355,
        0.6838359633869787,
        0.4902483909308118,
        0.8816775639420614,
        0.8665558337605546,
        0.6904266234781657,
        0.7730547591969178,
        0.7743527318367949,
        0.665500499909706,
        0.32934493512956386,
        0.2753330123153993,
        0.145548252114173,
        0.693482578845283,
        0.7647993778721004,
        0.7655941075919315,
        0.896036466736048,
        0.7035124205750809,
        0.8427914353737354,
        0.3702488865240646,
        0.7561128793934885,
        0.8426471742162752,
        0.37950580503851405,
        0.17425698047924226,
        0.26225167587249765,
        0.8601450011524807,
        0.240447081645935,
        0.9047880338376687,
        0.6873995908541075,
        0.43668744394247483,
        0.4687820328688061,
        0.9382378887818493,
        0.9376299631234004,
        0.9338774090566669,
        0.8283706985386549,
        0.9116886889961101,
        0.9072212039967645,
        0.45856014656957733,
        0.3796194511995054,
        0.4702192342824828,
        0.8141114900804672,
        0.9093618927711293,
        0.7605954246406122,
        0.6556995415921245,
        0.607636487392022,
        0.5423404470632357,
        0.7298424217414692,
        0.23987616963800554,
        0.5526968936937756,
        0.9627852956785194,
        0.9592842898528203,
        0.9583776796915568,
        0.8081272276095415,
        0.8797203521063615,
        0.7244961417111708,
        0.9179214491942121,
        0.47247678250135594,
        0.9615913741345138,
        0.712080004744672,
        0.7014569384831455,
        0.7168516612154405,
        0.12280336309803241,
        0.46955606260289734,
        0.1690082524261627,
        0.6717384527978056,
        0.14237613999401788,
        0.5640585635517243,
        0.5918917678576142,
        0.803705220451714,
        0.5883251342004877
    ],
    "eval_rms_calibration_error": 0.2692420890346802
}