GENIES/results/evaluations/cra/llama-13b-code/counterfactual_python.json

{
    "model_dir": "models/llama-13b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-16 02:55:57",
    "eval_score": -0.19940564615118447,
    "eval_brier_score": 0.19940564615118447,
    "eval_average_probability": 0.6224957085237571,
    "eval_accuracy": 0.6866666666666666,
    "eval_probabilities": [
        0.09344779967149998,
        0.33588168666754636,
        0.1718395176827795,
        0.5589462419099489,
        0.2956991217503502,
        0.6350655852764985,
        0.9626341483234846,
        0.9594646228268371,
        0.9627275222578623,
        0.350033083415764,
        0.4219282981234941,
        0.5742947699314447,
        0.6191987846104259,
        0.3608908819206269,
        0.4335262508011233,
        0.4314621762987986,
        0.7230018490660439,
        0.7588221575766209,
        0.8623346589587517,
        0.7782165906326343,
        0.8834141966893051,
        0.5678555441259491,
        0.49574139070659506,
        0.6707132459393299,
        0.8206727043772027,
        0.703115846453824,
        0.1594041226376553,
        0.9265729837981316,
        0.9323702385269571,
        0.9585451858315833,
        0.6742655150627253,
        0.19973904960439037,
        0.4401267158246399,
        0.4497189229930416,
        0.3349235932651621,
        0.17016114560009787,
        0.8618913071295968,
        0.9455284417896311,
        0.9316843189510383,
        0.812194791720381,
        0.37855195792758267,
        0.9257286262246142,
        0.35026470398039866,
        0.4947269387508273,
        0.5573893112049144,
        0.21718548937919518,
        0.7525670524361489,
        0.17791261946945114,
        0.7669981322593139,
        0.444660796714397,
        0.2824601099972529,
        0.4799684006469732,
        0.4893535095830771,
        0.6083630975001779,
        0.25518540101278464,
        0.2639101328394432,
        0.21542219231325999,
        0.740894597710868,
        0.5525499584617628,
        0.7677461170312021,
        0.8712961691638318,
        0.6999281007795369,
        0.5998807794700801,
        0.3233408152901983,
        0.7507497337254139,
        0.31627592475868516,
        0.5153962447872468,
        0.5957662685335481,
        0.399374838060509,
        0.8816838492194832,
        0.7151610973394005,
        0.8866900259438903,
        0.4281725298500172,
        0.8090725868926818,
        0.6774689310691829,
        0.6114650447443296,
        0.3233285597952076,
        0.5424081930353163,
        0.07649459955812143,
        0.7070947560065856,
        0.2230293143906492,
        0.6412541171350641,
        0.6982674909027116,
        0.818323254620965,
        0.29185129882269717,
        0.8166806740808625,
        0.11267166122848171,
        0.8219690249858521,
        0.7938941688066261,
        0.7033879674085227,
        0.7060650130577772,
        0.14763631297425706,
        0.8377378663714654,
        0.1106313995934131,
        0.9210489686698227,
        0.5550498806906923,
        0.8154758717060259,
        0.8604940425921764,
        0.8761791315468259,
        0.38131429026610664,
        0.31762359310022525,
        0.3526541285471819,
        0.5596796554346086,
        0.4858023605389746,
        0.5076086965909133,
        0.9492940487004488,
        0.9468439455176215,
        0.9593610687332786,
        0.9816285979029845,
        0.9783484364329331,
        0.9814649627961766,
        0.5053621242692674,
        0.6790648167496715,
        0.7871617505070717,
        0.3447672990143904,
        0.13193021959899184,
        0.8408843276071232,
        0.8033528804785522,
        0.7887241919569841,
        0.8821486399148601,
        0.4533726902243902,
        0.936865740387347,
        0.32946257802686096,
        0.8795000113339739,
        0.7410731312490069,
        0.8223744380227984,
        0.7152509384319883,
        0.30979471712055884,
        0.7129520504998241,
        0.7619358190662618,
        0.9557076692340402,
        0.955933581777696,
        0.6327363473121431,
        0.6486360975037508,
        0.7966709616346737,
        0.7822371908427735,
        0.7979682263698666,
        0.6739062151493486,
        0.441553814849967,
        0.3633367960515379,
        0.8207791717736295,
        0.7704844465323847,
        0.324839749722668,
        0.811236291547164,
        0.38060896201953864,
        0.3939091074311525,
        0.48153105716263156,
        0.6378048917096393,
        0.6811126409921007,
        0.755572987395525,
        0.21763014137151654,
        0.3025379628211751,
        0.33568862088908036,
        0.9555310532352137,
        0.9352631631553374,
        0.9833745498976548,
        0.28274544826114045,
        0.372977497313441,
        0.27407539779557305,
        0.6346802984237926,
        0.34448502541538945,
        0.8590113647111123,
        0.7606233523821117,
        0.7011288514094368,
        0.8809598550798401,
        0.29169405464898224,
        0.20206508475811424,
        0.13365824241421556,
        0.6857693834219963,
        0.26715390848696097,
        0.6106889834745327,
        0.8121860832366347,
        0.842727101924307,
        0.8305438132748276,
        0.5157769603638743,
        0.43486153653690557,
        0.42474890970046386,
        0.6172126621894911,
        0.7070682242929431,
        0.7273797605366612,
        0.4622661300911054,
        0.7947727696861161,
        0.6853766571482558,
        0.9018451097455454,
        0.5082247828399857,
        0.9224374036940697,
        0.26049421160959696,
        0.3885714261585742,
        0.5200029491172452,
        0.7679749472011146,
        0.4918563089651369,
        0.6403962122039064,
        0.7310467359994655,
        0.5851039192158076,
        0.7095231086895784,
        0.25760543545338277,
        0.10879441834434755,
        0.7662305437865057,
        0.4722835874128213,
        0.6393657102540481,
        0.260808587546595,
        0.5111508451217354,
        0.4986983042952958,
        0.5473512744509307,
        0.7771636460066251,
        0.7824228576053347,
        0.7398686121658999,
        0.4015119056157411,
        0.46408351655566626,
        0.7475410179643536,
        0.5744195878583203,
        0.6404742407515502,
        0.3659201133386213,
        0.8343772643307252,
        0.8331115056891809,
        0.6125240856480079,
        0.4681648649309843,
        0.9097042494711993,
        0.788471341590673,
        0.8490022480082453,
        0.9213800048848072,
        0.7762541177166726,
        0.2113261561934633,
        0.40713585143956066,
        0.22285427243810996,
        0.4772858977305129,
        0.4644396624263111,
        0.8043393468511484,
        0.26475792410498233,
        0.20483284246182232,
        0.13284145395005537,
        0.5554553391028269,
        0.3762237919454154,
        0.6772477768103492,
        0.8186264069850985,
        0.8495830659387287,
        0.8385472184968858,
        0.9177741126637436,
        0.8357647843093957,
        0.8867746630006194,
        0.9156990548941611,
        0.9132420923448301,
        0.8125798641468942,
        0.2530028977686066,
        0.846517847495775,
        0.536393015539019,
        0.5605570564633713,
        0.26632676351480183,
        0.8995695825479073,
        0.5526061487069529,
        0.20684832512867682,
        0.35503352927018317,
        0.43165643171236157,
        0.8070669189856208,
        0.5695443607297265,
        0.8199061403854238,
        0.541206209969287,
        0.6517195116846907,
        0.8205540419933833,
        0.5163165817471738,
        0.32899980905737725,
        0.3877431461515855,
        0.30694239896399167,
        0.25041305868473673,
        0.711915645413729,
        0.3591120572896288,
        0.5589460898698595,
        0.47580201829793095,
        0.5476378834319522,
        0.5111626227031526,
        0.8592563013897532,
        0.733017089584693,
        0.8655100555586177,
        0.585220724003237,
        0.42774158100949744,
        0.20222557919994355,
        0.46186365329778795,
        0.5898511621858912,
        0.810689525170423,
        0.722171492432954,
        0.6247150410430841,
        0.7979097187022944,
        0.5105748968856827,
        0.6486687448898968,
        0.5446273360531348,
        0.8875349517700013,
        0.5726683593181494,
        0.5379071164657736,
        0.5721177776676143,
        0.7978980947725702,
        0.11771624662336637,
        0.7636592986164455,
        0.7864406276015965,
        0.6194947266822024,
        0.9849912912226388,
        0.9829731738433806,
        0.9876490652845709,
        0.6890416670354941,
        0.7322604900527228,
        0.6972278541825149,
        0.8924216306339328,
        0.39803436036177253,
        0.9049141331096653,
        0.7716557194627036,
        0.8263605150125526,
        0.6535820901689993,
        0.7300124083344216,
        0.6148280749362078,
        0.6127994205034085,
        0.3683716358500124,
        0.3468720209103458,
        0.7496033158113385,
        0.6240877009218954,
        0.4964749759269344,
        0.1985612549229404,
        0.756142416219264,
        0.8333545202368586,
        0.8017415969683462,
        0.7268214097971143,
        0.6485452994084328,
        0.5802267602255372,
        0.3809220423766724,
        0.4937975331729783,
        0.5487282595335882,
        0.45624762869885815,
        0.32921269842677536,
        0.28849465253488027,
        0.6609009885373935,
        0.4171191113684296,
        0.516059503725023,
        0.494646673760881,
        0.860273557984475,
        0.5955175628898923,
        0.5123717097023381,
        0.37509222611300247,
        0.5747872131535017,
        0.7126005599752117,
        0.6417804452851483,
        0.40752734566177196,
        0.4942031246381433,
        0.2995835360338276,
        0.3758529868382086,
        0.5108985164332189,
        0.5097979715749362,
        0.4704720408736015,
        0.49499481843691345,
        0.43386890957632673,
        0.3734864597048363,
        0.5521774797584775,
        0.41467785976205496,
        0.2542566878949939,
        0.8546602274783249,
        0.8929417939198067,
        0.8308896349927787,
        0.4482222713324071,
        0.6632286764568575,
        0.18666751957640879,
        0.5733882837271019,
        0.8684531090894221,
        0.8478181805294744,
        0.7233941285721774,
        0.5272270740060595,
        0.5112954765947457,
        0.9459290577869722,
        0.8588267970111807,
        0.7400445434334093,
        0.5334447216790874,
        0.5451644501317864,
        0.8465168538521605,
        0.9491994851754609,
        0.9841853057123666,
        0.9062428031204898,
        0.9076595924565988,
        0.9095147798541658,
        0.9194132547962649,
        0.8570936772106189,
        0.838760606166192,
        0.8236543419963458,
        0.6400685442062143,
        0.7465717308410336,
        0.8076201672633123,
        0.2396184968949357,
        0.6835236802388249,
        0.29332094132709097,
        0.9638138287561449,
        0.9410186926518848,
        0.9711873315470371,
        0.5796360343513806,
        0.33571097719364446,
        0.3990932159839775,
        0.5274981458102925,
        0.17572464404601526,
        0.17539719484004734,
        0.5027286776367326,
        0.5512620990912691,
        0.5672252278104939,
        0.5411878550569593,
        0.5585312478968649,
        0.38069339301481653,
        0.6727272589384724,
        0.7579006894058828,
        0.9078602732369658,
        0.7188823072973438,
        0.772167774721211,
        0.8864803958880177,
        0.8697560929443937,
        0.8235465694998839,
        0.7782101988848045,
        0.9160364244127983,
        0.9264196571963794,
        0.9470164725359386,
        0.16055572648242142,
        0.4942856046639321,
        0.7431409282631372,
        0.8247377464123742,
        0.6587316249516262,
        0.5524164976465422,
        0.24000793566278145,
        0.6535414426517425,
        0.7586645417736996,
        0.6241516815821784,
        0.8400467251776316,
        0.5949088449965302,
        0.9833986585708602,
        0.9741606989586363,
        0.979760630615872,
        0.8260214917608392,
        0.8347712041804861,
        0.783649175097344,
        0.1356495871971407,
        0.21057836988193854,
        0.31326537301928903,
        0.5968328918693734,
        0.7946256961973313,
        0.7319552824532445,
        0.6928555443455031,
        0.7386469351013343,
        0.815160386334178,
        0.33019120032084204,
        0.5858514663009957,
        0.7246187499753909,
        0.2916143767874614,
        0.35860555714665826,
        0.49028861866855544,
        0.7527998709027656,
        0.7195103627591773,
        0.38370402037265583,
        0.8575003522304913,
        0.9096480892846281,
        0.9298211311719928,
        0.6256213918188605,
        0.8627696386157893,
        0.5423255323524709,
        0.1126650513232409,
        0.3029965103557127,
        0.7335656858482154,
        0.7446026874586447,
        0.8142684695225813,
        0.4964619664823108,
        0.2907687181442399,
        0.1891795853889625,
        0.3407938413266202,
        0.3405341288937166,
        0.5997137702865807,
        0.7180118817359469,
        0.7035666509606353,
        0.6544247860449968,
        0.6930247821950654,
        0.9573868573120781,
        0.9509911908215405,
        0.9758325170644538,
        0.6191634907972445,
        0.7749043113001646,
        0.716313809372902,
        0.7206459104217467,
        0.654182185443603,
        0.494117352609558,
        0.39646364819811675,
        0.1697461385768203,
        0.2633793609110227,
        0.865904665847136,
        0.8573318646467146,
        0.8229297994078574,
        0.7623032072655914,
        0.5682191703744194,
        0.5902563143024765,
        0.808016272824937,
        0.7136542577510536,
        0.7017491083493088,
        0.8925515089512518,
        0.8240252263195245,
        0.6360295329013443,
        0.6557873281546601,
        0.920884098957899,
        0.5410912155401061,
        0.6052171247847248,
        0.6929795218820537,
        0.7059289226045264,
        0.8314540093963538,
        0.6801725029809624,
        0.08555742772769688,
        0.7815343268592296,
        0.8752525937091528,
        0.31572910855400815,
        0.38835382562770043,
        0.6131289212729019,
        0.511963163318525,
        0.3381203836965635,
        0.32925447352410786,
        0.40922831566549667,
        0.8000934207501802,
        0.6375488098590837,
        0.24068019017919423,
        0.43228101811583974,
        0.28426407874790266,
        0.7929902275266465,
        0.8216924428040748,
        0.8308462767181002,
        0.6903872682562997,
        0.9188253678685468,
        0.9608810886371975,
        0.950693538844868,
        0.6639439081421177,
        0.2951193197550924,
        0.7794266242275376,
        0.8239583941133947,
        0.8606520619727788,
        0.8931814287130602,
        0.8120579057830862,
        0.8379584352498162,
        0.32923054376920935,
        0.9891041276433701,
        0.9878956234186866,
        0.9883883086326417,
        0.9568303826087353,
        0.921744618748538,
        0.9549945958331127,
        0.23708992210054025,
        0.2479236807282443,
        0.6953659674371371,
        0.8285308197289227,
        0.8137438948195703,
        0.8814522183750767,
        0.7309628030981876,
        0.6266948746018369,
        0.8656736199451128,
        0.5618661174244481,
        0.6472970253661682,
        0.49502420044817963,
        0.5995648541766663,
        0.6061501562735283,
        0.1574395049830165,
        0.5020839851812081,
        0.5387357486806588,
        0.5359614000509043,
        0.7460498070200247,
        0.9095543341272543,
        0.5813113429716185,
        0.8066393192394491,
        0.8192568673365808,
        0.9818792808750997,
        0.7362050886275462,
        0.7118027893543547,
        0.7392426020476355,
        0.7370405156262356,
        0.7189906707030468,
        0.7273098310791513,
        0.6359126768087593,
        0.340852808634563,
        0.5641793339735769,
        0.5568036117042887,
        0.8494579296701305,
        0.700790044102831,
        0.7203965218067567,
        0.7083038583778142,
        0.48210857896826875,
        0.723343045060501,
        0.5444436568835753,
        0.4925498248194309,
        0.17524955516441432,
        0.5477024886299997,
        0.5940016417947831,
        0.9676051077920272,
        0.9697685773160578,
        0.9686758503487334,
        0.15101074042419715,
        0.2792091794198909,
        0.6229898418282673,
        0.9400798771593858,
        0.7685907611745604,
        0.807945561428904,
        0.8517397526675596,
        0.8524658094169082,
        0.8019723831536021,
        0.6620476647618968,
        0.634270129691899,
        0.680356719664567,
        0.93983105635933,
        0.9437400547228941,
        0.9463005827687597,
        0.7832009251742088,
        0.3029885259595262,
        0.5322400340029744,
        0.22308007973183375,
        0.32385238113828957,
        0.8784107792758813,
        0.9540727572485451,
        0.9503887250604528,
        0.9411491000601885,
        0.36633397770279696,
        0.6140361731765678,
        0.7626798738176574,
        0.4975489402222367,
        0.2184954369515948,
        0.710650637898463,
        0.6461034998309487,
        0.6387476391967841,
        0.36261205032040245,
        0.6515790152371379,
        0.419059710618902,
        0.3727273433205032,
        0.8471822456415784,
        0.8632227262970896,
        0.8798063921661637,
        0.7740225368658169,
        0.8126453742531827,
        0.5330743498822927,
        0.9578441701923375,
        0.7797554520425806,
        0.9033192594296283,
        0.5919211648714021,
        0.11838839745180785,
        0.8523426545739379,
        0.8499612200568878,
        0.9885471018057362,
        0.9360390092118034,
        0.8934153009778834,
        0.8566766970895259,
        0.7981052769953074,
        0.8798671502060633,
        0.9588082724118959,
        0.9696620778272772,
        0.8535521335972678,
        0.936871171637882,
        0.8753652454316524,
        0.46994659097765057,
        0.10663256830277168,
        0.5798398535484057,
        0.9695708692205753,
        0.878547498323063,
        0.8768534938805277,
        0.6464515493336939,
        0.6075437644077455,
        0.5394757285644686,
        0.9529335802746429,
        0.9582745196722964,
        0.9771522574636969,
        0.8314369155454995,
        0.45699282429817367,
        0.3148220184065226,
        0.764073053026978,
        0.856578728295873,
        0.7898218550439132,
        0.7554574780973032,
        0.4328387925677256,
        0.5572065556600339,
        0.6018401478685597,
        0.3986253412937225,
        0.6308328358152584,
        0.6115763725301802,
        0.6719134640772073,
        0.5343649482416515,
        0.6744895451318866,
        0.2617676496114853,
        0.3605319588920833,
        0.9608239923438273,
        0.9531464642058992,
        0.9740454660465189,
        0.499329389227025,
        0.1204243922710924,
        0.2323591308005898,
        0.2252797010424492,
        0.19997899758113521,
        0.491997152725693,
        0.7946043661530732,
        0.8354913777892532,
        0.6850704135599197,
        0.6368859659855284,
        0.5402892407155975,
        0.3927686107655309,
        0.3755248562801943,
        0.3370529886666218,
        0.19302365376531805,
        0.44331422706033596,
        0.5883465503626623,
        0.7073256216657429,
        0.6583642852664431,
        0.5105388704893568,
        0.4588141579212054,
        0.2546884736065036,
        0.2630872892190692,
        0.3402396426880403,
        0.48618666361012775,
        0.28826013212785234,
        0.38247529817238357,
        0.4511870783387236,
        0.4793746078380059,
        0.7774642752028553,
        0.2937452079005358,
        0.36000546669243083,
        0.0853234000121668,
        0.8454427994874036,
        0.6459646986591201,
        0.8837532353053728,
        0.9319624692176958,
        0.9132471726555619,
        0.9236911325027173,
        0.4782344362032725,
        0.36995555500104976,
        0.6008207472972695,
        0.4167827675469883,
        0.43042119492342157,
        0.24382151989478748,
        0.09088723489997035,
        0.21000421668032168,
        0.3153011756190897,
        0.3057261644059507,
        0.8687208264275554,
        0.3159586243318467,
        0.9675833401051027,
        0.9731016514127239,
        0.9666377079163411,
        0.601808240769405,
        0.8673502623057381,
        0.5783529862851868,
        0.6285527768098711,
        0.6151536353183193,
        0.7558267454513903,
        0.7198953628378397,
        0.7418270621596101,
        0.5793547595551096,
        0.368438336860736,
        0.5556494250829725,
        0.07896659135849216,
        0.27122805998864397,
        0.27257878394376367,
        0.30466862503453906,
        0.9122363016653384,
        0.923332862207179,
        0.8099727740383413
    ],
    "eval_rms_calibration_error": 0.2745156752395569
}