GENIES/results/evaluations/mms/llama-30b-counterfactual_python/counterfactual_python.json

{
    "model_dir": "models/llama-30b",
    "distribution_id": "counterfactual_python",
    "timestamp": "2023-10-14 00:33:59",
    "eval_score": -0.12626091087768904,
    "eval_brier_score": 0.12626091087768904,
    "eval_average_probability": 0.7690080467139531,
    "eval_accuracy": 0.812,
    "eval_probabilities": [
        0.09707631364540659,
        0.1493529870376521,
        0.2575790612176362,
        0.8068149177564431,
        0.9605916559395961,
        0.8722189812412396,
        0.9949100434361289,
        0.9935696928367154,
        0.9944468358793717,
        0.9548071729487445,
        0.9977564545381801,
        0.9993285944179837,
        0.8266953113788562,
        0.45113998506512376,
        0.8457970371066048,
        0.8932391025858696,
        0.8669514036352957,
        0.9661818089564158,
        0.493247332906379,
        0.8043974909567232,
        0.9474808480638564,
        0.9909802328640595,
        0.9875775196399492,
        0.9773998867521875,
        0.40193960757949876,
        0.8590936482415235,
        0.037015678109410664,
        0.6382941428127759,
        0.7270429827216445,
        0.675644247467304,
        0.9567345894901282,
        0.44238047448795886,
        0.9621592163165092,
        0.9533088685710993,
        0.9343457184479244,
        0.7181225880199363,
        0.6184699789837935,
        0.9402438880266063,
        0.959558367918333,
        0.9883113695866208,
        0.41552355862348067,
        0.14936191303797955,
        0.32859786107261796,
        0.42218840939883984,
        0.23162313853874275,
        0.9440113296680771,
        0.8931344364608053,
        0.9728469390750751,
        0.511666635090093,
        0.11018448284352607,
        0.4750767507178223,
        0.8581963850830249,
        0.9752247602997663,
        0.45691904013209117,
        0.5202064339432643,
        0.5021749229909855,
        0.34189788227996654,
        0.9974391271793318,
        0.9930935544960878,
        0.9975051654400106,
        0.9983496306290346,
        0.9992608372295323,
        0.987156255240143,
        0.20453446900825478,
        0.9919786796540587,
        0.9916308241616694,
        0.9202540843533222,
        0.6948431156416388,
        0.6351874535800494,
        0.93926636825715,
        0.6152423282701873,
        0.23739531474930303,
        0.8183364995314193,
        0.9942965924218856,
        0.9916563711893374,
        0.9736872541538172,
        0.9926249619152416,
        0.33663090598535383,
        0.07332444490600545,
        0.9997507625794186,
        0.011907165046909779,
        0.852501839289723,
        0.9914156412256243,
        0.9919201765367033,
        0.8663321325565755,
        0.9942447340357033,
        0.8778070215101712,
        0.2986074232437034,
        0.2878384744795608,
        0.44145080439807927,
        0.9990139099133939,
        0.23588862477552974,
        0.9986827440553137,
        0.14251656818831,
        0.9285907596266435,
        0.7808052465100364,
        0.7338973969100406,
        0.90670212082773,
        0.8677143993481498,
        0.9982336563344768,
        0.8588614485967128,
        0.9894270384149633,
        0.9121848961614982,
        0.743109707203103,
        0.892512208304683,
        0.9477889238068903,
        0.8953075269045164,
        0.8867244706437739,
        0.7160442911335854,
        0.7754158848806941,
        0.512282929668627,
        0.9990317099069909,
        0.9914477657523133,
        0.9780713273059698,
        0.14409713656977385,
        0.978240099840246,
        0.4565008034916873,
        0.9327910203967337,
        0.9415486023031012,
        0.9058881838003395,
        0.9917836722982635,
        0.9928588439573276,
        0.9878282599084065,
        0.9989788956090386,
        0.9994250858059878,
        0.9996351924758249,
        0.7559969441004704,
        0.10228732272987293,
        0.9825510457195524,
        0.9079338135467103,
        0.8502740251679554,
        0.6774942910298711,
        0.8879525446222649,
        0.9974441022015166,
        0.9929232970197994,
        0.5849063353409262,
        0.3521783366523119,
        0.540423216671794,
        0.16248494199798655,
        0.9247142450332243,
        0.4556942550163401,
        0.3369132756448113,
        0.9998349844607897,
        0.9938849808079128,
        0.36499860752307994,
        0.973877934035688,
        0.17909437293301347,
        0.9639998226256867,
        0.9969575179629975,
        0.9916798930526954,
        0.42625231044499146,
        0.40216434967013404,
        0.998237001004998,
        0.9915284269732674,
        0.8688923623759673,
        0.9049744213284107,
        0.13916479031695883,
        0.17731694964455114,
        0.16848019969863715,
        0.7704712060749501,
        0.228526855291871,
        0.8590279665482673,
        0.7713736410370591,
        0.8129101061363249,
        0.9595075414497394,
        0.928147792899557,
        0.6269541217628096,
        0.7856255539827308,
        0.757602954772881,
        0.41575236950894234,
        0.9649316457379525,
        0.7218897396013647,
        0.8168700135455047,
        0.696908624623504,
        0.716041090426227,
        0.36461980368458236,
        0.5692222451502107,
        0.6318449883445587,
        0.997195955728812,
        0.6020167080076648,
        0.8631809530808797,
        0.9205825307272881,
        0.9558999600856997,
        0.9976545201972362,
        0.9999390017592809,
        0.988596498508063,
        0.2988032546282391,
        0.3263944445675452,
        0.39756675315321194,
        0.966272099510271,
        0.8429268615072737,
        0.9931526187163838,
        0.3058891305712412,
        0.3231667144270779,
        0.5714507286538411,
        0.1390464601855559,
        0.4793148488923489,
        0.8893247154837501,
        0.9508416480803303,
        0.9428950577448872,
        0.6154736190757679,
        0.9285826509368444,
        0.9842312558333931,
        0.9806099419203965,
        0.9925776077310768,
        0.9965538806110358,
        0.9829028898971902,
        0.9874483455483734,
        0.5316318394710082,
        0.8584305314765192,
        0.26276679884104764,
        0.9173428853458241,
        0.7746067806483149,
        0.9692714354159883,
        0.9110680016626609,
        0.5440645596369724,
        0.9067906890956587,
        0.9426162785426885,
        0.6737253430402026,
        0.9047194068417123,
        0.9924295333896633,
        0.9976965795679578,
        0.7435569763954908,
        0.8417464827482389,
        0.7250463149175728,
        0.9968043767986283,
        0.9957439275436439,
        0.9976956526551667,
        0.5147713312648121,
        0.29437631973314127,
        0.32382736142714275,
        0.6537262354089346,
        0.6876289453703532,
        0.6312791991444403,
        0.9114449115426246,
        0.9899518024974829,
        0.9464914308945952,
        0.9974339968988813,
        0.9941776430817536,
        0.9881393320045815,
        0.9999606694073593,
        0.9999676366367559,
        0.9998639297666608,
        0.07741190590119784,
        0.8912733258303949,
        0.9612443447313184,
        0.9870246807675808,
        0.9997794011016558,
        0.9991503976246423,
        0.7212275169961057,
        0.11479926863779977,
        0.09834081825524527,
        0.8302467275198613,
        0.9266705359383758,
        0.98251904595745,
        0.9844290097090114,
        0.9781134132071143,
        0.7402435957807749,
        0.7760293513947369,
        0.9327196260029551,
        0.9700135121466977,
        0.9843140885935715,
        0.9578728330638413,
        0.9916490641262203,
        0.4842835217375645,
        0.8146173429544472,
        0.9432175921447322,
        0.9957053689180874,
        0.9983669212698714,
        0.9873237722395682,
        0.9998865121466892,
        0.9631473491051983,
        0.9998906445268455,
        0.35606902807622104,
        0.17091298631624188,
        0.37531574473425133,
        0.9942507523227344,
        0.9727140704435232,
        0.8408420575531385,
        0.5181391393714528,
        0.7198142172884636,
        0.8652769498969989,
        0.8874309055854074,
        0.8556148338043775,
        0.9695650167064082,
        0.9991686700516728,
        0.9674678171382793,
        0.9795475658924315,
        0.9863968950763055,
        0.9996314883124053,
        0.9413978078836458,
        0.9883842889621021,
        0.9795819375001715,
        0.7413282998995973,
        0.9928550137953406,
        0.9789359131099898,
        0.9912393974511533,
        0.9388883927579047,
        0.9741991722307198,
        0.8054639424382244,
        0.9736290217501473,
        0.047350769512420676,
        0.998383724266908,
        0.7102825795107864,
        0.9501330877594276,
        0.6937428393490819,
        0.9073897339400667,
        0.34681889833650525,
        0.7980230054813106,
        0.9754738565030003,
        0.9592989381761513,
        0.9194872170104441,
        0.9995158103533758,
        0.8971019637932491,
        0.9910674615589754,
        0.9940003846237029,
        0.9830292638840311,
        0.9913815552540096,
        0.9915744602439198,
        0.9985848281744811,
        0.97616658305294,
        0.5205724320968066,
        0.7883941512986812,
        0.7020155848969601,
        0.9997101894767456,
        0.9999304525577368,
        0.4175244248309654,
        0.9942802742236688,
        0.7509182292784591,
        0.583413820031303,
        0.6241642822509901,
        0.9841043138824123,
        0.9890562975774818,
        0.9731766830042258,
        0.8696189934548166,
        0.9466405041690903,
        0.9824234946120214,
        0.9838277665148961,
        0.8821159318720778,
        0.9782586254586003,
        0.9768917921172272,
        0.7375710871253567,
        0.15800365938590522,
        0.679199503239163,
        0.07840602099880356,
        0.9741546680402448,
        0.9911615801112751,
        0.9969983137114697,
        0.8939493845992662,
        0.2631465213206892,
        0.6459917193076329,
        0.36281255819624425,
        0.1902884266975855,
        0.10911380384143221,
        0.2776952232408998,
        0.6878377652620256,
        0.2362594358369133,
        0.9841394702361734,
        0.9017304530476176,
        0.7864692644562357,
        0.15786677185847175,
        0.3545977834108986,
        0.5493289146789548,
        0.9937308520951093,
        0.4482461508471828,
        0.28046990052578447,
        0.9776015125218097,
        0.09545944653442295,
        0.963698991776297,
        0.7364294583287193,
        0.8878367120873117,
        0.5873309041913981,
        0.5687548247322276,
        0.5989019083354654,
        0.5051526821918355,
        0.9943570573180921,
        0.9966055227847573,
        0.9450761337688701,
        0.40724565481608616,
        0.8741387533103557,
        0.5756957391029308,
        0.7493144314656807,
        0.8690693816419436,
        0.9924543207328314,
        0.9911829292454838,
        0.9761847760649002,
        0.9745758275789763,
        0.2993728837887618,
        0.6322598068302993,
        0.8020264645011942,
        0.7805419320277832,
        0.40961782993060736,
        0.7107729724891633,
        0.5266568793050561,
        0.9222428462136111,
        0.9521080649553758,
        0.5799175044935768,
        0.9156591798044714,
        0.5042088634960745,
        0.9907502318903616,
        0.9952304040296853,
        0.9874725153280582,
        0.3536018708013284,
        0.8046355395685506,
        0.9606285391023384,
        0.9990783148701413,
        0.9977413109913933,
        0.9949246047851787,
        0.3759220754438341,
        0.39571267303921287,
        0.2817262008060838,
        0.18457629594062877,
        0.21947528062998012,
        0.7667247689358693,
        0.9883209195375364,
        0.8840987708772121,
        0.9897537886236281,
        0.8998229819019667,
        0.7603406462040827,
        0.6312552463552827,
        0.3509753123436662,
        0.37934630790486223,
        0.38151516309360434,
        0.9467075630684223,
        0.9953468849167743,
        0.9828406119980494,
        0.9937033406836021,
        0.7663511725903601,
        0.7856865252944134,
        0.9632478783058701,
        0.9984494265986467,
        0.9854005647385018,
        0.999646193190282,
        0.9397820353208389,
        0.9988232388270439,
        0.9961382678590681,
        0.9953112155340956,
        0.9986423863280111,
        0.43008486948843494,
        0.2972326386722969,
        0.8794780508210489,
        0.3566628462418096,
        0.48002447578665536,
        0.3223305832747997,
        0.9536791505531561,
        0.9934348697798668,
        0.9402842881251635,
        0.9923099625933913,
        0.8934950747561944,
        0.9705116174232368,
        0.9969852687282537,
        0.9994650876501906,
        0.9361985588634457,
        0.8124795194258794,
        0.940082987767594,
        0.8724184502910823,
        0.766193293753968,
        0.8595625007942995,
        0.8402456110130151,
        0.9475372802428057,
        0.5888902283874768,
        0.9905907982280208,
        0.7860154341644496,
        0.9835588683141815,
        0.9166101873720995,
        0.9971552084186551,
        0.9811092126831796,
        0.9984395758714273,
        0.9714196280307249,
        0.948906091984758,
        0.6605820348327613,
        0.15102661264555806,
        0.16746783974641022,
        0.07974600279854338,
        0.9947081253777386,
        0.9753367436433207,
        0.9996687431059235,
        0.9893124846685822,
        0.27733104385643736,
        0.9822223114875718,
        0.992150708963981,
        0.9998238944905266,
        0.998485836234062,
        0.7904688201494776,
        0.9850903448511689,
        0.7166500679265202,
        0.33690705371699253,
        0.2969304632417338,
        0.38676843319459075,
        0.9705117913641012,
        0.9856969822221182,
        0.5433940421820701,
        0.9493834445897346,
        0.9991088660564239,
        0.9031294011942066,
        0.8488449458583529,
        0.900870100911653,
        0.9702668425529198,
        0.20514470713946598,
        0.27993235947999456,
        0.2641180754390646,
        0.7906376415519807,
        0.9522489438891006,
        0.8851901460247066,
        0.9906849094429365,
        0.9809975065949824,
        0.9962734511949866,
        0.7964570832932529,
        0.7631523063425144,
        0.8977346909981005,
        0.5713647285273165,
        0.8487143903904257,
        0.5236732175838132,
        0.47512899839344225,
        0.2610086940960882,
        0.8618266462599043,
        0.9026822710982091,
        0.8998917273445286,
        0.9098970160741026,
        0.6576137047344671,
        0.752188240126285,
        0.6905092402530154,
        0.09952280806201272,
        0.16329516210345074,
        0.5870638069555216,
        0.9992036407824303,
        0.9990693005919642,
        0.9998056876393031,
        0.999347300103306,
        0.9348593549614775,
        0.9650824902385012,
        0.7971143061412893,
        0.8025899691775386,
        0.9235317480403982,
        0.9814602344313352,
        0.9655160850306385,
        0.97334100182183,
        0.3550950988242153,
        0.9714344839008336,
        0.9989796068126054,
        0.7017728578342007,
        0.9402671518143442,
        0.7262160341584359,
        0.9968346281784207,
        0.9874711374208299,
        0.8161984670507784,
        0.8640475503478984,
        0.9270573162771427,
        0.9507248074866774,
        0.9006722495526966,
        0.9795762446639144,
        0.049068436053112396,
        0.9651918475002743,
        0.4149672514883649,
        0.9540065383663736,
        0.012985775802310329,
        0.8017101825214474,
        0.7914570650169543,
        0.15744224240171845,
        0.20520099016706764,
        0.9190903700599001,
        0.9993893393139311,
        0.9608691870024085,
        0.990089452328163,
        0.8068983260243284,
        0.7688863653588145,
        0.8797929271333947,
        0.9993752536716721,
        0.9834271572440291,
        0.9853582423942262,
        0.8953631291789088,
        0.9667166485770973,
        0.9827026516096411,
        0.9993475006925552,
        0.9994673822072312,
        0.9997318448489851,
        0.940010031947647,
        0.9463008756388372,
        0.931311440912532,
        0.9701232793865736,
        0.9743697042572914,
        0.9994608356731064,
        0.6874125036514986,
        0.8096985127119647,
        0.5667450827086239,
        0.919854440834506,
        0.1713210644773546,
        0.971559184457492,
        0.5536613796843896,
        0.5117389412935908,
        0.5347925099268337,
        0.998943681181361,
        0.998553884978758,
        0.9937279093421101,
        0.9045863380659703,
        0.8835454763768827,
        0.9112406263528917,
        0.8781874581653004,
        0.968929213284744,
        0.9920974386784155,
        0.9744615646485181,
        0.9058531830975484,
        0.9001993665393194,
        0.6675313621930562,
        0.5298169115728485,
        0.6244519895381471,
        0.7979212993028372,
        0.7221837994544191,
        0.7024667850192259,
        0.15922018013362782,
        0.9161250206809028,
        0.9887920128691228,
        0.8376041898871638,
        0.6043950371092871,
        0.8540072569167872,
        0.8933260268360665,
        0.9186245998571487,
        0.7310940434749378,
        0.13974227679979156,
        0.7592168146555253,
        0.7846841069648043,
        0.7938797502424876,
        0.6491830055139838,
        0.7398997766761448,
        0.99048269534217,
        0.5354142578825354,
        0.024427374766956153,
        0.9648730858856674,
        0.49642955411999923,
        0.758346903874309,
        0.7589087702742848,
        0.4853947164111987,
        0.5401925401184722,
        0.8391841252322549,
        0.8987865357442285,
        0.32406206325388454,
        0.9999743629857982,
        0.9999344452415749,
        0.9991204106720253,
        0.8284155802481972,
        0.9683846784797027,
        0.9515967942310349,
        0.961749243397762,
        0.9997143932388702,
        0.9179183053654651,
        0.9036468344359938,
        0.2944387025760363,
        0.21213237181449743,
        0.4105089348013406,
        0.6437804760962582,
        0.5636401417000182,
        0.5612218143353969,
        0.49520723001110484,
        0.2943380937416964,
        0.9951059110995617,
        0.9606206161454576,
        0.9796129875666946,
        0.7638666137544313,
        0.1748903260650848,
        0.2102348116674906,
        0.7019274257940665,
        0.7524145622334861,
        0.20964873602787545,
        0.943642181885478,
        0.985506570133235,
        0.9782388883422884,
        0.8811780221224044,
        0.8276646862399527,
        0.356196747987124,
        0.9938064218566337,
        0.4054229285721124,
        0.3547901269364012,
        0.23428514454156446,
        0.4543901376002555,
        0.9665963595155715,
        0.9567197003115138,
        0.8832577282934077,
        0.912611139242919,
        0.988221757893683,
        0.9857567127070026,
        0.9976372092905403,
        0.9963235453335268,
        0.9911291776739343,
        0.9854682796431795,
        0.9737338428887214,
        0.9271688439374645,
        0.9699156519040106,
        0.9712248028311752,
        0.9632628782623334,
        0.9215977396601767,
        0.24707441133620786,
        0.9775277354632392,
        0.7872223313220186,
        0.8970394647871913,
        0.6941334938369127,
        0.6220852651709281,
        0.9864959427784473,
        0.9690168765621744,
        0.9620083243939835,
        0.7526393929886094,
        0.7089364036798333,
        0.4150995502788369,
        0.5624991414576194,
        0.6744473502375403,
        0.5933639637671901,
        0.9697126012833616,
        0.9771148013519741,
        0.23261099337656932,
        0.9933572184257078,
        0.7541873679419147,
        0.6468270109291353,
        0.9327799346305994,
        0.8339002345838575,
        0.843567255323104,
        0.3456164720446852,
        0.4009288045059597,
        0.7852126976921832,
        0.7448368087115094,
        0.6313448695348642,
        0.8715868518883664,
        0.8975474425846012,
        0.9644445385725985,
        0.9169343025478089,
        0.5636788524849337,
        0.46870647490293504,
        0.9651675927204505,
        0.8338717228714817,
        0.5463617721789731,
        0.9792269928269492,
        0.9405738202441639,
        0.5655059822803163,
        0.9584943895827902,
        0.9813586075172945,
        0.9951862120686092,
        0.70921966471776,
        0.6205413565837872,
        0.9979545571519053,
        0.984397739614432,
        0.9507841106098599,
        0.9337711289014871,
        0.9414912637873702,
        0.9955146722901976,
        0.9961964954808037,
        0.9979525589851406,
        0.9985962810158815,
        0.9981102535968489,
        0.993380852141166,
        0.29414446399267746,
        0.8264913729724969,
        0.8359604587841043
    ]
}