Files
GENIES/results/evaluations/mms/llama-13b-code/counterfactual_python.json
T
2023-11-11 19:44:09 +00:00

762 lines
21 KiB
JSON

{
"model_dir": "models/llama-13b",
"distribution_id": "counterfactual_python",
"timestamp": "2023-10-14 00:15:02",
"eval_score": -0.1498359961936773,
"eval_brier_score": 0.1498359961936773,
"eval_average_probability": 0.7262082472191324,
"eval_accuracy": 0.7706666666666667,
"eval_probabilities": [
0.5321437057296083,
0.7523073621895225,
0.5416386957418207,
0.9964154385322935,
0.9286889144346034,
0.9787842744339728,
0.9794758650641628,
0.9623648328045664,
0.9660353996876058,
0.9874545548264746,
0.9940746870584459,
0.998683482926758,
0.6066972486340345,
0.6339868018357984,
0.6967710813467866,
0.481137174404888,
0.7947025846081094,
0.5205534584308296,
0.5861641980227741,
0.8620001057647008,
0.8900558816848283,
0.9608653938332872,
0.9705074599942657,
0.9974138266644968,
0.6225217623841957,
0.9225283236610567,
0.29624852265355583,
0.7401834468786094,
0.8151589983294492,
0.8762922474475872,
0.9820562260988562,
0.20888189858621664,
0.9914185100636583,
0.9638586710420166,
0.6963085532405354,
0.2920148570542163,
0.7891646429867054,
0.8438699688597507,
0.7772870965375203,
0.7991586565475399,
0.5343978348426586,
0.2337972745936518,
0.7415182055403986,
0.4898791834336534,
0.406957784932964,
0.39128740627020075,
0.2175147065310838,
0.9593441663908088,
0.15987131252799902,
0.6603277666293442,
0.941139371380797,
0.18465856312885254,
0.5499478805043478,
0.3826684776288768,
0.5184429474665382,
0.16242123023145882,
0.04583491428471169,
0.9953378324550288,
0.8253002356921246,
0.9051212762158957,
0.9960233727645017,
0.9893220626522674,
0.777722594899702,
0.53246377400244,
0.9894092191335541,
0.9945145597235021,
0.9093793979020149,
0.6383444776641216,
0.27876567169833794,
0.9164587934780217,
0.3034306881894985,
0.4338509472616884,
0.902474045905746,
0.9780226871433538,
0.8227589949941531,
0.39897834576870267,
0.8618198885774251,
0.6555622653592019,
0.195979427421571,
0.9886075638528768,
0.32431709294158517,
0.9803317017592226,
0.9662077545237737,
0.9952332179331477,
0.6625215869948327,
0.9277729292274218,
0.7176122556054381,
0.3755041125206511,
0.3720588492798575,
0.3901358935264609,
0.996213445396134,
0.5512919383964797,
0.9957388670630262,
0.1888542811968974,
0.6287602572877318,
0.7277833450109558,
0.9474799473412744,
0.8103694959835831,
0.9518730897815092,
0.9970874633029091,
0.11164855951491648,
0.984592306117654,
0.5881386771939707,
0.3929462131864776,
0.3243409667176705,
0.6986037062983769,
0.7936832711538645,
0.7281319627093199,
0.8929978628535974,
0.8465417669078463,
0.891510769201278,
0.994429689684301,
0.9767661539654054,
0.8064282725114025,
0.6528502678845984,
0.9505143719025557,
0.47737102565812545,
0.84566582811388,
0.7404483825218381,
0.6779383997449111,
0.9678625332294739,
0.9960811957092193,
0.8842026932851998,
0.9927386934922611,
0.998316083127517,
0.9942465264913065,
0.8201126968970655,
0.16601786921686665,
0.7765434750022189,
0.8271594578586767,
0.9675008032512663,
0.7577986925168054,
0.9055797300450935,
0.9260839505676403,
0.9738612490950824,
0.2867298595862837,
0.742479545645562,
0.28532110700309415,
0.09891153909767926,
0.7452594133774461,
0.3034119243161138,
0.6177649488392023,
0.9950113530463509,
0.9937327016288163,
0.8658784850898685,
0.8476319230744451,
0.7161919296779101,
0.8546210495273698,
0.9715694875465697,
0.6396076794542163,
0.023048840211654542,
0.00538893358072889,
0.9076001969093693,
0.8520333700073334,
0.6709238816059915,
0.9688002237562289,
0.3941715343424579,
0.5614642720100157,
0.37778378303116045,
0.5492824960875163,
0.4393779813482811,
0.8468234177369729,
0.6357168627053837,
0.45583579211255326,
0.9317240661014583,
0.8069996072667336,
0.28845942678546577,
0.29919023276493156,
0.5526150200100461,
0.2295426344321424,
0.9790501817987727,
0.7980381545910162,
0.9415096266559801,
0.8702359285521526,
0.24367005902464353,
0.42768927675757884,
0.6167577261941768,
0.8485808948169674,
0.9849556457198487,
0.7119803956646722,
0.8749830114637789,
0.7216045509083838,
0.5062923895566623,
0.9516194756256406,
0.9577774941295494,
0.9731410201169121,
0.447886292338019,
0.7496261658154897,
0.40079437551173497,
0.38127594842061885,
0.9635095553314663,
0.9940914646780837,
0.5985163581031284,
0.4910974830732844,
0.5172380729899199,
0.6461010814700975,
0.8432085585365385,
0.7729673861293682,
0.9596947425306334,
0.9655236311326321,
0.8570534209821261,
0.7065147319211331,
0.9628825773319067,
0.9715640148842902,
0.9798061473201368,
0.9879214129801631,
0.9606146844724539,
0.9881102958169454,
0.4334709951135701,
0.48744283744366435,
0.45773443723202345,
0.10841690222471927,
0.32557813451323914,
0.6714869798958963,
0.7291257137366067,
0.31612827182173586,
0.9687119969659561,
0.3593192166949496,
0.9508341546962947,
0.9586966954530433,
0.9896761221967985,
0.9917940040613996,
0.8718624191237813,
0.9174969530306738,
0.8832812818489443,
0.8963178868696985,
0.9912279153053094,
0.9682889822546598,
0.2339115627097633,
0.3963258855970886,
0.8463329308872877,
0.21104776786742308,
0.7126408056358581,
0.11297945228808859,
0.8942698865667542,
0.9732243971883917,
0.8636029257375031,
0.9936372311413965,
0.9815003944220255,
0.9658368726454256,
0.9995357170473664,
0.9997768379355896,
0.9957374856667149,
0.016810138659267615,
0.625744079550866,
0.41649911470161444,
0.9481137088156273,
0.9973748383630704,
0.9856314253374556,
0.8683564112012547,
0.13587292232457782,
0.48287810554135346,
0.9697897556140078,
0.29718553164971584,
0.6564333773308374,
0.9621622145002665,
0.9200143413602964,
0.9058070734788324,
0.09500699717436334,
0.7967833177488765,
0.9168183020835509,
0.9898260248161947,
0.9866923025428642,
0.9778250739476811,
0.38494780717884985,
0.9694554112433617,
0.8669249254361029,
0.985602852304723,
0.9801755725600301,
0.9451065445787897,
0.9985191763862198,
0.626511380742125,
0.9989997050961694,
0.7132328171752407,
0.5803617005069888,
0.4050910031408489,
0.9743283994705809,
0.9955204040025749,
0.9847230565962176,
0.23946705939340718,
0.2504595429933524,
0.32331275733832965,
0.7781319321408697,
0.6603465726287439,
0.7376846155197625,
0.9943965342561184,
0.9546931365361399,
0.9957236802982894,
0.9977175734434807,
0.9985807823293543,
0.7515264127530763,
0.9659956755245168,
0.9745518480700819,
0.89709750075,
0.9951115002544422,
0.9750828179238081,
0.9911220896292745,
0.8539024647122831,
0.9602605023481634,
0.6779960936986112,
0.9870908818744203,
0.07569753727576126,
0.9908876170202917,
0.7519834060471163,
0.5114350261448876,
0.29128089250421907,
0.8439033235845357,
0.11835061960640826,
0.05142425554640246,
0.4851485675765131,
0.25571695232900615,
0.36699037628691045,
0.9429203711619776,
0.08797856172733964,
0.8923756831976206,
0.9206148816291977,
0.9472473859611692,
0.9099002322055597,
0.9898406209397985,
0.9937410590428625,
0.9599516809182959,
0.35492801429857107,
0.4454408892104093,
0.19555350480711955,
0.9678616774619412,
0.9854832490213918,
0.8759801125669525,
0.7714229801494253,
0.017255562562099896,
0.4703334960496476,
0.9565109781338792,
0.981750435809722,
0.9775479875088297,
0.8000892462748606,
0.8505259775857414,
0.8948057512598694,
0.987303213680041,
0.9934076715746827,
0.8391112110418082,
0.9828639102950764,
0.9846710071135923,
0.9875577781920457,
0.06725026925077299,
0.3885377961837856,
0.11270028613871352,
0.9809428289643898,
0.9669615905288483,
0.9940606290955467,
0.988534098948984,
0.5137326598745545,
0.8846914676235581,
0.38269751526055124,
0.6210908911258916,
0.6398987755482596,
0.5407275451342715,
0.751764107315217,
0.5223639993256783,
0.9716861571791324,
0.8726272838704159,
0.2238181159542863,
0.4217106289397944,
0.41599457426648023,
0.575432139486906,
0.995595527714981,
0.9566774041659716,
0.9716515454592538,
0.9763288401463325,
0.7050887989598534,
0.8619893491539881,
0.7608844288027707,
0.8122585598995326,
0.5536000683558873,
0.948123332345878,
0.9416075549779391,
0.9293513291881285,
0.7398940481536427,
0.7924856783815974,
0.8113061526058998,
0.6013952348834256,
0.691708487558749,
0.9248179578707516,
0.8312816968813779,
0.9358454337943473,
0.9317098935350827,
0.9430046593931884,
0.8162560791099516,
0.954576636697253,
0.439308732967973,
0.8962769335599134,
0.940401822583704,
0.31568974425714913,
0.43030011202180807,
0.26310010004970297,
0.9167883316822287,
0.9515803281624097,
0.8360233157636554,
0.9015373643337079,
0.8136710737486825,
0.7392216910538758,
0.8776621303223235,
0.9669783409278728,
0.9429268067171138,
0.8572498904694127,
0.723024969415799,
0.9224449168370622,
0.9970892006609751,
0.9973569906150177,
0.9832262809223327,
0.49194628922234585,
0.6372032578913698,
0.6135692629979467,
0.8618122008861239,
0.28737215644816066,
0.8569275432609433,
0.9517954562992481,
0.6806874441628525,
0.9068167334646873,
0.7758646987765705,
0.7635660685534643,
0.40113575398901524,
0.6112440398802479,
0.6587465335982683,
0.7955952713735832,
0.5634560016097039,
0.9529549938984965,
0.5437452669972838,
0.9696063304224071,
0.9184674167850452,
0.9320927176193902,
0.9354970609911056,
0.9915934879833751,
0.876761183037267,
0.9900225765635357,
0.9545802197760636,
0.9790160887802662,
0.9306778505582937,
0.985018902823512,
0.9760834033797646,
0.7205792162753969,
0.8594802901929696,
0.5242229661368457,
0.44649469368766936,
0.5120810910710759,
0.7950652562494949,
0.9392617045897996,
0.9907991733633991,
0.9837407896785503,
0.9946223494193724,
0.9968529072228426,
0.9940168895595957,
0.9959014305421839,
0.9901511990929318,
0.3549938887645508,
0.7282867616027753,
0.5257927855880565,
0.46975195689321547,
0.8787559095691746,
0.8076193530345719,
0.801760526915581,
0.10502970743930287,
0.1525899328115804,
0.4394337727645906,
0.9052592981441459,
0.9793161827793283,
0.9800944185671205,
0.9582144485850829,
0.9359365286175415,
0.990322535040572,
0.7576182823012233,
0.2740037249550037,
0.414812212690019,
0.36189685850258596,
0.5515611487083081,
0.7518334381529295,
0.9622009204096525,
0.8736581970619268,
0.9953474750623695,
0.9413614243894218,
0.6425353675507897,
0.9904226257938854,
0.9909197291003409,
0.9993525879530054,
0.9955634442710826,
0.9240378744752276,
0.7035042085256847,
0.7925865701623052,
0.6946924133629835,
0.6109461165384681,
0.6730250823493366,
0.9117306951477845,
0.9572976139379301,
0.7258138347320341,
0.8891775131915631,
0.997593270308027,
0.8972847822320656,
0.4029255534625007,
0.09800457929188197,
0.4535783956238163,
0.4103399039188098,
0.440651597594101,
0.36278306000694344,
0.8198331193758167,
0.8305093459110063,
0.7833912403638319,
0.9376142788558895,
0.8757203217254313,
0.9394309641309907,
0.24621630012120155,
0.2329248229877242,
0.16386859093047357,
0.2402527822019943,
0.8911374859553858,
0.4970947596942347,
0.4922039278032683,
0.321644993540934,
0.9706274172247725,
0.9212738033432863,
0.9044671341444614,
0.9334156108715012,
0.8072865575269,
0.695519150000823,
0.7639137454868329,
0.16544583672961174,
0.7277043534933016,
0.6791081963033953,
0.9974102640865764,
0.9982258306271593,
0.9987791938344731,
0.9962194934154089,
0.9393603915616203,
0.9381509302729449,
0.8102610892870675,
0.7040834219809597,
0.7964514278273164,
0.9156293317016251,
0.9601596094085282,
0.9162437767558725,
0.1314831303380887,
0.9918757092044086,
0.9844952745824859,
0.1865179344210453,
0.7750158088965149,
0.2895394215656055,
0.9793374185705449,
0.9893853241263466,
0.787854829394387,
0.9120938986218953,
0.8334039501068973,
0.9201885948447117,
0.9569010607569196,
0.947715447927881,
0.16626989170332676,
0.9228081266231472,
0.3003014743063662,
0.8979566089990949,
0.23732899078328018,
0.9027083525427378,
0.0436543764326391,
0.25657572074637497,
0.4839725576286698,
0.6781985878927624,
0.9985645555634688,
0.8988814825277909,
0.9774243233798147,
0.3098017013984377,
0.3668530852438893,
0.7686100341981551,
0.9870172802715474,
0.963553908248085,
0.9834965331263684,
0.5630285142522454,
0.8321303711049843,
0.654963975827348,
0.9980636407312687,
0.9977994828168923,
0.9990784305187614,
0.9876084257857589,
0.9880549138268854,
0.9875427444220128,
0.9800431400571367,
0.951890617405156,
0.9977588706524987,
0.6662611448788041,
0.7211497891575512,
0.7577257424127956,
0.468618506571523,
0.30817828402073766,
0.501314153544726,
0.9315554678970348,
0.8067255417337292,
0.8652874675518349,
0.9952231677138761,
0.9905804799168594,
0.9968792094440576,
0.4063551143669025,
0.4183668817411378,
0.30673193229186063,
0.683170503087152,
0.976279539500537,
0.9915454590058567,
0.9893398710827954,
0.20840625410422134,
0.7522087036330882,
0.6639065360644159,
0.651487441047674,
0.31049535807230283,
0.8717689022727245,
0.6608783642250562,
0.510837617674006,
0.8910265528913042,
0.7157498718393817,
0.9604436025327114,
0.6094070361815838,
0.9281394777829985,
0.9648827332810419,
0.14233109824233492,
0.2474942468008552,
0.07269726523813358,
0.2056561313377807,
0.2874256918675092,
0.696163122092173,
0.36098277277698504,
0.3529657980392793,
0.519858440735209,
0.9772222053939466,
0.9555958907200673,
0.3110236412025751,
0.9765244817951015,
0.9785487048609468,
0.9865569234161923,
0.42168055922550884,
0.09337233927465326,
0.5282677124817843,
0.8305172140794185,
0.9927577963354649,
0.916036506545004,
0.9994840297845874,
0.9990368805680914,
0.9973496054864804,
0.14480820764204297,
0.3107893288641862,
0.3225797927871926,
0.9313284103962411,
0.9911681824770397,
0.930125516571835,
0.7568167450888392,
0.0751427861877052,
0.11870544628800594,
0.321331833802722,
0.39953828706873595,
0.5086638901043136,
0.551196054230811,
0.39295376853995845,
0.5825392584055363,
0.9519409780982964,
0.8801219326495143,
0.9580701823823895,
0.9596615340404726,
0.8344892917975639,
0.4470904111532461,
0.9460954885183918,
0.9705938153012319,
0.7001957334711773,
0.7058727587691398,
0.9152584990294215,
0.9487284444974211,
0.5727625733345967,
0.17162266972018972,
0.08563897734566425,
0.5361720112632444,
0.12676441100249985,
0.15391776859901907,
0.4931384600265347,
0.6652788855318079,
0.9259653300322171,
0.5366045567645233,
0.6177283654410551,
0.646267970893549,
0.8309311008663018,
0.8274970709880519,
0.9366343764101264,
0.8658343747710204,
0.8551584534887833,
0.9787688640346135,
0.993710762091379,
0.9596979856335146,
0.9357697602070487,
0.6901383584699947,
0.7367146677997757,
0.583256341896293,
0.07953772461243483,
0.36667047181023504,
0.4773358474825552,
0.62142047696734,
0.6489321100715572,
0.6313306062727888,
0.9390236025360768,
0.9663987029926713,
0.653385587921173,
0.7018635280103412,
0.35002119616626204,
0.830319094379947,
0.2930320033432952,
0.24383037559298504,
0.33878668906777143,
0.9585674749652207,
0.9338671403453905,
0.6042152222986289,
0.9980021118423141,
0.5356783872981559,
0.9762191288104468,
0.5816611217939401,
0.5705520746936923,
0.9255292818355814,
0.43565171588171225,
0.33311285138822916,
0.6874243918541141,
0.6138098486136232,
0.8403034956127288,
0.8339779800771578,
0.828568087094341,
0.5985609983766822,
0.7825118751061773,
0.7856019931355573,
0.8945556439518909,
0.921384120487391,
0.7492548613321668,
0.7881101289968927,
0.9339227596651893,
0.7451249405810754,
0.860431401107631,
0.75011569296917,
0.964014123723442,
0.9906402139363188,
0.9857397583752495,
0.18754546572219363,
0.9912290411987539,
0.9298603326230516,
0.9122439222409425,
0.9663955389186705,
0.9632241850355402,
0.9473479319636873,
0.9167449261741369,
0.9288688430210013,
0.991091274666395,
0.9226729900647876,
0.9591421355322669,
0.7395582105352451,
0.9477415508668295,
0.9325250582689044
],
"eval_rms_calibration_error": 0.3022047826091495
}