Files
GENIES/results/evaluations/cra/llama-30b-code/counterfactual_python.json
T
2023-11-11 19:44:09 +00:00

762 lines
21 KiB
JSON

{
"model_dir": "models/llama-30b",
"distribution_id": "counterfactual_python",
"timestamp": "2023-10-16 05:02:38",
"eval_score": -0.1602959858560005,
"eval_brier_score": 0.1602959858560005,
"eval_average_probability": 0.7153305772141936,
"eval_accuracy": 0.756,
"eval_probabilities": [
0.3482837292998797,
0.3783023810703453,
0.34406304589137116,
0.8852480913936146,
0.6681274406585637,
0.8500176736518327,
0.9665866666127438,
0.9561804019659782,
0.9807953440967611,
0.18831737576956908,
0.12242655374602998,
0.26740214401059,
0.880038145406822,
0.8052538621579634,
0.630639587350845,
0.41109749132686335,
0.7413337254007217,
0.2992694352813535,
0.7696056344381557,
0.7766334084528915,
0.4873436125382436,
0.928503848645545,
0.7351840743750679,
0.46734931174079575,
0.6339005977667397,
0.8526291600948045,
0.1786043450571611,
0.9616284482809195,
0.9558227226997268,
0.9587585737980441,
0.9373760104304291,
0.5801834064721548,
0.8272531994010429,
0.2116238092572872,
0.5309095468354046,
0.1025723132430989,
0.9701110552723634,
0.8371009811263762,
0.9288307120313521,
0.9711763996832323,
0.458705323302918,
0.7135146586741994,
0.9848092663104608,
0.9554936668613504,
0.8332116279714664,
0.7905046089943973,
0.5567507026317117,
0.9489541711820604,
0.9301455895002383,
0.8464996496459468,
0.9004826625560879,
0.7208559153171017,
0.8464612353312494,
0.8276526131450594,
0.1558288262483752,
0.3485243981224758,
0.15980296462573462,
0.9292745760731886,
0.35744574422626707,
0.6061029618631597,
0.9960385629983888,
0.9476978022125558,
0.819740932950565,
0.03587666663690403,
0.9474383609471961,
0.26036954975411064,
0.9931442695621624,
0.9787663017476513,
0.7507425430179797,
0.929139074510726,
0.7405343680590921,
0.9714817344365474,
0.7955279806225356,
0.935970239360868,
0.7780281847431593,
0.2799703823957431,
0.41536723726778124,
0.5932816007185956,
0.03365123627982236,
0.9897061390004118,
0.2718470845931489,
0.8796234049326078,
0.9886268302338611,
0.9044284134833441,
0.16580912752730415,
0.9328960723769664,
0.2637685013683311,
0.8206042090874732,
0.8189278915043761,
0.4755349058449232,
0.9978040186460166,
0.24198121837387887,
0.9985129492608306,
0.7912216436133522,
0.7816732935667001,
0.3317669805868085,
0.5178242160424771,
0.8731469539158524,
0.6564760492388417,
0.8825078218550447,
0.6075638524454983,
0.8136129248500024,
0.9360148252585867,
0.8908431013443432,
0.8832936705948122,
0.9950257549249668,
0.9959231927115826,
0.9930887520382322,
0.9978908096666151,
0.998194903400527,
0.9961161856307522,
0.9741935529545909,
0.7100506504257222,
0.7927435434525484,
0.8312118033505079,
0.7000629037890058,
0.9754502172485313,
0.9801819251450693,
0.958578314944936,
0.9865325093613755,
0.27783587542761545,
0.9849419778044982,
0.5762200411740057,
0.9705428164121723,
0.8924753510396473,
0.982846068175998,
0.7816658909217868,
0.7589410802941363,
0.475017592700799,
0.4773647537322373,
0.9902569349084885,
0.9670223956719262,
0.9482286655748237,
0.8841851699490643,
0.9896666357786182,
0.4371720388629631,
0.47788360733262997,
0.7118399828990372,
0.5700085763400243,
0.9185192446759636,
0.9874131753602005,
0.8953480638605148,
0.9952433723547541,
0.4036071017733496,
0.9568859685239114,
0.43855371327296644,
0.4657743711989162,
0.7819312284791321,
0.7723597540920757,
0.8472502415588077,
0.5640077518066451,
0.4672561172313241,
0.47434407898650666,
0.9818992848163407,
0.9553553066899774,
0.993775220271461,
0.42785605321182063,
0.4580973848196294,
0.5058844286196256,
0.680684575403971,
0.20624146838811502,
0.9877147298162767,
0.9851662886030403,
0.6891276366772126,
0.9147106636489923,
0.7797938954621293,
0.4621184340959983,
0.28652817447531975,
0.7485447919135981,
0.16663489203198345,
0.16563512671012043,
0.9373133237491007,
0.9000801450433846,
0.8267582453356539,
0.918764118315305,
0.9122092102045225,
0.9219828806912368,
0.8339107655574973,
0.9566984387372508,
0.699205569995832,
0.5642406447350026,
0.7321117026351822,
0.07041075891173643,
0.44118307606845525,
0.9935080414575415,
0.5838243075073661,
0.07602892239871589,
0.6842252515450816,
0.13643477972850354,
0.9533623374560461,
0.5102010660137288,
0.9624148080079036,
0.5827833659593701,
0.3937173457214009,
0.4183655532041177,
0.8155786242128972,
0.9666890639219299,
0.9136034824504835,
0.8026972935256482,
0.6915380179537132,
0.5438234708420076,
0.41813126006040396,
0.6145922944533997,
0.9128724636282429,
0.9878613434354494,
0.9985944810122943,
0.9968112985741862,
0.927573280614265,
0.6777335256933413,
0.398975676164364,
0.6604935281265021,
0.41376350453866095,
0.503725895822542,
0.6873639730796464,
0.8948202362158899,
0.8262095425832101,
0.9461241721203385,
0.9682656850252713,
0.9961109192292251,
0.9963729706829286,
0.9986378738499114,
0.6691563448315321,
0.9065326901742689,
0.9348192157833585,
0.8996075085337669,
0.981544617966517,
0.7929050544667985,
0.7966620412293048,
0.2541401240388722,
0.9636662885855094,
0.8562910633123442,
0.3168834775747087,
0.42141123849693585,
0.16654552393771627,
0.9410682448051426,
0.9580714923462634,
0.9517394456331092,
0.8976951103933799,
0.9393489526621494,
0.9851142497136983,
0.9992470124892179,
0.9990014831824566,
0.9988728397120907,
0.8926704694625638,
0.9664791557182622,
0.6768583560357694,
0.9097465361027998,
0.9837506808003315,
0.9063756792759121,
0.7571433561443484,
0.6021767215901802,
0.7897829266670828,
0.35205442770073936,
0.5180170751437194,
0.4184453186267005,
0.8439254538265742,
0.5218118757444076,
0.9453575720164186,
0.6206628035807287,
0.7989924702478353,
0.9677665123487394,
0.9247984070052898,
0.6847115433661146,
0.7144194384106587,
0.8997856910963092,
0.47772495746196975,
0.8578525246164402,
0.9133032353125251,
0.9162668729807256,
0.927711367167412,
0.9906704527602238,
0.28605681332993027,
0.9069367998969399,
0.3815427990142765,
0.5372943912341306,
0.4823988378646796,
0.8054457952380122,
0.7919206349372578,
0.3737484340012249,
0.08366829016073585,
0.10267466289943385,
0.08500515739896708,
0.963806778687632,
0.9235918434139467,
0.9027713553777498,
0.9982842238304499,
0.8548779173814132,
0.9781676215102154,
0.892052549744723,
0.9896081054272707,
0.6261460929221344,
0.9899996275547724,
0.9684557216445062,
0.516341474464463,
0.995282974874602,
0.975666711848419,
0.9927104232697546,
0.9906089362426832,
0.9777618121094027,
0.9693751774129954,
0.9918600961975593,
0.0377481437300933,
0.9976095630248185,
0.505780326433546,
0.8898887773452625,
0.6357283364072068,
0.40875969216319424,
0.6663470170747248,
0.7899569234638015,
0.31029487449349646,
0.3069318096634421,
0.6852885312600903,
0.9973518791349223,
0.04427084237341456,
0.8859488371361542,
0.9687756022596398,
0.9750742228357443,
0.8130214260846955,
0.9708704196605836,
0.9893659127500364,
0.7605260376798695,
0.3580599928885142,
0.23546160488616225,
0.1855033715574513,
0.857124172891642,
0.9267283150066444,
0.43210684804592764,
0.9518728918864202,
0.8040334345041391,
0.8571381716662386,
0.11263091262236591,
0.7374689658357767,
0.5280373228086105,
0.8295297948904461,
0.7214054262771518,
0.703110447333888,
0.9879274373276764,
0.934247404557961,
0.908032109433639,
0.28911812046922303,
0.9139501182899412,
0.12223000944383344,
0.4085585732757162,
0.3520942848094857,
0.23281864646673922,
0.6675647204043917,
0.735537513265502,
0.5210359240281881,
0.9741745556726085,
0.9120914120840438,
0.9316589482145893,
0.049531263247419924,
0.08681656534706828,
0.029420399201542692,
0.9866142447271007,
0.9107147642562972,
0.7755309917352194,
0.9909432861714187,
0.997914978817138,
0.3787970043529012,
0.9089822212808241,
0.0866431078472829,
0.9172063914781335,
0.9900191308021457,
0.7112877936181379,
0.7982271987774979,
0.6375864699883405,
0.4588315649782437,
0.6244544342047518,
0.7474920564962062,
0.9925794002858057,
0.2938496124386301,
0.8860945024893111,
0.8874132590206638,
0.8145132053081475,
0.701640819489057,
0.9551181720348829,
0.7750723572440175,
0.34404258909281815,
0.43751986803906306,
0.8712432824093016,
0.7323986242682926,
0.9836424308689056,
0.9671515600270906,
0.9952925956521708,
0.9829821907596669,
0.9849750737446379,
0.7381354285604097,
0.7673242214899141,
0.9791028277692181,
0.8153915289934034,
0.7828098418353894,
0.6116846843821868,
0.8746914063679969,
0.6676950196975383,
0.9211319917911136,
0.843351852107736,
0.5927711419194429,
0.39282366799203094,
0.9854056644103267,
0.974432951332648,
0.18146905870865593,
0.9956913088644569,
0.9895649272909052,
0.979412396602162,
0.9980977121070569,
0.9980948428289234,
0.9978630034026087,
0.8754219072045079,
0.7237186945380356,
0.8012164539613912,
0.1064429673955648,
0.8624930912553788,
0.22007552055825905,
0.9614415321025308,
0.285787790586001,
0.821475432850232,
0.6219046050024402,
0.8584198658089781,
0.6850818110160728,
0.10761860671322909,
0.06165651125371614,
0.2049824627686472,
0.9712588060295666,
0.9749626609346688,
0.943403386097038,
0.9118743971490628,
0.9568620614207388,
0.16246123250466543,
0.9312831030143861,
0.9446830787728298,
0.9257449759943558,
0.8050504460798991,
0.917578632289532,
0.9443167239631106,
0.28464660707901446,
0.46673259601461337,
0.8014598821997417,
0.5147390035742745,
0.37415778572404645,
0.942917733264327,
0.7240810317847948,
0.6183443639613802,
0.35696945792135715,
0.6557271590179223,
0.45794998353655353,
0.6391857390666649,
0.8409977752111826,
0.9976680381691225,
0.9995023421354746,
0.4271278170441859,
0.9643744929176656,
0.8678590947851383,
0.6282732017882603,
0.4946485418478618,
0.5775089482493851,
0.7818754503331515,
0.8603114895446637,
0.13315727246869313,
0.6321722579375965,
0.8806676062116489,
0.9592025750291225,
0.5178934190471527,
0.4362419755471012,
0.7674834347222856,
0.8965901558651117,
0.9044666718236636,
0.8502662114797137,
0.9521185285701248,
0.9760859425760409,
0.9957880891502178,
0.060517663534970234,
0.1584316059948152,
0.09766770200071839,
0.9960378424259719,
0.7490796269126034,
0.9983802590459592,
0.7010124540221994,
0.21540509642592554,
0.883273084643376,
0.8458353935083158,
0.9921991966075455,
0.9565989555209453,
0.6861781895221926,
0.23124415482061655,
0.2789927721773747,
0.974519244782859,
0.965410852385001,
0.9766730723857964,
0.7369681125218612,
0.9679781553514502,
0.7106550298400295,
0.750640018088822,
0.6305999534336778,
0.9197171242112517,
0.9345018357441529,
0.9482761089777015,
0.9253211516795594,
0.4566901952947894,
0.6046445626218383,
0.1883754148419593,
0.8580602496365893,
0.9271839955181358,
0.18745653755059288,
0.9853308689673338,
0.4362687916638279,
0.991692617773852,
0.7322590464953957,
0.8124020488509772,
0.741404506251846,
0.5548726623030186,
0.8307258638487562,
0.5408931179542475,
0.2570031218776686,
0.4121389944191537,
0.8776081653273926,
0.9628764230300787,
0.6122037934628579,
0.35562809518633476,
0.9635674042740044,
0.9795898091045022,
0.9780240256716629,
0.23114900688028447,
0.5112820719755945,
0.8392862084148448,
0.9991709355587056,
0.9994758403630871,
0.999530622448215,
0.9883949174410971,
0.9954918818169072,
0.9988087361806876,
0.9172298321684752,
0.512215258979031,
0.9063306342488172,
0.9845954694132131,
0.9480394462288949,
0.9632393396629543,
0.9467055817669626,
0.970532073661207,
0.9911634858977529,
0.9973815644115188,
0.9977348868375615,
0.9960273560522643,
0.9322927364068022,
0.8918734212298634,
0.8904655515842633,
0.21435258602311014,
0.9790947227553685,
0.9739347380678925,
0.526866420177302,
0.841239520848795,
0.18309783386210932,
0.5622534888457946,
0.7900044380331482,
0.5746581576271784,
0.6397493816022859,
0.8109994451592443,
0.796815857785305,
0.9403537244529927,
0.9492766617412843,
0.9833465280075316,
0.9544636556171896,
0.9021959447617447,
0.9148972115428702,
0.9910671479889225,
0.9899890022839585,
0.9930623104068543,
0.4969016866985283,
0.9172211049574555,
0.7783653613433845,
0.8841484421134813,
0.8916847672929814,
0.9789772449022675,
0.9990727653948631,
0.9949320899660549,
0.9898901056788,
0.24978144084448406,
0.4133948642066546,
0.6972243142993134,
0.9731488979825651,
0.834597765003432,
0.6018544156331703,
0.982285222602242,
0.9560370115179697,
0.9932556074457511,
0.7583974946458854,
0.23596131694064193,
0.2197768935469634,
0.9562244143266814,
0.7958346721137389,
0.9081527153549205,
0.9945878193175554,
0.9951273511594135,
0.945946208493047,
0.6871990774081982,
0.5824570471422151,
0.7118946822519394,
0.4759519993255025,
0.9230557143905683,
0.8074626439458131,
0.09810225405019363,
0.10134026057957979,
0.782808725070329,
0.40320019630960735,
0.32779011449426765,
0.5222271816255813,
0.9952365450873182,
0.9961426807840965,
0.985471359008213,
0.6149514756440467,
0.42474502388249014,
0.8226944749450196,
0.7685998020808963,
0.9140494543596499,
0.7817881678223233,
0.6161427841445997,
0.3837997373943493,
0.6002983047856918,
0.38454472278404417,
0.4985733279590814,
0.2422422850252357,
0.5446234436315016,
0.42753417588064074,
0.5161790766257691,
0.9609282119792704,
0.8639248387466316,
0.12785604504681003,
0.983496325748218,
0.8272292654356762,
0.9977938252669634,
0.7553393436044342,
0.4126629811565957,
0.7281638369921986,
0.9206849554983553,
0.9905974783430489,
0.3014905928785945,
0.9997128734792569,
0.9991569649319902,
0.9992783000270095,
0.9802326182302509,
0.9937580660375622,
0.9935135932069823,
0.9016142391749976,
0.9819307083170147,
0.9333983594085686,
0.728604626438466,
0.6713452511421085,
0.4672459737821737,
0.8278760551292996,
0.45741424858898866,
0.5006177678253302,
0.6610538936153391,
0.5340768871342858,
0.4158468013925058,
0.9804250762260913,
0.985007555209467,
0.9939921328609622,
0.221119938856445,
0.18899783646431587,
0.17853401111787828,
0.8108773930647952,
0.6660307689297544,
0.2017453005202136,
0.3521495842095583,
0.7173561662285606,
0.14597703518952176,
0.9898170176193182,
0.9361496658587538,
0.9664149371528107,
0.9946006070448458,
0.9799789949514319,
0.9936367800759488,
0.4381788539490673,
0.07613396328078038,
0.1854667531922786,
0.9910419685720075,
0.9840435622329676,
0.9973076202051944,
0.7444396347122462,
0.055166600623406496,
0.24547194504678435,
0.9248807746474513,
0.8857111491683518,
0.8648557138887764,
0.9964391919726295,
0.999082034732223,
0.9728338116049408,
0.14387617296238087,
0.31708583866237117,
0.09513128934107463,
0.1098194879167499,
0.11453909392731654,
0.10718072077787433,
0.6830309198340987,
0.37179905441509076,
0.6817178953326635,
0.7889507323475526,
0.7186031884974836,
0.9688451491058756,
0.7851481101629425,
0.8284649324206429,
0.4479530890573225,
0.0924434720821522,
0.17646518175237505,
0.28010082669048764,
0.5168660449390295,
0.8834294318064987,
0.9485971759060231,
0.35482351194951356,
0.12165056530807905,
0.8442152387946288,
0.9962755331008303,
0.990110965377304,
0.9968792103899246,
0.9301925197224078,
0.8975385239732893,
0.9086177377643376,
0.3905846693832626,
0.4967158124272359,
0.8360636206812255,
0.48947458468343485,
0.8285822320062416,
0.5478548354015778,
0.6138542401190111,
0.8110672986075866,
0.5488907488735744,
0.6090397172382643,
0.7559246811859499,
0.3256539818282496,
0.9970160977439657,
0.9958518459651241,
0.9928567009807621,
0.2661115861123205,
0.771429319613574,
0.18659085755352856,
0.9114752756359458,
0.8213147923072197,
0.9939077003138168,
0.7287378679031513,
0.5933865152465665,
0.4741298600881667,
0.9029202360633348,
0.9581313474472939,
0.9212961780259411,
0.10183784164260279,
0.29301017333950874,
0.0636904089680359,
0.677679432658763,
0.9267657734019976,
0.9930394016121549
],
"eval_rms_calibration_error": 0.30141389487139975
}