Files
GENIES/results/evaluations/mms/llama-30b-code/counterfactual_python.json
T
2023-11-11 19:44:09 +00:00

762 lines
21 KiB
JSON

{
"model_dir": "models/llama-30b",
"distribution_id": "counterfactual_python",
"timestamp": "2023-10-14 00:57:16",
"eval_score": -0.1210494198466278,
"eval_brier_score": 0.1210494198466278,
"eval_average_probability": 0.7955275027653708,
"eval_accuracy": 0.832,
"eval_probabilities": [
0.4994596227312894,
0.9830697457422535,
0.6761181276527769,
0.9790940284747771,
0.9879866314551482,
0.992985854082541,
0.9502179146537891,
0.9497422251341852,
0.9546358835847337,
0.9968242706367211,
0.9997380743350573,
0.9996931645569691,
0.7671404717509651,
0.8913773688938686,
0.9417579753791495,
0.7681098910690609,
0.9396219086738937,
0.9528335664135146,
0.5641415834793536,
0.8353329300564969,
0.988036171330343,
0.9485565435459689,
0.9267750591834006,
0.9998928374510772,
0.8203272191129436,
0.9324975566567333,
0.13703210173128957,
0.020140390441617716,
0.014653087493839409,
0.05451768974020772,
0.5709146406172967,
0.6070816301931146,
0.9878284744604555,
0.9901647298392862,
0.9895837407488343,
0.7598045090184362,
0.9576724240395799,
0.9845560738555056,
0.9894395136517675,
0.9898724070042736,
0.828424761121092,
0.2761430476790979,
0.6318488566611999,
0.6730816450147721,
0.3659183241910513,
0.9862528525827025,
0.4091206188704494,
0.9804355043332443,
0.9128473647661373,
0.43032117521471946,
0.8675103765255833,
0.5953024490760344,
0.940065730450544,
0.676326725993841,
0.5268557783486632,
0.07323645556542045,
0.18512136210720942,
0.9964968887101457,
0.9945419246908965,
0.9973912758299442,
0.9999210396764112,
0.9999222345888596,
0.9992371842417129,
0.0994601247475223,
0.9722797490965173,
0.9504697589950296,
0.9842340297200961,
0.8270811968727609,
0.7931700208303757,
0.8808779672512941,
0.8874067816636343,
0.7318958978383047,
0.799310004193758,
0.9938766334144225,
0.9976557074373981,
0.9842812044170387,
0.9852110106942475,
0.23632035721145983,
0.013003004969846632,
0.9651285966666996,
0.006894405246874791,
0.8767204340940182,
0.9700136808764578,
0.9793041061364903,
0.9117229944602866,
0.99593821971211,
0.9947919100935937,
0.3512196788387602,
0.3016034595137677,
0.5030260936047187,
0.9996862313773067,
0.08371546125313649,
0.9938457192876816,
0.1319352510857782,
0.9747329194358147,
0.9553321486054664,
0.9936898099663497,
0.9908800019919889,
0.9922627574046745,
0.9999349921280685,
0.5435457515015849,
0.9973801528749103,
0.9582364349310426,
0.9669051063409018,
0.9824253151860294,
0.9911647606774308,
0.9943651125839111,
0.9870539023348615,
0.9585714628879751,
0.9817355905261983,
0.873279065660405,
0.9858619242414177,
0.8772203503481911,
0.8596403555031827,
0.9210677379599786,
0.9939013896107223,
0.38959532990465273,
0.5415586609224285,
0.7918501168228502,
0.7130661276294178,
0.9866300439007052,
0.9981917546614624,
0.9992177239659739,
0.9986593445857919,
0.999768791868441,
0.9997690671601693,
0.957700624871202,
0.2587060718348465,
0.9184534978055107,
0.9304300606835381,
0.8947882653847251,
0.37776499573321787,
0.37617453478311846,
0.9990720483326291,
0.9979045233621215,
0.8826035617773077,
0.744988056577687,
0.359266762781577,
0.4224447549380833,
0.8838697811161604,
0.9717947411073223,
0.9109812674591865,
0.9839652825755203,
0.9932723465237259,
0.8017443389368863,
0.5571450874142521,
0.3355016689855085,
0.9972974452053748,
0.9985645841474265,
0.9879651156810261,
0.3296389884442189,
0.2989957126046423,
0.9813051869544073,
0.9945708959733337,
0.5773725602193246,
0.9675427268341327,
0.259726984021507,
0.28914549048214994,
0.27724887739672555,
0.9123948573093279,
0.12112116266253803,
0.8819786312018454,
0.9572541027775873,
0.9895315441319088,
0.9832628346071525,
0.9970192814704294,
0.6860523415059302,
0.9323506291632724,
0.9746348209351275,
0.8305502505050111,
0.9928698002887824,
0.8764756811423818,
0.8525776368697664,
0.9721154559309492,
0.9669750598840086,
0.9344781812191871,
0.9279148808928738,
0.5257508964109086,
0.9988479416004146,
0.9904823492199282,
0.9925166666359793,
0.6505802799852038,
0.7753416579626443,
0.8610055818942869,
0.99425230087933,
0.9168586973608489,
0.5840936233795815,
0.31992430934385874,
0.12826346967413738,
0.9891739771611423,
0.9975288357140529,
0.9999182211680955,
0.5063836559020124,
0.25625142282695756,
0.6848393576421102,
0.631847745770684,
0.7549018358845171,
0.9221000589724078,
0.9963790002155264,
0.9890266389371603,
0.7590074348775725,
0.9114945120475614,
0.9643663092359718,
0.9519022650551944,
0.9956489685717416,
0.992752019803463,
0.9895502767448131,
0.992182009544004,
0.12859730610683417,
0.4969986791304286,
0.4182244332909274,
0.9570340660185019,
0.1387020403279665,
0.958869328901663,
0.9428402103441009,
0.7250961976676368,
0.9918005917574786,
0.9934237614399088,
0.9873782169885076,
0.8983309925231601,
0.9902222323772172,
0.9917312994661381,
0.9913633925302272,
0.9872034105830768,
0.9913705951551955,
0.9991480603852438,
0.9503085940668056,
0.9980242431063966,
0.8373861894977186,
0.6922487588145558,
0.8407738573238164,
0.5729163588955296,
0.9742548673997029,
0.6794800222998123,
0.7857118394036027,
0.9742661845791998,
0.5790482679307691,
0.998984621396015,
0.9979168974693984,
0.9949825790901744,
0.9999941500163853,
0.9999930346739063,
0.9999510052928648,
0.19963642030989126,
0.8980868161475201,
0.9677743493695382,
0.9904299722343738,
0.9999526854267965,
0.9978495382020129,
0.989345477442509,
0.059325587346786945,
0.11093229579336257,
0.48682423940755654,
0.8365651450611851,
0.9220337496056964,
0.9982127849055668,
0.9993566026201467,
0.9622668976571563,
0.9210323164559797,
0.9873213420773848,
0.9835230117900708,
0.999368297174878,
0.9989474110457099,
0.9988218945413124,
0.7951513527899927,
0.6017435930152407,
0.5982650150944514,
0.9979014953402534,
0.999524559878509,
0.9996214813344124,
0.9999286516308747,
0.9014622074152558,
0.9998393589861535,
0.8359527596640343,
0.7923852560855424,
0.3793507832949896,
0.9998579052802673,
0.9984925746766918,
0.14545121938120073,
0.16879295874424513,
0.12822517633328043,
0.8079333881575312,
0.7914994262062567,
0.7144092328886074,
0.9736919659671964,
0.9999823722842521,
0.9986821318577798,
0.9999061252370731,
0.9995334268470633,
0.9998795218793681,
0.9632511743219205,
0.9816651973772433,
0.9998698717492964,
0.9374615891028052,
0.9693424495251817,
0.9725108760411346,
0.9445230928901587,
0.99837169030731,
0.9987737841269977,
0.9745540857942334,
0.855490507272628,
0.06385247893572144,
0.9921712038101476,
0.9962052421497394,
0.9996488021343731,
0.9893302752585678,
0.9752269685229631,
0.21504493273277037,
0.5510558305322699,
0.7627107983803905,
0.6647391726506962,
0.9033757052702264,
0.9998146893561298,
0.9695317045152977,
0.983834899327194,
0.9993368016312958,
0.9992971182977198,
0.999832911190766,
0.9989734356819956,
0.9999191562273905,
0.9941330812840301,
0.8982263728693693,
0.9468131966186877,
0.9213485110108337,
0.9561975991987521,
0.9895566808682941,
0.9175986031537867,
0.9954369132737195,
0.9281868948158041,
0.7344643288079493,
0.2823763028487887,
0.999664139807425,
0.9988050201987348,
0.9529414057816404,
0.5590122263104166,
0.9215480380518278,
0.999088042767089,
0.9985912187845195,
0.9819376030789391,
0.995960923503627,
0.9804060774519536,
0.9637928665383247,
0.005362730480562138,
0.03279568986692227,
0.011365415866797304,
0.978829027136805,
0.9828088088915727,
0.9873309930868529,
0.9816755950407534,
0.4556489263276,
0.5147597685054144,
0.11825122610038648,
0.1045294204322107,
0.1709585667836797,
0.28551063212338645,
0.6009072588645462,
0.023660384199321958,
0.9960744189163548,
0.9402373610557605,
0.9822553375771655,
0.5058942265791592,
0.11601285651922078,
0.9441707701172497,
0.9939341056608547,
0.8544203860021633,
0.9828135911516505,
0.9991335820629043,
0.267382217813112,
0.9841814046565897,
0.6926769204428156,
0.9378606317957211,
0.5637323410781037,
0.9320221040950836,
0.8869811675952127,
0.942897055922611,
0.8749298954388647,
0.9719690967131298,
0.9246598334024948,
0.25212943717863967,
0.9062174907959029,
0.9957314155411302,
0.130603705361389,
0.971390354919085,
0.9952020015982946,
0.989897881134638,
0.9746725145559471,
0.9917012376585405,
0.17910739749732546,
0.709919143685499,
0.874747936007634,
0.10485484637789189,
0.06459136282693023,
0.3804091599231914,
0.6962564014538197,
0.8289308892474065,
0.9302922874862793,
0.46396039207611117,
0.6696546270872941,
0.6969114722251775,
0.990109685875292,
0.9991375899876487,
0.9995935838323702,
0.8074044713871185,
0.4603066305039317,
0.9125430418135543,
0.9987361281292336,
0.9984610738956342,
0.9989852412593464,
0.7194806606202337,
0.483166129308867,
0.48697483492119253,
0.5423591712219109,
0.511437885743547,
0.7637933912970598,
0.9494088305630697,
0.923470828305564,
0.9070091694658742,
0.9609772512682367,
0.7284985398550509,
0.8148197175241988,
0.5403179389580307,
0.26832510565019835,
0.8575780196700898,
0.9856111925761528,
0.9968361354062576,
0.9927374947273158,
0.9999243162376192,
0.9964580701028789,
0.9902658875586852,
0.9701997788690508,
0.9996389252311962,
0.9835411503031783,
0.9990735336604923,
0.9996443203495764,
0.999466213067142,
0.9857689508055011,
0.9537857164738222,
0.9972784056209165,
0.1454516485712931,
0.9107590971698029,
0.9124152668490042,
0.9427812367885453,
0.7224541044354882,
0.526154541494199,
0.979515859487405,
0.9998525165937996,
0.9648377251692121,
0.9992162602174641,
0.685226302767673,
0.9569770324353712,
0.9998442726463724,
0.998975576092651,
0.9564438576221018,
0.5787636268386083,
0.9939351205230448,
0.9592067793346521,
0.8889384655612487,
0.9248988550987424,
0.7740329509989787,
0.9724332389893767,
0.5546792546801443,
0.9928869282707287,
0.9804373338804535,
0.9770892074835196,
0.9935964809040931,
0.9900390684343244,
0.9893403653581155,
0.9925009369210628,
0.7734380903378199,
0.6370781251505362,
0.9205083334588738,
0.1880705102434626,
0.0942439821356672,
0.20436214276296893,
0.9975999673800343,
0.9832985351170362,
0.9970682063661617,
0.9916745290976255,
0.5812258829109805,
0.9837258556800742,
0.991648011111408,
0.9998905474549817,
0.9981371199560866,
0.9070712710257014,
0.9871669033195472,
0.4149988548414743,
0.8777222519741323,
0.8262483841340911,
0.8501196836340204,
0.8961756463577774,
0.9562508740524144,
0.9409438049682759,
0.9286942351750606,
0.9997058528484168,
0.9996726872076542,
0.9519489252336635,
0.9020242440032263,
0.9269767107617276,
0.18196683049596177,
0.16654498468234996,
0.575511036612419,
0.8999355463840633,
0.9496884015294393,
0.760727584090432,
0.9975230756684007,
0.9983928551955056,
0.9988885586613732,
0.24181245544553434,
0.2390471310698037,
0.32398489033268507,
0.3189598047618828,
0.8860421988057242,
0.853564530694866,
0.47841721114336766,
0.3895825839836688,
0.9575390370056415,
0.9927904638537923,
0.992602218620355,
0.9869795447653177,
0.5534016817675818,
0.3235408080580336,
0.46404683088065857,
0.002851254280834372,
0.024905883953449726,
0.39990390854153474,
0.999530688364502,
0.9998094536896738,
0.9996380203822094,
0.9992292681027609,
0.9846261884859975,
0.9843683173291385,
0.9319543646803198,
0.9170596189672418,
0.9728333044598035,
0.9992997299214013,
0.9972760747096359,
0.9988081839228107,
0.8498876996983833,
0.9968624771472792,
0.9994135289793166,
0.9848631850843171,
0.9985300316174996,
0.7564066112045231,
0.9954385182173201,
0.994566722687248,
0.9689380439687482,
0.5023321533779329,
0.9608283328135182,
0.9374035299026235,
0.9840383359285051,
0.9596917705731755,
0.8171007994187419,
0.9608984665993764,
0.5497942947822017,
0.9527579548388186,
0.04617030261254118,
0.9361969524346834,
0.9420143454168347,
0.1417488698129017,
0.49112497272401284,
0.7039803082669375,
0.9999527590905988,
0.9963314646600797,
0.9992761628221183,
0.7481979770535292,
0.5191601673315707,
0.9904455649676348,
0.9980941878382126,
0.9218524615040332,
0.9997591541485196,
0.961965194300712,
0.9954455651346685,
0.9738775433522722,
0.9985733601630767,
0.9998707753080811,
0.9997693968472537,
0.993674309494479,
0.9592150940903098,
0.9792534724922768,
0.985364054551646,
0.9977620562079882,
0.9999692456143087,
0.9872023229731008,
0.991869316725319,
0.9962050936911019,
0.787705370714308,
0.0178077421125159,
0.9578555577703501,
0.6785812561544085,
0.5659399950196848,
0.7385476772650172,
0.9989188694029626,
0.9973588815489944,
0.9937565997066082,
0.9190222606099704,
0.9278025266820563,
0.9463751378700623,
0.6983606042032569,
0.9930404504988846,
0.9971888659200164,
0.9991459002159911,
0.9472274821584504,
0.888405884087169,
0.9714718650952786,
0.8474046588320773,
0.9488858144112831,
0.964240968152332,
0.8516789139362468,
0.7938445399149058,
0.9738893862720738,
0.9566232592798625,
0.9549393440267667,
0.995144336374552,
0.8787848707023568,
0.9713699385032467,
0.9481692100075596,
0.9993627823252337,
0.8793315727744835,
0.5604991846327643,
0.795734536767755,
0.320232465960549,
0.39397172568490463,
0.24516974383726156,
0.24134344081151549,
0.8107810078932584,
0.7823097300161064,
0.017369299294356044,
0.9970380665596932,
0.9198700114527215,
0.965970017328499,
0.9666555236649039,
0.6207846665595698,
0.876765590933591,
0.4219927594432938,
0.709676193841456,
0.39104488425389383,
0.9999639758510622,
0.9999757185409412,
0.9996348071876237,
0.05419384940388408,
0.12796075914416188,
0.09457880468798834,
0.9941146690397518,
0.9995774946982351,
0.9407161670890178,
0.9001482218832239,
0.012376781828172591,
0.011241224104543783,
0.9695471481418317,
0.9736840743739285,
0.9738817952581478,
0.29423763997247687,
0.4135231197658542,
0.2330329890302893,
0.9010353839396608,
0.8076802057488411,
0.9105008458469138,
0.9513327868282017,
0.10783537726324832,
0.08139828711973805,
0.9557556381075983,
0.9195714321980801,
0.9027001749721512,
0.2822551358458568,
0.9096837646003915,
0.9247181908184882,
0.7440233528471061,
0.28725860037944984,
0.16984314512778034,
0.9900493065996413,
0.3705244244492758,
0.5195279694343415,
0.32241324616306166,
0.8773419445382564,
0.9450042678288851,
0.8779198443641277,
0.7263542659819937,
0.8330829823196638,
0.9953630861569137,
0.3948965395019294,
0.9442005932322844,
0.9996445274445508,
0.9996278771871652,
0.9966245431513535,
0.9901810510119423,
0.9490353593842783,
0.9568774860218565,
0.8783095694417801,
0.7517447860947909,
0.5997041051934197,
0.40224410746650974,
0.9932457278936375,
0.8614479373255706,
0.8900901201382344,
0.8170893798533427,
0.22314076967868837,
0.9992154736046637,
0.9434240681109025,
0.9153536154518171,
0.7311129083035066,
0.6127342868760424,
0.5448517510325463,
0.9303545875038128,
0.8911703770989511,
0.8173276484954181,
0.9933391986370118,
0.9979324846057183,
0.7034762371725323,
0.9997818150735389,
0.9706293350756902,
0.9062959969784968,
0.8487907775927394,
0.5753310977021052,
0.5518662086789322,
0.47920496831145776,
0.48708224667767847,
0.7176735328588735,
0.9919025910471428,
0.9957163945322187,
0.9988585416022305,
0.9676708588626063,
0.9401718504641617,
0.9778531643821716,
0.48482962829542536,
0.19653116639778245,
0.9874157166665435,
0.8741415575191687,
0.7380184774850397,
0.9792270659549401,
0.7002239559682926,
0.3738324160015515,
0.9291940146434118,
0.9942908163089287,
0.998492137744469,
0.5775191238958343,
0.38037876894708894,
0.9996204250364483,
0.9977303874867348,
0.993708026995922,
0.9846900531998343,
0.9878225029508765,
0.9708127457906005,
0.980030564142893,
0.9689754645752103,
0.9920249070632933,
0.9763775362047861,
0.9318956723456827,
0.6664271705082677,
0.20289381880652546,
0.991116779314202
],
"eval_rms_calibration_error": 0.31651198418281457
}