Fix

2b855b70c4py/evaluation/generation/2b855b70c4py-results_lm-eval_global_step52452_2022-12-28-11-33-12.csv

@@ -1,21 +0,0 @@
-task,metric,value,err,version
-anli_r1,acc,0.333,0.014910846164229868,0
-anli_r2,acc,0.334,0.014922019523732961,0
-anli_r3,acc,0.3333333333333333,0.0136139500102256,0
-arc_challenge,acc,0.24658703071672355,0.012595726268790129,0
-arc_challenge,acc_norm,0.27986348122866894,0.013119040897725927,0
-arc_easy,acc,0.5526094276094277,0.010202832385415646,0
-arc_easy,acc_norm,0.5008417508417509,0.010259768981815234,0
-boolq,acc,0.5966360856269113,0.008580168554889729,1
-cb,acc,0.39285714285714285,0.0658538889806635,1
-cb,f1,0.18803418803418803,,1
-copa,acc,0.72,0.04512608598542126,0
-hellaswag,acc,0.42033459470225054,0.0049260381977145285,0
-hellaswag,acc_norm,0.5456084445329615,0.004968979259738329,0
-piqa,acc,0.720892274211099,0.010465657948498228,0
-piqa,acc_norm,0.7366702937976061,0.010276185322196764,0
-rte,acc,0.5270758122743683,0.030052303463143706,0
-sciq,acc,0.828,0.011939788882495321,0
-sciq,acc_norm,0.745,0.013790038620872832,0
-storycloze_2016,acc,0.6761090326028861,0.010821488046867113,0
-winogrande,acc,0.5438042620363063,0.013998453610924324,0

2b855b70c4py/evaluation/generation/2b855b70c4py-results_lm-eval_global_step52452_2022-12-28-11-33-12.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.333,
-            "acc_stderr": 0.014910846164229868
-        },
-        "anli_r2": {
-            "acc": 0.334,
-            "acc_stderr": 0.014922019523732961
-        },
-        "anli_r3": {
-            "acc": 0.3333333333333333,
-            "acc_stderr": 0.0136139500102256
-        },
-        "cb": {
-            "acc": 0.39285714285714285,
-            "acc_stderr": 0.0658538889806635,
-            "f1": 0.18803418803418803
-        },
-        "copa": {
-            "acc": 0.72,
-            "acc_stderr": 0.04512608598542126
-        },
-        "hellaswag": {
-            "acc": 0.42033459470225054,
-            "acc_stderr": 0.0049260381977145285,
-            "acc_norm": 0.5456084445329615,
-            "acc_norm_stderr": 0.004968979259738329
-        },
-        "rte": {
-            "acc": 0.5270758122743683,
-            "acc_stderr": 0.030052303463143706
-        },
-        "winogrande": {
-            "acc": 0.5438042620363063,
-            "acc_stderr": 0.013998453610924324
-        },
-        "storycloze_2016": {
-            "acc": 0.6761090326028861,
-            "acc_stderr": 0.010821488046867113
-        },
-        "boolq": {
-            "acc": 0.5966360856269113,
-            "acc_stderr": 0.008580168554889729
-        },
-        "arc_easy": {
-            "acc": 0.5526094276094277,
-            "acc_stderr": 0.010202832385415646,
-            "acc_norm": 0.5008417508417509,
-            "acc_norm_stderr": 0.010259768981815234
-        },
-        "arc_challenge": {
-            "acc": 0.24658703071672355,
-            "acc_stderr": 0.012595726268790129,
-            "acc_norm": 0.27986348122866894,
-            "acc_norm_stderr": 0.013119040897725927
-        },
-        "sciq": {
-            "acc": 0.828,
-            "acc_stderr": 0.011939788882495321,
-            "acc_norm": 0.745,
-            "acc_norm_stderr": 0.013790038620872832
-        },
-        "piqa": {
-            "acc": 0.720892274211099,
-            "acc_stderr": 0.010465657948498228,
-            "acc_norm": 0.7366702937976061,
-            "acc_norm_stderr": 0.010276185322196764
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

2b855b70c4py/evaluation/generation/merged.csv

@@ -1,5 +1,53 @@
-dataset,prompt,metric,value
-e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.21617228257685636
-gem_xsum,article_DOC_summary,rouge2_fmeasure,0.030464878974703208
-web_nlg_en,PALM_prompt,rouge2_fmeasure,0.07413326891469442
-wiki_lingua_en,tldr_en,rouge2_fmeasure,0.04467897015503159
+dataset,fewshots,prompt,metric,value
+e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.014284904001998958
+e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.014284904001998958
+e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.21617228257685636
+e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.21617228257685636
+e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.23824922321774356
+e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.23824922321774356
+e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.24424957066759415
+e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.24424957066759415
+e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.24557008443757672
+e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.24557008443757672
+e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.25083601181824494
+e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.25083601181824494
+e2e_nlg_cleaned,5,average,multiple,0.20156034612000245
+gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.043025444615171264
+gem_xsum,0,median,rouge2_fmeasure,0.043025444615171264
+gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.030464878974703208
+gem_xsum,1,median,rouge2_fmeasure,0.030464878974703208
+gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.027647787238487997
+gem_xsum,2,median,rouge2_fmeasure,0.027647787238487997
+gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.025369930503724297
+gem_xsum,3,median,rouge2_fmeasure,0.025369930503724297
+gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.007343327369807664
+gem_xsum,4,median,rouge2_fmeasure,0.007343327369807664
+gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.00016742834067019316
+gem_xsum,5,median,rouge2_fmeasure,0.00016742834067019316
+gem_xsum,5,average,multiple,0.02233646617376077
+web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.05460161519782302
+web_nlg_en,0,median,rouge2_fmeasure,0.05460161519782302
+web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.07413326891469442
+web_nlg_en,1,median,rouge2_fmeasure,0.07413326891469442
+web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.08955322431476385
+web_nlg_en,2,median,rouge2_fmeasure,0.08955322431476385
+web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.09627244148305192
+web_nlg_en,3,median,rouge2_fmeasure,0.09627244148305192
+web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.09929599169717132
+web_nlg_en,4,median,rouge2_fmeasure,0.09929599169717132
+web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.09910715685843922
+web_nlg_en,5,median,rouge2_fmeasure,0.09910715685843922
+web_nlg_en,5,average,multiple,0.08549394974432396
+wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.03992178847672173
+wiki_lingua_en,0,median,rouge2_fmeasure,0.03992178847672173
+wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.04467897015503159
+wiki_lingua_en,1,median,rouge2_fmeasure,0.04467897015503159
+wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.060201951887495
+wiki_lingua_en,2,median,rouge2_fmeasure,0.060201951887495
+wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.050540511866150155
+wiki_lingua_en,3,median,rouge2_fmeasure,0.050540511866150155
+wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.0175773824397642
+wiki_lingua_en,4,median,rouge2_fmeasure,0.0175773824397642
+wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.002750872542308398
+wiki_lingua_en,5,median,rouge2_fmeasure,0.002750872542308398
+wiki_lingua_en,5,average,multiple,0.03594524622791184

2b855b70c4py/evaluation/generation/merged.json

@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.36177222640850887, "bleu_stderr": 0.0320694142508483, "rouge1_fmeasure": 0.1260914844194479, "rouge1_fmeasure_stderr": 0.0021622839860059584, "rouge1_precision": 0.08191223286755775, "rouge1_precision_stderr": 0.001627521975692058, "rouge1_recall": 0.3742173507209863, "rouge1_recall_stderr": 0.004991592786505357, "rouge2_fmeasure": 0.05460161519782302, "rouge2_fmeasure_stderr": 0.001306882844687894, "rouge2_precision": 0.035351987582242776, "rouge2_precision_stderr": 0.0009387018858353787, "rouge2_recall": 0.1661676178025246, "rouge2_recall_stderr": 0.0035708234167462403, "rougeL_fmeasure": 0.11814914005010795, "rougeL_fmeasure_stderr": 0.0019029778378069377, "rougeL_precision": 0.0763516967113459, "rougeL_precision_stderr": 0.0014168315437976147, "rougeL_recall": 0.3566988120259588, "rougeL_recall_stderr": 0.004806387899927629, "rougeLsum_fmeasure": 0.114927147129508, "rougeLsum_fmeasure_stderr": 0.001976052266411515, "rougeLsum_precision": 0.07476607280600958, "rougeLsum_precision_stderr": 0.0014909447748365661, "rougeLsum_recall": 0.3403400611882494, "rougeLsum_recall_stderr": 0.004531326876643143}}, "1": {"PALM_prompt": {"bleu": 0.4579494195095615, "bleu_stderr": 0.03272477515499559, "rouge1_fmeasure": 0.14911355096418907, "rouge1_fmeasure_stderr": 0.0034810683292200908, "rouge1_precision": 0.14660657504374577, "rouge1_precision_stderr": 0.004891491213595761, "rouge1_recall": 0.28685628423906706, "rouge1_recall_stderr": 0.005114922627348603, "rouge2_fmeasure": 0.07413326891469442, "rouge2_fmeasure_stderr": 0.002330546710842158, "rouge2_precision": 0.07500514669920703, "rouge2_precision_stderr": 0.003437628004318021, "rouge2_recall": 0.14606361566773116, "rouge2_recall_stderr": 0.003505093933771862, "rougeL_fmeasure": 0.1377016687791557, "rougeL_fmeasure_stderr": 0.0030883419576430114, "rougeL_precision": 0.13479729623556277, "rougeL_precision_stderr": 0.004491122191495393, "rougeL_recall": 0.27166090970530216, "rougeL_recall_stderr": 0.004815625198315023, "rougeLsum_fmeasure": 0.13928799013475338, "rougeLsum_fmeasure_stderr": 0.003148720380821768, "rougeLsum_precision": 0.13665733041276623, "rougeLsum_precision_stderr": 0.004554100040041119, "rougeLsum_recall": 0.27228940452192685, "rougeLsum_recall_stderr": 0.004791073548058962}}, "2": {"PALM_prompt": {"bleu": 0.5832437265326549, "bleu_stderr": 0.04073326044681886, "rouge1_fmeasure": 0.17444972738437425, "rouge1_fmeasure_stderr": 0.003926235082715643, "rouge1_precision": 0.1684119036089443, "rouge1_precision_stderr": 0.005400417284045452, "rouge1_recall": 0.32649209286182873, "rouge1_recall_stderr": 0.004843923820854222, "rouge2_fmeasure": 0.08955322431476385, "rouge2_fmeasure_stderr": 0.0026915677154337792, "rouge2_precision": 0.08962453361046427, "rouge2_precision_stderr": 0.0037370812866666775, "rouge2_recall": 0.16908597685060872, "rouge2_recall_stderr": 0.0034964640407355997, "rougeL_fmeasure": 0.1586317889080828, "rougeL_fmeasure_stderr": 0.0034095706493399254, "rougeL_precision": 0.1514534471488523, "rougeL_precision_stderr": 0.004799241736665215, "rougeL_recall": 0.30623625936643073, "rougeL_recall_stderr": 0.00449614371260453, "rougeLsum_fmeasure": 0.16163214401579332, "rougeLsum_fmeasure_stderr": 0.0035025180986267, "rougeLsum_precision": 0.1550872804167069, "rougeLsum_precision_stderr": 0.004938299505990902, "rougeLsum_recall": 0.3092382977470476, "rougeLsum_recall_stderr": 0.004535623957080562}}, "3": {"PALM_prompt": {"bleu": 0.670717543600456, "bleu_stderr": 0.023906388833018523, "rouge1_fmeasure": 0.1824987111154357, "rouge1_fmeasure_stderr": 0.004158865936964165, "rouge1_precision": 0.1686724373174789, "rouge1_precision_stderr": 0.0052508533039634115, "rouge1_recall": 0.33800174590565385, "rouge1_recall_stderr": 0.005001248230138175, "rouge2_fmeasure": 0.09627244148305192, "rouge2_fmeasure_stderr": 0.0028603458702883048, "rouge2_precision": 0.09191616981832372, "rouge2_precision_stderr": 0.0036099234275133243, "rouge2_recall": 0.17964872144023064, "rouge2_recall_stderr": 0.003699653610649366, "rougeL_fmeasure": 0.16384914354476587, "rougeL_fmeasure_stderr": 0.0035444518800591593, "rougeL_precision": 0.149076088380221, "rougeL_precision_stderr": 0.004530093531628108, "rougeL_recall": 0.3151823249965444, "rougeL_recall_stderr": 0.004608050839568957, "rougeLsum_fmeasure": 0.1677216835527187, "rougeLsum_fmeasure_stderr": 0.003661865723321482, "rougeLsum_precision": 0.15392727300048414, "rougeLsum_precision_stderr": 0.004737142858950911, "rougeLsum_recall": 0.31921175675769126, "rougeLsum_recall_stderr": 0.0046568458145587605}}, "4": {"PALM_prompt": {"bleu": 0.7604227444018716, "bleu_stderr": 0.049611845083858346, "rouge1_fmeasure": 0.18958910173700302, "rouge1_fmeasure_stderr": 0.0041678022897180525, "rouge1_precision": 0.17491295099951767, "rouge1_precision_stderr": 0.005253411705645377, "rouge1_recall": 0.3600949376320361, "rouge1_recall_stderr": 0.005028738929998684, "rouge2_fmeasure": 0.09929599169717132, "rouge2_fmeasure_stderr": 0.002821858249179042, "rouge2_precision": 0.09345056897454469, "rouge2_precision_stderr": 0.003463104935384006, "rouge2_recall": 0.19108543444993117, "rouge2_recall_stderr": 0.003763798465807593, "rougeL_fmeasure": 0.16935131924738828, "rougeL_fmeasure_stderr": 0.003509123055792932, "rougeL_precision": 0.15392591338310865, "rougeL_precision_stderr": 0.004493312696520681, "rougeL_recall": 0.33438402667017314, "rougeL_recall_stderr": 0.004627350404813471, "rougeLsum_fmeasure": 0.17405170191643468, "rougeLsum_fmeasure_stderr": 0.0036605271554700065, "rougeLsum_precision": 0.15927668290565722, "rougeLsum_precision_stderr": 0.004690076718573134, "rougeLsum_recall": 0.3392069471680405, "rougeLsum_recall_stderr": 0.00467404502921642}}, "5": {"PALM_prompt": {"bleu": 0.7621826571266548, "bleu_stderr": 0.034082068767903304, "rouge1_fmeasure": 0.18798952549969697, "rouge1_fmeasure_stderr": 0.004017693486744882, "rouge1_precision": 0.17544469444117916, "rouge1_precision_stderr": 0.005271215330525362, "rouge1_recall": 0.36005918323820457, "rouge1_recall_stderr": 0.0049301056252259974, "rouge2_fmeasure": 0.09910715685843922, "rouge2_fmeasure_stderr": 0.002771423885727132, "rouge2_precision": 0.09636800122415268, "rouge2_precision_stderr": 0.0037076389079263405, "rouge2_recall": 0.1913319305812655, "rouge2_recall_stderr": 0.003701472164105202, "rougeL_fmeasure": 0.16829466070473165, "rougeL_fmeasure_stderr": 0.0033963222345664555, "rougeL_precision": 0.15484498220143258, "rougeL_precision_stderr": 0.0045664207500154454, "rougeL_recall": 0.3350376880318819, "rougeL_recall_stderr": 0.004558748695717041, "rougeLsum_fmeasure": 0.17344470237675785, "rougeLsum_fmeasure_stderr": 0.0035582976005115313, "rougeLsum_precision": 0.16088355278033994, "rougeLsum_precision_stderr": 0.004789870853627559, "rougeLsum_recall": 0.34025511491593396, "rougeLsum_recall_stderr": 0.004604758323409871}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 1.7690388806815223, "bleu_stderr": 0.061199166238469514, "rouge1_fmeasure": 0.18578054428380605, "rouge1_fmeasure_stderr": 0.0019205485546632257, "rouge1_precision": 0.1586659747034313, "rouge1_precision_stderr": 0.0019639881989730484, "rouge1_recall": 0.26889887264288703, "rouge1_recall_stderr": 0.002721960316556434, "rouge2_fmeasure": 0.03992178847672173, "rouge2_fmeasure_stderr": 0.0008843717904267976, "rouge2_precision": 0.03376360542922583, "rouge2_precision_stderr": 0.0007902842443148329, "rouge2_recall": 0.05985225408994821, "rouge2_recall_stderr": 0.0014389022143521143, "rougeL_fmeasure": 0.14470616664417285, "rougeL_fmeasure_stderr": 0.001363115967259926, "rougeL_precision": 0.12203086378040674, "rougeL_precision_stderr": 0.0013591980647123959, "rougeL_recall": 0.21465672514706977, "rougeL_recall_stderr": 0.002192357545236951, "rougeLsum_fmeasure": 0.16993997476607003, "rougeLsum_fmeasure_stderr": 0.0017414418379303176, "rougeLsum_precision": 0.14487665706610783, "rougeLsum_precision_stderr": 0.001777791368403861, "rougeLsum_recall": 0.2471259123994469, "rougeLsum_recall_stderr": 0.002526603551557684}}, "1": {"tldr_en": {"bleu": 2.4968553627699, "bleu_stderr": 0.09756973664186838, "rouge1_fmeasure": 0.18361213099108636, "rouge1_fmeasure_stderr": 0.0020874146685115865, "rouge1_precision": 0.28476915880295867, "rouge1_precision_stderr": 0.004346267072633461, "rouge1_recall": 0.1966486553119253, "rouge1_recall_stderr": 0.002777916166707439, "rouge2_fmeasure": 0.04467897015503159, "rouge2_fmeasure_stderr": 0.0012239448647545761, "rouge2_precision": 0.08399669886018704, "rouge2_precision_stderr": 0.0028732020875856046, "rouge2_recall": 0.04602176376593877, "rouge2_recall_stderr": 0.0013889720058457467, "rougeL_fmeasure": 0.14197877658343508, "rougeL_fmeasure_stderr": 0.0016059057943586031, "rougeL_precision": 0.22924568046824617, "rougeL_precision_stderr": 0.0038326984433139123, "rougeL_recall": 0.15093870382423633, "rougeL_recall_stderr": 0.0021331476051670113, "rougeLsum_fmeasure": 0.1722743001980724, "rougeLsum_fmeasure_stderr": 0.001960905869131214, "rougeLsum_precision": 0.2689115570397548, "rougeLsum_precision_stderr": 0.004193019127000938, "rougeLsum_recall": 0.1845417723091065, "rougeLsum_recall_stderr": 0.002610038331896122}}, "2": {"tldr_en": {"bleu": 2.922575032906168, "bleu_stderr": 0.13964542153174686, "rouge1_fmeasure": 0.20724330644269431, "rouge1_fmeasure_stderr": 0.0022178542025854306, "rouge1_precision": 0.36849455870341613, "rouge1_precision_stderr": 0.004696471251423989, "rouge1_recall": 0.198356311762661, "rouge1_recall_stderr": 0.0027208655512888976, "rouge2_fmeasure": 0.060201951887495, "rouge2_fmeasure_stderr": 0.0014302793495886324, "rouge2_precision": 0.12687075022312783, "rouge2_precision_stderr": 0.003502964376813627, "rouge2_recall": 0.05539640617987267, "rouge2_recall_stderr": 0.0014611009523242283, "rougeL_fmeasure": 0.16401138649005056, "rougeL_fmeasure_stderr": 0.0018003373635147922, "rougeL_precision": 0.3011673052622378, "rougeL_precision_stderr": 0.004195098117529287, "rougeL_recall": 0.1558957943390216, "rougeL_recall_stderr": 0.002171237078736034, "rougeLsum_fmeasure": 0.19579525136400283, "rougeLsum_fmeasure_stderr": 0.002106514818847442, "rougeLsum_precision": 0.35061620943313776, "rougeLsum_precision_stderr": 0.004571035478479939, "rougeLsum_recall": 0.18723986196379075, "rougeLsum_recall_stderr": 0.002581923039738377}}, "3": {"tldr_en": {"bleu": 1.8257415143103009, "bleu_stderr": 0.08734762740975269, "rouge1_fmeasure": 0.17461240553839358, "rouge1_fmeasure_stderr": 0.002478093094080404, "rouge1_precision": 0.32265400875788397, "rouge1_precision_stderr": 0.0049669120272902096, "rouge1_recall": 0.16042394155909814, "rouge1_recall_stderr": 0.002820925891763586, "rouge2_fmeasure": 0.050540511866150155, "rouge2_fmeasure_stderr": 0.0013872245176174876, "rouge2_precision": 0.107874775674973, "rouge2_precision_stderr": 0.003296476831098696, "rouge2_recall": 0.04569986496835191, "rouge2_recall_stderr": 0.0014530200024000586, "rougeL_fmeasure": 0.14014552433738017, "rougeL_fmeasure_stderr": 0.0020067864333084837, "rougeL_precision": 0.2666081331310461, "rougeL_precision_stderr": 0.004364289520862936, "rougeL_recall": 0.1276751653966867, "rougeL_recall_stderr": 0.002254922647475454, "rougeLsum_fmeasure": 0.164724510834769, "rougeLsum_fmeasure_stderr": 0.0023387789009413997, "rougeLsum_precision": 0.30691838318130493, "rougeLsum_precision_stderr": 0.004787442631412461, "rougeLsum_recall": 0.15091705594866833, "rougeLsum_recall_stderr": 0.0026557157205465065}}, "4": {"tldr_en": {"bleu": 0.016903004488006267, "bleu_stderr": 0.0028101662399068147, "rouge1_fmeasure": 0.05804722897961801, "rouge1_fmeasure_stderr": 0.002077858056151942, "rouge1_precision": 0.10847689841656194, "rouge1_precision_stderr": 0.004061910441788624, "rouge1_recall": 0.053633126486665336, "rouge1_recall_stderr": 0.002149979403931979, "rouge2_fmeasure": 0.0175773824397642, "rouge2_fmeasure_stderr": 0.0009837792991547632, "rouge2_precision": 0.03855518604488764, "rouge2_precision_stderr": 0.0023529087379367455, "rouge2_recall": 0.01614225432921013, "rouge2_recall_stderr": 0.0010421268634975495, "rougeL_fmeasure": 0.04715756446982826, "rougeL_fmeasure_stderr": 0.0017147765607776458, "rougeL_precision": 0.09089807331243692, "rougeL_precision_stderr": 0.0035498729989687154, "rougeL_recall": 0.04325203366143314, "rougeL_recall_stderr": 0.0017626307116868716, "rougeLsum_fmeasure": 0.05488512798106204, "rougeLsum_fmeasure_stderr": 0.0019703022359280485, "rougeLsum_precision": 0.10329569903261887, "rougeLsum_precision_stderr": 0.0038912749121635375, "rougeLsum_recall": 0.05066452016539995, "rougeLsum_recall_stderr": 0.002043184509473453}}, "5": {"tldr_en": {"bleu": 1.1114806623820518e-18, "bleu_stderr": 1.0674262114653223e-16, "rouge1_fmeasure": 0.009181416179164, "rouge1_fmeasure_stderr": 0.000933405053140164, "rouge1_precision": 0.017988007003326284, "rouge1_precision_stderr": 0.0018872272206102072, "rouge1_recall": 0.00836018091720164, "rouge1_recall_stderr": 0.0009018936819318786, "rouge2_fmeasure": 0.002750872542308398, "rouge2_fmeasure_stderr": 0.0003989808322430752, "rouge2_precision": 0.006456325182786169, "rouge2_precision_stderr": 0.0010453481707116332, "rouge2_recall": 0.0023547790199948226, "rouge2_recall_stderr": 0.00034531560604633216, "rougeL_fmeasure": 0.007283367178434869, "rougeL_fmeasure_stderr": 0.0007272796244337632, "rougeL_precision": 0.015091952889941246, "rougeL_precision_stderr": 0.001647661153103192, "rougeL_recall": 0.00658810184172878, "rougeL_recall_stderr": 0.0006985979897792559, "rougeLsum_fmeasure": 0.008713180070907066, "rougeLsum_fmeasure_stderr": 0.0008870694064056895, "rougeLsum_precision": 0.01719152417959649, "rougeLsum_precision_stderr": 0.001813610331535302, "rougeLsum_recall": 0.00793952061153923, "rougeLsum_recall_stderr": 0.0008635571673921616}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 0.4233111592647398, "bleu_stderr": 0.056213807495910775, "rouge1_fmeasure": 0.10312010578118724, "rouge1_fmeasure_stderr": 0.0018054029747208828, "rouge1_precision": 0.0959560109077337, "rouge1_precision_stderr": 0.0018157211121353064, "rouge1_recall": 0.1199990662922811, "rouge1_recall_stderr": 0.0020910548635598427, "rouge2_fmeasure": 0.014284904001998958, "rouge2_fmeasure_stderr": 0.0006945802854540839, "rouge2_precision": 0.013480670444417035, "rouge2_precision_stderr": 0.0006569348736147822, "rouge2_recall": 0.01668721594245476, "rouge2_recall_stderr": 0.0008431828362766337, "rougeL_fmeasure": 0.08685489319716908, "rougeL_fmeasure_stderr": 0.0012885783089700282, "rougeL_precision": 0.07972797895653407, "rougeL_precision_stderr": 0.001237662366013737, "rougeL_recall": 0.10255406543582875, "rougeL_recall_stderr": 0.0016093095192510158, "rougeLsum_fmeasure": 0.08683518574704205, "rougeLsum_fmeasure_stderr": 0.0014988299833317504, "rougeLsum_precision": 0.0810757461545558, "rougeLsum_precision_stderr": 0.0015236783097646303, "rougeLsum_recall": 0.10067083728837142, "rougeLsum_recall_stderr": 0.0017257744389301488}}, "1": {"generate_text_restaurant": {"bleu": 11.589568419584984, "bleu_stderr": 0.15818750812352242, "rouge1_fmeasure": 0.4605036165611307, "rouge1_fmeasure_stderr": 0.002434571850871462, "rouge1_precision": 0.5614202943712363, "rouge1_precision_stderr": 0.003318770856775173, "rouge1_recall": 0.42912912417917104, "rouge1_recall_stderr": 0.0030404477523093163, "rouge2_fmeasure": 0.21617228257685636, "rouge2_fmeasure_stderr": 0.002039507810232155, "rouge2_precision": 0.2675425882548481, "rouge2_precision_stderr": 0.002711297954229281, "rouge2_recall": 0.2010664393209872, "rouge2_recall_stderr": 0.002161675592274865, "rougeL_fmeasure": 0.33149764018760386, "rougeL_fmeasure_stderr": 0.0021141320222871364, "rougeL_precision": 0.4076583038091403, "rougeL_precision_stderr": 0.0030149802333527724, "rougeL_recall": 0.3076957265069834, "rougeL_recall_stderr": 0.0024338237770882296, "rougeLsum_fmeasure": 0.3739925854562877, "rougeLsum_fmeasure_stderr": 0.002366249747872058, "rougeLsum_precision": 0.45761577531758296, "rougeLsum_precision_stderr": 0.0032239507636290044, "rougeLsum_recall": 0.34773928423273576, "rougeLsum_recall_stderr": 0.0027415488284097105}}, "2": {"generate_text_restaurant": {"bleu": 13.008789130965523, "bleu_stderr": 0.1799190974976631, "rouge1_fmeasure": 0.48199853488294636, "rouge1_fmeasure_stderr": 0.002353301394536953, "rouge1_precision": 0.5973831735637382, "rouge1_precision_stderr": 0.003350271201610617, "rouge1_recall": 0.44146117945666813, "rouge1_recall_stderr": 0.0029428987733678574, "rouge2_fmeasure": 0.23824922321774356, "rouge2_fmeasure_stderr": 0.002141436188136717, "rouge2_precision": 0.3009766842869326, "rouge2_precision_stderr": 0.0029351461940240317, "rouge2_recall": 0.21761987007296849, "rouge2_recall_stderr": 0.0022456925015639823, "rougeL_fmeasure": 0.3548343173414083, "rougeL_fmeasure_stderr": 0.002151680724029199, "rougeL_precision": 0.442936738434032, "rougeL_precision_stderr": 0.003142988468903537, "rougeL_recall": 0.3241138729829191, "rougeL_recall_stderr": 0.002451970272560583, "rougeLsum_fmeasure": 0.39860999022543026, "rougeLsum_fmeasure_stderr": 0.0023775946895355487, "rougeLsum_precision": 0.4957063607574493, "rougeLsum_precision_stderr": 0.003355764806049463, "rougeLsum_recall": 0.36448362846302285, "rougeLsum_recall_stderr": 0.002726932732570614}}, "3": {"generate_text_restaurant": {"bleu": 13.507983624600893, "bleu_stderr": 0.17932423948606033, "rouge1_fmeasure": 0.4861979453866817, "rouge1_fmeasure_stderr": 0.0023120206596950028, "rouge1_precision": 0.6023145145540068, "rouge1_precision_stderr": 0.003357218702794606, "rouge1_recall": 0.4430032717076908, "rouge1_recall_stderr": 0.0028619583263082317, "rouge2_fmeasure": 0.24424957066759415, "rouge2_fmeasure_stderr": 0.0021696590244700328, "rouge2_precision": 0.30876542499257, "rouge2_precision_stderr": 0.002988307824368767, "rouge2_recall": 0.22164980130742637, "rouge2_recall_stderr": 0.0022444834890213796, "rougeL_fmeasure": 0.36106087394333214, "rougeL_fmeasure_stderr": 0.0021772157570658234, "rougeL_precision": 0.45038320374960633, "rougeL_precision_stderr": 0.00320380867953722, "rougeL_recall": 0.32791450453708637, "rougeL_recall_stderr": 0.00242583586505486, "rougeLsum_fmeasure": 0.40444978709611884, "rougeLsum_fmeasure_stderr": 0.002397811942978674, "rougeLsum_precision": 0.5019751635364414, "rougeLsum_precision_stderr": 0.003374739966256141, "rougeLsum_recall": 0.36812641537416585, "rougeLsum_recall_stderr": 0.0027125234155325464}}, "4": {"generate_text_restaurant": {"bleu": 13.64220572288601, "bleu_stderr": 0.2348095844031076, "rouge1_fmeasure": 0.4855676897407202, "rouge1_fmeasure_stderr": 0.002325672844344846, "rouge1_precision": 0.6012745712030935, "rouge1_precision_stderr": 0.003329850986103485, "rouge1_recall": 0.441947195116862, "rouge1_recall_stderr": 0.002855010332415042, "rouge2_fmeasure": 0.24557008443757672, "rouge2_fmeasure_stderr": 0.0022219909269245116, "rouge2_precision": 0.309624048092463, "rouge2_precision_stderr": 0.003021455740523968, "rouge2_recall": 0.222818180236328, "rouge2_recall_stderr": 0.0022866973072367784, "rougeL_fmeasure": 0.36281803085576364, "rougeL_fmeasure_stderr": 0.0021844043393582217, "rougeL_precision": 0.4522453023128288, "rougeL_precision_stderr": 0.003171940144444519, "rougeL_recall": 0.32920947876209405, "rougeL_recall_stderr": 0.0024330787773457, "rougeLsum_fmeasure": 0.4068386061276735, "rougeLsum_fmeasure_stderr": 0.0024069167685560468, "rougeLsum_precision": 0.5043582361954958, "rougeLsum_precision_stderr": 0.0033439129021371745, "rougeLsum_recall": 0.37015137720860586, "rougeLsum_recall_stderr": 0.002729934190393941}}, "5": {"generate_text_restaurant": {"bleu": 14.040728762331781, "bleu_stderr": 0.18368071436682032, "rouge1_fmeasure": 0.4906476375857304, "rouge1_fmeasure_stderr": 0.002258429732872185, "rouge1_precision": 0.6023052616478523, "rouge1_precision_stderr": 0.0033392854580359945, "rouge1_recall": 0.4494101884830406, "rouge1_recall_stderr": 0.002808452250254566, "rouge2_fmeasure": 0.25083601181824494, "rouge2_fmeasure_stderr": 0.002180280674951652, "rouge2_precision": 0.3135823113503061, "rouge2_precision_stderr": 0.002993506877326475, "rouge2_recall": 0.2291093091752363, "rouge2_recall_stderr": 0.0022515483842061547, "rougeL_fmeasure": 0.3687169204441277, "rougeL_fmeasure_stderr": 0.0021999773391684816, "rougeL_precision": 0.45425345554218605, "rougeL_precision_stderr": 0.0031671018948055663, "rougeL_recall": 0.33738151554389173, "rougeL_recall_stderr": 0.0024756900889837995, "rougeLsum_fmeasure": 0.41305866802958763, "rougeLsum_fmeasure_stderr": 0.0023825281838228195, "rougeLsum_precision": 0.507265450785699, "rougeLsum_precision_stderr": 0.0033535663524657837, "rougeLsum_recall": 0.3784706698018511, "rougeLsum_recall_stderr": 0.002725756506434112}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 1.7752764026405226, "bleu_stderr": 0.09352910406654073, "rouge1_fmeasure": 0.2023965191186876, "rouge1_fmeasure_stderr": 0.002625325708684963, "rouge1_precision": 0.15847933547085274, "rouge1_precision_stderr": 0.0023327288742325315, "rouge1_recall": 0.31956752062331983, "rouge1_recall_stderr": 0.004482151283790898, "rouge2_fmeasure": 0.043025444615171264, "rouge2_fmeasure_stderr": 0.0015169748683593926, "rouge2_precision": 0.0326952070515575, "rouge2_precision_stderr": 0.0012330337155625067, "rouge2_recall": 0.07127283557070531, "rouge2_recall_stderr": 0.002583794531003543, "rougeL_fmeasure": 0.15134216028667954, "rougeL_fmeasure_stderr": 0.0019385483298069402, "rougeL_precision": 0.11848589984320053, "rougeL_precision_stderr": 0.0017475639736686283, "rougeL_recall": 0.2402965313417492, "rougeL_recall_stderr": 0.003442350521152835, "rougeLsum_fmeasure": 0.1588275114813706, "rougeLsum_fmeasure_stderr": 0.0021864431145321725, "rougeLsum_precision": 0.12389994301009628, "rougeLsum_precision_stderr": 0.00187691202173923, "rougeLsum_recall": 0.2529890876991462, "rougeLsum_recall_stderr": 0.0038955276273026384}}, "1": {"article_DOC_summary": {"bleu": 1.2676096297388244, "bleu_stderr": 0.06284753273943922, "rouge1_fmeasure": 0.16743465983237138, "rouge1_fmeasure_stderr": 0.002278466926262833, "rouge1_precision": 0.11851885868955787, "rouge1_precision_stderr": 0.0016941400640940385, "rouge1_recall": 0.29716687533329084, "rouge1_recall_stderr": 0.003950873581063913, "rouge2_fmeasure": 0.030464878974703208, "rouge2_fmeasure_stderr": 0.0012937549720197195, "rouge2_precision": 0.021317145387635697, "rouge2_precision_stderr": 0.0009018047306137615, "rouge2_recall": 0.05599324476647025, "rouge2_recall_stderr": 0.0025087115664945824, "rougeL_fmeasure": 0.13025708325925467, "rougeL_fmeasure_stderr": 0.001706858442684011, "rougeL_precision": 0.09196569277047886, "rougeL_precision_stderr": 0.0012516735137150042, "rougeL_recall": 0.23292336721242884, "rougeL_recall_stderr": 0.0031169638835663468, "rougeLsum_fmeasure": 0.13397484270431992, "rougeLsum_fmeasure_stderr": 0.0018804296968347048, "rougeLsum_precision": 0.09460079453068865, "rougeLsum_precision_stderr": 0.00137724116805024, "rougeLsum_recall": 0.2395072926927437, "rougeLsum_recall_stderr": 0.0034157832925364425}}, "2": {"article_DOC_summary": {"bleu": 1.06979874835557, "bleu_stderr": 0.08149792810913956, "rouge1_fmeasure": 0.16081716241600977, "rouge1_fmeasure_stderr": 0.002177595968109337, "rouge1_precision": 0.11388138730186037, "rouge1_precision_stderr": 0.001606419574593934, "rouge1_recall": 0.2854453345158146, "rouge1_recall_stderr": 0.003827451963289913, "rouge2_fmeasure": 0.027647787238487997, "rouge2_fmeasure_stderr": 0.0011954609601607634, "rouge2_precision": 0.019361353467349008, "rouge2_precision_stderr": 0.0008311624880424274, "rouge2_recall": 0.050458200832752174, "rouge2_recall_stderr": 0.0022925869570692686, "rougeL_fmeasure": 0.12662749957188985, "rougeL_fmeasure_stderr": 0.0016570599361094096, "rougeL_precision": 0.0895425941692775, "rougeL_precision_stderr": 0.0012150929902347222, "rougeL_recall": 0.22598192318482266, "rougeL_recall_stderr": 0.003005160725079529, "rougeLsum_fmeasure": 0.12986946202714258, "rougeLsum_fmeasure_stderr": 0.0018058886777436965, "rougeLsum_precision": 0.09173827393320932, "rougeLsum_precision_stderr": 0.0013100459640905703, "rougeLsum_recall": 0.23230910795202298, "rougeLsum_recall_stderr": 0.0033445430701823918}}, "3": {"article_DOC_summary": {"bleu": 1.030752065110202, "bleu_stderr": 0.11093121713483114, "rouge1_fmeasure": 0.15415215162015383, "rouge1_fmeasure_stderr": 0.002344556587021401, "rouge1_precision": 0.11208466938339205, "rouge1_precision_stderr": 0.0019110195832627874, "rouge1_recall": 0.26903762113492224, "rouge1_recall_stderr": 0.004076034996775704, "rouge2_fmeasure": 0.025369930503724297, "rouge2_fmeasure_stderr": 0.0012002800860865547, "rouge2_precision": 0.018142257555870216, "rouge2_precision_stderr": 0.0008952105922829451, "rouge2_recall": 0.045863753219686845, "rouge2_recall_stderr": 0.002264116072454382, "rougeL_fmeasure": 0.1221302140131779, "rougeL_fmeasure_stderr": 0.0017740949474691632, "rougeL_precision": 0.0884278868800365, "rougeL_precision_stderr": 0.001417385201165191, "rougeL_recall": 0.21466328266616594, "rougeL_recall_stderr": 0.003222758387666387, "rougeLsum_fmeasure": 0.1242195060885606, "rougeLsum_fmeasure_stderr": 0.0019408070844620941, "rougeLsum_precision": 0.08988092160648196, "rougeLsum_precision_stderr": 0.001535288507296881, "rougeLsum_recall": 0.2186418716421356, "rougeLsum_recall_stderr": 0.0035178357911136654}}, "4": {"article_DOC_summary": {"bleu": 0.6007209821745559, "bleu_stderr": 0.0779549704506759, "rouge1_fmeasure": 0.043520050953035065, "rouge1_fmeasure_stderr": 0.0024154294129010256, "rouge1_precision": 0.03758157400761349, "rouge1_precision_stderr": 0.002401756298175864, "rouge1_recall": 0.06939220481127047, "rouge1_recall_stderr": 0.003969979147736475, "rouge2_fmeasure": 0.007343327369807664, "rouge2_fmeasure_stderr": 0.000748307052109553, "rouge2_precision": 0.005816294730946609, "rouge2_precision_stderr": 0.0006899080852548144, "rouge2_recall": 0.0126988593850261, "rouge2_recall_stderr": 0.0014250878564603699, "rougeL_fmeasure": 0.03463795183456895, "rougeL_fmeasure_stderr": 0.0018955259736367848, "rougeL_precision": 0.03033330126825888, "rougeL_precision_stderr": 0.0020311049629855695, "rougeL_recall": 0.055677943315720915, "rougeL_recall_stderr": 0.0031784910856630434, "rougeLsum_fmeasure": 0.03619059807864272, "rougeLsum_fmeasure_stderr": 0.0020150546965075787, "rougeLsum_precision": 0.031585124703076374, "rougeLsum_precision_stderr": 0.002102940128233958, "rougeLsum_recall": 0.05793050251080939, "rougeLsum_recall_stderr": 0.0033582110137027866}}, "5": {"article_DOC_summary": {"bleu": 7.75010450442393e-39, "bleu_stderr": 1.7777115133494914e-33, "rouge1_fmeasure": 0.00218055448143642, "rouge1_fmeasure_stderr": 0.0006060691557906891, "rouge1_precision": 0.00242279261132582, "rouge1_precision_stderr": 0.0007024882428106118, "rouge1_recall": 0.0020628291327855837, "rouge1_recall_stderr": 0.0005601655561491828, "rouge2_fmeasure": 0.00016742834067019316, "rouge2_fmeasure_stderr": 9.863168498418184e-05, "rouge2_precision": 0.0001905850962454736, "rouge2_precision_stderr": 0.00011662554684056579, "rouge2_recall": 0.00015519072122845707, "rouge2_recall_stderr": 9.051149629358608e-05, "rougeL_fmeasure": 0.0016602182096702625, "rougeL_fmeasure_stderr": 0.00043790007549847083, "rougeL_precision": 0.0018324360991864934, "rougeL_precision_stderr": 0.0004958519741726313, "rougeL_recall": 0.001586241494857876, "rougeL_recall_stderr": 0.0004161211305315624, "rougeLsum_fmeasure": 0.001667805964006388, "rougeLsum_fmeasure_stderr": 0.000453360536479583, "rougeLsum_precision": 0.0018271256785480745, "rougeLsum_precision_stderr": 0.0005070137233618809, "rougeLsum_recall": 0.0016030699598752222, "rougeLsum_recall_stderr": 0.0004341160681349645}}}}