| { | |
| "cn_college_listen_mcq_test": { | |
| "llama3_70b_judge": { | |
| "gpt-4o-audio": 90.88507265521797, | |
| "Qwen-Audio-Chat": 63.232056362835756, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.50726552179657, | |
| "Qwen2-Audio-7B-Instruct": 74.7247908410392, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 91.85380889476001, | |
| "gemini-1.5-flash": 89.25583443416997, | |
| "phi_4_multimodal_instruct": 73.18361955085865, | |
| "WavLLM_fairseq": 66.31439894319684, | |
| "SALMONN_7B": 50.99075297225891, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 85.2928225451343 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894, | |
| "Qwen2-Audio-7B-Instruct": 79.0 | |
| } | |
| }, | |
| "mmau_mini": { | |
| "string_match": { | |
| "gpt-4o-audio": 0.0, | |
| "Qwen-Audio-Chat": 38.5, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.5, | |
| "Qwen2-Audio-7B-Instruct": 44.4, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 46.7, | |
| "gemini-1.5-flash": 31.4, | |
| "phi_4_multimodal_instruct": 54.50000000000001, | |
| "seallms_audio_7b": 51.5, | |
| "SALMONN_7B": 40.5, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 51.0 | |
| }, | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 53.6, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.60000000000001, | |
| "Qwen2-Audio-7B-Instruct": 58.9, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 52.7, | |
| "gemini-1.5-flash": 58.199999999999996, | |
| "phi_4_multimodal_instruct": 59.4, | |
| "seallms_audio_7b": 60.199999999999996, | |
| "SALMONN_7B": 48.4, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9, | |
| "Qwen2-Audio-7B-Instruct": 53.0 | |
| } | |
| }, | |
| "mmau_mini_music": { | |
| "string_match": { | |
| "gpt-4o-audio": 0.0, | |
| "Qwen-Audio-Chat": 0.4311377245508982, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6077844311377245, | |
| "Qwen2-Audio-7B-Instruct": 0.45808383233532934, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.39520958083832336, | |
| "gemini-1.5-flash": 0.2904191616766467, | |
| "phi_4_multimodal_instruct": 0.6377245508982036, | |
| "seallms_audio_7b": 0.6047904191616766, | |
| "SALMONN_7B": 0.4820359281437126, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5 | |
| }, | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 0.5958083832335329, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6437125748502994, | |
| "Qwen2-Audio-7B-Instruct": 0.6017964071856288, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.44011976047904194, | |
| "gemini-1.5-flash": 0.5868263473053892, | |
| "phi_4_multimodal_instruct": 0.688622754491018, | |
| "seallms_audio_7b": 0.6646706586826348, | |
| "SALMONN_7B": 0.5598802395209581, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557, | |
| "Qwen2-Audio-7B-Instruct": 0.5473684210526316 | |
| } | |
| }, | |
| "mmau_mini_sound": { | |
| "string_match": { | |
| "gpt-4o-audio": 0.0, | |
| "Qwen-Audio-Chat": 0.43543543543543545, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6606606606606606, | |
| "Qwen2-Audio-7B-Instruct": 0.4744744744744745, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.45045045045045046, | |
| "gemini-1.5-flash": 0.3483483483483483, | |
| "phi_4_multimodal_instruct": 0.5975975975975976, | |
| "seallms_audio_7b": 0.5165165165165165, | |
| "SALMONN_7B": 0.4594594594594595, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.46546546546546547 | |
| }, | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 0.5945945945945946, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.7027027027027027, | |
| "Qwen2-Audio-7B-Instruct": 0.6306306306306306, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5345345345345346, | |
| "gemini-1.5-flash": 0.5885885885885885, | |
| "phi_4_multimodal_instruct": 0.6456456456456456, | |
| "seallms_audio_7b": 0.6486486486486487, | |
| "SALMONN_7B": 0.5105105105105106, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997, | |
| "Qwen2-Audio-7B-Instruct": 0.5980392156862745 | |
| } | |
| }, | |
| "mmau_mini_speech": { | |
| "string_match": { | |
| "gpt-4o-audio": 0.0, | |
| "Qwen-Audio-Chat": 0.2882882882882883, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5465465465465466, | |
| "Qwen2-Audio-7B-Instruct": 0.3993993993993994, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5555555555555556, | |
| "gemini-1.5-flash": 0.3033033033033033, | |
| "phi_4_multimodal_instruct": 0.3993993993993994, | |
| "seallms_audio_7b": 0.42342342342342343, | |
| "SALMONN_7B": 0.2732732732732733, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5645645645645646 | |
| }, | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 0.4174174174174174, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5915915915915916, | |
| "Qwen2-Audio-7B-Instruct": 0.5345345345345346, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.6066066066066066, | |
| "gemini-1.5-flash": 0.5705705705705706, | |
| "phi_4_multimodal_instruct": 0.44744744744744747, | |
| "seallms_audio_7b": 0.4924924924924925, | |
| "SALMONN_7B": 0.3813813813813814, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826, | |
| "Qwen2-Audio-7B-Instruct": 0.44660194174757284 | |
| } | |
| }, | |
| "slue_p2_sqa5_test": { | |
| "llama3_70b_judge": { | |
| "gpt-4o-audio": 89.41176470588235, | |
| "Qwen-Audio-Chat": 79.36274509803921, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 86.76470588235293, | |
| "Qwen2-Audio-7B-Instruct": 80.04901960784315, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 88.57843137254902, | |
| "phi_4_multimodal_instruct": 88.33333333333334, | |
| "seallms_audio_7b": 83.52941176470588, | |
| "WavLLM_fairseq": 83.92156862745098, | |
| "SALMONN_7B": 83.48039215686273, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 82.99019607843137 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.23529411764707, | |
| "Qwen2-Audio-7B-Instruct": 84.86666666666666, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 87.79411764705883 | |
| } | |
| }, | |
| "voxceleb_accent_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 48.05088223225277, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 47.01682396389003, | |
| "Qwen2-Audio-7B-Instruct": 29.187525646286417, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 24.640951990151827, | |
| "phi_4_multimodal_instruct": 26.815757078375054, | |
| "seallms_audio_7b": 8.658186294624539, | |
| "WavLLM_fairseq": 39.96717275338531, | |
| "SALMONN_7B": 34.222404595814524, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 39.32704144439885 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 14.813295034878948, | |
| "Qwen2-Audio-7B-Instruct": 22.666666666666664, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 39.462453836684446 | |
| } | |
| }, | |
| "imda_part4_30s_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 1.1764312018747907, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.0, | |
| "Qwen2-Audio-7B-Instruct": 0.5685405990059489, | |
| "whisper_large_v3": 0.8294532718704128, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.4757667842702995, | |
| "phi_4_multimodal_instruct": 1.3868687388941825, | |
| "seallms_audio_7b": 1.8960881769720068, | |
| "WavLLM_fairseq": 1.2058793232211378, | |
| "SALMONN_7B": 0.7757204295537071, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.4715562308464886 | |
| } | |
| }, | |
| "wavcaps_test": { | |
| "meteor": { | |
| "Qwen-Audio-Chat": 0.2355106805560457, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.3175511907248581, | |
| "Qwen2-Audio-7B-Instruct": 0.21342294856199182, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.120421856260385, | |
| "phi_4_multimodal_instruct": 0.24508284335582894, | |
| "seallms_audio_7b": 0.1444387454989207, | |
| "WavLLM_fairseq": 0.06399522524688675, | |
| "SALMONN_7B": 0.17175112770658157, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.1388630786594543 | |
| }, | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 32.9364161849711, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 33.97687861271676, | |
| "Qwen2-Audio-7B-Instruct": 33.78034682080925, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 6.3468208092485545, | |
| "phi_4_multimodal_instruct": 21.884393063583815, | |
| "WavLLM_fairseq": 6.901734104046243, | |
| "SALMONN_7B": 23.76878612716763, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 3.445086705202312 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 31.641618497109825, | |
| "Qwen2-Audio-7B-Instruct": 34.86666666666667, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 4.61271676300578 | |
| } | |
| }, | |
| "covost2_zh_en_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 9.898238298955656, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 18.76473995941838, | |
| "Qwen2-Audio-7B-Instruct": 16.466557744958333, | |
| "whisper_large_v3": 14.673689493155793, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 15.209998552437538, | |
| "phi_4_multimodal_instruct": 22.678131781242936, | |
| "seallms_audio_7b": 18.79451062979056, | |
| "WavLLM_fairseq": 2.368659001743569, | |
| "SALMONN_7B": 5.296039450108202, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 14.154700735606419 | |
| } | |
| }, | |
| "imda_part6_30s_sqa_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 51.4, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 67.2, | |
| "Qwen2-Audio-7B-Instruct": 53.6, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 71.6, | |
| "phi_4_multimodal_instruct": 66.2, | |
| "seallms_audio_7b": 58.2, | |
| "WavLLM_fairseq": 62.199999999999996, | |
| "SALMONN_7B": 46.8, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 64.0 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 71.6, | |
| "Qwen2-Audio-7B-Instruct": 56.0, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 67.0 | |
| } | |
| }, | |
| "ukusnews_short_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.10399586086125925, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.10144869855926132, | |
| "Qwen2-Audio-7B-Instruct": 0.1194380323171217, | |
| "whisper_large_v3": 0.06168908700151238, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.06877338215394412, | |
| "WavLLM_fairseq": 0.2066783411605508, | |
| "SALMONN_7B": 0.09042426172092653, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.0700867627159118 | |
| } | |
| }, | |
| "imda_part6_30s_ds_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 43.84, | |
| "Qwen2-Audio-7B-Instruct": 48.38, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 65.6, | |
| "SALMONN_7B": 27.12, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 59.2 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 67.58 | |
| } | |
| }, | |
| "muchomusic_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 59.0564448188711, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 57.7927548441449, | |
| "Qwen2-Audio-7B-Instruct": 71.60909856781802, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 51.727042965459134, | |
| "phi_4_multimodal_instruct": 54.422914911541696, | |
| "seallms_audio_7b": 63.184498736310026, | |
| "WavLLM_fairseq": 44.3133951137321, | |
| "SALMONN_7B": 50.88458298230834, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 56.44481887110362 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 57.87700084245998, | |
| "Qwen2-Audio-7B-Instruct": 64.66666666666666 | |
| } | |
| }, | |
| "imda_30s_sqa_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 42.199999999999996, | |
| "Qwen2-Audio-7B-Instruct": 47.1, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 62.95, | |
| "SALMONN_7B": 42.300000000000004, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 55.7 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 61.550000000000004 | |
| } | |
| }, | |
| "imda_part2_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.45479263046830615, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.048088629169710254, | |
| "Qwen2-Audio-7B-Instruct": 0.1905689473257041, | |
| "whisper_large_v3": 0.3171008846684522, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.32988393799204613, | |
| "phi_4_multimodal_instruct": 0.3470091713334957, | |
| "seallms_audio_7b": 0.290236182128074, | |
| "WavLLM_fairseq": 0.4463923382842302, | |
| "SALMONN_7B": 0.42346400454508565, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.31912994075156237 | |
| } | |
| }, | |
| "earnings21_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.2655529121410546, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.13488732754499672, | |
| "Qwen2-Audio-7B-Instruct": 0.18872219319407232, | |
| "whisper_large_v3": 0.11863959266711877, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.11416493424197618, | |
| "phi_4_multimodal_instruct": 0.15921168191570967, | |
| "seallms_audio_7b": 0.5115646296316884, | |
| "WavLLM_fairseq": 0.6447482518259942, | |
| "SALMONN_7B": 0.2577708974886327, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.11773910240019567 | |
| } | |
| }, | |
| "parliament_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.26279685873781816, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.058922319992430694, | |
| "Qwen2-Audio-7B-Instruct": 0.23270886555019396, | |
| "whisper_large_v3": 0.0753619074652285, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.06282524363705176, | |
| "WavLLM_fairseq": 0.5216434856656259, | |
| "SALMONN_7B": 0.3010928186204939, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.07517267480367111 | |
| } | |
| }, | |
| "librispeech_test_other": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.043467569561352074, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.041576030415949455, | |
| "Qwen2-Audio-7B-Instruct": 0.060415760304159495, | |
| "whisper_large_v3": 0.03660128246354058, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.05307658841999735, | |
| "phi_4_multimodal_instruct": 0.03879546787220762, | |
| "seallms_audio_7b": 0.09453912648722265, | |
| "WavLLM_fairseq": 0.04798834811886432, | |
| "SALMONN_7B": 0.09671439650443565, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.03714982881570734 | |
| } | |
| }, | |
| "librispeech_test_clean": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.020258799562379748, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.022918474365262006, | |
| "Qwen2-Audio-7B-Instruct": 0.035141660693401744, | |
| "whisper_large_v3": 0.01878749009695552, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.032349945297468596, | |
| "phi_4_multimodal_instruct": 0.0167502923755989, | |
| "seallms_audio_7b": 0.0509676689176444, | |
| "WavLLM_fairseq": 0.02103218017882069, | |
| "SALMONN_7B": 0.10270871845172973, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.018334779492209605 | |
| } | |
| }, | |
| "imda_part6_30s_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.31394240863063033, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.11292172031202054, | |
| "Qwen2-Audio-7B-Instruct": 0.2245352799625317, | |
| "whisper_large_v3": 0.1698509342851144, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.1789273082575623, | |
| "phi_4_multimodal_instruct": 0.14552883606001388, | |
| "seallms_audio_7b": 0.6259629515980555, | |
| "WavLLM_fairseq": 0.42541061709652933, | |
| "SALMONN_7B": 0.24872817713464365, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.17467982364056267 | |
| } | |
| }, | |
| "openhermes_audio_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 10.600000000000001, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 65.6, | |
| "Qwen2-Audio-7B-Instruct": 44.800000000000004, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 72.2, | |
| "phi_4_multimodal_instruct": 30.8, | |
| "seallms_audio_7b": 63.8, | |
| "WavLLM_fairseq": 19.2, | |
| "SALMONN_7B": 15.8, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 63.0 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 74.2, | |
| "Qwen2-Audio-7B-Instruct": 57.199999999999996, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 75.0 | |
| } | |
| }, | |
| "iemocap_emotion_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 29.382470119521916, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 48.505976095617534, | |
| "Qwen2-Audio-7B-Instruct": 53.98406374501992, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 44.322709163346616, | |
| "phi_4_multimodal_instruct": 41.03585657370518, | |
| "WavLLM_fairseq": 59.76095617529881, | |
| "SALMONN_7B": 23.804780876494025, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 46.713147410358566 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 37.45019920318725, | |
| "Qwen2-Audio-7B-Instruct": 35.333333333333336 | |
| } | |
| }, | |
| "public_sg_speech_qa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 63.16860465116279, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 59.7093023255814, | |
| "Qwen2-Audio-7B-Instruct": 58.31395348837209, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 73.11046511627907, | |
| "phi_4_multimodal_instruct": 68.40116279069767, | |
| "seallms_audio_7b": 57.587209302325576, | |
| "WavLLM_fairseq": 58.54651162790698, | |
| "SALMONN_7B": 59.24418604651163, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 64.94186046511628 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.18604651162791, | |
| "Qwen2-Audio-7B-Instruct": 62.733333333333334, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 73.02325581395348 | |
| } | |
| }, | |
| "mediacorp_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.4498529892192094, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.170859196341065, | |
| "Qwen2-Audio-7B-Instruct": 0.18694870957203527, | |
| "whisper_large_v3": 0.12054884024828487, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.12455080039202875, | |
| "WavLLM_fairseq": 0.3595230316889905, | |
| "SALMONN_7B": 0.32089186540346293, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.13598497223129696 | |
| } | |
| }, | |
| "common_voice_15_en_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.11272421128398918, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.07811646454714301, | |
| "Qwen2-Audio-7B-Instruct": 0.11438872500819404, | |
| "whisper_large_v3": 0.10001863741235596, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.10600831614192711, | |
| "phi_4_multimodal_instruct": 0.08262800367606891, | |
| "seallms_audio_7b": 0.14429855849255468, | |
| "WavLLM_fairseq": 0.14533325621300636, | |
| "SALMONN_7B": 0.3062255383962828, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.09876543209876543 | |
| } | |
| }, | |
| "spoken_squad_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 64.8327415436367, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 73.66473556344609, | |
| "Qwen2-Audio-7B-Instruct": 64.86264249672958, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 88.61894972902262, | |
| "phi_4_multimodal_instruct": 77.58549803774996, | |
| "seallms_audio_7b": 67.73313399364605, | |
| "WavLLM_fairseq": 77.64903756307233, | |
| "SALMONN_7B": 66.39506634273968, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 83.81984675761541 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 74.99159035694262, | |
| "Qwen2-Audio-7B-Instruct": 65.6, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 90.12521024107643 | |
| } | |
| }, | |
| "seame_dev_sge": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 1.05567969634822, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.35550521901496834, | |
| "Qwen2-Audio-7B-Instruct": 0.5486546879304539, | |
| "whisper_large_v3": 0.5377268970583734, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5840399155162387, | |
| "gemini-1.5-flash": 1.1100431601824359, | |
| "phi_4_multimodal_instruct": 0.8529492791331231, | |
| "seallms_audio_7b": 1.7106737273868193, | |
| "WavLLM_fairseq": 1.2204842511249197, | |
| "SALMONN_7B": 1.0189782362484312, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.507882090054792 | |
| } | |
| }, | |
| "meld_sentiment_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 44.90421455938697, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 46.206896551724135, | |
| "Qwen2-Audio-7B-Instruct": 53.9463601532567, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 56.59003831417625, | |
| "phi_4_multimodal_instruct": 51.609195402298845, | |
| "seallms_audio_7b": 52.1455938697318, | |
| "WavLLM_fairseq": 51.072796934865906, | |
| "SALMONN_7B": 41.7624521072797, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 45.593869731800766 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 47.356321839080465, | |
| "Qwen2-Audio-7B-Instruct": 57.666666666666664 | |
| } | |
| }, | |
| "imda_part4_30s_sqa_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 37.8, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 53.2, | |
| "Qwen2-Audio-7B-Instruct": 39.6, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 66.0, | |
| "phi_4_multimodal_instruct": 43.8, | |
| "seallms_audio_7b": 45.0, | |
| "WavLLM_fairseq": 46.6, | |
| "SALMONN_7B": 36.6, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 53.8 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.0, | |
| "Qwen2-Audio-7B-Instruct": 43.4, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 61.4 | |
| } | |
| }, | |
| "voxceleb_gender_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 70.5990972507181, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 99.75379565038982, | |
| "Qwen2-Audio-7B-Instruct": 99.1177677472302, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 34.94050061551087, | |
| "phi_4_multimodal_instruct": 94.54247025030776, | |
| "WavLLM_fairseq": 69.61427985227739, | |
| "SALMONN_7B": 88.79770209273697, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 42.921624948707425 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 99.46655724251129, | |
| "Qwen2-Audio-7B-Instruct": 99.66666666666667 | |
| } | |
| }, | |
| "imda_gr_dialogue": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 37.2, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 93.76666666666667, | |
| "Qwen2-Audio-7B-Instruct": 61.56666666666667, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 19.6, | |
| "phi_4_multimodal_instruct": 36.833333333333336, | |
| "seallms_audio_7b": 30.5, | |
| "WavLLM_fairseq": 46.766666666666666, | |
| "SALMONN_7B": 42.733333333333334, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 25.433333333333337 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 93.86666666666666, | |
| "Qwen2-Audio-7B-Instruct": 61.0 | |
| } | |
| }, | |
| "imda_30s_ds_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 30.65, | |
| "Qwen2-Audio-7B-Instruct": 37.599999999999994, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 50.15, | |
| "SALMONN_7B": 16.15, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 43.849999999999994 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 54.65 | |
| } | |
| }, | |
| "imda_part5_30s_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.3016882870525747, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.17694182194919086, | |
| "Qwen2-Audio-7B-Instruct": 0.27856006770658537, | |
| "whisper_large_v3": 0.2143555471246589, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.22881615619208825, | |
| "phi_4_multimodal_instruct": 0.22801359968481416, | |
| "seallms_audio_7b": 0.5812260145043848, | |
| "WavLLM_fairseq": 0.39796588405247263, | |
| "SALMONN_7B": 0.34868891450584405, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.22004640235805695 | |
| } | |
| }, | |
| "imda_part4_30s_sqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 41.92, | |
| "Qwen2-Audio-7B-Instruct": 50.279999999999994, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 66.34, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 61.980000000000004 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 64.9 | |
| } | |
| }, | |
| "earnings22_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.3664994875132684, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.1652245056860175, | |
| "Qwen2-Audio-7B-Instruct": 0.23542555661330924, | |
| "whisper_large_v3": 0.15887899737116104, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.1448629161356777, | |
| "phi_4_multimodal_instruct": 0.24134627375003423, | |
| "seallms_audio_7b": 0.5738685499413504, | |
| "WavLLM_fairseq": 0.6671766188447099, | |
| "SALMONN_7B": 0.3597423676988383, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.15611126487402763 | |
| } | |
| }, | |
| "idpc_short_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.6008025988916491, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.24918784635964075, | |
| "Qwen2-Audio-7B-Instruct": 0.21326199120963119, | |
| "whisper_large_v3": 0.1662526275558953, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.16931014714313014, | |
| "WavLLM_fairseq": 0.36728454041658704, | |
| "SALMONN_7B": 0.26313777947639977, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.15803554366520162 | |
| } | |
| }, | |
| "cna_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.19753284203780838, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.15924383210509452, | |
| "Qwen2-Audio-7B-Instruct": 0.2067713339741536, | |
| "whisper_large_v3": 0.13841717398269784, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.15171419416853574, | |
| "WavLLM_fairseq": 0.26946491509131687, | |
| "SALMONN_7B": 0.15395706504325538, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.13798996048275125 | |
| } | |
| }, | |
| "covost2_id_en_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 0.45648619714728844, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 44.43289180618449, | |
| "Qwen2-Audio-7B-Instruct": 6.326113431899141, | |
| "whisper_large_v3": 46.01512198258627, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 46.80524126004861, | |
| "phi_4_multimodal_instruct": 0.36465303013961253, | |
| "seallms_audio_7b": 43.98074943006231, | |
| "WavLLM_fairseq": 5.933522277713613, | |
| "SALMONN_7B": 26.89649039333571, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 46.79924664837527 | |
| } | |
| }, | |
| "imda_part3_30s_sqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 51.08, | |
| "Qwen2-Audio-7B-Instruct": 60.620000000000005, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 70.17999999999999, | |
| "SALMONN_7B": 50.8, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 70.28 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 73.0 | |
| } | |
| }, | |
| "idpc_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.7710863986313088, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.30008554319931563, | |
| "Qwen2-Audio-7B-Instruct": 0.19093242087254064, | |
| "whisper_large_v3": 0.19880239520958085, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.16766467065868262, | |
| "WavLLM_fairseq": 0.7686911890504705, | |
| "SALMONN_7B": 0.4550898203592814, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.17741659538066723 | |
| } | |
| }, | |
| "gigaspeech_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.13018910022587737, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.14457154747310655, | |
| "Qwen2-Audio-7B-Instruct": 0.11723812890302816, | |
| "whisper_large_v3": 0.09459022434812692, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.09948381629977261, | |
| "phi_4_multimodal_instruct": 0.09672866386388193, | |
| "seallms_audio_7b": 0.13672725996455154, | |
| "WavLLM_fairseq": 0.15491778414546403, | |
| "SALMONN_7B": 0.10765150204693537, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.09515429104337297 | |
| } | |
| }, | |
| "mediacorp_short_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.2548909377108163, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.13301101866426804, | |
| "Qwen2-Audio-7B-Instruct": 0.17180121430177647, | |
| "whisper_large_v3": 0.11715763436024286, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.14571621317742298, | |
| "WavLLM_fairseq": 0.2621992354396222, | |
| "SALMONN_7B": 0.1751742747919946, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.11434675061839443 | |
| } | |
| }, | |
| "imda_part3_30s_ds_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 16.4, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 48.4, | |
| "Qwen2-Audio-7B-Instruct": 33.8, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 45.4, | |
| "phi_4_multimodal_instruct": 41.2, | |
| "seallms_audio_7b": 43.0, | |
| "WavLLM_fairseq": 31.6, | |
| "SALMONN_7B": 9.0, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 37.400000000000006 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 59.2, | |
| "Qwen2-Audio-7B-Instruct": 43.2, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 47.400000000000006 | |
| } | |
| }, | |
| "imda_ar_dialogue": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 0.6666666666666667, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 77.83333333333333, | |
| "Qwen2-Audio-7B-Instruct": 0.9666666666666667, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 7.633333333333334, | |
| "phi_4_multimodal_instruct": 0.5333333333333333, | |
| "seallms_audio_7b": 15.633333333333333, | |
| "WavLLM_fairseq": 0.23333333333333336, | |
| "SALMONN_7B": 0.06666666666666667, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 9.666666666666666 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 77.13333333333333, | |
| "Qwen2-Audio-7B-Instruct": 0.33333333333333337 | |
| } | |
| }, | |
| "iemocap_gender_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 50.0996015936255, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 93.48605577689243, | |
| "Qwen2-Audio-7B-Instruct": 92.80876494023903, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 15.737051792828685, | |
| "phi_4_multimodal_instruct": 59.46215139442231, | |
| "seallms_audio_7b": 66.43426294820716, | |
| "WavLLM_fairseq": 51.932270916334666, | |
| "SALMONN_7B": 81.31474103585658, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 44.22310756972111 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 98.20717131474103, | |
| "Qwen2-Audio-7B-Instruct": 98.33333333333333 | |
| } | |
| }, | |
| "ytb_asr_batch2": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.4315277327278625, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.15162720294085846, | |
| "Qwen2-Audio-7B-Instruct": 0.2080008649583739, | |
| "whisper_large_v3": 0.17210509244242622, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.2192622950819672, | |
| "phi_4_multimodal_instruct": 0.23849064763758243, | |
| "WavLLM_fairseq": 0.48091685587631094, | |
| "SALMONN_7B": 0.3238620391393664, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.23561466104443723 | |
| } | |
| }, | |
| "ytb_pqa_batch1": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 37.16117216117216, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 40.97069597069597, | |
| "Qwen2-Audio-7B-Instruct": 36.97802197802198, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 55.01831501831502, | |
| "gemini-1.5-flash": 49.908424908424905, | |
| "WavLLM_fairseq": 40.95238095238095, | |
| "SALMONN_7B": 32.124542124542124, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 52.252747252747255 | |
| } | |
| }, | |
| "dream_tts_mcq_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 59.749085206481965, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 84.31782540512285, | |
| "Qwen2-Audio-7B-Instruct": 66.49242028227914, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 89.33612127548353, | |
| "phi_4_multimodal_instruct": 72.60846837428123, | |
| "seallms_audio_7b": 75.6926293779404, | |
| "WavLLM_fairseq": 66.5446941975954, | |
| "SALMONN_7B": 56.455828541557764, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 86.4610559330894 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 84.31782540512285, | |
| "Qwen2-Audio-7B-Instruct": 68.66666666666667 | |
| } | |
| }, | |
| "imda_part5_30s_ds_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 28.2, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 57.0, | |
| "Qwen2-Audio-7B-Instruct": 40.4, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 58.0, | |
| "phi_4_multimodal_instruct": 52.199999999999996, | |
| "seallms_audio_7b": 49.400000000000006, | |
| "WavLLM_fairseq": 45.199999999999996, | |
| "SALMONN_7B": 17.2, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 49.0 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.0, | |
| "Qwen2-Audio-7B-Instruct": 50.8, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 56.8 | |
| } | |
| }, | |
| "aishell_asr_zh_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.9469917443725129, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.13165449110094832, | |
| "Qwen2-Audio-7B-Instruct": 0.09260359129694522, | |
| "whisper_large_v3": 0.12359684029221357, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.20886539565639167, | |
| "phi_4_multimodal_instruct": 0.07466690423868068, | |
| "seallms_audio_7b": 0.11804359446457208, | |
| "WavLLM_fairseq": 0.7054601967888183, | |
| "SALMONN_7B": 0.8259290055631446, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.12450753301261111 | |
| } | |
| }, | |
| "imda_part3_30s_sqa_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 32.2, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 51.4, | |
| "Qwen2-Audio-7B-Instruct": 42.0, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 56.0, | |
| "phi_4_multimodal_instruct": 43.8, | |
| "seallms_audio_7b": 45.599999999999994, | |
| "WavLLM_fairseq": 45.199999999999996, | |
| "SALMONN_7B": 40.599999999999994, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 49.0 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.599999999999994, | |
| "Qwen2-Audio-7B-Instruct": 50.199999999999996, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 52.800000000000004 | |
| } | |
| }, | |
| "imda_part4_30s_ds_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 16.0, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 46.4, | |
| "Qwen2-Audio-7B-Instruct": 24.8, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 44.0, | |
| "phi_4_multimodal_instruct": 37.0, | |
| "seallms_audio_7b": 35.4, | |
| "WavLLM_fairseq": 31.6, | |
| "SALMONN_7B": 7.0, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 36.0 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 55.199999999999996, | |
| "Qwen2-Audio-7B-Instruct": 35.8, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 48.2 | |
| } | |
| }, | |
| "ytb_sds_batch1": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 43.878954607977995, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 53.97524071526823, | |
| "Qwen2-Audio-7B-Instruct": 51.5818431911967, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 64.12654745529574, | |
| "gemini-1.5-flash": 65.9697386519945, | |
| "WavLLM_fairseq": 55.625859697386524, | |
| "SALMONN_7B": 31.279229711141674, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 59.44979367262724 | |
| } | |
| }, | |
| "audiocaps_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 47.04090909090909, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 38.00454545454545, | |
| "Qwen2-Audio-7B-Instruct": 40.77727272727273, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 3.0954545454545457, | |
| "phi_4_multimodal_instruct": 26.386363636363637, | |
| "seallms_audio_7b": 53.20909090909091, | |
| "WavLLM_fairseq": 5.5, | |
| "SALMONN_7B": 37.445454545454545, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 2.4727272727272727 | |
| }, | |
| "meteor": { | |
| "Qwen-Audio-Chat": 0.27553015076950976, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.24920047034353812, | |
| "Qwen2-Audio-7B-Instruct": 0.19891712076314283, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.05796819723943051, | |
| "phi_4_multimodal_instruct": 0.1757379026471828, | |
| "seallms_audio_7b": 0.30423899385222564, | |
| "WavLLM_fairseq": 0.041732965094428545, | |
| "SALMONN_7B": 0.20994052484339956, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.07953048457785493 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 39.29545454545455, | |
| "Qwen2-Audio-7B-Instruct": 41.53333333333333, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 4.868181818181818 | |
| } | |
| }, | |
| "imda_ar_sentence": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 3.933333333333333, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 7.816666666666666, | |
| "Qwen2-Audio-7B-Instruct": 2.55, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 26.016666666666666, | |
| "phi_4_multimodal_instruct": 3.5166666666666666, | |
| "seallms_audio_7b": 3.5833333333333335, | |
| "WavLLM_fairseq": 2.6833333333333336, | |
| "SALMONN_7B": 2.5166666666666666, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 12.416666666666666 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 10.116666666666667, | |
| "Qwen2-Audio-7B-Instruct": 4.666666666666667 | |
| } | |
| }, | |
| "imda_part6_30s_sqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 63.040000000000006, | |
| "Qwen2-Audio-7B-Instruct": 69.42, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 83.08, | |
| "SALMONN_7B": 66.86, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 80.60000000000001 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 81.8 | |
| } | |
| }, | |
| "covost2_ta_en_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 0.01699144301093184, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 5.023057608950299, | |
| "Qwen2-Audio-7B-Instruct": 0.04425838146050298, | |
| "whisper_large_v3": 2.451098639578599, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 2.8327095799289337, | |
| "phi_4_multimodal_instruct": 0.053138495633157125, | |
| "seallms_audio_7b": 0.06475917031217593, | |
| "WavLLM_fairseq": 0.1695522548322915, | |
| "SALMONN_7B": 0.3649023706010388, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 2.4245628096245917 | |
| } | |
| }, | |
| "covost2_en_id_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 4.102230932924371, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 37.60224687716629, | |
| "Qwen2-Audio-7B-Instruct": 16.325186897428104, | |
| "whisper_large_v3": 1.600581653970121, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 27.620150160643625, | |
| "phi_4_multimodal_instruct": 15.012558278964478, | |
| "seallms_audio_7b": 27.583542512329426, | |
| "WavLLM_fairseq": 13.841886973016162, | |
| "SALMONN_7B": 14.102682915273142, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 10.930203684508578 | |
| } | |
| }, | |
| "clotho_aqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 61.934856587263, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.15021876519203, | |
| "Qwen2-Audio-7B-Instruct": 50.919591292758774, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 24.647544968400585, | |
| "phi_4_multimodal_instruct": 47.86582401555663, | |
| "seallms_audio_7b": 53.03840544482256, | |
| "WavLLM_fairseq": 43.01199466903598, | |
| "SALMONN_7B": 57.75401069518716, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 29.47134606841404 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.09333981526495, | |
| "Qwen2-Audio-7B-Instruct": 56.86666666666667, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 28.076410484229232 | |
| } | |
| }, | |
| "ytb_asr_batch1": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.2297764461857571, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.11484981178458939, | |
| "Qwen2-Audio-7B-Instruct": 0.16843358684796805, | |
| "whisper_large_v3": 0.12226319428439733, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.1400092187139894, | |
| "gemini-1.5-flash": 0.1089344703080587, | |
| "phi_4_multimodal_instruct": 0.16175001920565416, | |
| "WavLLM_fairseq": 0.41876008296842593, | |
| "SALMONN_7B": 0.21487285856956287, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.12579703464700007 | |
| } | |
| }, | |
| "imda_part3_30s_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.6412550574306894, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.2919053954978684, | |
| "Qwen2-Audio-7B-Instruct": 0.35076166942732234, | |
| "whisper_large_v3": 0.27026366524560785, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.3035544573275043, | |
| "phi_4_multimodal_instruct": 0.44227061666711925, | |
| "seallms_audio_7b": 1.0837293290249002, | |
| "WavLLM_fairseq": 0.7540934640345399, | |
| "SALMONN_7B": 0.6569229098215983, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.29992939962527493 | |
| } | |
| }, | |
| "alpaca_audio_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 9.8, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 74.80000000000001, | |
| "Qwen2-Audio-7B-Instruct": 52.599999999999994, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 73.8, | |
| "phi_4_multimodal_instruct": 25.8, | |
| "WavLLM_fairseq": 21.6, | |
| "SALMONN_7B": 17.2, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 70.8 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 78.60000000000001, | |
| "Qwen2-Audio-7B-Instruct": 61.6, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 77.8 | |
| } | |
| }, | |
| "imda_30s_sqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 54.669999999999995, | |
| "Qwen2-Audio-7B-Instruct": 62.190000000000005, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 75.09, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 72.475 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 75.11999999999999 | |
| } | |
| }, | |
| "ytb_sqa_batch1": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 60.827586206896555, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.51231527093596, | |
| "Qwen2-Audio-7B-Instruct": 60.453201970443345, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 70.18719211822659, | |
| "gemini-1.5-flash": 78.06896551724138, | |
| "WavLLM_fairseq": 60.70935960591133, | |
| "SALMONN_7B": 55.665024630541865, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 67.3103448275862 | |
| } | |
| }, | |
| "audiocaps_qa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 50.22364217252396, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 49.77635782747604, | |
| "Qwen2-Audio-7B-Instruct": 45.75079872204473, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 18.466453674121407, | |
| "phi_4_multimodal_instruct": 38.466453674121404, | |
| "seallms_audio_7b": 53.73801916932908, | |
| "WavLLM_fairseq": 29.840255591054312, | |
| "SALMONN_7B": 50.287539936102235, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 17.380191693290733 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 50.60702875399361, | |
| "Qwen2-Audio-7B-Instruct": 50.599999999999994, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 14.63258785942492 | |
| } | |
| }, | |
| "imda_part1_asr_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.10550313315290274, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.042254894789457, | |
| "Qwen2-Audio-7B-Instruct": 0.07197717796796138, | |
| "whisper_large_v3": 0.06844171360300393, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.07041669714480775, | |
| "phi_4_multimodal_instruct": 0.05739643527661961, | |
| "seallms_audio_7b": 0.17813863896813206, | |
| "WavLLM_fairseq": 0.10077292565771828, | |
| "SALMONN_7B": 0.0925804013361617, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.06922195401458074 | |
| } | |
| }, | |
| "peoples_speech_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.31419144746723354, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.21050407754683692, | |
| "Qwen2-Audio-7B-Instruct": 0.2165498391593041, | |
| "whisper_large_v3": 0.14602420615337386, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.20140159998943682, | |
| "seallms_audio_7b": 0.369768551146351, | |
| "WavLLM_fairseq": 0.3792176325635977, | |
| "SALMONN_7B": 0.23699946689025367, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.14540692118393275 | |
| } | |
| }, | |
| "covost2_en_ta_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 0.03451483807236294, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 14.407399367512914, | |
| "Qwen2-Audio-7B-Instruct": 0.03245972071872916, | |
| "whisper_large_v3": 0.02107778621423822, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 8.433062902024755, | |
| "phi_4_multimodal_instruct": 0.19835914151649442, | |
| "seallms_audio_7b": 0.012334972259958572, | |
| "WavLLM_fairseq": 0.0033159224040994286, | |
| "SALMONN_7B": 0.00046745670226766583, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 1.0368044741318085 | |
| } | |
| }, | |
| "wavcaps_qa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 42.69736842105263, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 46.31578947368421, | |
| "Qwen2-Audio-7B-Instruct": 44.473684210526315, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 18.88157894736842, | |
| "phi_4_multimodal_instruct": 35.13157894736842, | |
| "seallms_audio_7b": 42.10526315789473, | |
| "WavLLM_fairseq": 26.25, | |
| "SALMONN_7B": 47.30263157894737, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 16.710526315789473 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 46.31578947368421, | |
| "Qwen2-Audio-7B-Instruct": 48.2, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 14.736842105263158 | |
| } | |
| }, | |
| "parliament_short_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.09347360821020603, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.056935097083623425, | |
| "Qwen2-Audio-7B-Instruct": 0.08416492612361723, | |
| "whisper_large_v3": 0.05543951935226013, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.07325752301384698, | |
| "WavLLM_fairseq": 0.09512390087929656, | |
| "SALMONN_7B": 0.08676929424202573, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.05742502771975968 | |
| } | |
| }, | |
| "covost2_en_zh_test": { | |
| "bleu": { | |
| "Qwen-Audio-Chat": 15.330641138043728, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 43.941098854450516, | |
| "Qwen2-Audio-7B-Instruct": 25.765420247070075, | |
| "whisper_large_v3": 0.16408986541757878, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 35.274306071307024, | |
| "phi_4_multimodal_instruct": 45.295964957544776, | |
| "seallms_audio_7b": 36.4496678966979, | |
| "WavLLM_fairseq": 31.96381187282953, | |
| "SALMONN_7B": 33.88941292215531, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 5.987143868370054 | |
| } | |
| }, | |
| "imda_part5_30s_sqa_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 61.260000000000005, | |
| "Qwen2-Audio-7B-Instruct": 68.52000000000001, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 80.34, | |
| "SALMONN_7B": 62.62, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 76.56 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 80.36 | |
| } | |
| }, | |
| "imda_gr_sentence": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 57.550000000000004, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.13333333333333, | |
| "Qwen2-Audio-7B-Instruct": 68.38333333333333, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 26.35, | |
| "phi_4_multimodal_instruct": 51.68333333333334, | |
| "seallms_audio_7b": 50.083333333333336, | |
| "WavLLM_fairseq": 49.06666666666666, | |
| "SALMONN_7B": 59.766666666666666, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 36.016666666666666 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.9, | |
| "Qwen2-Audio-7B-Instruct": 66.33333333333333 | |
| } | |
| }, | |
| "tedlium3_long_form_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.2911540507002305, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.10228682857649353, | |
| "Qwen2-Audio-7B-Instruct": 0.08739585179932637, | |
| "whisper_large_v3": 0.03208650948413402, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.04396383619925545, | |
| "phi_4_multimodal_instruct": 0.0381847190214501, | |
| "seallms_audio_7b": 0.3208650948413402, | |
| "WavLLM_fairseq": 0.4536784258110264, | |
| "SALMONN_7B": 0.14231519234178336, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.04754476156709803 | |
| } | |
| }, | |
| "meld_emotion_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 50.72796934865901, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 36.36015325670498, | |
| "Qwen2-Audio-7B-Instruct": 41.60919540229885, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 47.356321839080465, | |
| "phi_4_multimodal_instruct": 43.524904214559385, | |
| "seallms_audio_7b": 51.11111111111111, | |
| "WavLLM_fairseq": 41.57088122605364, | |
| "SALMONN_7B": 30.536398467432953, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 36.81992337164751 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 36.206896551724135, | |
| "Qwen2-Audio-7B-Instruct": 39.0 | |
| } | |
| }, | |
| "tedlium3_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.04052375714133636, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.07884745040985061, | |
| "Qwen2-Audio-7B-Instruct": 0.06114048472375004, | |
| "whisper_large_v3": 0.037649480146197796, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.04900464852205386, | |
| "phi_4_multimodal_instruct": 0.028494375643163834, | |
| "seallms_audio_7b": 0.04829495049856286, | |
| "WavLLM_fairseq": 0.06621482559171073, | |
| "SALMONN_7B": 0.0459884319222171, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.038146268762641496 | |
| } | |
| }, | |
| "seame_dev_man": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.8783373786407767, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.388282092772384, | |
| "Qwen2-Audio-7B-Instruct": 0.5522518878101402, | |
| "whisper_large_v3": 0.7225930420711975, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.7824973031283711, | |
| "gemini-1.5-flash": 0.9690871089536138, | |
| "phi_4_multimodal_instruct": 0.7126483279395901, | |
| "seallms_audio_7b": 1.0639495685005393, | |
| "WavLLM_fairseq": 1.2913969795037756, | |
| "SALMONN_7B": 1.2721817691477886, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6848705501618123 | |
| } | |
| }, | |
| "imda_30s_ds_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 31.295, | |
| "Qwen2-Audio-7B-Instruct": 38.915, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 54.515, | |
| "SALMONN_7B": 18.345, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 48.269999999999996 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 57.99 | |
| } | |
| }, | |
| "imda_part5_30s_sqa_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 47.800000000000004, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.80000000000001, | |
| "Qwen2-Audio-7B-Instruct": 51.6, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 74.0, | |
| "phi_4_multimodal_instruct": 49.0, | |
| "seallms_audio_7b": 54.2, | |
| "WavLLM_fairseq": 50.8, | |
| "SALMONN_7B": 44.6, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 57.800000000000004 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.8, | |
| "Qwen2-Audio-7B-Instruct": 58.0, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 64.80000000000001 | |
| } | |
| }, | |
| "imda_part6_30s_ds_human_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 40.4, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 62.599999999999994, | |
| "Qwen2-Audio-7B-Instruct": 46.2, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 65.4, | |
| "phi_4_multimodal_instruct": 52.599999999999994, | |
| "seallms_audio_7b": 53.0, | |
| "WavLLM_fairseq": 49.400000000000006, | |
| "SALMONN_7B": 24.2, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 57.199999999999996 | |
| }, | |
| "gpt4o_judge": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 70.0, | |
| "Qwen2-Audio-7B-Instruct": 51.4, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 64.4 | |
| } | |
| }, | |
| "imda_part4_30s_ds_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 18.060000000000002, | |
| "Qwen2-Audio-7B-Instruct": 25.019999999999996, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 43.4, | |
| "SALMONN_7B": 9.399999999999999, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 37.879999999999995 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 47.74 | |
| } | |
| }, | |
| "ukusnews_test": { | |
| "wer": { | |
| "Qwen-Audio-Chat": 0.3158631121194933, | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.12554358101720553, | |
| "Qwen2-Audio-7B-Instruct": 0.13843826810361126, | |
| "whisper_large_v3": 0.07135564378899603, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.07388920400831915, | |
| "WavLLM_fairseq": 0.5911892607298166, | |
| "SALMONN_7B": 0.18918510115333712, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.07642276422764227 | |
| } | |
| }, | |
| "imda_part3_30s_ds_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 25.22, | |
| "Qwen2-Audio-7B-Instruct": 35.54, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 48.339999999999996, | |
| "WavLLM_fairseq": 36.5, | |
| "SALMONN_7B": 12.82, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 42.32 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 52.38 | |
| } | |
| }, | |
| "imda_part5_30s_ds_test": { | |
| "llama3_70b_judge": { | |
| "Qwen-Audio-Chat": 39.14, | |
| "Qwen2-Audio-7B-Instruct": 45.38, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 61.48, | |
| "SALMONN_7B": 24.340000000000003, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 54.379999999999995 | |
| }, | |
| "gpt4o_judge": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 63.68000000000001 | |
| } | |
| }, | |
| "gigaspeech2_viet": { | |
| "wer": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.4960741822016732, | |
| "Qwen2-Audio-7B-Instruct": 1.5011671350211242 | |
| } | |
| }, | |
| "gigaspeech2_thai": { | |
| "wer": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.8901628256099774, | |
| "Qwen2-Audio-7B-Instruct": 1.2449725324578913, | |
| "seallms_audio_7b": 0.3332398502070376 | |
| } | |
| }, | |
| "gigaspeech2_indo": { | |
| "wer": { | |
| "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5087211232500294, | |
| "Qwen2-Audio-7B-Instruct": 0.8956121130400584 | |
| } | |
| }, | |
| "imda_30s_ar_test": { | |
| "llama3_70b_judge": { | |
| "Qwen2-Audio-7B-Instruct": 5.106666666666667, | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 15.773333333333333, | |
| "SALMONN_7B": 5.673333333333334, | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 27.186666666666667 | |
| } | |
| }, | |
| "imda_30s_gr_test": { | |
| "llama3_70b_judge": { | |
| "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 18.46666666666667 | |
| } | |
| }, | |
| "nlb_asr_test": { | |
| "wer": { | |
| "cascade_whisper_large_v3_llama_3_8b_instruct": 0.2796380263880551 | |
| } | |
| } | |
| } |