Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 0223_0351_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_1ad6e5/vsibench.json filter=lfs diff=lfs merge=lfs -text
 0223_2055_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_2229cf/vsibench.json filter=lfs diff=lfs merge=lfs -text
 0223_2055_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_3f39f5/vsibench.json filter=lfs diff=lfs merge=lfs -text
+vsibench/0227_2109_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_2b3696/vsibench.json filter=lfs diff=lfs merge=lfs -text

vsibench/0227_2109_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_2b3696/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 33.12872055166613,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002c6720940>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002c6721000>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002c67217e0>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002c6722290>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002c67227a0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 5130,
+            "effective": 5130
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_base_lora_ep3,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2123"
+}

vsibench/0227_2109_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_2b3696/vsibench.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f25180c5ef080b415e72591ed1418a06ebc911b4752f37d839834f515cc706bd
+size 13369488

vsibench/0227_2119_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_4ffbe0/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 35.480714020333025,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002e6f6b0a0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002e6f6b760>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002e6f6bd90>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002e5c14a60>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002e5c152d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_cut3r_lora_ep3,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2135"
+}

vsibench/0227_2125_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_13af5f/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 35.378964434086534,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002e42670a0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002e4267760>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002e4267d90>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002e4520a60>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002e45212d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_cut3r_lora_diff_lr_ep3,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2140"
+}

vsibench/0227_2133_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_4ca234/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 36.41748572972608,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002bf8788b0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002bf878f70>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002bf879750>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002bf87a200>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002bf87a710>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_base_lora_ep3/checkpoint-385,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2144"
+}

vsibench/0227_2134_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_0a55b4/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 32.436417063532275,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002d8d588b0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002d8d58f70>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002d8d59750>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002d8d5a200>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002d8d5a710>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_base_lora_ep3/checkpoint-192,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2145"
+}

vsibench/0227_2139_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_c76152/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 30.28599655486878,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002c50930a0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002c5093760>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002c5093d90>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002c3d3ca60>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002c3d3d2d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_cut3r_lora_ep3/checkpoint-192,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2155"
+}

vsibench/0227_2143_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_62e9eb/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 30.28599655486878,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002e407b0a0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002e407b760>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002e407bd90>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002e4c3ca60>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002e4c3d2d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_cut3r_lora_ep3/checkpoint-385,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2159"
+}

vsibench/0227_2147_llava_one_vision_llava_qwen_lora_ov_32f_llava_onevision_model_args_eb18c4/results.json

@@ -0,0 +1,112 @@
+{
+    "results": {
+        "vsibench": {
+            "alias": "vsibench",
+            "vsibench_score,none": 30.28599655486878,
+            "vsibench_score_stderr,none": "N/A"
+        }
+    },
+    "group_subtasks": {
+        "vsibench": []
+    },
+    "configs": {
+        "vsibench": {
+            "task": "vsibench",
+            "dataset_path": "nyu-visionx/VSI-Bench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "full_docs": false,
+            "process_results_use_image": false,
+            "process_docs": "<function process_docs at 0x4002c2bc70a0>",
+            "doc_to_visual": "<function vsibench_doc_to_visual at 0x4002c2bc7760>",
+            "doc_to_text": "<function vsibench_doc_to_text at 0x4002c2bc7d90>",
+            "doc_to_target": "ground_truth",
+            "process_results": "<function vsibench_process_results at 0x4002c377ca60>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "num_fewshot": 0,
+            "metric_list": [
+                {
+                    "metric": "vsibench_score",
+                    "aggregation": "<function vsibench_aggregate_results at 0x4002c377d2d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "lmms_eval_specific_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Please answer the question using a single word or phrase."
+                },
+                "gemini_api": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                    "na_post_prompt": "Do not response anything other than a single number!"
+                },
+                "pre_prompt": "",
+                "mca_post_prompt": "Answer with the option's letter from the given choices directly.",
+                "na_post_prompt": "Please answer the question using a single word or phrase."
+            }
+        }
+    },
+    "versions": {
+        "vsibench": "Yaml"
+    },
+    "n-shot": {
+        "vsibench": 0
+    },
+    "higher_is_better": {
+        "vsibench": {
+            "vsibench_score": true
+        }
+    },
+    "n-samples": {
+        "vsibench": {
+            "original": 1544,
+            "effective": 1544
+        }
+    },
+    "config": {
+        "model": "llava_onevision",
+        "model_args": "pretrained=/scratch/08367/zhiwen/work_dirs/llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-02_27_cut3r_lora_diff_lr_ep3/checkpoint-385,attn_implementation=flash_attention_2,conv_template=qwen_1_5,model_name=llava_qwen_lora,max_frames_num=32,model_base=LLaVA-NeXT/checkpoints/LLaVA-Video-7B-Qwen2",
+        "batch_size": "1",
+        "batch_sizes": [],
+        "device": null,
+        "use_cache": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": "",
+        "random_seed": 0,
+        "numpy_seed": 1234,
+        "torch_seed": 1234,
+        "fewshot_seed": 1234
+    },
+    "git_hash": "e374ddc",
+    "date": "0227_2203"
+}