Delete eval

eval/__init__.py

@@ -1,2 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0

eval/gen/gen_images_mp.py

@@ -1,238 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-import json
-import argparse
-from safetensors.torch import load_file
-
-import torch
-import torch.distributed as dist
-from data.data_utils import add_special_tokens
-from modeling.bagel import (
-    BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel
-)
-from modeling.qwen2 import Qwen2Tokenizer
-from modeling.autoencoder import load_ae
-
-from PIL import Image
-from modeling.bagel.qwen2_navit import NaiveCache
-
-
-def move_generation_input_to_device(generation_input, device):
-    # Utility to move all tensors in generation_input to device
-    for k, v in generation_input.items():
-        if isinstance(v, torch.Tensor):
-            generation_input[k] = v.to(device)
-    return generation_input
-
-
-def setup_distributed():
-    dist.init_process_group(backend="nccl")
-    torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
-
-
-def generate_image(prompt, num_timesteps=50, cfg_scale=10.0, cfg_interval=[0, 1.0], cfg_renorm_min=0., timestep_shift=1.0, num_images=4, resolution=512, device=None):  # 添加device参数
-    past_key_values = NaiveCache(gen_model.config.llm_config.num_hidden_layers)
-    newlens = [0] * num_images
-    new_rope = [0] * num_images
-
-    generation_input, newlens, new_rope = gen_model.prepare_prompts(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        prompts=[prompt] * num_images,
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-
-    with torch.no_grad():
-        with torch.amp.autocast("cuda", enabled=True, dtype=torch.float16):
-            past_key_values = gen_model.forward_cache_update_text(past_key_values, **generation_input)
-
-    generation_input = gen_model.prepare_vae_latent(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        image_sizes=[(resolution, resolution)] * num_images, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-
-    cfg_past_key_values = NaiveCache(gen_model.config.llm_config.num_hidden_layers)
-    cfg_newlens = [0] * num_images
-    cfg_new_rope = [0] * num_images
-
-    generation_input_cfg = model.prepare_vae_latent_cfg(
-        curr_kvlens=cfg_newlens,
-        curr_rope=cfg_new_rope, 
-        image_sizes=[(resolution, resolution)] * num_images,
-    )
-    generation_input_cfg = move_generation_input_to_device(generation_input_cfg, device)
-
-    with torch.no_grad():
-        with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-            unpacked_latent = gen_model.generate_image(
-                past_key_values=past_key_values,
-                num_timesteps=num_timesteps,
-                cfg_text_scale=cfg_scale,
-                cfg_interval=cfg_interval,
-                cfg_renorm_min=cfg_renorm_min,
-                timestep_shift=timestep_shift,
-                cfg_text_past_key_values=cfg_past_key_values,
-                cfg_text_packed_position_ids=generation_input_cfg["cfg_packed_position_ids"],
-                cfg_text_key_values_lens=generation_input_cfg["cfg_key_values_lens"],
-                cfg_text_packed_query_indexes=generation_input_cfg["cfg_packed_query_indexes"],
-                cfg_text_packed_key_value_indexes=generation_input_cfg["cfg_packed_key_value_indexes"],
-                **generation_input,
-            )
-
-    image_list = []
-    for latent in unpacked_latent:
-        latent = latent.reshape(1, resolution//16, resolution//16, 2, 2, 16)
-        latent = torch.einsum("nhwpqc->nchpwq", latent)
-        latent = latent.reshape(1, 16, resolution//8, resolution//8)
-        image = vae_model.decode(latent.to(device))
-        tmpimage = ((image * 0.5 + 0.5).clamp(0, 1)[0].permute(1, 2, 0) * 255).to(torch.uint8).cpu().numpy()
-        tmpimage = Image.fromarray(tmpimage)
-        image_list.append(tmpimage)
-
-    return image_list
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Generate images using Bagel model.")
-    parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated images.")
-    parser.add_argument("--metadata_file", type=str, required=True, help="JSONL file containing lines of metadata for each prompt.")
-    parser.add_argument("--num_images", type=int, default=4)
-    parser.add_argument("--batch_size", type=int, default=4)
-    parser.add_argument("--cfg_scale", type=float, default=4)
-    parser.add_argument("--resolution", type=int, default=1024)
-    parser.add_argument("--max_latent_size", type=int, default=64)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-    
-    seed = 42
-    if seed is not None:
-        import random
-        import numpy as np
-        random.seed(seed)
-        np.random.seed(seed)
-        torch.manual_seed(seed)
-        if torch.cuda.is_available():
-            torch.cuda.manual_seed(seed)
-            torch.cuda.manual_seed_all(seed)
-        torch.backends.cudnn.deterministic = True
-        torch.backends.cudnn.benchmark = False
-
-    setup_distributed()
-    rank = dist.get_rank()
-    world_size = dist.get_world_size()
-    device = f"cuda:{rank}"
-    
-    output_dir = args.output_dir
-    os.makedirs(output_dir, exist_ok=True)
-    if rank == 0:
-        print(f"Output images are saved in {output_dir}")
-
-    llm_config = Qwen2Config.from_json_file(os.path.join(args.model_path, "llm_config.json"))
-    llm_config.qk_norm = True
-    llm_config.tie_word_embeddings = False
-    llm_config.layer_module = "Qwen2MoTDecoderLayer"
-
-    vit_config = SiglipVisionConfig.from_json_file(os.path.join(args.model_path, "vit_config.json"))
-    vit_config.rope = False
-    vit_config.num_hidden_layers = vit_config.num_hidden_layers - 1
-
-    vae_model, vae_config = load_ae(local_path=os.path.join(args.model_path, "ae.safetensors"))
-
-    config = BagelConfig(
-        visual_gen=True,
-        visual_und=True,
-        llm_config=llm_config, 
-        vit_config=vit_config,
-        vae_config=vae_config,
-        vit_max_num_patch_per_side=70,
-        connector_act='gelu_pytorch_tanh',
-        latent_patch_size=2,
-        max_latent_size=args.max_latent_size,
-    )
-    language_model = Qwen2ForCausalLM(llm_config)
-    vit_model = SiglipVisionModel(vit_config)
-    model = Bagel(language_model, vit_model, config)
-    model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config)
-
-    tokenizer = Qwen2Tokenizer.from_pretrained(args.model_path)
-    tokenizer, new_token_ids, _ = add_special_tokens(tokenizer)
-
-    model_state_dict_path = os.path.join(args.model_path, "ema.safetensors")
-    model_state_dict = load_file(model_state_dict_path, device="cpu")
-    msg = model.load_state_dict(model_state_dict, strict=False)
-    if rank == 0:
-        print(msg)
-    del model_state_dict
-
-    model = model.to(device).eval()
-    vae_model = vae_model.to(device).eval()
-    gen_model = model
-
-    cfg_scale = args.cfg_scale
-    cfg_interval = [0, 1.0]
-    timestep_shift = 3.0
-    num_timesteps = 50
-    cfg_renorm_min = 0.0
-
-    with open(args.metadata_file, "r", encoding="utf-8") as fp:
-        metadatas = [json.loads(line) for line in fp]
-    total_metadatas = len(metadatas)
-    
-    prompts_per_gpu = (total_metadatas + world_size - 1) // world_size
-    start = rank * prompts_per_gpu
-    end = min(start + prompts_per_gpu, total_metadatas)
-    print(f"GPU {rank}: Processing {end - start} prompts (indices {start} to {end - 1})")
-
-    for idx in range(start, end):
-        metadata = metadatas[idx]
-        outpath = os.path.join(output_dir, f"{idx:0>5}")
-        os.makedirs(outpath, exist_ok=True)
-        prompt = metadata['prompt']
-        print(f"GPU {rank} processing prompt {idx - start + 1}/{end - start}: '{prompt}'")
-
-        sample_path = os.path.join(outpath, "samples")
-        os.makedirs(sample_path, exist_ok=True)
-
-        flag = True
-        for idx in range(args.num_images):
-            if not os.path.exists(os.path.join(sample_path, f"{idx:05}.png")):
-                flag = False
-                break
-        if flag:
-            print(f"GPU {rank} skipping generation for prompt: {prompt}")
-            continue
-
-        with open(os.path.join(outpath, "metadata.jsonl"), "w", encoding="utf-8") as fp:
-            json.dump(metadata, fp)
-
-        image_list = []
-
-        for i in range(args.num_images // args.batch_size):
-            tmp_image_list = generate_image(
-                prompt=prompt,
-                cfg_scale=cfg_scale, 
-                cfg_interval=cfg_interval, 
-                cfg_renorm_min=cfg_renorm_min,
-                timestep_shift=timestep_shift, 
-                num_timesteps=num_timesteps,
-                num_images=args.batch_size,
-                resolution=args.resolution,
-                device=device,
-            )
-            image_list.extend(tmp_image_list)
-
-        sample_count = 0
-        for sample in image_list:
-            sample = sample.crop(sample.getbbox())
-            sample.save(os.path.join(sample_path, f"{sample_count:05}.png"))
-            sample_count += 1
-
-    print(f"GPU {rank} has completed all tasks")
-    dist.barrier()

eval/gen/gen_images_mp_wise.py

@@ -1,365 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-import json
-import argparse
-from safetensors.torch import load_file
-
-import torch
-import torch.distributed as dist
-from data.data_utils import add_special_tokens
-from modeling.bagel import (
-    BagelConfig, Bagel, Qwen2Config, Qwen2ForCausalLM, SiglipVisionConfig, SiglipVisionModel
-)
-from modeling.qwen2 import Qwen2Tokenizer
-from modeling.autoencoder import load_ae
-
-import copy
-from PIL import Image
-from modeling.bagel.qwen2_navit import NaiveCache
-
-
-def setup_distributed():
-    dist.init_process_group(backend="nccl")
-    torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
-
-
-SYSTEM_PROMPT = '''You should first think about the planning process in the mind and then generate the image. 
-The planning process is enclosed within <think> </think> tags, i.e. <think> planning process here </think> image here'''
-
-
-def move_generation_input_to_device(generation_input, device):
-    # Utility to move all tensors in generation_input to device
-    for k, v in generation_input.items():
-        if isinstance(v, torch.Tensor):
-            generation_input[k] = v.to(device)
-    return generation_input
-
-
-def generate_image_with_think(
-    prompt, num_timesteps=50, cfg_scale=4.0, cfg_interval=[0, 1.0], cfg_renorm_min=0., timestep_shift=4.0, resolution=1024,
-    max_length=2048, simple_think=False, device=None
-):
-    h, w = resolution, resolution
-
-    past_key_values = NaiveCache(model.config.llm_config.num_hidden_layers)
-    newlens = [0]
-    new_rope = [0]
-    
-    # system prompt
-    generation_input, newlens, new_rope = model.prepare_prompts(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        prompts=[SYSTEM_PROMPT],
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        past_key_values = model.forward_cache_update_text(past_key_values, **generation_input)  
-        
-    ##########  cfg
-    generation_input_cfg = model.prepare_vae_latent_cfg(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        image_sizes=[(h, w)], 
-    )
-    generation_input_cfg = move_generation_input_to_device(generation_input_cfg, device)
-    ##########  cfg
-    
-    generation_input, newlens, new_rope = model.prepare_prompts(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        prompts=[prompt],
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        past_key_values = model.forward_cache_update_text(past_key_values, **generation_input)  
-        
-    ########## think
-    tmp_past_key_values = copy.deepcopy(past_key_values)
-    tmp_newlens = copy.deepcopy(newlens)
-    tmp_new_rope = copy.deepcopy(new_rope)
-    tmp_generation_input, tmp_newlens, tmp_new_rope = model.prepare_prompts(
-        curr_kvlens=tmp_newlens,
-        curr_rope=tmp_new_rope, 
-        prompts=[prompt],
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    tmp_generation_input = move_generation_input_to_device(tmp_generation_input, device)
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        tmp_past_key_values = model.forward_cache_update_text(tmp_past_key_values, **tmp_generation_input)  
-    
-    tmp_generation_input = model.prepare_start_tokens(tmp_newlens, tmp_new_rope, new_token_ids)
-    tmp_generation_input = move_generation_input_to_device(tmp_generation_input, device)
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        unpacked_latent = model.generate_text(
-            past_key_values=tmp_past_key_values,
-            max_length=max_length,
-            do_sample=True,
-            temperature=0.3,
-            end_token_id=new_token_ids['eos_token_id'],
-            **tmp_generation_input,
-            )
-        output = tokenizer.decode(unpacked_latent[:,0])
-        think_output = output.split('<|im_end|>')[0].split('<|im_start|>')[1]  
-        
-    print("="*30, "original think", "="*30)
-    print(think_output) 
-    if simple_think:
-        think_output_list = think_output.split("</think>")
-        if think_output_list[1] != "":
-            think_output = think_output_list[1].strip()
-        print("="*30, "processed think", "="*30)
-        print(think_output) 
-    ########## think
-    
-    generation_input, newlens, new_rope = model.prepare_prompts(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        prompts=[think_output],
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        past_key_values = model.forward_cache_update_text(past_key_values, **generation_input)
-
-    generation_input = model.prepare_vae_latent(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        image_sizes=[(h, w)], 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-
-    ########## generate image
-    with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-        unpacked_latent = model.generate_image(
-            past_key_values=past_key_values,
-            num_timesteps=num_timesteps,
-            cfg_text_scale=cfg_scale, 
-            cfg_interval=cfg_interval,
-            timestep_shift=timestep_shift,
-            cfg_renorm_min=cfg_renorm_min,
-            cfg_renorm_type="global",
-            cfg_text_past_key_values=None,
-            cfg_text_packed_position_ids=generation_input_cfg["cfg_packed_position_ids"],
-            cfg_text_key_values_lens=generation_input_cfg["cfg_key_values_lens"],
-            cfg_text_packed_query_indexes=generation_input_cfg["cfg_packed_query_indexes"],
-            cfg_text_packed_key_value_indexes=generation_input_cfg["cfg_packed_key_value_indexes"],
-            **generation_input,
-        )
-    
-    latent0 = unpacked_latent[0]
-    latent0 = latent0.reshape(1, h//16, w//16, 2, 2, 16)
-    latent0 = torch.einsum("nhwpqc->nchpwq", latent0)
-    latent0 = latent0.reshape(1, 16, h//8, w//8)
-    image = vae_model.decode(latent0.to(device))
-    tmpimage = ((image * 0.5 + 0.5).clamp(0, 1)[0].permute(1, 2, 0) * 255).to(torch.uint8).cpu().numpy()
-    tmpimage = Image.fromarray(tmpimage)
-    
-    return tmpimage, think_output
-
-
-def generate_image(prompt, num_timesteps=50, cfg_scale=4.0, cfg_interval=[0, 1.0], cfg_renorm_min=0., timestep_shift=1.0, resolution=1024, device=None):
-    past_key_values = NaiveCache(gen_model.config.llm_config.num_hidden_layers)
-    newlens = [0]
-    new_rope = [0]
-
-    generation_input, newlens, new_rope = gen_model.prepare_prompts(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        prompts=[prompt],
-        tokenizer=tokenizer, 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-
-    with torch.no_grad():
-        with torch.amp.autocast("cuda", enabled=True, dtype=torch.float16):
-            past_key_values = gen_model.forward_cache_update_text(past_key_values, **generation_input)
-
-    generation_input = gen_model.prepare_vae_latent(
-        curr_kvlens=newlens,
-        curr_rope=new_rope, 
-        image_sizes=[(resolution, resolution)], 
-        new_token_ids=new_token_ids,
-    )
-    generation_input = move_generation_input_to_device(generation_input, device)
-
-    cfg_past_key_values = NaiveCache(gen_model.config.llm_config.num_hidden_layers)
-    cfg_newlens = [0]
-    cfg_new_rope = [0]
-
-    generation_input_cfg = model.prepare_vae_latent_cfg(
-        curr_kvlens=cfg_newlens,
-        curr_rope=cfg_new_rope, 
-        image_sizes=[(resolution, resolution)],
-    )
-    generation_input_cfg = move_generation_input_to_device(generation_input_cfg, device)
-    with torch.no_grad():
-        with torch.amp.autocast("cuda", enabled=True, dtype=torch.bfloat16):
-            unpacked_latent = gen_model.generate_image(
-                past_key_values=past_key_values,
-                num_timesteps=num_timesteps,
-                cfg_text_scale=cfg_scale,
-                cfg_interval=cfg_interval,
-                cfg_renorm_min=cfg_renorm_min,
-                timestep_shift=timestep_shift,
-                cfg_text_past_key_values=cfg_past_key_values,
-                cfg_text_packed_position_ids=generation_input_cfg["cfg_packed_position_ids"],
-                cfg_text_key_values_lens=generation_input_cfg["cfg_key_values_lens"],
-                cfg_text_packed_query_indexes=generation_input_cfg["cfg_packed_query_indexes"],
-                cfg_text_packed_key_value_indexes=generation_input_cfg["cfg_packed_key_value_indexes"],
-                **generation_input,
-            )
-
-    latent = unpacked_latent[0]
-    latent = latent.reshape(1, resolution//16, resolution//16, 2, 2, 16)
-    latent = torch.einsum("nhwpqc->nchpwq", latent)
-    latent = latent.reshape(1, 16, resolution//8, resolution//8)
-    image = vae_model.decode(latent.to(device))
-    tmpimage = ((image * 0.5 + 0.5).clamp(0, 1)[0].permute(1, 2, 0) * 255).to(torch.uint8).cpu().numpy()
-    tmpimage = Image.fromarray(tmpimage)
-
-    return tmpimage
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Generate images using Bagel model.")
-    parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated images.")
-    parser.add_argument("--metadata_file", type=str, required=True, help="JSON file containing lines of metadata for each prompt.")
-    parser.add_argument("--cfg_scale", type=float, default=4)
-    parser.add_argument("--resolution", type=int, default=1024)
-    parser.add_argument("--max_latent_size", type=int, default=64)
-    parser.add_argument("--think", action="store_true")
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-    
-    seed = 42
-    if seed is not None:
-        import random
-        import numpy as np
-        random.seed(seed)
-        np.random.seed(seed)
-        torch.manual_seed(seed)
-        if torch.cuda.is_available():
-            torch.cuda.manual_seed(seed)
-            torch.cuda.manual_seed_all(seed)
-        torch.backends.cudnn.deterministic = True
-        torch.backends.cudnn.benchmark = False
-
-    setup_distributed()
-    rank = dist.get_rank()
-    world_size = dist.get_world_size()
-    device = f"cuda:{rank}"
-    
-    output_dir = args.output_dir
-    os.makedirs(output_dir, exist_ok=True)
-    if rank == 0:
-        print(f"Output images are saved in {output_dir}")
-
-    llm_config = Qwen2Config.from_json_file(os.path.join(args.model_path, "llm_config.json"))
-    llm_config.qk_norm = True
-    llm_config.tie_word_embeddings = False
-    llm_config.layer_module = "Qwen2MoTDecoderLayer"
-
-    vit_config = SiglipVisionConfig.from_json_file(os.path.join(args.model_path, "vit_config.json"))
-    vit_config.rope = False
-    vit_config.num_hidden_layers = vit_config.num_hidden_layers - 1
-
-    vae_model, vae_config = load_ae(local_path=os.path.join(args.model_path, "ae.safetensors"))
-
-    config = BagelConfig(
-        visual_gen=True,
-        visual_und=True,
-        llm_config=llm_config, 
-        vit_config=vit_config,
-        vae_config=vae_config,
-        vit_max_num_patch_per_side=70,
-        connector_act='gelu_pytorch_tanh',
-        latent_patch_size=2,
-        max_latent_size=args.max_latent_size,
-    )
-    language_model = Qwen2ForCausalLM(llm_config)
-    vit_model = SiglipVisionModel(vit_config)
-    model = Bagel(language_model, vit_model, config)
-    model.vit_model.vision_model.embeddings.convert_conv2d_to_linear(vit_config)
-
-    tokenizer = Qwen2Tokenizer.from_pretrained(args.model_path)
-    tokenizer, new_token_ids, _ = add_special_tokens(tokenizer)
-
-    model_state_dict_path = os.path.join(args.model_path, "ema.safetensors")
-    model_state_dict = load_file(model_state_dict_path, device="cpu")
-    msg = model.load_state_dict(model_state_dict, strict=False)
-    if rank == 0:
-        print(msg)
-
-    del model_state_dict
-    model = model.to(device).eval()
-    vae_model = vae_model.to(device).eval()
-    gen_model = model
-
-    cfg_scale = args.cfg_scale
-    cfg_interval = [0.4, 1.0]
-    timestep_shift = 3.0
-    num_timesteps = 50
-    cfg_renorm_min = 0.0
-
-    with open(args.metadata_file, "r") as f:
-        metadatas = json.load(f)
-    total_metadatas = len(metadatas)
-    
-    prompts_per_gpu = (total_metadatas + world_size - 1) // world_size
-    start = rank * prompts_per_gpu
-    end = min(start + prompts_per_gpu, total_metadatas)
-    print(f"GPU {rank}: Processing {end - start} prompts (indices {start} to {end - 1})")
-
-    for idx in range(start, end):
-        metadata = metadatas[idx]
-        prompt = metadata['Prompt']
-        prompt_id = metadata['prompt_id']
-        outpath = os.path.join(output_dir, f"{prompt_id}.png")
-        print(f"GPU {rank} processing prompt {idx - start + 1}/{end - start}: '{prompt}'")
-
-        if os.path.exists(outpath):
-            print(f"GPU {rank} skipping generation for prompt: {prompt}")
-            continue
-        
-        if args.think:
-            tmpimage, think_output = generate_image_with_think(
-                prompt=prompt,
-                cfg_scale=cfg_scale, 
-                cfg_interval=cfg_interval, 
-                cfg_renorm_min=cfg_renorm_min,
-                timestep_shift=timestep_shift, 
-                num_timesteps=num_timesteps,
-                resolution=args.resolution,
-                max_length=2048, 
-                simple_think=False, 
-                device=device,
-            )
-            with open(outpath.replace(".png", ".txt"), "w") as f:
-                f.write(think_output)
-        else:
-            tmpimage = generate_image(
-                prompt=prompt,
-                cfg_scale=cfg_scale, 
-                cfg_interval=cfg_interval, 
-                cfg_renorm_min=cfg_renorm_min,
-                timestep_shift=timestep_shift, 
-                num_timesteps=num_timesteps,
-                resolution=args.resolution,
-                device=device,
-            )
-
-        tmpimage = tmpimage.crop(tmpimage.getbbox())
-        tmpimage.save(outpath)
-
-    print(f"GPU {rank} has completed all tasks")
-    dist.barrier()

eval/gen/geneval/evaluation/download_models.sh

@@ -1,20 +0,0 @@
-# Copyright (c) 2023 Dhruba Ghosh
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/djghosh13/geneval/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-#!/bin/bash
-
-# Download Mask2Former object detection config and weights
-
-if [ ! -z "$1" ]
-then
-    mkdir -p "$1"
-    wget https://download.openmmlab.com/mmdetection/v2.0/mask2former/mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco/mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco_20220504_001756-743b7d99.pth -O "$1/mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco.pth"
-fi

eval/gen/geneval/evaluation/evaluate_images.py

@@ -1,304 +0,0 @@
-# Copyright (c) 2023 Dhruba Ghosh
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/djghosh13/geneval/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-"""
-Evaluate generated images using Mask2Former (or other object detector model)
-"""
-
-import argparse
-import json
-import os
-import re
-import sys
-import time
-from tqdm import tqdm
-
-import warnings
-warnings.filterwarnings("ignore")
-
-import numpy as np
-import pandas as pd
-from PIL import Image, ImageOps
-import torch
-import mmdet
-from mmdet.apis import inference_detector, init_detector
-
-import open_clip
-from clip_benchmark.metrics import zeroshot_classification as zsc
-zsc.tqdm = lambda it, *args, **kwargs: it
-
-# Get directory path
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("imagedir", type=str)
-    parser.add_argument("--outfile", type=str, default="results.jsonl")
-    parser.add_argument("--model-config", type=str, default=None)
-    parser.add_argument("--model-path", type=str, default="./")
-    # Other arguments
-    parser.add_argument("--options", nargs="*", type=str, default=[])
-    args = parser.parse_args()
-    args.options = dict(opt.split("=", 1) for opt in args.options)
-    if args.model_config is None:
-        args.model_config = os.path.join(
-            os.path.dirname(mmdet.__file__),
-            "../configs/mask2former/mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco.py"
-        )
-    return args
-
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-assert DEVICE == "cuda"
-
-def timed(fn):
-    def wrapper(*args, **kwargs):
-        startt = time.time()
-        result = fn(*args, **kwargs)
-        endt = time.time()
-        print(f'Function {fn.__name__!r} executed in {endt - startt:.3f}s', file=sys.stderr)
-        return result
-    return wrapper
-
-# Load models
-
-@timed
-def load_models(args):
-    CONFIG_PATH = args.model_config
-    OBJECT_DETECTOR = args.options.get('model', "mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco")
-    CKPT_PATH = os.path.join(args.model_path, f"{OBJECT_DETECTOR}.pth")
-    object_detector = init_detector(CONFIG_PATH, CKPT_PATH, device=DEVICE)
-
-    clip_arch = args.options.get('clip_model', "ViT-L-14")
-    clip_model, _, transform = open_clip.create_model_and_transforms(clip_arch, pretrained="openai", device=DEVICE)
-    tokenizer = open_clip.get_tokenizer(clip_arch)
-
-    with open(os.path.join(os.path.dirname(__file__), "object_names.txt")) as cls_file:
-        classnames = [line.strip() for line in cls_file]
-
-    return object_detector, (clip_model, transform, tokenizer), classnames
-
-
-COLORS = ["red", "orange", "yellow", "green", "blue", "purple", "pink", "brown", "black", "white"]
-COLOR_CLASSIFIERS = {}
-
-# Evaluation parts
-
-class ImageCrops(torch.utils.data.Dataset):
-    def __init__(self, image: Image.Image, objects):
-        self._image = image.convert("RGB")
-        bgcolor = args.options.get('bgcolor', "#999")
-        if bgcolor == "original":
-            self._blank = self._image.copy()
-        else:
-            self._blank = Image.new("RGB", image.size, color=bgcolor)
-        self._objects = objects
-
-    def __len__(self):
-        return len(self._objects)
-
-    def __getitem__(self, index):
-        box, mask = self._objects[index]
-        if mask is not None:
-            assert tuple(self._image.size[::-1]) == tuple(mask.shape), (index, self._image.size[::-1], mask.shape)
-            image = Image.composite(self._image, self._blank, Image.fromarray(mask))
-        else:
-            image = self._image
-        if args.options.get('crop', '1') == '1':
-            image = image.crop(box[:4])
-        # if args.save:
-        #     base_count = len(os.listdir(args.save))
-        #     image.save(os.path.join(args.save, f"cropped_{base_count:05}.png"))
-        return (transform(image), 0)
-
-
-def color_classification(image, bboxes, classname):
-    if classname not in COLOR_CLASSIFIERS:
-        COLOR_CLASSIFIERS[classname] = zsc.zero_shot_classifier(
-            clip_model, tokenizer, COLORS,
-            [
-                f"a photo of a {{c}} {classname}",
-                f"a photo of a {{c}}-colored {classname}",
-                f"a photo of a {{c}} object"
-            ],
-            DEVICE
-        )
-    clf = COLOR_CLASSIFIERS[classname]
-    dataloader = torch.utils.data.DataLoader(
-        ImageCrops(image, bboxes),
-        batch_size=16, num_workers=4
-    )
-    with torch.no_grad():
-        pred, _ = zsc.run_classification(clip_model, clf, dataloader, DEVICE)
-        return [COLORS[index.item()] for index in pred.argmax(1)]
-
-
-def compute_iou(box_a, box_b):
-    area_fn = lambda box: max(box[2] - box[0] + 1, 0) * max(box[3] - box[1] + 1, 0)
-    i_area = area_fn([
-        max(box_a[0], box_b[0]), max(box_a[1], box_b[1]),
-        min(box_a[2], box_b[2]), min(box_a[3], box_b[3])
-    ])
-    u_area = area_fn(box_a) + area_fn(box_b) - i_area
-    return i_area / u_area if u_area else 0
-
-
-def relative_position(obj_a, obj_b):
-    """Give position of A relative to B, factoring in object dimensions"""
-    boxes = np.array([obj_a[0], obj_b[0]])[:, :4].reshape(2, 2, 2)
-    center_a, center_b = boxes.mean(axis=-2)
-    dim_a, dim_b = np.abs(np.diff(boxes, axis=-2))[..., 0, :]
-    offset = center_a - center_b
-    #
-    revised_offset = np.maximum(np.abs(offset) - POSITION_THRESHOLD * (dim_a + dim_b), 0) * np.sign(offset)
-    if np.all(np.abs(revised_offset) < 1e-3):
-        return set()
-    #
-    dx, dy = revised_offset / np.linalg.norm(offset)
-    relations = set()
-    if dx < -0.5: relations.add("left of")
-    if dx > 0.5: relations.add("right of")
-    if dy < -0.5: relations.add("above")
-    if dy > 0.5: relations.add("below")
-    return relations
-
-
-def evaluate(image, objects, metadata):
-    """
-    Evaluate given image using detected objects on the global metadata specifications.
-    Assumptions:
-    * Metadata combines 'include' clauses with AND, and 'exclude' clauses with OR
-    * All clauses are independent, i.e., duplicating a clause has no effect on the correctness
-    * CHANGED: Color and position will only be evaluated on the most confidently predicted objects;
-        therefore, objects are expected to appear in sorted order
-    """
-    correct = True
-    reason = []
-    matched_groups = []
-    # Check for expected objects
-    for req in metadata.get('include', []):
-        classname = req['class']
-        matched = True
-        found_objects = objects.get(classname, [])[:req['count']]
-        if len(found_objects) < req['count']:
-            correct = matched = False
-            reason.append(f"expected {classname}>={req['count']}, found {len(found_objects)}")
-        else:
-            if 'color' in req:
-                # Color check
-                colors = color_classification(image, found_objects, classname)
-                if colors.count(req['color']) < req['count']:
-                    correct = matched = False
-                    reason.append(
-                        f"expected {req['color']} {classname}>={req['count']}, found " +
-                        f"{colors.count(req['color'])} {req['color']}; and " +
-                        ", ".join(f"{colors.count(c)} {c}" for c in COLORS if c in colors)
-                    )
-            if 'position' in req and matched:
-                # Relative position check
-                expected_rel, target_group = req['position']
-                if matched_groups[target_group] is None:
-                    correct = matched = False
-                    reason.append(f"no target for {classname} to be {expected_rel}")
-                else:
-                    for obj in found_objects:
-                        for target_obj in matched_groups[target_group]:
-                            true_rels = relative_position(obj, target_obj)
-                            if expected_rel not in true_rels:
-                                correct = matched = False
-                                reason.append(
-                                    f"expected {classname} {expected_rel} target, found " +
-                                    f"{' and '.join(true_rels)} target"
-                                )
-                                break
-                        if not matched:
-                            break
-        if matched:
-            matched_groups.append(found_objects)
-        else:
-            matched_groups.append(None)
-    # Check for non-expected objects
-    for req in metadata.get('exclude', []):
-        classname = req['class']
-        if len(objects.get(classname, [])) >= req['count']:
-            correct = False
-            reason.append(f"expected {classname}<{req['count']}, found {len(objects[classname])}")
-    return correct, "\n".join(reason)
-
-
-def evaluate_image(filepath, metadata):
-    result = inference_detector(object_detector, filepath)
-    bbox = result[0] if isinstance(result, tuple) else result
-    segm = result[1] if isinstance(result, tuple) and len(result) > 1 else None
-    image = ImageOps.exif_transpose(Image.open(filepath))
-    detected = {}
-    # Determine bounding boxes to keep
-    confidence_threshold = THRESHOLD if metadata['tag'] != "counting" else COUNTING_THRESHOLD
-    for index, classname in enumerate(classnames):
-        ordering = np.argsort(bbox[index][:, 4])[::-1]
-        ordering = ordering[bbox[index][ordering, 4] > confidence_threshold] # Threshold
-        ordering = ordering[:MAX_OBJECTS].tolist() # Limit number of detected objects per class
-        detected[classname] = []
-        while ordering:
-            max_obj = ordering.pop(0)
-            detected[classname].append((bbox[index][max_obj], None if segm is None else segm[index][max_obj]))
-            ordering = [
-                obj for obj in ordering
-                if NMS_THRESHOLD == 1 or compute_iou(bbox[index][max_obj], bbox[index][obj]) < NMS_THRESHOLD
-            ]
-        if not detected[classname]:
-            del detected[classname]
-    # Evaluate
-    is_correct, reason = evaluate(image, detected, metadata)
-    return {
-        'filename': filepath,
-        'tag': metadata['tag'],
-        'prompt': metadata['prompt'],
-        'correct': is_correct,
-        'reason': reason,
-        'metadata': json.dumps(metadata),
-        'details': json.dumps({
-            key: [box.tolist() for box, _ in value]
-            for key, value in detected.items()
-        })
-    }
-
-
-def main(args):
-    full_results = []
-    for subfolder in tqdm(os.listdir(args.imagedir)):
-        folderpath = os.path.join(args.imagedir, subfolder)
-        if not os.path.isdir(folderpath) or not subfolder.isdigit():
-            continue
-        with open(os.path.join(folderpath, "metadata.jsonl")) as fp:
-            metadata = json.load(fp)
-        # Evaluate each image
-        for imagename in os.listdir(os.path.join(folderpath, "samples")):
-            imagepath = os.path.join(folderpath, "samples", imagename)
-            if not os.path.isfile(imagepath) or not re.match(r"\d+\.png", imagename):
-                continue
-            result = evaluate_image(imagepath, metadata)
-            full_results.append(result)
-    # Save results
-    if os.path.dirname(args.outfile):
-        os.makedirs(os.path.dirname(args.outfile), exist_ok=True)
-    with open(args.outfile, "w") as fp:
-        pd.DataFrame(full_results).to_json(fp, orient="records", lines=True)
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    object_detector, (clip_model, transform, tokenizer), classnames = load_models(args)
-    THRESHOLD = float(args.options.get('threshold', 0.3))
-    COUNTING_THRESHOLD = float(args.options.get('counting_threshold', 0.9))
-    MAX_OBJECTS = int(args.options.get('max_objects', 16))
-    NMS_THRESHOLD = float(args.options.get('max_overlap', 1.0))
-    POSITION_THRESHOLD = float(args.options.get('position_threshold', 0.1))
-
-    main(args)

eval/gen/geneval/evaluation/evaluate_images_mp.py

@@ -1,332 +0,0 @@
-# Copyright (c) 2023 Dhruba Ghosh
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/djghosh13/geneval/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import json
-import os
-import re
-import sys
-import time
-from tqdm import tqdm
-
-import warnings
-warnings.filterwarnings("ignore")
-
-import numpy as np
-import pandas as pd
-from PIL import Image, ImageOps
-import torch
-import torch.distributed as dist
-import mmdet
-from mmdet.apis import inference_detector, init_detector
-
-import open_clip
-from clip_benchmark.metrics import zeroshot_classification as zsc
-zsc.tqdm = lambda it, *args, **kwargs: it
-
-
-def setup_distributed():
-    """初始化分布式环境"""
-    dist.init_process_group(backend="nccl")
-    torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
-
-
-# Get directory path
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("imagedir", type=str)
-    parser.add_argument("--outfile", type=str, default="results.jsonl")
-    parser.add_argument("--model-config", type=str, default=None)
-    parser.add_argument("--model-path", type=str, default="./")
-    # Other arguments
-    parser.add_argument("--options", nargs="*", type=str, default=[])
-    args = parser.parse_args()
-    args.options = dict(opt.split("=", 1) for opt in args.options)
-    if args.model_config is None:
-        args.model_config = os.path.join(
-            os.path.dirname(mmdet.__file__),
-            "../configs/mask2former/mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco.py"
-        )
-    return args
-
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-assert DEVICE == "cuda"
-
-def timed(fn):
-    def wrapper(*args, **kwargs):
-        startt = time.time()
-        result = fn(*args, **kwargs)
-        endt = time.time()
-        print(f'Function {fn.__name__!r} executed in {endt - startt:.3f}s', file=sys.stderr)
-        return result
-    return wrapper
-
-# Load models
-
-@timed
-def load_models(args):
-    CONFIG_PATH = args.model_config
-    OBJECT_DETECTOR = args.options.get('model', "mask2former_swin-s-p4-w7-224_lsj_8x2_50e_coco")
-    CKPT_PATH = os.path.join(args.model_path, f"{OBJECT_DETECTOR}.pth")
-    object_detector = init_detector(CONFIG_PATH, CKPT_PATH, device=DEVICE)
-
-    clip_arch = args.options.get('clip_model', "ViT-L-14")
-    clip_model, _, transform = open_clip.create_model_and_transforms(clip_arch, pretrained="openai", device=DEVICE)
-    tokenizer = open_clip.get_tokenizer(clip_arch)
-
-    with open(os.path.join(os.path.dirname(__file__), "object_names.txt")) as cls_file:
-        classnames = [line.strip() for line in cls_file]
-
-    return object_detector, (clip_model, transform, tokenizer), classnames
-
-
-COLORS = ["red", "orange", "yellow", "green", "blue", "purple", "pink", "brown", "black", "white"]
-COLOR_CLASSIFIERS = {}
-
-# Evaluation parts
-
-class ImageCrops(torch.utils.data.Dataset):
-    def __init__(self, image: Image.Image, objects):
-        self._image = image.convert("RGB")
-        bgcolor = args.options.get('bgcolor', "#999")
-        if bgcolor == "original":
-            self._blank = self._image.copy()
-        else:
-            self._blank = Image.new("RGB", image.size, color=bgcolor)
-        self._objects = objects
-
-    def __len__(self):
-        return len(self._objects)
-
-    def __getitem__(self, index):
-        box, mask = self._objects[index]
-        if mask is not None:
-            assert tuple(self._image.size[::-1]) == tuple(mask.shape), (index, self._image.size[::-1], mask.shape)
-            image = Image.composite(self._image, self._blank, Image.fromarray(mask))
-        else:
-            image = self._image
-        if args.options.get('crop', '1') == '1':
-            image = image.crop(box[:4])
-        # if args.save:
-        #     base_count = len(os.listdir(args.save))
-        #     image.save(os.path.join(args.save, f"cropped_{base_count:05}.png"))
-        return (transform(image), 0)
-
-
-def color_classification(image, bboxes, classname):
-    if classname not in COLOR_CLASSIFIERS:
-        COLOR_CLASSIFIERS[classname] = zsc.zero_shot_classifier(
-            clip_model, tokenizer, COLORS,
-            [
-                f"a photo of a {{c}} {classname}",
-                f"a photo of a {{c}}-colored {classname}",
-                f"a photo of a {{c}} object"
-            ],
-            DEVICE
-        )
-    clf = COLOR_CLASSIFIERS[classname]
-    dataloader = torch.utils.data.DataLoader(
-        ImageCrops(image, bboxes),
-        batch_size=16, num_workers=4
-    )
-    with torch.no_grad():
-        pred, _ = zsc.run_classification(clip_model, clf, dataloader, DEVICE)
-        return [COLORS[index.item()] for index in pred.argmax(1)]
-
-
-def compute_iou(box_a, box_b):
-    area_fn = lambda box: max(box[2] - box[0] + 1, 0) * max(box[3] - box[1] + 1, 0)
-    i_area = area_fn([
-        max(box_a[0], box_b[0]), max(box_a[1], box_b[1]),
-        min(box_a[2], box_b[2]), min(box_a[3], box_b[3])
-    ])
-    u_area = area_fn(box_a) + area_fn(box_b) - i_area
-    return i_area / u_area if u_area else 0
-
-
-def relative_position(obj_a, obj_b):
-    """Give position of A relative to B, factoring in object dimensions"""
-    boxes = np.array([obj_a[0], obj_b[0]])[:, :4].reshape(2, 2, 2)
-    center_a, center_b = boxes.mean(axis=-2)
-    dim_a, dim_b = np.abs(np.diff(boxes, axis=-2))[..., 0, :]
-    offset = center_a - center_b
-    #
-    revised_offset = np.maximum(np.abs(offset) - POSITION_THRESHOLD * (dim_a + dim_b), 0) * np.sign(offset)
-    if np.all(np.abs(revised_offset) < 1e-3):
-        return set()
-    #
-    dx, dy = revised_offset / np.linalg.norm(offset)
-    relations = set()
-    if dx < -0.5: relations.add("left of")
-    if dx > 0.5: relations.add("right of")
-    if dy < -0.5: relations.add("above")
-    if dy > 0.5: relations.add("below")
-    return relations
-
-
-def evaluate(image, objects, metadata):
-    """
-    Evaluate given image using detected objects on the global metadata specifications.
-    Assumptions:
-    * Metadata combines 'include' clauses with AND, and 'exclude' clauses with OR
-    * All clauses are independent, i.e., duplicating a clause has no effect on the correctness
-    * CHANGED: Color and position will only be evaluated on the most confidently predicted objects;
-        therefore, objects are expected to appear in sorted order
-    """
-    correct = True
-    reason = []
-    matched_groups = []
-    # Check for expected objects
-    for req in metadata.get('include', []):
-        classname = req['class']
-        matched = True
-        found_objects = objects.get(classname, [])[:req['count']]
-        if len(found_objects) < req['count']:
-            correct = matched = False
-            reason.append(f"expected {classname}>={req['count']}, found {len(found_objects)}")
-        else:
-            if 'color' in req:
-                # Color check
-                colors = color_classification(image, found_objects, classname)
-                if colors.count(req['color']) < req['count']:
-                    correct = matched = False
-                    reason.append(
-                        f"expected {req['color']} {classname}>={req['count']}, found " +
-                        f"{colors.count(req['color'])} {req['color']}; and " +
-                        ", ".join(f"{colors.count(c)} {c}" for c in COLORS if c in colors)
-                    )
-            if 'position' in req and matched:
-                # Relative position check
-                expected_rel, target_group = req['position']
-                if matched_groups[target_group] is None:
-                    correct = matched = False
-                    reason.append(f"no target for {classname} to be {expected_rel}")
-                else:
-                    for obj in found_objects:
-                        for target_obj in matched_groups[target_group]:
-                            true_rels = relative_position(obj, target_obj)
-                            if expected_rel not in true_rels:
-                                correct = matched = False
-                                reason.append(
-                                    f"expected {classname} {expected_rel} target, found " +
-                                    f"{' and '.join(true_rels)} target"
-                                )
-                                break
-                        if not matched:
-                            break
-        if matched:
-            matched_groups.append(found_objects)
-        else:
-            matched_groups.append(None)
-    # Check for non-expected objects
-    for req in metadata.get('exclude', []):
-        classname = req['class']
-        if len(objects.get(classname, [])) >= req['count']:
-            correct = False
-            reason.append(f"expected {classname}<{req['count']}, found {len(objects[classname])}")
-    return correct, "\n".join(reason)
-
-
-def evaluate_image(filepath, metadata):
-    result = inference_detector(object_detector, filepath)
-    bbox = result[0] if isinstance(result, tuple) else result
-    segm = result[1] if isinstance(result, tuple) and len(result) > 1 else None
-    image = ImageOps.exif_transpose(Image.open(filepath))
-    detected = {}
-    # Determine bounding boxes to keep
-    confidence_threshold = THRESHOLD if metadata['tag'] != "counting" else COUNTING_THRESHOLD
-    for index, classname in enumerate(classnames):
-        ordering = np.argsort(bbox[index][:, 4])[::-1]
-        ordering = ordering[bbox[index][ordering, 4] > confidence_threshold] # Threshold
-        ordering = ordering[:MAX_OBJECTS].tolist() # Limit number of detected objects per class
-        detected[classname] = []
-        while ordering:
-            max_obj = ordering.pop(0)
-            detected[classname].append((bbox[index][max_obj], None if segm is None else segm[index][max_obj]))
-            ordering = [
-                obj for obj in ordering
-                if NMS_THRESHOLD == 1 or compute_iou(bbox[index][max_obj], bbox[index][obj]) < NMS_THRESHOLD
-            ]
-        if not detected[classname]:
-            del detected[classname]
-    # Evaluate
-    is_correct, reason = evaluate(image, detected, metadata)
-    return {
-        'filename': filepath,
-        'tag': metadata['tag'],
-        'prompt': metadata['prompt'],
-        'correct': is_correct,
-        'reason': reason,
-        'metadata': json.dumps(metadata),
-        'details': json.dumps({
-            key: [box.tolist() for box, _ in value]
-            for key, value in detected.items()
-        })
-    }
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    THRESHOLD = float(args.options.get('threshold', 0.3))
-    COUNTING_THRESHOLD = float(args.options.get('counting_threshold', 0.9))
-    MAX_OBJECTS = int(args.options.get('max_objects', 16))
-    NMS_THRESHOLD = float(args.options.get('max_overlap', 1.0))
-    POSITION_THRESHOLD = float(args.options.get('position_threshold', 0.1))
-
-    # Initialize distributed environment
-    setup_distributed()
-    rank = dist.get_rank()
-    world_size = dist.get_world_size()
-    device = f"cuda:{rank}"
-
-    # Load models
-    if rank == 0:
-        print(f"[Rank 0] Loading model...")
-    object_detector, (clip_model, transform, tokenizer), classnames = load_models(args)
-
-    full_results = []
-    subfolders = [f for f in os.listdir(args.imagedir) if os.path.isdir(os.path.join(args.imagedir, f)) and f.isdigit()]
-    total_subfolders = len(subfolders)
-    # Divide subfolders to process by GPU
-    subfolders_per_gpu = (total_subfolders + world_size - 1) // world_size
-    start = rank * subfolders_per_gpu
-    end = min(start + subfolders_per_gpu, total_subfolders)
-    print(f"GPU {rank}: Processing {end - start} subfolders (index {start} to {end - 1})")
-
-    for subfolder in tqdm(subfolders[start:end]):
-        folderpath = os.path.join(args.imagedir, subfolder)
-        with open(os.path.join(folderpath, "metadata.jsonl")) as fp:
-            metadata = json.load(fp)
-        # Evaluate each image
-        for imagename in os.listdir(os.path.join(folderpath, "samples")):
-            imagepath = os.path.join(folderpath, "samples", imagename)
-            if not os.path.isfile(imagepath) or not re.match(r"\d+\.png", imagename):
-                continue
-            result = evaluate_image(imagepath, metadata)
-            full_results.append(result)
-
-    # Synchronize results from all GPUs
-    all_results = [None] * world_size
-    dist.all_gather_object(all_results, full_results)
-    if rank == 0:
-        # Merge results from all GPUs
-        final_results = []
-        for results in all_results:
-            final_results.extend(results)
-        # Save results
-        if os.path.dirname(args.outfile):
-            os.makedirs(os.path.dirname(args.outfile), exist_ok=True)
-        with open(args.outfile, "w") as fp:
-            pd.DataFrame(final_results).to_json(fp, orient="records", lines=True)
-        print("All GPUs have completed their tasks and the final results have been saved.")
-    else:
-        print(f"GPU {rank} has completed all tasks")

eval/gen/geneval/evaluation/object_names.txt

@@ -1,80 +0,0 @@
-person
-bicycle
-car
-motorcycle
-airplane
-bus
-train
-truck
-boat
-traffic light
-fire hydrant
-stop sign
-parking meter
-bench
-bird
-cat
-dog
-horse
-sheep
-cow
-elephant
-bear
-zebra
-giraffe
-backpack
-umbrella
-handbag
-tie
-suitcase
-frisbee
-skis
-snowboard
-sports ball
-kite
-baseball bat
-baseball glove
-skateboard
-surfboard
-tennis racket
-bottle
-wine glass
-cup
-fork
-knife
-spoon
-bowl
-banana
-apple
-sandwich
-orange
-broccoli
-carrot
-hot dog
-pizza
-donut
-cake
-chair
-couch
-potted plant
-bed
-dining table
-toilet
-tv
-laptop
-computer mouse
-tv remote
-computer keyboard
-cell phone
-microwave
-oven
-toaster
-sink
-refrigerator
-book
-clock
-vase
-scissors
-teddy bear
-hair drier
-toothbrush

eval/gen/geneval/evaluation/summary_scores.py

@@ -1,64 +0,0 @@
-# Copyright (c) 2023 Dhruba Ghosh
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/djghosh13/geneval/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import os
-
-import numpy as np
-import pandas as pd
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("filename", type=str)
-args = parser.parse_args()
-
-# Load classnames
-
-with open(os.path.join(os.path.dirname(__file__), "object_names.txt")) as cls_file:
-    classnames = [line.strip() for line in cls_file]
-    cls_to_idx = {"_".join(cls.split()):idx for idx, cls in enumerate(classnames)}
-
-# Load results
-
-df = pd.read_json(args.filename, orient="records", lines=True)
-
-# Measure overall success
-
-print("Summary")
-print("=======")
-print(f"Total images: {len(df)}")
-print(f"Total prompts: {len(df.groupby('metadata'))}")
-print(f"% correct images: {df['correct'].mean():.2%}")
-print(f"% correct prompts: {df.groupby('metadata')['correct'].any().mean():.2%}")
-print()
-
-# By group
-
-task_scores = []
-
-print("Task breakdown")
-print("==============")
-for tag, task_df in df.groupby('tag', sort=False):
-    task_scores.append(task_df['correct'].mean())
-    print(f"{tag:<16} = {task_df['correct'].mean():.2%} ({task_df['correct'].sum()} / {len(task_df)})")
-print()
-
-print(f"Overall score (avg. over tasks): {np.mean(task_scores):.5f}")
-
-
-print("\n\n==============")
-output_info = "SO   TO   CT   CL   POS  ATTR ALL\n"
-for score in task_scores:
-    output_info += f"{score:.2f} "
-output_info += f"{np.mean(task_scores):.2f}" + "\n"
-print(output_info)
-with open(os.path.join(os.path.dirname(args.filename), "geneval_results.txt"), "w") as f:
-    f.write(output_info)

eval/gen/geneval/prompts/create_prompts.py

@@ -1,194 +0,0 @@
-# Copyright (c) 2023 Dhruba Ghosh
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/djghosh13/geneval/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-"""
-Generate prompts for evaluation
-"""
-
-import argparse
-import json
-import os
-import yaml
-
-import numpy as np
-
-# Load classnames
-
-with open("object_names.txt") as cls_file:
-    classnames = [line.strip() for line in cls_file]
-
-# Proper a vs an
-
-def with_article(name: str):
-    if name[0] in "aeiou":
-        return f"an {name}"
-    return f"a {name}"
-
-# Proper plural
-
-def make_plural(name: str):
-    if name[-1] in "s":
-        return f"{name}es"
-    return f"{name}s"
-
-# Generates single object samples
-
-def generate_single_object_sample(rng: np.random.Generator, size: int = None):
-    TAG = "single_object"
-    if size > len(classnames):
-        size = len(classnames)
-        print(f"Not enough distinct classes, generating only {size} samples")
-    return_scalar = size is None
-    size = size or 1
-    idxs = rng.choice(len(classnames), size=size, replace=False)
-    samples = [dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx], "count": 1}
-        ],
-        prompt=f"a photo of {with_article(classnames[idx])}"
-    ) for idx in idxs]
-    if return_scalar:
-        return samples[0]
-    return samples
-
-# Generate two object samples
-
-def generate_two_object_sample(rng: np.random.Generator):
-    TAG = "two_object"
-    idx_a, idx_b = rng.choice(len(classnames), size=2, replace=False)
-    return dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx_a], "count": 1},
-            {"class": classnames[idx_b], "count": 1}
-        ],
-        prompt=f"a photo of {with_article(classnames[idx_a])} and {with_article(classnames[idx_b])}"
-    )
-
-# Generate counting samples
-
-numbers = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]
-
-def generate_counting_sample(rng: np.random.Generator, max_count=4):
-    TAG = "counting"
-    idx = rng.choice(len(classnames))
-    num = int(rng.integers(2, max_count, endpoint=True))
-    return dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx], "count": num}
-        ],
-        exclude=[
-            {"class": classnames[idx], "count": num + 1}
-        ],
-        prompt=f"a photo of {numbers[num]} {make_plural(classnames[idx])}"
-    )
-
-# Generate color samples
-
-colors = ["red", "orange", "yellow", "green", "blue", "purple", "pink", "brown", "black", "white"]
-
-def generate_color_sample(rng: np.random.Generator):
-    TAG = "colors"
-    idx = rng.choice(len(classnames) - 1) + 1
-    idx = (idx + classnames.index("person")) % len(classnames) # No "[COLOR] person" prompts
-    color = colors[rng.choice(len(colors))]
-    return dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx], "count": 1, "color": color}
-        ],
-        prompt=f"a photo of {with_article(color)} {classnames[idx]}"
-    )
-
-# Generate position samples
-
-positions = ["left of", "right of", "above", "below"]
-
-def generate_position_sample(rng: np.random.Generator):
-    TAG = "position"
-    idx_a, idx_b = rng.choice(len(classnames), size=2, replace=False)
-    position = positions[rng.choice(len(positions))]
-    return dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx_b], "count": 1},
-            {"class": classnames[idx_a], "count": 1, "position": (position, 0)}
-        ],
-        prompt=f"a photo of {with_article(classnames[idx_a])} {position} {with_article(classnames[idx_b])}"
-    )
-
-# Generate color attribution samples
-
-def generate_color_attribution_sample(rng: np.random.Generator):
-    TAG = "color_attr"
-    idxs = rng.choice(len(classnames) - 1, size=2, replace=False) + 1
-    idx_a, idx_b = (idxs + classnames.index("person")) % len(classnames) # No "[COLOR] person" prompts
-    cidx_a, cidx_b = rng.choice(len(colors), size=2, replace=False)
-    return dict(
-        tag=TAG,
-        include=[
-            {"class": classnames[idx_a], "count": 1, "color": colors[cidx_a]},
-            {"class": classnames[idx_b], "count": 1, "color": colors[cidx_b]}
-        ],
-        prompt=f"a photo of {with_article(colors[cidx_a])} {classnames[idx_a]} and {with_article(colors[cidx_b])} {classnames[idx_b]}"
-    )
-
-
-# Generate evaluation suite
-
-def generate_suite(rng: np.random.Generator, n: int = 100, output_path: str = ""):
-    samples = []
-    # Generate single object samples for all COCO classnames
-    samples.extend(generate_single_object_sample(rng, size=len(classnames)))
-    # Generate two object samples (~100)
-    for _ in range(n):
-        samples.append(generate_two_object_sample(rng))
-    # Generate counting samples
-    for _ in range(n):
-        samples.append(generate_counting_sample(rng, max_count=4))
-    # Generate color samples
-    for _ in range(n):
-        samples.append(generate_color_sample(rng))
-    # Generate position samples
-    for _ in range(n):
-        samples.append(generate_position_sample(rng))
-    # Generate color attribution samples
-    for _ in range(n):
-        samples.append(generate_color_attribution_sample(rng))
-    # De-duplicate
-    unique_samples, used_samples = [], set()
-    for sample in samples:
-        sample_text = yaml.safe_dump(sample)
-        if sample_text not in used_samples:
-            unique_samples.append(sample)
-            used_samples.add(sample_text)
-
-    # Write to files
-    os.makedirs(output_path, exist_ok=True)
-    with open(os.path.join(output_path, "generation_prompts.txt"), "w") as fp:
-        for sample in unique_samples:
-            print(sample['prompt'], file=fp)
-    with open(os.path.join(output_path, "evaluation_metadata.jsonl"), "w") as fp:
-        for sample in unique_samples:
-            print(json.dumps(sample), file=fp)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--seed", type=int, default=43, help="generation seed (default: 43)")
-    parser.add_argument("--num-prompts", "-n", type=int, default=100, help="number of prompts per task (default: 100)")
-    parser.add_argument("--output-path", "-o", type=str, default="prompts", help="output folder for prompts and metadata (default: 'prompts/')")
-    args = parser.parse_args()
-    rng = np.random.default_rng(args.seed)
-    generate_suite(rng, args.num_prompts, args.output_path)
-

eval/gen/geneval/prompts/evaluation_metadata.jsonl

@@ -1,553 +0,0 @@
-{"tag": "single_object", "include": [{"class": "bench", "count": 1}], "prompt": "a photo of a bench"}
-{"tag": "single_object", "include": [{"class": "cow", "count": 1}], "prompt": "a photo of a cow"}
-{"tag": "single_object", "include": [{"class": "bicycle", "count": 1}], "prompt": "a photo of a bicycle"}
-{"tag": "single_object", "include": [{"class": "clock", "count": 1}], "prompt": "a photo of a clock"}
-{"tag": "single_object", "include": [{"class": "carrot", "count": 1}], "prompt": "a photo of a carrot"}
-{"tag": "single_object", "include": [{"class": "suitcase", "count": 1}], "prompt": "a photo of a suitcase"}
-{"tag": "single_object", "include": [{"class": "fork", "count": 1}], "prompt": "a photo of a fork"}
-{"tag": "single_object", "include": [{"class": "surfboard", "count": 1}], "prompt": "a photo of a surfboard"}
-{"tag": "single_object", "include": [{"class": "refrigerator", "count": 1}], "prompt": "a photo of a refrigerator"}
-{"tag": "single_object", "include": [{"class": "cup", "count": 1}], "prompt": "a photo of a cup"}
-{"tag": "single_object", "include": [{"class": "microwave", "count": 1}], "prompt": "a photo of a microwave"}
-{"tag": "single_object", "include": [{"class": "potted plant", "count": 1}], "prompt": "a photo of a potted plant"}
-{"tag": "single_object", "include": [{"class": "snowboard", "count": 1}], "prompt": "a photo of a snowboard"}
-{"tag": "single_object", "include": [{"class": "zebra", "count": 1}], "prompt": "a photo of a zebra"}
-{"tag": "single_object", "include": [{"class": "parking meter", "count": 1}], "prompt": "a photo of a parking meter"}
-{"tag": "single_object", "include": [{"class": "spoon", "count": 1}], "prompt": "a photo of a spoon"}
-{"tag": "single_object", "include": [{"class": "skateboard", "count": 1}], "prompt": "a photo of a skateboard"}
-{"tag": "single_object", "include": [{"class": "car", "count": 1}], "prompt": "a photo of a car"}
-{"tag": "single_object", "include": [{"class": "motorcycle", "count": 1}], "prompt": "a photo of a motorcycle"}
-{"tag": "single_object", "include": [{"class": "traffic light", "count": 1}], "prompt": "a photo of a traffic light"}
-{"tag": "single_object", "include": [{"class": "book", "count": 1}], "prompt": "a photo of a book"}
-{"tag": "single_object", "include": [{"class": "couch", "count": 1}], "prompt": "a photo of a couch"}
-{"tag": "single_object", "include": [{"class": "backpack", "count": 1}], "prompt": "a photo of a backpack"}
-{"tag": "single_object", "include": [{"class": "computer keyboard", "count": 1}], "prompt": "a photo of a computer keyboard"}
-{"tag": "single_object", "include": [{"class": "toaster", "count": 1}], "prompt": "a photo of a toaster"}
-{"tag": "single_object", "include": [{"class": "bird", "count": 1}], "prompt": "a photo of a bird"}
-{"tag": "single_object", "include": [{"class": "bowl", "count": 1}], "prompt": "a photo of a bowl"}
-{"tag": "single_object", "include": [{"class": "dog", "count": 1}], "prompt": "a photo of a dog"}
-{"tag": "single_object", "include": [{"class": "tie", "count": 1}], "prompt": "a photo of a tie"}
-{"tag": "single_object", "include": [{"class": "laptop", "count": 1}], "prompt": "a photo of a laptop"}
-{"tag": "single_object", "include": [{"class": "computer mouse", "count": 1}], "prompt": "a photo of a computer mouse"}
-{"tag": "single_object", "include": [{"class": "sandwich", "count": 1}], "prompt": "a photo of a sandwich"}
-{"tag": "single_object", "include": [{"class": "baseball bat", "count": 1}], "prompt": "a photo of a baseball bat"}
-{"tag": "single_object", "include": [{"class": "train", "count": 1}], "prompt": "a photo of a train"}
-{"tag": "single_object", "include": [{"class": "cell phone", "count": 1}], "prompt": "a photo of a cell phone"}
-{"tag": "single_object", "include": [{"class": "chair", "count": 1}], "prompt": "a photo of a chair"}
-{"tag": "single_object", "include": [{"class": "tv", "count": 1}], "prompt": "a photo of a tv"}
-{"tag": "single_object", "include": [{"class": "broccoli", "count": 1}], "prompt": "a photo of a broccoli"}
-{"tag": "single_object", "include": [{"class": "bed", "count": 1}], "prompt": "a photo of a bed"}
-{"tag": "single_object", "include": [{"class": "skis", "count": 1}], "prompt": "a photo of a skis"}
-{"tag": "single_object", "include": [{"class": "handbag", "count": 1}], "prompt": "a photo of a handbag"}
-{"tag": "single_object", "include": [{"class": "pizza", "count": 1}], "prompt": "a photo of a pizza"}
-{"tag": "single_object", "include": [{"class": "frisbee", "count": 1}], "prompt": "a photo of a frisbee"}
-{"tag": "single_object", "include": [{"class": "scissors", "count": 1}], "prompt": "a photo of a scissors"}
-{"tag": "single_object", "include": [{"class": "bottle", "count": 1}], "prompt": "a photo of a bottle"}
-{"tag": "single_object", "include": [{"class": "elephant", "count": 1}], "prompt": "a photo of an elephant"}
-{"tag": "single_object", "include": [{"class": "toilet", "count": 1}], "prompt": "a photo of a toilet"}
-{"tag": "single_object", "include": [{"class": "oven", "count": 1}], "prompt": "a photo of an oven"}
-{"tag": "single_object", "include": [{"class": "orange", "count": 1}], "prompt": "a photo of an orange"}
-{"tag": "single_object", "include": [{"class": "person", "count": 1}], "prompt": "a photo of a person"}
-{"tag": "single_object", "include": [{"class": "teddy bear", "count": 1}], "prompt": "a photo of a teddy bear"}
-{"tag": "single_object", "include": [{"class": "vase", "count": 1}], "prompt": "a photo of a vase"}
-{"tag": "single_object", "include": [{"class": "banana", "count": 1}], "prompt": "a photo of a banana"}
-{"tag": "single_object", "include": [{"class": "toothbrush", "count": 1}], "prompt": "a photo of a toothbrush"}
-{"tag": "single_object", "include": [{"class": "tv remote", "count": 1}], "prompt": "a photo of a tv remote"}
-{"tag": "single_object", "include": [{"class": "dining table", "count": 1}], "prompt": "a photo of a dining table"}
-{"tag": "single_object", "include": [{"class": "stop sign", "count": 1}], "prompt": "a photo of a stop sign"}
-{"tag": "single_object", "include": [{"class": "sheep", "count": 1}], "prompt": "a photo of a sheep"}
-{"tag": "single_object", "include": [{"class": "fire hydrant", "count": 1}], "prompt": "a photo of a fire hydrant"}
-{"tag": "single_object", "include": [{"class": "airplane", "count": 1}], "prompt": "a photo of an airplane"}
-{"tag": "single_object", "include": [{"class": "giraffe", "count": 1}], "prompt": "a photo of a giraffe"}
-{"tag": "single_object", "include": [{"class": "horse", "count": 1}], "prompt": "a photo of a horse"}
-{"tag": "single_object", "include": [{"class": "cat", "count": 1}], "prompt": "a photo of a cat"}
-{"tag": "single_object", "include": [{"class": "donut", "count": 1}], "prompt": "a photo of a donut"}
-{"tag": "single_object", "include": [{"class": "boat", "count": 1}], "prompt": "a photo of a boat"}
-{"tag": "single_object", "include": [{"class": "baseball glove", "count": 1}], "prompt": "a photo of a baseball glove"}
-{"tag": "single_object", "include": [{"class": "hair drier", "count": 1}], "prompt": "a photo of a hair drier"}
-{"tag": "single_object", "include": [{"class": "sink", "count": 1}], "prompt": "a photo of a sink"}
-{"tag": "single_object", "include": [{"class": "cake", "count": 1}], "prompt": "a photo of a cake"}
-{"tag": "single_object", "include": [{"class": "wine glass", "count": 1}], "prompt": "a photo of a wine glass"}
-{"tag": "single_object", "include": [{"class": "apple", "count": 1}], "prompt": "a photo of an apple"}
-{"tag": "single_object", "include": [{"class": "bus", "count": 1}], "prompt": "a photo of a bus"}
-{"tag": "single_object", "include": [{"class": "tennis racket", "count": 1}], "prompt": "a photo of a tennis racket"}
-{"tag": "single_object", "include": [{"class": "knife", "count": 1}], "prompt": "a photo of a knife"}
-{"tag": "single_object", "include": [{"class": "hot dog", "count": 1}], "prompt": "a photo of a hot dog"}
-{"tag": "single_object", "include": [{"class": "truck", "count": 1}], "prompt": "a photo of a truck"}
-{"tag": "single_object", "include": [{"class": "umbrella", "count": 1}], "prompt": "a photo of an umbrella"}
-{"tag": "single_object", "include": [{"class": "sports ball", "count": 1}], "prompt": "a photo of a sports ball"}
-{"tag": "single_object", "include": [{"class": "bear", "count": 1}], "prompt": "a photo of a bear"}
-{"tag": "single_object", "include": [{"class": "kite", "count": 1}], "prompt": "a photo of a kite"}
-{"tag": "two_object", "include": [{"class": "bench", "count": 1}, {"class": "sports ball", "count": 1}], "prompt": "a photo of a bench and a sports ball"}
-{"tag": "two_object", "include": [{"class": "toothbrush", "count": 1}, {"class": "snowboard", "count": 1}], "prompt": "a photo of a toothbrush and a snowboard"}
-{"tag": "two_object", "include": [{"class": "toaster", "count": 1}, {"class": "oven", "count": 1}], "prompt": "a photo of a toaster and an oven"}
-{"tag": "two_object", "include": [{"class": "broccoli", "count": 1}, {"class": "vase", "count": 1}], "prompt": "a photo of a broccoli and a vase"}
-{"tag": "two_object", "include": [{"class": "tennis racket", "count": 1}, {"class": "wine glass", "count": 1}], "prompt": "a photo of a tennis racket and a wine glass"}
-{"tag": "two_object", "include": [{"class": "fork", "count": 1}, {"class": "knife", "count": 1}], "prompt": "a photo of a fork and a knife"}
-{"tag": "two_object", "include": [{"class": "hair drier", "count": 1}, {"class": "cake", "count": 1}], "prompt": "a photo of a hair drier and a cake"}
-{"tag": "two_object", "include": [{"class": "horse", "count": 1}, {"class": "giraffe", "count": 1}], "prompt": "a photo of a horse and a giraffe"}
-{"tag": "two_object", "include": [{"class": "horse", "count": 1}, {"class": "computer keyboard", "count": 1}], "prompt": "a photo of a horse and a computer keyboard"}
-{"tag": "two_object", "include": [{"class": "toothbrush", "count": 1}, {"class": "carrot", "count": 1}], "prompt": "a photo of a toothbrush and a carrot"}
-{"tag": "two_object", "include": [{"class": "cake", "count": 1}, {"class": "zebra", "count": 1}], "prompt": "a photo of a cake and a zebra"}
-{"tag": "two_object", "include": [{"class": "hair drier", "count": 1}, {"class": "bear", "count": 1}], "prompt": "a photo of a hair drier and a bear"}
-{"tag": "two_object", "include": [{"class": "knife", "count": 1}, {"class": "zebra", "count": 1}], "prompt": "a photo of a knife and a zebra"}
-{"tag": "two_object", "include": [{"class": "couch", "count": 1}, {"class": "wine glass", "count": 1}], "prompt": "a photo of a couch and a wine glass"}
-{"tag": "two_object", "include": [{"class": "frisbee", "count": 1}, {"class": "vase", "count": 1}], "prompt": "a photo of a frisbee and a vase"}
-{"tag": "two_object", "include": [{"class": "book", "count": 1}, {"class": "laptop", "count": 1}], "prompt": "a photo of a book and a laptop"}
-{"tag": "two_object", "include": [{"class": "dining table", "count": 1}, {"class": "bear", "count": 1}], "prompt": "a photo of a dining table and a bear"}
-{"tag": "two_object", "include": [{"class": "frisbee", "count": 1}, {"class": "couch", "count": 1}], "prompt": "a photo of a frisbee and a couch"}
-{"tag": "two_object", "include": [{"class": "couch", "count": 1}, {"class": "horse", "count": 1}], "prompt": "a photo of a couch and a horse"}
-{"tag": "two_object", "include": [{"class": "toilet", "count": 1}, {"class": "computer mouse", "count": 1}], "prompt": "a photo of a toilet and a computer mouse"}
-{"tag": "two_object", "include": [{"class": "bottle", "count": 1}, {"class": "refrigerator", "count": 1}], "prompt": "a photo of a bottle and a refrigerator"}
-{"tag": "two_object", "include": [{"class": "potted plant", "count": 1}, {"class": "backpack", "count": 1}], "prompt": "a photo of a potted plant and a backpack"}
-{"tag": "two_object", "include": [{"class": "skateboard", "count": 1}, {"class": "cake", "count": 1}], "prompt": "a photo of a skateboard and a cake"}
-{"tag": "two_object", "include": [{"class": "broccoli", "count": 1}, {"class": "parking meter", "count": 1}], "prompt": "a photo of a broccoli and a parking meter"}
-{"tag": "two_object", "include": [{"class": "zebra", "count": 1}, {"class": "bed", "count": 1}], "prompt": "a photo of a zebra and a bed"}
-{"tag": "two_object", "include": [{"class": "oven", "count": 1}, {"class": "bed", "count": 1}], "prompt": "a photo of an oven and a bed"}
-{"tag": "two_object", "include": [{"class": "baseball bat", "count": 1}, {"class": "fork", "count": 1}], "prompt": "a photo of a baseball bat and a fork"}
-{"tag": "two_object", "include": [{"class": "vase", "count": 1}, {"class": "spoon", "count": 1}], "prompt": "a photo of a vase and a spoon"}
-{"tag": "two_object", "include": [{"class": "skateboard", "count": 1}, {"class": "sink", "count": 1}], "prompt": "a photo of a skateboard and a sink"}
-{"tag": "two_object", "include": [{"class": "pizza", "count": 1}, {"class": "bench", "count": 1}], "prompt": "a photo of a pizza and a bench"}
-{"tag": "two_object", "include": [{"class": "bowl", "count": 1}, {"class": "pizza", "count": 1}], "prompt": "a photo of a bowl and a pizza"}
-{"tag": "two_object", "include": [{"class": "tennis racket", "count": 1}, {"class": "bird", "count": 1}], "prompt": "a photo of a tennis racket and a bird"}
-{"tag": "two_object", "include": [{"class": "wine glass", "count": 1}, {"class": "bear", "count": 1}], "prompt": "a photo of a wine glass and a bear"}
-{"tag": "two_object", "include": [{"class": "fork", "count": 1}, {"class": "book", "count": 1}], "prompt": "a photo of a fork and a book"}
-{"tag": "two_object", "include": [{"class": "scissors", "count": 1}, {"class": "bowl", "count": 1}], "prompt": "a photo of a scissors and a bowl"}
-{"tag": "two_object", "include": [{"class": "laptop", "count": 1}, {"class": "carrot", "count": 1}], "prompt": "a photo of a laptop and a carrot"}
-{"tag": "two_object", "include": [{"class": "stop sign", "count": 1}, {"class": "bottle", "count": 1}], "prompt": "a photo of a stop sign and a bottle"}
-{"tag": "two_object", "include": [{"class": "microwave", "count": 1}, {"class": "truck", "count": 1}], "prompt": "a photo of a microwave and a truck"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "bear", "count": 1}], "prompt": "a photo of a person and a bear"}
-{"tag": "two_object", "include": [{"class": "frisbee", "count": 1}, {"class": "cell phone", "count": 1}], "prompt": "a photo of a frisbee and a cell phone"}
-{"tag": "two_object", "include": [{"class": "parking meter", "count": 1}, {"class": "teddy bear", "count": 1}], "prompt": "a photo of a parking meter and a teddy bear"}
-{"tag": "two_object", "include": [{"class": "tennis racket", "count": 1}, {"class": "bicycle", "count": 1}], "prompt": "a photo of a tennis racket and a bicycle"}
-{"tag": "two_object", "include": [{"class": "stop sign", "count": 1}, {"class": "motorcycle", "count": 1}], "prompt": "a photo of a stop sign and a motorcycle"}
-{"tag": "two_object", "include": [{"class": "fire hydrant", "count": 1}, {"class": "tennis racket", "count": 1}], "prompt": "a photo of a fire hydrant and a tennis racket"}
-{"tag": "two_object", "include": [{"class": "scissors", "count": 1}, {"class": "sandwich", "count": 1}], "prompt": "a photo of a scissors and a sandwich"}
-{"tag": "two_object", "include": [{"class": "pizza", "count": 1}, {"class": "book", "count": 1}], "prompt": "a photo of a pizza and a book"}
-{"tag": "two_object", "include": [{"class": "giraffe", "count": 1}, {"class": "computer mouse", "count": 1}], "prompt": "a photo of a giraffe and a computer mouse"}
-{"tag": "two_object", "include": [{"class": "stop sign", "count": 1}, {"class": "toaster", "count": 1}], "prompt": "a photo of a stop sign and a toaster"}
-{"tag": "two_object", "include": [{"class": "computer mouse", "count": 1}, {"class": "zebra", "count": 1}], "prompt": "a photo of a computer mouse and a zebra"}
-{"tag": "two_object", "include": [{"class": "chair", "count": 1}, {"class": "bench", "count": 1}], "prompt": "a photo of a chair and a bench"}
-{"tag": "two_object", "include": [{"class": "tv", "count": 1}, {"class": "carrot", "count": 1}], "prompt": "a photo of a tv and a carrot"}
-{"tag": "two_object", "include": [{"class": "surfboard", "count": 1}, {"class": "suitcase", "count": 1}], "prompt": "a photo of a surfboard and a suitcase"}
-{"tag": "two_object", "include": [{"class": "computer keyboard", "count": 1}, {"class": "laptop", "count": 1}], "prompt": "a photo of a computer keyboard and a laptop"}
-{"tag": "two_object", "include": [{"class": "computer keyboard", "count": 1}, {"class": "microwave", "count": 1}], "prompt": "a photo of a computer keyboard and a microwave"}
-{"tag": "two_object", "include": [{"class": "scissors", "count": 1}, {"class": "bird", "count": 1}], "prompt": "a photo of a scissors and a bird"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "snowboard", "count": 1}], "prompt": "a photo of a person and a snowboard"}
-{"tag": "two_object", "include": [{"class": "cow", "count": 1}, {"class": "horse", "count": 1}], "prompt": "a photo of a cow and a horse"}
-{"tag": "two_object", "include": [{"class": "handbag", "count": 1}, {"class": "refrigerator", "count": 1}], "prompt": "a photo of a handbag and a refrigerator"}
-{"tag": "two_object", "include": [{"class": "chair", "count": 1}, {"class": "laptop", "count": 1}], "prompt": "a photo of a chair and a laptop"}
-{"tag": "two_object", "include": [{"class": "toothbrush", "count": 1}, {"class": "bench", "count": 1}], "prompt": "a photo of a toothbrush and a bench"}
-{"tag": "two_object", "include": [{"class": "book", "count": 1}, {"class": "baseball bat", "count": 1}], "prompt": "a photo of a book and a baseball bat"}
-{"tag": "two_object", "include": [{"class": "horse", "count": 1}, {"class": "train", "count": 1}], "prompt": "a photo of a horse and a train"}
-{"tag": "two_object", "include": [{"class": "bench", "count": 1}, {"class": "vase", "count": 1}], "prompt": "a photo of a bench and a vase"}
-{"tag": "two_object", "include": [{"class": "traffic light", "count": 1}, {"class": "backpack", "count": 1}], "prompt": "a photo of a traffic light and a backpack"}
-{"tag": "two_object", "include": [{"class": "sports ball", "count": 1}, {"class": "cow", "count": 1}], "prompt": "a photo of a sports ball and a cow"}
-{"tag": "two_object", "include": [{"class": "computer mouse", "count": 1}, {"class": "spoon", "count": 1}], "prompt": "a photo of a computer mouse and a spoon"}
-{"tag": "two_object", "include": [{"class": "tv", "count": 1}, {"class": "bicycle", "count": 1}], "prompt": "a photo of a tv and a bicycle"}
-{"tag": "two_object", "include": [{"class": "bench", "count": 1}, {"class": "snowboard", "count": 1}], "prompt": "a photo of a bench and a snowboard"}
-{"tag": "two_object", "include": [{"class": "toothbrush", "count": 1}, {"class": "toilet", "count": 1}], "prompt": "a photo of a toothbrush and a toilet"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "apple", "count": 1}], "prompt": "a photo of a person and an apple"}
-{"tag": "two_object", "include": [{"class": "sink", "count": 1}, {"class": "sports ball", "count": 1}], "prompt": "a photo of a sink and a sports ball"}
-{"tag": "two_object", "include": [{"class": "stop sign", "count": 1}, {"class": "dog", "count": 1}], "prompt": "a photo of a stop sign and a dog"}
-{"tag": "two_object", "include": [{"class": "knife", "count": 1}, {"class": "stop sign", "count": 1}], "prompt": "a photo of a knife and a stop sign"}
-{"tag": "two_object", "include": [{"class": "wine glass", "count": 1}, {"class": "handbag", "count": 1}], "prompt": "a photo of a wine glass and a handbag"}
-{"tag": "two_object", "include": [{"class": "bowl", "count": 1}, {"class": "skis", "count": 1}], "prompt": "a photo of a bowl and a skis"}
-{"tag": "two_object", "include": [{"class": "frisbee", "count": 1}, {"class": "apple", "count": 1}], "prompt": "a photo of a frisbee and an apple"}
-{"tag": "two_object", "include": [{"class": "computer keyboard", "count": 1}, {"class": "cell phone", "count": 1}], "prompt": "a photo of a computer keyboard and a cell phone"}
-{"tag": "two_object", "include": [{"class": "stop sign", "count": 1}, {"class": "fork", "count": 1}], "prompt": "a photo of a stop sign and a fork"}
-{"tag": "two_object", "include": [{"class": "potted plant", "count": 1}, {"class": "boat", "count": 1}], "prompt": "a photo of a potted plant and a boat"}
-{"tag": "two_object", "include": [{"class": "tv", "count": 1}, {"class": "cell phone", "count": 1}], "prompt": "a photo of a tv and a cell phone"}
-{"tag": "two_object", "include": [{"class": "tie", "count": 1}, {"class": "broccoli", "count": 1}], "prompt": "a photo of a tie and a broccoli"}
-{"tag": "two_object", "include": [{"class": "potted plant", "count": 1}, {"class": "donut", "count": 1}], "prompt": "a photo of a potted plant and a donut"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "sink", "count": 1}], "prompt": "a photo of a person and a sink"}
-{"tag": "two_object", "include": [{"class": "couch", "count": 1}, {"class": "snowboard", "count": 1}], "prompt": "a photo of a couch and a snowboard"}
-{"tag": "two_object", "include": [{"class": "fork", "count": 1}, {"class": "baseball glove", "count": 1}], "prompt": "a photo of a fork and a baseball glove"}
-{"tag": "two_object", "include": [{"class": "apple", "count": 1}, {"class": "toothbrush", "count": 1}], "prompt": "a photo of an apple and a toothbrush"}
-{"tag": "two_object", "include": [{"class": "bus", "count": 1}, {"class": "baseball glove", "count": 1}], "prompt": "a photo of a bus and a baseball glove"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "stop sign", "count": 1}], "prompt": "a photo of a person and a stop sign"}
-{"tag": "two_object", "include": [{"class": "carrot", "count": 1}, {"class": "couch", "count": 1}], "prompt": "a photo of a carrot and a couch"}
-{"tag": "two_object", "include": [{"class": "baseball bat", "count": 1}, {"class": "bear", "count": 1}], "prompt": "a photo of a baseball bat and a bear"}
-{"tag": "two_object", "include": [{"class": "fire hydrant", "count": 1}, {"class": "train", "count": 1}], "prompt": "a photo of a fire hydrant and a train"}
-{"tag": "two_object", "include": [{"class": "baseball glove", "count": 1}, {"class": "carrot", "count": 1}], "prompt": "a photo of a baseball glove and a carrot"}
-{"tag": "two_object", "include": [{"class": "microwave", "count": 1}, {"class": "bench", "count": 1}], "prompt": "a photo of a microwave and a bench"}
-{"tag": "two_object", "include": [{"class": "cake", "count": 1}, {"class": "stop sign", "count": 1}], "prompt": "a photo of a cake and a stop sign"}
-{"tag": "two_object", "include": [{"class": "car", "count": 1}, {"class": "computer mouse", "count": 1}], "prompt": "a photo of a car and a computer mouse"}
-{"tag": "two_object", "include": [{"class": "suitcase", "count": 1}, {"class": "dining table", "count": 1}], "prompt": "a photo of a suitcase and a dining table"}
-{"tag": "two_object", "include": [{"class": "person", "count": 1}, {"class": "traffic light", "count": 1}], "prompt": "a photo of a person and a traffic light"}
-{"tag": "two_object", "include": [{"class": "cell phone", "count": 1}, {"class": "horse", "count": 1}], "prompt": "a photo of a cell phone and a horse"}
-{"tag": "two_object", "include": [{"class": "baseball bat", "count": 1}, {"class": "giraffe", "count": 1}], "prompt": "a photo of a baseball bat and a giraffe"}
-{"tag": "counting", "include": [{"class": "clock", "count": 2}], "exclude": [{"class": "clock", "count": 3}], "prompt": "a photo of two clocks"}
-{"tag": "counting", "include": [{"class": "backpack", "count": 2}], "exclude": [{"class": "backpack", "count": 3}], "prompt": "a photo of two backpacks"}
-{"tag": "counting", "include": [{"class": "handbag", "count": 4}], "exclude": [{"class": "handbag", "count": 5}], "prompt": "a photo of four handbags"}
-{"tag": "counting", "include": [{"class": "frisbee", "count": 2}], "exclude": [{"class": "frisbee", "count": 3}], "prompt": "a photo of two frisbees"}
-{"tag": "counting", "include": [{"class": "sports ball", "count": 3}], "exclude": [{"class": "sports ball", "count": 4}], "prompt": "a photo of three sports balls"}
-{"tag": "counting", "include": [{"class": "bear", "count": 2}], "exclude": [{"class": "bear", "count": 3}], "prompt": "a photo of two bears"}
-{"tag": "counting", "include": [{"class": "tie", "count": 2}], "exclude": [{"class": "tie", "count": 3}], "prompt": "a photo of two ties"}
-{"tag": "counting", "include": [{"class": "sink", "count": 4}], "exclude": [{"class": "sink", "count": 5}], "prompt": "a photo of four sinks"}
-{"tag": "counting", "include": [{"class": "toothbrush", "count": 2}], "exclude": [{"class": "toothbrush", "count": 3}], "prompt": "a photo of two toothbrushs"}
-{"tag": "counting", "include": [{"class": "person", "count": 3}], "exclude": [{"class": "person", "count": 4}], "prompt": "a photo of three persons"}
-{"tag": "counting", "include": [{"class": "tennis racket", "count": 3}], "exclude": [{"class": "tennis racket", "count": 4}], "prompt": "a photo of three tennis rackets"}
-{"tag": "counting", "include": [{"class": "bowl", "count": 4}], "exclude": [{"class": "bowl", "count": 5}], "prompt": "a photo of four bowls"}
-{"tag": "counting", "include": [{"class": "vase", "count": 4}], "exclude": [{"class": "vase", "count": 5}], "prompt": "a photo of four vases"}
-{"tag": "counting", "include": [{"class": "cup", "count": 3}], "exclude": [{"class": "cup", "count": 4}], "prompt": "a photo of three cups"}
-{"tag": "counting", "include": [{"class": "computer keyboard", "count": 4}], "exclude": [{"class": "computer keyboard", "count": 5}], "prompt": "a photo of four computer keyboards"}
-{"tag": "counting", "include": [{"class": "sink", "count": 3}], "exclude": [{"class": "sink", "count": 4}], "prompt": "a photo of three sinks"}
-{"tag": "counting", "include": [{"class": "oven", "count": 2}], "exclude": [{"class": "oven", "count": 3}], "prompt": "a photo of two ovens"}
-{"tag": "counting", "include": [{"class": "toilet", "count": 2}], "exclude": [{"class": "toilet", "count": 3}], "prompt": "a photo of two toilets"}
-{"tag": "counting", "include": [{"class": "bicycle", "count": 2}], "exclude": [{"class": "bicycle", "count": 3}], "prompt": "a photo of two bicycles"}
-{"tag": "counting", "include": [{"class": "train", "count": 2}], "exclude": [{"class": "train", "count": 3}], "prompt": "a photo of two trains"}
-{"tag": "counting", "include": [{"class": "orange", "count": 3}], "exclude": [{"class": "orange", "count": 4}], "prompt": "a photo of three oranges"}
-{"tag": "counting", "include": [{"class": "bus", "count": 3}], "exclude": [{"class": "bus", "count": 4}], "prompt": "a photo of three buses"}
-{"tag": "counting", "include": [{"class": "handbag", "count": 3}], "exclude": [{"class": "handbag", "count": 4}], "prompt": "a photo of three handbags"}
-{"tag": "counting", "include": [{"class": "snowboard", "count": 3}], "exclude": [{"class": "snowboard", "count": 4}], "prompt": "a photo of three snowboards"}
-{"tag": "counting", "include": [{"class": "snowboard", "count": 2}], "exclude": [{"class": "snowboard", "count": 3}], "prompt": "a photo of two snowboards"}
-{"tag": "counting", "include": [{"class": "dog", "count": 4}], "exclude": [{"class": "dog", "count": 5}], "prompt": "a photo of four dogs"}
-{"tag": "counting", "include": [{"class": "apple", "count": 3}], "exclude": [{"class": "apple", "count": 4}], "prompt": "a photo of three apples"}
-{"tag": "counting", "include": [{"class": "sheep", "count": 2}], "exclude": [{"class": "sheep", "count": 3}], "prompt": "a photo of two sheeps"}
-{"tag": "counting", "include": [{"class": "hot dog", "count": 3}], "exclude": [{"class": "hot dog", "count": 4}], "prompt": "a photo of three hot dogs"}
-{"tag": "counting", "include": [{"class": "zebra", "count": 3}], "exclude": [{"class": "zebra", "count": 4}], "prompt": "a photo of three zebras"}
-{"tag": "counting", "include": [{"class": "kite", "count": 3}], "exclude": [{"class": "kite", "count": 4}], "prompt": "a photo of three kites"}
-{"tag": "counting", "include": [{"class": "apple", "count": 4}], "exclude": [{"class": "apple", "count": 5}], "prompt": "a photo of four apples"}
-{"tag": "counting", "include": [{"class": "cell phone", "count": 3}], "exclude": [{"class": "cell phone", "count": 4}], "prompt": "a photo of three cell phones"}
-{"tag": "counting", "include": [{"class": "baseball glove", "count": 4}], "exclude": [{"class": "baseball glove", "count": 5}], "prompt": "a photo of four baseball gloves"}
-{"tag": "counting", "include": [{"class": "computer keyboard", "count": 3}], "exclude": [{"class": "computer keyboard", "count": 4}], "prompt": "a photo of three computer keyboards"}
-{"tag": "counting", "include": [{"class": "bed", "count": 2}], "exclude": [{"class": "bed", "count": 3}], "prompt": "a photo of two beds"}
-{"tag": "counting", "include": [{"class": "tv remote", "count": 2}], "exclude": [{"class": "tv remote", "count": 3}], "prompt": "a photo of two tv remotes"}
-{"tag": "counting", "include": [{"class": "fire hydrant", "count": 3}], "exclude": [{"class": "fire hydrant", "count": 4}], "prompt": "a photo of three fire hydrants"}
-{"tag": "counting", "include": [{"class": "book", "count": 3}], "exclude": [{"class": "book", "count": 4}], "prompt": "a photo of three books"}
-{"tag": "counting", "include": [{"class": "giraffe", "count": 4}], "exclude": [{"class": "giraffe", "count": 5}], "prompt": "a photo of four giraffes"}
-{"tag": "counting", "include": [{"class": "vase", "count": 2}], "exclude": [{"class": "vase", "count": 3}], "prompt": "a photo of two vases"}
-{"tag": "counting", "include": [{"class": "donut", "count": 4}], "exclude": [{"class": "donut", "count": 5}], "prompt": "a photo of four donuts"}
-{"tag": "counting", "include": [{"class": "chair", "count": 4}], "exclude": [{"class": "chair", "count": 5}], "prompt": "a photo of four chairs"}
-{"tag": "counting", "include": [{"class": "baseball bat", "count": 3}], "exclude": [{"class": "baseball bat", "count": 4}], "prompt": "a photo of three baseball bats"}
-{"tag": "counting", "include": [{"class": "stop sign", "count": 4}], "exclude": [{"class": "stop sign", "count": 5}], "prompt": "a photo of four stop signs"}
-{"tag": "counting", "include": [{"class": "pizza", "count": 2}], "exclude": [{"class": "pizza", "count": 3}], "prompt": "a photo of two pizzas"}
-{"tag": "counting", "include": [{"class": "refrigerator", "count": 3}], "exclude": [{"class": "refrigerator", "count": 4}], "prompt": "a photo of three refrigerators"}
-{"tag": "counting", "include": [{"class": "fire hydrant", "count": 2}], "exclude": [{"class": "fire hydrant", "count": 3}], "prompt": "a photo of two fire hydrants"}
-{"tag": "counting", "include": [{"class": "giraffe", "count": 3}], "exclude": [{"class": "giraffe", "count": 4}], "prompt": "a photo of three giraffes"}
-{"tag": "counting", "include": [{"class": "tv", "count": 4}], "exclude": [{"class": "tv", "count": 5}], "prompt": "a photo of four tvs"}
-{"tag": "counting", "include": [{"class": "wine glass", "count": 3}], "exclude": [{"class": "wine glass", "count": 4}], "prompt": "a photo of three wine glasses"}
-{"tag": "counting", "include": [{"class": "broccoli", "count": 4}], "exclude": [{"class": "broccoli", "count": 5}], "prompt": "a photo of four broccolis"}
-{"tag": "counting", "include": [{"class": "truck", "count": 3}], "exclude": [{"class": "truck", "count": 4}], "prompt": "a photo of three trucks"}
-{"tag": "counting", "include": [{"class": "truck", "count": 2}], "exclude": [{"class": "truck", "count": 3}], "prompt": "a photo of two trucks"}
-{"tag": "counting", "include": [{"class": "carrot", "count": 2}], "exclude": [{"class": "carrot", "count": 3}], "prompt": "a photo of two carrots"}
-{"tag": "counting", "include": [{"class": "sandwich", "count": 2}], "exclude": [{"class": "sandwich", "count": 3}], "prompt": "a photo of two sandwichs"}
-{"tag": "counting", "include": [{"class": "traffic light", "count": 4}], "exclude": [{"class": "traffic light", "count": 5}], "prompt": "a photo of four traffic lights"}
-{"tag": "counting", "include": [{"class": "clock", "count": 4}], "exclude": [{"class": "clock", "count": 5}], "prompt": "a photo of four clocks"}
-{"tag": "counting", "include": [{"class": "car", "count": 2}], "exclude": [{"class": "car", "count": 3}], "prompt": "a photo of two cars"}
-{"tag": "counting", "include": [{"class": "banana", "count": 2}], "exclude": [{"class": "banana", "count": 3}], "prompt": "a photo of two bananas"}
-{"tag": "counting", "include": [{"class": "wine glass", "count": 2}], "exclude": [{"class": "wine glass", "count": 3}], "prompt": "a photo of two wine glasses"}
-{"tag": "counting", "include": [{"class": "pizza", "count": 3}], "exclude": [{"class": "pizza", "count": 4}], "prompt": "a photo of three pizzas"}
-{"tag": "counting", "include": [{"class": "knife", "count": 4}], "exclude": [{"class": "knife", "count": 5}], "prompt": "a photo of four knifes"}
-{"tag": "counting", "include": [{"class": "suitcase", "count": 3}], "exclude": [{"class": "suitcase", "count": 4}], "prompt": "a photo of three suitcases"}
-{"tag": "counting", "include": [{"class": "zebra", "count": 4}], "exclude": [{"class": "zebra", "count": 5}], "prompt": "a photo of four zebras"}
-{"tag": "counting", "include": [{"class": "teddy bear", "count": 2}], "exclude": [{"class": "teddy bear", "count": 3}], "prompt": "a photo of two teddy bears"}
-{"tag": "counting", "include": [{"class": "skateboard", "count": 4}], "exclude": [{"class": "skateboard", "count": 5}], "prompt": "a photo of four skateboards"}
-{"tag": "counting", "include": [{"class": "hot dog", "count": 4}], "exclude": [{"class": "hot dog", "count": 5}], "prompt": "a photo of four hot dogs"}
-{"tag": "counting", "include": [{"class": "bird", "count": 3}], "exclude": [{"class": "bird", "count": 4}], "prompt": "a photo of three birds"}
-{"tag": "counting", "include": [{"class": "boat", "count": 4}], "exclude": [{"class": "boat", "count": 5}], "prompt": "a photo of four boats"}
-{"tag": "counting", "include": [{"class": "microwave", "count": 4}], "exclude": [{"class": "microwave", "count": 5}], "prompt": "a photo of four microwaves"}
-{"tag": "counting", "include": [{"class": "hair drier", "count": 2}], "exclude": [{"class": "hair drier", "count": 3}], "prompt": "a photo of two hair driers"}
-{"tag": "counting", "include": [{"class": "laptop", "count": 3}], "exclude": [{"class": "laptop", "count": 4}], "prompt": "a photo of three laptops"}
-{"tag": "counting", "include": [{"class": "cow", "count": 3}], "exclude": [{"class": "cow", "count": 4}], "prompt": "a photo of three cows"}
-{"tag": "counting", "include": [{"class": "parking meter", "count": 2}], "exclude": [{"class": "parking meter", "count": 3}], "prompt": "a photo of two parking meters"}
-{"tag": "counting", "include": [{"class": "bench", "count": 4}], "exclude": [{"class": "bench", "count": 5}], "prompt": "a photo of four benchs"}
-{"tag": "counting", "include": [{"class": "bench", "count": 3}], "exclude": [{"class": "bench", "count": 4}], "prompt": "a photo of three benchs"}
-{"tag": "counting", "include": [{"class": "frisbee", "count": 4}], "exclude": [{"class": "frisbee", "count": 5}], "prompt": "a photo of four frisbees"}
-{"tag": "counting", "include": [{"class": "book", "count": 4}], "exclude": [{"class": "book", "count": 5}], "prompt": "a photo of four books"}
-{"tag": "counting", "include": [{"class": "bus", "count": 4}], "exclude": [{"class": "bus", "count": 5}], "prompt": "a photo of four buses"}
-{"tag": "colors", "include": [{"class": "fire hydrant", "count": 1, "color": "blue"}], "prompt": "a photo of a blue fire hydrant"}
-{"tag": "colors", "include": [{"class": "car", "count": 1, "color": "pink"}], "prompt": "a photo of a pink car"}
-{"tag": "colors", "include": [{"class": "cup", "count": 1, "color": "purple"}], "prompt": "a photo of a purple cup"}
-{"tag": "colors", "include": [{"class": "cow", "count": 1, "color": "blue"}], "prompt": "a photo of a blue cow"}
-{"tag": "colors", "include": [{"class": "boat", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow boat"}
-{"tag": "colors", "include": [{"class": "umbrella", "count": 1, "color": "blue"}], "prompt": "a photo of a blue umbrella"}
-{"tag": "colors", "include": [{"class": "elephant", "count": 1, "color": "blue"}], "prompt": "a photo of a blue elephant"}
-{"tag": "colors", "include": [{"class": "elephant", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow elephant"}
-{"tag": "colors", "include": [{"class": "bicycle", "count": 1, "color": "red"}], "prompt": "a photo of a red bicycle"}
-{"tag": "colors", "include": [{"class": "suitcase", "count": 1, "color": "purple"}], "prompt": "a photo of a purple suitcase"}
-{"tag": "colors", "include": [{"class": "hair drier", "count": 1, "color": "purple"}], "prompt": "a photo of a purple hair drier"}
-{"tag": "colors", "include": [{"class": "sandwich", "count": 1, "color": "white"}], "prompt": "a photo of a white sandwich"}
-{"tag": "colors", "include": [{"class": "elephant", "count": 1, "color": "purple"}], "prompt": "a photo of a purple elephant"}
-{"tag": "colors", "include": [{"class": "microwave", "count": 1, "color": "green"}], "prompt": "a photo of a green microwave"}
-{"tag": "colors", "include": [{"class": "zebra", "count": 1, "color": "red"}], "prompt": "a photo of a red zebra"}
-{"tag": "colors", "include": [{"class": "apple", "count": 1, "color": "red"}], "prompt": "a photo of a red apple"}
-{"tag": "colors", "include": [{"class": "tv remote", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow tv remote"}
-{"tag": "colors", "include": [{"class": "toilet", "count": 1, "color": "blue"}], "prompt": "a photo of a blue toilet"}
-{"tag": "colors", "include": [{"class": "orange", "count": 1, "color": "orange"}], "prompt": "a photo of an orange orange"}
-{"tag": "colors", "include": [{"class": "donut", "count": 1, "color": "black"}], "prompt": "a photo of a black donut"}
-{"tag": "colors", "include": [{"class": "vase", "count": 1, "color": "red"}], "prompt": "a photo of a red vase"}
-{"tag": "colors", "include": [{"class": "pizza", "count": 1, "color": "purple"}], "prompt": "a photo of a purple pizza"}
-{"tag": "colors", "include": [{"class": "skateboard", "count": 1, "color": "pink"}], "prompt": "a photo of a pink skateboard"}
-{"tag": "colors", "include": [{"class": "skateboard", "count": 1, "color": "green"}], "prompt": "a photo of a green skateboard"}
-{"tag": "colors", "include": [{"class": "bear", "count": 1, "color": "purple"}], "prompt": "a photo of a purple bear"}
-{"tag": "colors", "include": [{"class": "chair", "count": 1, "color": "brown"}], "prompt": "a photo of a brown chair"}
-{"tag": "colors", "include": [{"class": "computer keyboard", "count": 1, "color": "brown"}], "prompt": "a photo of a brown computer keyboard"}
-{"tag": "colors", "include": [{"class": "cow", "count": 1, "color": "orange"}], "prompt": "a photo of an orange cow"}
-{"tag": "colors", "include": [{"class": "skis", "count": 1, "color": "brown"}], "prompt": "a photo of a brown skis"}
-{"tag": "colors", "include": [{"class": "kite", "count": 1, "color": "white"}], "prompt": "a photo of a white kite"}
-{"tag": "colors", "include": [{"class": "dog", "count": 1, "color": "red"}], "prompt": "a photo of a red dog"}
-{"tag": "colors", "include": [{"class": "couch", "count": 1, "color": "green"}], "prompt": "a photo of a green couch"}
-{"tag": "colors", "include": [{"class": "airplane", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow airplane"}
-{"tag": "colors", "include": [{"class": "tv", "count": 1, "color": "orange"}], "prompt": "a photo of an orange tv"}
-{"tag": "colors", "include": [{"class": "scissors", "count": 1, "color": "white"}], "prompt": "a photo of a white scissors"}
-{"tag": "colors", "include": [{"class": "cell phone", "count": 1, "color": "pink"}], "prompt": "a photo of a pink cell phone"}
-{"tag": "colors", "include": [{"class": "surfboard", "count": 1, "color": "green"}], "prompt": "a photo of a green surfboard"}
-{"tag": "colors", "include": [{"class": "fire hydrant", "count": 1, "color": "white"}], "prompt": "a photo of a white fire hydrant"}
-{"tag": "colors", "include": [{"class": "bicycle", "count": 1, "color": "black"}], "prompt": "a photo of a black bicycle"}
-{"tag": "colors", "include": [{"class": "carrot", "count": 1, "color": "purple"}], "prompt": "a photo of a purple carrot"}
-{"tag": "colors", "include": [{"class": "dining table", "count": 1, "color": "black"}], "prompt": "a photo of a black dining table"}
-{"tag": "colors", "include": [{"class": "potted plant", "count": 1, "color": "purple"}], "prompt": "a photo of a purple potted plant"}
-{"tag": "colors", "include": [{"class": "backpack", "count": 1, "color": "purple"}], "prompt": "a photo of a purple backpack"}
-{"tag": "colors", "include": [{"class": "train", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow train"}
-{"tag": "colors", "include": [{"class": "potted plant", "count": 1, "color": "pink"}], "prompt": "a photo of a pink potted plant"}
-{"tag": "colors", "include": [{"class": "giraffe", "count": 1, "color": "red"}], "prompt": "a photo of a red giraffe"}
-{"tag": "colors", "include": [{"class": "bear", "count": 1, "color": "brown"}], "prompt": "a photo of a brown bear"}
-{"tag": "colors", "include": [{"class": "train", "count": 1, "color": "black"}], "prompt": "a photo of a black train"}
-{"tag": "colors", "include": [{"class": "laptop", "count": 1, "color": "orange"}], "prompt": "a photo of an orange laptop"}
-{"tag": "colors", "include": [{"class": "hot dog", "count": 1, "color": "green"}], "prompt": "a photo of a green hot dog"}
-{"tag": "colors", "include": [{"class": "parking meter", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow parking meter"}
-{"tag": "colors", "include": [{"class": "potted plant", "count": 1, "color": "red"}], "prompt": "a photo of a red potted plant"}
-{"tag": "colors", "include": [{"class": "traffic light", "count": 1, "color": "green"}], "prompt": "a photo of a green traffic light"}
-{"tag": "colors", "include": [{"class": "tv", "count": 1, "color": "blue"}], "prompt": "a photo of a blue tv"}
-{"tag": "colors", "include": [{"class": "refrigerator", "count": 1, "color": "brown"}], "prompt": "a photo of a brown refrigerator"}
-{"tag": "colors", "include": [{"class": "tv remote", "count": 1, "color": "black"}], "prompt": "a photo of a black tv remote"}
-{"tag": "colors", "include": [{"class": "scissors", "count": 1, "color": "purple"}], "prompt": "a photo of a purple scissors"}
-{"tag": "colors", "include": [{"class": "orange", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow orange"}
-{"tag": "colors", "include": [{"class": "toaster", "count": 1, "color": "brown"}], "prompt": "a photo of a brown toaster"}
-{"tag": "colors", "include": [{"class": "parking meter", "count": 1, "color": "red"}], "prompt": "a photo of a red parking meter"}
-{"tag": "colors", "include": [{"class": "orange", "count": 1, "color": "brown"}], "prompt": "a photo of a brown orange"}
-{"tag": "colors", "include": [{"class": "clock", "count": 1, "color": "green"}], "prompt": "a photo of a green clock"}
-{"tag": "colors", "include": [{"class": "sheep", "count": 1, "color": "white"}], "prompt": "a photo of a white sheep"}
-{"tag": "colors", "include": [{"class": "oven", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow oven"}
-{"tag": "colors", "include": [{"class": "vase", "count": 1, "color": "green"}], "prompt": "a photo of a green vase"}
-{"tag": "colors", "include": [{"class": "teddy bear", "count": 1, "color": "black"}], "prompt": "a photo of a black teddy bear"}
-{"tag": "colors", "include": [{"class": "carrot", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow carrot"}
-{"tag": "colors", "include": [{"class": "hot dog", "count": 1, "color": "black"}], "prompt": "a photo of a black hot dog"}
-{"tag": "colors", "include": [{"class": "scissors", "count": 1, "color": "red"}], "prompt": "a photo of a red scissors"}
-{"tag": "colors", "include": [{"class": "teddy bear", "count": 1, "color": "white"}], "prompt": "a photo of a white teddy bear"}
-{"tag": "colors", "include": [{"class": "skis", "count": 1, "color": "black"}], "prompt": "a photo of a black skis"}
-{"tag": "colors", "include": [{"class": "dining table", "count": 1, "color": "blue"}], "prompt": "a photo of a blue dining table"}
-{"tag": "colors", "include": [{"class": "refrigerator", "count": 1, "color": "black"}], "prompt": "a photo of a black refrigerator"}
-{"tag": "colors", "include": [{"class": "dog", "count": 1, "color": "white"}], "prompt": "a photo of a white dog"}
-{"tag": "colors", "include": [{"class": "scissors", "count": 1, "color": "orange"}], "prompt": "a photo of an orange scissors"}
-{"tag": "colors", "include": [{"class": "cell phone", "count": 1, "color": "red"}], "prompt": "a photo of a red cell phone"}
-{"tag": "colors", "include": [{"class": "orange", "count": 1, "color": "white"}], "prompt": "a photo of a white orange"}
-{"tag": "colors", "include": [{"class": "clock", "count": 1, "color": "blue"}], "prompt": "a photo of a blue clock"}
-{"tag": "colors", "include": [{"class": "carrot", "count": 1, "color": "blue"}], "prompt": "a photo of a blue carrot"}
-{"tag": "colors", "include": [{"class": "motorcycle", "count": 1, "color": "green"}], "prompt": "a photo of a green motorcycle"}
-{"tag": "colors", "include": [{"class": "stop sign", "count": 1, "color": "pink"}], "prompt": "a photo of a pink stop sign"}
-{"tag": "colors", "include": [{"class": "vase", "count": 1, "color": "black"}], "prompt": "a photo of a black vase"}
-{"tag": "colors", "include": [{"class": "backpack", "count": 1, "color": "black"}], "prompt": "a photo of a black backpack"}
-{"tag": "colors", "include": [{"class": "car", "count": 1, "color": "red"}], "prompt": "a photo of a red car"}
-{"tag": "colors", "include": [{"class": "computer mouse", "count": 1, "color": "green"}], "prompt": "a photo of a green computer mouse"}
-{"tag": "colors", "include": [{"class": "backpack", "count": 1, "color": "red"}], "prompt": "a photo of a red backpack"}
-{"tag": "colors", "include": [{"class": "bus", "count": 1, "color": "green"}], "prompt": "a photo of a green bus"}
-{"tag": "colors", "include": [{"class": "toaster", "count": 1, "color": "orange"}], "prompt": "a photo of an orange toaster"}
-{"tag": "colors", "include": [{"class": "fork", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow fork"}
-{"tag": "colors", "include": [{"class": "parking meter", "count": 1, "color": "pink"}], "prompt": "a photo of a pink parking meter"}
-{"tag": "colors", "include": [{"class": "book", "count": 1, "color": "blue"}], "prompt": "a photo of a blue book"}
-{"tag": "colors", "include": [{"class": "broccoli", "count": 1, "color": "yellow"}], "prompt": "a photo of a yellow broccoli"}
-{"tag": "colors", "include": [{"class": "computer mouse", "count": 1, "color": "orange"}], "prompt": "a photo of an orange computer mouse"}
-{"tag": "colors", "include": [{"class": "cake", "count": 1, "color": "red"}], "prompt": "a photo of a red cake"}
-{"tag": "position", "include": [{"class": "teddy bear", "count": 1}, {"class": "dog", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a dog right of a teddy bear"}
-{"tag": "position", "include": [{"class": "kite", "count": 1}, {"class": "wine glass", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a wine glass above a kite"}
-{"tag": "position", "include": [{"class": "cup", "count": 1}, {"class": "couch", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a couch below a cup"}
-{"tag": "position", "include": [{"class": "cow", "count": 1}, {"class": "laptop", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a laptop left of a cow"}
-{"tag": "position", "include": [{"class": "hair drier", "count": 1}, {"class": "fork", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a fork above a hair drier"}
-{"tag": "position", "include": [{"class": "baseball bat", "count": 1}, {"class": "tie", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a tie right of a baseball bat"}
-{"tag": "position", "include": [{"class": "fork", "count": 1}, {"class": "stop sign", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a stop sign above a fork"}
-{"tag": "position", "include": [{"class": "skateboard", "count": 1}, {"class": "bird", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a bird below a skateboard"}
-{"tag": "position", "include": [{"class": "tv", "count": 1}, {"class": "apple", "count": 1, "position": ["above", 0]}], "prompt": "a photo of an apple above a tv"}
-{"tag": "position", "include": [{"class": "potted plant", "count": 1}, {"class": "train", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a train above a potted plant"}
-{"tag": "position", "include": [{"class": "refrigerator", "count": 1}, {"class": "truck", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a truck left of a refrigerator"}
-{"tag": "position", "include": [{"class": "cow", "count": 1}, {"class": "tv remote", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a tv remote below a cow"}
-{"tag": "position", "include": [{"class": "train", "count": 1}, {"class": "bottle", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a bottle right of a train"}
-{"tag": "position", "include": [{"class": "cow", "count": 1}, {"class": "dog", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a dog above a cow"}
-{"tag": "position", "include": [{"class": "person", "count": 1}, {"class": "skateboard", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a skateboard above a person"}
-{"tag": "position", "include": [{"class": "umbrella", "count": 1}, {"class": "baseball glove", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a baseball glove below an umbrella"}
-{"tag": "position", "include": [{"class": "oven", "count": 1}, {"class": "dining table", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a dining table right of an oven"}
-{"tag": "position", "include": [{"class": "suitcase", "count": 1}, {"class": "hot dog", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a hot dog left of a suitcase"}
-{"tag": "position", "include": [{"class": "toothbrush", "count": 1}, {"class": "bus", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a bus below a toothbrush"}
-{"tag": "position", "include": [{"class": "sandwich", "count": 1}, {"class": "backpack", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a backpack right of a sandwich"}
-{"tag": "position", "include": [{"class": "baseball bat", "count": 1}, {"class": "cake", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a cake below a baseball bat"}
-{"tag": "position", "include": [{"class": "tie", "count": 1}, {"class": "dog", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a dog right of a tie"}
-{"tag": "position", "include": [{"class": "boat", "count": 1}, {"class": "suitcase", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a suitcase right of a boat"}
-{"tag": "position", "include": [{"class": "clock", "count": 1}, {"class": "bear", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a bear above a clock"}
-{"tag": "position", "include": [{"class": "umbrella", "count": 1}, {"class": "tv remote", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a tv remote left of an umbrella"}
-{"tag": "position", "include": [{"class": "umbrella", "count": 1}, {"class": "sports ball", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a sports ball left of an umbrella"}
-{"tag": "position", "include": [{"class": "dining table", "count": 1}, {"class": "train", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a train right of a dining table"}
-{"tag": "position", "include": [{"class": "elephant", "count": 1}, {"class": "hair drier", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a hair drier below an elephant"}
-{"tag": "position", "include": [{"class": "spoon", "count": 1}, {"class": "tennis racket", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a tennis racket right of a spoon"}
-{"tag": "position", "include": [{"class": "hot dog", "count": 1}, {"class": "wine glass", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a wine glass right of a hot dog"}
-{"tag": "position", "include": [{"class": "bench", "count": 1}, {"class": "computer mouse", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a computer mouse left of a bench"}
-{"tag": "position", "include": [{"class": "orange", "count": 1}, {"class": "carrot", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a carrot left of an orange"}
-{"tag": "position", "include": [{"class": "toothbrush", "count": 1}, {"class": "kite", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a kite above a toothbrush"}
-{"tag": "position", "include": [{"class": "traffic light", "count": 1}, {"class": "toaster", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a toaster below a traffic light"}
-{"tag": "position", "include": [{"class": "baseball glove", "count": 1}, {"class": "cat", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a cat below a baseball glove"}
-{"tag": "position", "include": [{"class": "zebra", "count": 1}, {"class": "skis", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a skis right of a zebra"}
-{"tag": "position", "include": [{"class": "chair", "count": 1}, {"class": "stop sign", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a stop sign above a chair"}
-{"tag": "position", "include": [{"class": "parking meter", "count": 1}, {"class": "stop sign", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a stop sign above a parking meter"}
-{"tag": "position", "include": [{"class": "skateboard", "count": 1}, {"class": "hot dog", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a hot dog right of a skateboard"}
-{"tag": "position", "include": [{"class": "computer keyboard", "count": 1}, {"class": "pizza", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a pizza below a computer keyboard"}
-{"tag": "position", "include": [{"class": "toilet", "count": 1}, {"class": "hair drier", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a hair drier left of a toilet"}
-{"tag": "position", "include": [{"class": "stop sign", "count": 1}, {"class": "cow", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a cow left of a stop sign"}
-{"tag": "position", "include": [{"class": "skis", "count": 1}, {"class": "suitcase", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a suitcase above a skis"}
-{"tag": "position", "include": [{"class": "laptop", "count": 1}, {"class": "book", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a book above a laptop"}
-{"tag": "position", "include": [{"class": "pizza", "count": 1}, {"class": "toothbrush", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a toothbrush below a pizza"}
-{"tag": "position", "include": [{"class": "kite", "count": 1}, {"class": "toilet", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a toilet left of a kite"}
-{"tag": "position", "include": [{"class": "sink", "count": 1}, {"class": "tie", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a tie above a sink"}
-{"tag": "position", "include": [{"class": "couch", "count": 1}, {"class": "bird", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a bird left of a couch"}
-{"tag": "position", "include": [{"class": "sports ball", "count": 1}, {"class": "bed", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a bed right of a sports ball"}
-{"tag": "position", "include": [{"class": "surfboard", "count": 1}, {"class": "elephant", "count": 1, "position": ["below", 0]}], "prompt": "a photo of an elephant below a surfboard"}
-{"tag": "position", "include": [{"class": "motorcycle", "count": 1}, {"class": "frisbee", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a frisbee right of a motorcycle"}
-{"tag": "position", "include": [{"class": "fire hydrant", "count": 1}, {"class": "vase", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a vase above a fire hydrant"}
-{"tag": "position", "include": [{"class": "elephant", "count": 1}, {"class": "zebra", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a zebra left of an elephant"}
-{"tag": "position", "include": [{"class": "bear", "count": 1}, {"class": "bench", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a bench left of a bear"}
-{"tag": "position", "include": [{"class": "bench", "count": 1}, {"class": "donut", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a donut right of a bench"}
-{"tag": "position", "include": [{"class": "horse", "count": 1}, {"class": "frisbee", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a frisbee below a horse"}
-{"tag": "position", "include": [{"class": "snowboard", "count": 1}, {"class": "computer keyboard", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a computer keyboard above a snowboard"}
-{"tag": "position", "include": [{"class": "cow", "count": 1}, {"class": "tv", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a tv below a cow"}
-{"tag": "position", "include": [{"class": "horse", "count": 1}, {"class": "elephant", "count": 1, "position": ["below", 0]}], "prompt": "a photo of an elephant below a horse"}
-{"tag": "position", "include": [{"class": "banana", "count": 1}, {"class": "suitcase", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a suitcase left of a banana"}
-{"tag": "position", "include": [{"class": "airplane", "count": 1}, {"class": "train", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a train below an airplane"}
-{"tag": "position", "include": [{"class": "backpack", "count": 1}, {"class": "cat", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a cat below a backpack"}
-{"tag": "position", "include": [{"class": "cake", "count": 1}, {"class": "backpack", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a backpack below a cake"}
-{"tag": "position", "include": [{"class": "knife", "count": 1}, {"class": "sandwich", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a sandwich below a knife"}
-{"tag": "position", "include": [{"class": "parking meter", "count": 1}, {"class": "bicycle", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a bicycle above a parking meter"}
-{"tag": "position", "include": [{"class": "suitcase", "count": 1}, {"class": "knife", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a knife right of a suitcase"}
-{"tag": "position", "include": [{"class": "knife", "count": 1}, {"class": "hot dog", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a hot dog above a knife"}
-{"tag": "position", "include": [{"class": "parking meter", "count": 1}, {"class": "zebra", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a zebra right of a parking meter"}
-{"tag": "position", "include": [{"class": "zebra", "count": 1}, {"class": "chair", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a chair left of a zebra"}
-{"tag": "position", "include": [{"class": "airplane", "count": 1}, {"class": "cow", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a cow below an airplane"}
-{"tag": "position", "include": [{"class": "umbrella", "count": 1}, {"class": "cup", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a cup left of an umbrella"}
-{"tag": "position", "include": [{"class": "computer keyboard", "count": 1}, {"class": "zebra", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a zebra below a computer keyboard"}
-{"tag": "position", "include": [{"class": "broccoli", "count": 1}, {"class": "zebra", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a zebra below a broccoli"}
-{"tag": "position", "include": [{"class": "sports ball", "count": 1}, {"class": "laptop", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a laptop below a sports ball"}
-{"tag": "position", "include": [{"class": "baseball bat", "count": 1}, {"class": "truck", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a truck left of a baseball bat"}
-{"tag": "position", "include": [{"class": "baseball bat", "count": 1}, {"class": "refrigerator", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a refrigerator above a baseball bat"}
-{"tag": "position", "include": [{"class": "baseball bat", "count": 1}, {"class": "tv", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a tv above a baseball bat"}
-{"tag": "position", "include": [{"class": "bear", "count": 1}, {"class": "baseball glove", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a baseball glove right of a bear"}
-{"tag": "position", "include": [{"class": "scissors", "count": 1}, {"class": "refrigerator", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a refrigerator below a scissors"}
-{"tag": "position", "include": [{"class": "suitcase", "count": 1}, {"class": "dining table", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a dining table above a suitcase"}
-{"tag": "position", "include": [{"class": "broccoli", "count": 1}, {"class": "parking meter", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a parking meter above a broccoli"}
-{"tag": "position", "include": [{"class": "truck", "count": 1}, {"class": "frisbee", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a frisbee above a truck"}
-{"tag": "position", "include": [{"class": "banana", "count": 1}, {"class": "pizza", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a pizza right of a banana"}
-{"tag": "position", "include": [{"class": "boat", "count": 1}, {"class": "bus", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a bus above a boat"}
-{"tag": "position", "include": [{"class": "tennis racket", "count": 1}, {"class": "cell phone", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a cell phone left of a tennis racket"}
-{"tag": "position", "include": [{"class": "broccoli", "count": 1}, {"class": "horse", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a horse right of a broccoli"}
-{"tag": "position", "include": [{"class": "bottle", "count": 1}, {"class": "broccoli", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a broccoli above a bottle"}
-{"tag": "position", "include": [{"class": "horse", "count": 1}, {"class": "vase", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a vase right of a horse"}
-{"tag": "position", "include": [{"class": "spoon", "count": 1}, {"class": "bear", "count": 1, "position": ["above", 0]}], "prompt": "a photo of a bear above a spoon"}
-{"tag": "position", "include": [{"class": "bed", "count": 1}, {"class": "zebra", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a zebra right of a bed"}
-{"tag": "position", "include": [{"class": "laptop", "count": 1}, {"class": "cow", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a cow right of a laptop"}
-{"tag": "position", "include": [{"class": "frisbee", "count": 1}, {"class": "bed", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a bed right of a frisbee"}
-{"tag": "position", "include": [{"class": "motorcycle", "count": 1}, {"class": "tie", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a tie right of a motorcycle"}
-{"tag": "position", "include": [{"class": "tv", "count": 1}, {"class": "laptop", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a laptop right of a tv"}
-{"tag": "position", "include": [{"class": "chair", "count": 1}, {"class": "cell phone", "count": 1, "position": ["right of", 0]}], "prompt": "a photo of a cell phone right of a chair"}
-{"tag": "position", "include": [{"class": "potted plant", "count": 1}, {"class": "couch", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a couch below a potted plant"}
-{"tag": "position", "include": [{"class": "tv", "count": 1}, {"class": "clock", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a clock below a tv"}
-{"tag": "position", "include": [{"class": "vase", "count": 1}, {"class": "couch", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a couch below a vase"}
-{"tag": "position", "include": [{"class": "cat", "count": 1}, {"class": "donut", "count": 1, "position": ["below", 0]}], "prompt": "a photo of a donut below a cat"}
-{"tag": "position", "include": [{"class": "toaster", "count": 1}, {"class": "couch", "count": 1, "position": ["left of", 0]}], "prompt": "a photo of a couch left of a toaster"}
-{"tag": "color_attr", "include": [{"class": "wine glass", "count": 1, "color": "purple"}, {"class": "apple", "count": 1, "color": "black"}], "prompt": "a photo of a purple wine glass and a black apple"}
-{"tag": "color_attr", "include": [{"class": "bus", "count": 1, "color": "green"}, {"class": "microwave", "count": 1, "color": "purple"}], "prompt": "a photo of a green bus and a purple microwave"}
-{"tag": "color_attr", "include": [{"class": "skis", "count": 1, "color": "green"}, {"class": "airplane", "count": 1, "color": "brown"}], "prompt": "a photo of a green skis and a brown airplane"}
-{"tag": "color_attr", "include": [{"class": "computer keyboard", "count": 1, "color": "yellow"}, {"class": "sink", "count": 1, "color": "black"}], "prompt": "a photo of a yellow computer keyboard and a black sink"}
-{"tag": "color_attr", "include": [{"class": "oven", "count": 1, "color": "pink"}, {"class": "motorcycle", "count": 1, "color": "green"}], "prompt": "a photo of a pink oven and a green motorcycle"}
-{"tag": "color_attr", "include": [{"class": "parking meter", "count": 1, "color": "purple"}, {"class": "laptop", "count": 1, "color": "red"}], "prompt": "a photo of a purple parking meter and a red laptop"}
-{"tag": "color_attr", "include": [{"class": "skateboard", "count": 1, "color": "yellow"}, {"class": "computer mouse", "count": 1, "color": "orange"}], "prompt": "a photo of a yellow skateboard and an orange computer mouse"}
-{"tag": "color_attr", "include": [{"class": "skis", "count": 1, "color": "red"}, {"class": "tie", "count": 1, "color": "brown"}], "prompt": "a photo of a red skis and a brown tie"}
-{"tag": "color_attr", "include": [{"class": "skateboard", "count": 1, "color": "pink"}, {"class": "train", "count": 1, "color": "black"}], "prompt": "a photo of a pink skateboard and a black train"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "white"}, {"class": "bed", "count": 1, "color": "purple"}], "prompt": "a photo of a white handbag and a purple bed"}
-{"tag": "color_attr", "include": [{"class": "elephant", "count": 1, "color": "purple"}, {"class": "sports ball", "count": 1, "color": "brown"}], "prompt": "a photo of a purple elephant and a brown sports ball"}
-{"tag": "color_attr", "include": [{"class": "dog", "count": 1, "color": "purple"}, {"class": "dining table", "count": 1, "color": "black"}], "prompt": "a photo of a purple dog and a black dining table"}
-{"tag": "color_attr", "include": [{"class": "dining table", "count": 1, "color": "white"}, {"class": "car", "count": 1, "color": "red"}], "prompt": "a photo of a white dining table and a red car"}
-{"tag": "color_attr", "include": [{"class": "cell phone", "count": 1, "color": "blue"}, {"class": "apple", "count": 1, "color": "green"}], "prompt": "a photo of a blue cell phone and a green apple"}
-{"tag": "color_attr", "include": [{"class": "car", "count": 1, "color": "red"}, {"class": "potted plant", "count": 1, "color": "orange"}], "prompt": "a photo of a red car and an orange potted plant"}
-{"tag": "color_attr", "include": [{"class": "carrot", "count": 1, "color": "brown"}, {"class": "potted plant", "count": 1, "color": "white"}], "prompt": "a photo of a brown carrot and a white potted plant"}
-{"tag": "color_attr", "include": [{"class": "kite", "count": 1, "color": "black"}, {"class": "bear", "count": 1, "color": "green"}], "prompt": "a photo of a black kite and a green bear"}
-{"tag": "color_attr", "include": [{"class": "laptop", "count": 1, "color": "blue"}, {"class": "bear", "count": 1, "color": "brown"}], "prompt": "a photo of a blue laptop and a brown bear"}
-{"tag": "color_attr", "include": [{"class": "teddy bear", "count": 1, "color": "green"}, {"class": "kite", "count": 1, "color": "brown"}], "prompt": "a photo of a green teddy bear and a brown kite"}
-{"tag": "color_attr", "include": [{"class": "stop sign", "count": 1, "color": "yellow"}, {"class": "potted plant", "count": 1, "color": "blue"}], "prompt": "a photo of a yellow stop sign and a blue potted plant"}
-{"tag": "color_attr", "include": [{"class": "snowboard", "count": 1, "color": "orange"}, {"class": "cat", "count": 1, "color": "green"}], "prompt": "a photo of an orange snowboard and a green cat"}
-{"tag": "color_attr", "include": [{"class": "truck", "count": 1, "color": "orange"}, {"class": "sink", "count": 1, "color": "pink"}], "prompt": "a photo of an orange truck and a pink sink"}
-{"tag": "color_attr", "include": [{"class": "hot dog", "count": 1, "color": "brown"}, {"class": "pizza", "count": 1, "color": "purple"}], "prompt": "a photo of a brown hot dog and a purple pizza"}
-{"tag": "color_attr", "include": [{"class": "couch", "count": 1, "color": "green"}, {"class": "umbrella", "count": 1, "color": "orange"}], "prompt": "a photo of a green couch and an orange umbrella"}
-{"tag": "color_attr", "include": [{"class": "bed", "count": 1, "color": "brown"}, {"class": "cell phone", "count": 1, "color": "pink"}], "prompt": "a photo of a brown bed and a pink cell phone"}
-{"tag": "color_attr", "include": [{"class": "broccoli", "count": 1, "color": "black"}, {"class": "cake", "count": 1, "color": "yellow"}], "prompt": "a photo of a black broccoli and a yellow cake"}
-{"tag": "color_attr", "include": [{"class": "train", "count": 1, "color": "red"}, {"class": "bear", "count": 1, "color": "purple"}], "prompt": "a photo of a red train and a purple bear"}
-{"tag": "color_attr", "include": [{"class": "tennis racket", "count": 1, "color": "purple"}, {"class": "sink", "count": 1, "color": "black"}], "prompt": "a photo of a purple tennis racket and a black sink"}
-{"tag": "color_attr", "include": [{"class": "vase", "count": 1, "color": "blue"}, {"class": "banana", "count": 1, "color": "black"}], "prompt": "a photo of a blue vase and a black banana"}
-{"tag": "color_attr", "include": [{"class": "clock", "count": 1, "color": "blue"}, {"class": "cup", "count": 1, "color": "white"}], "prompt": "a photo of a blue clock and a white cup"}
-{"tag": "color_attr", "include": [{"class": "umbrella", "count": 1, "color": "red"}, {"class": "couch", "count": 1, "color": "blue"}], "prompt": "a photo of a red umbrella and a blue couch"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "white"}, {"class": "giraffe", "count": 1, "color": "red"}], "prompt": "a photo of a white handbag and a red giraffe"}
-{"tag": "color_attr", "include": [{"class": "tv remote", "count": 1, "color": "pink"}, {"class": "airplane", "count": 1, "color": "blue"}], "prompt": "a photo of a pink tv remote and a blue airplane"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "pink"}, {"class": "scissors", "count": 1, "color": "black"}], "prompt": "a photo of a pink handbag and a black scissors"}
-{"tag": "color_attr", "include": [{"class": "car", "count": 1, "color": "brown"}, {"class": "hair drier", "count": 1, "color": "pink"}], "prompt": "a photo of a brown car and a pink hair drier"}
-{"tag": "color_attr", "include": [{"class": "bus", "count": 1, "color": "black"}, {"class": "cell phone", "count": 1, "color": "brown"}], "prompt": "a photo of a black bus and a brown cell phone"}
-{"tag": "color_attr", "include": [{"class": "sheep", "count": 1, "color": "purple"}, {"class": "banana", "count": 1, "color": "pink"}], "prompt": "a photo of a purple sheep and a pink banana"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "blue"}, {"class": "cell phone", "count": 1, "color": "white"}], "prompt": "a photo of a blue handbag and a white cell phone"}
-{"tag": "color_attr", "include": [{"class": "pizza", "count": 1, "color": "white"}, {"class": "umbrella", "count": 1, "color": "green"}], "prompt": "a photo of a white pizza and a green umbrella"}
-{"tag": "color_attr", "include": [{"class": "tie", "count": 1, "color": "white"}, {"class": "skateboard", "count": 1, "color": "purple"}], "prompt": "a photo of a white tie and a purple skateboard"}
-{"tag": "color_attr", "include": [{"class": "sports ball", "count": 1, "color": "yellow"}, {"class": "boat", "count": 1, "color": "green"}], "prompt": "a photo of a yellow sports ball and a green boat"}
-{"tag": "color_attr", "include": [{"class": "wine glass", "count": 1, "color": "white"}, {"class": "giraffe", "count": 1, "color": "brown"}], "prompt": "a photo of a white wine glass and a brown giraffe"}
-{"tag": "color_attr", "include": [{"class": "bowl", "count": 1, "color": "yellow"}, {"class": "baseball glove", "count": 1, "color": "white"}], "prompt": "a photo of a yellow bowl and a white baseball glove"}
-{"tag": "color_attr", "include": [{"class": "microwave", "count": 1, "color": "orange"}, {"class": "spoon", "count": 1, "color": "black"}], "prompt": "a photo of an orange microwave and a black spoon"}
-{"tag": "color_attr", "include": [{"class": "skateboard", "count": 1, "color": "orange"}, {"class": "bowl", "count": 1, "color": "pink"}], "prompt": "a photo of an orange skateboard and a pink bowl"}
-{"tag": "color_attr", "include": [{"class": "toilet", "count": 1, "color": "blue"}, {"class": "suitcase", "count": 1, "color": "white"}], "prompt": "a photo of a blue toilet and a white suitcase"}
-{"tag": "color_attr", "include": [{"class": "boat", "count": 1, "color": "white"}, {"class": "hot dog", "count": 1, "color": "orange"}], "prompt": "a photo of a white boat and an orange hot dog"}
-{"tag": "color_attr", "include": [{"class": "dining table", "count": 1, "color": "yellow"}, {"class": "dog", "count": 1, "color": "pink"}], "prompt": "a photo of a yellow dining table and a pink dog"}
-{"tag": "color_attr", "include": [{"class": "cake", "count": 1, "color": "red"}, {"class": "chair", "count": 1, "color": "purple"}], "prompt": "a photo of a red cake and a purple chair"}
-{"tag": "color_attr", "include": [{"class": "tie", "count": 1, "color": "blue"}, {"class": "dining table", "count": 1, "color": "pink"}], "prompt": "a photo of a blue tie and a pink dining table"}
-{"tag": "color_attr", "include": [{"class": "cow", "count": 1, "color": "blue"}, {"class": "computer keyboard", "count": 1, "color": "black"}], "prompt": "a photo of a blue cow and a black computer keyboard"}
-{"tag": "color_attr", "include": [{"class": "pizza", "count": 1, "color": "yellow"}, {"class": "oven", "count": 1, "color": "green"}], "prompt": "a photo of a yellow pizza and a green oven"}
-{"tag": "color_attr", "include": [{"class": "laptop", "count": 1, "color": "red"}, {"class": "car", "count": 1, "color": "brown"}], "prompt": "a photo of a red laptop and a brown car"}
-{"tag": "color_attr", "include": [{"class": "computer keyboard", "count": 1, "color": "purple"}, {"class": "scissors", "count": 1, "color": "blue"}], "prompt": "a photo of a purple computer keyboard and a blue scissors"}
-{"tag": "color_attr", "include": [{"class": "surfboard", "count": 1, "color": "green"}, {"class": "oven", "count": 1, "color": "orange"}], "prompt": "a photo of a green surfboard and an orange oven"}
-{"tag": "color_attr", "include": [{"class": "parking meter", "count": 1, "color": "yellow"}, {"class": "refrigerator", "count": 1, "color": "pink"}], "prompt": "a photo of a yellow parking meter and a pink refrigerator"}
-{"tag": "color_attr", "include": [{"class": "computer mouse", "count": 1, "color": "brown"}, {"class": "bottle", "count": 1, "color": "purple"}], "prompt": "a photo of a brown computer mouse and a purple bottle"}
-{"tag": "color_attr", "include": [{"class": "umbrella", "count": 1, "color": "red"}, {"class": "cow", "count": 1, "color": "green"}], "prompt": "a photo of a red umbrella and a green cow"}
-{"tag": "color_attr", "include": [{"class": "giraffe", "count": 1, "color": "red"}, {"class": "cell phone", "count": 1, "color": "black"}], "prompt": "a photo of a red giraffe and a black cell phone"}
-{"tag": "color_attr", "include": [{"class": "oven", "count": 1, "color": "brown"}, {"class": "train", "count": 1, "color": "purple"}], "prompt": "a photo of a brown oven and a purple train"}
-{"tag": "color_attr", "include": [{"class": "baseball bat", "count": 1, "color": "blue"}, {"class": "book", "count": 1, "color": "pink"}], "prompt": "a photo of a blue baseball bat and a pink book"}
-{"tag": "color_attr", "include": [{"class": "cup", "count": 1, "color": "green"}, {"class": "bowl", "count": 1, "color": "yellow"}], "prompt": "a photo of a green cup and a yellow bowl"}
-{"tag": "color_attr", "include": [{"class": "suitcase", "count": 1, "color": "yellow"}, {"class": "bus", "count": 1, "color": "brown"}], "prompt": "a photo of a yellow suitcase and a brown bus"}
-{"tag": "color_attr", "include": [{"class": "motorcycle", "count": 1, "color": "orange"}, {"class": "donut", "count": 1, "color": "pink"}], "prompt": "a photo of an orange motorcycle and a pink donut"}
-{"tag": "color_attr", "include": [{"class": "giraffe", "count": 1, "color": "orange"}, {"class": "baseball glove", "count": 1, "color": "white"}], "prompt": "a photo of an orange giraffe and a white baseball glove"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "orange"}, {"class": "carrot", "count": 1, "color": "green"}], "prompt": "a photo of an orange handbag and a green carrot"}
-{"tag": "color_attr", "include": [{"class": "bottle", "count": 1, "color": "black"}, {"class": "refrigerator", "count": 1, "color": "white"}], "prompt": "a photo of a black bottle and a white refrigerator"}
-{"tag": "color_attr", "include": [{"class": "dog", "count": 1, "color": "white"}, {"class": "potted plant", "count": 1, "color": "blue"}], "prompt": "a photo of a white dog and a blue potted plant"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "orange"}, {"class": "car", "count": 1, "color": "red"}], "prompt": "a photo of an orange handbag and a red car"}
-{"tag": "color_attr", "include": [{"class": "stop sign", "count": 1, "color": "red"}, {"class": "book", "count": 1, "color": "blue"}], "prompt": "a photo of a red stop sign and a blue book"}
-{"tag": "color_attr", "include": [{"class": "car", "count": 1, "color": "yellow"}, {"class": "toothbrush", "count": 1, "color": "orange"}], "prompt": "a photo of a yellow car and an orange toothbrush"}
-{"tag": "color_attr", "include": [{"class": "potted plant", "count": 1, "color": "black"}, {"class": "toilet", "count": 1, "color": "yellow"}], "prompt": "a photo of a black potted plant and a yellow toilet"}
-{"tag": "color_attr", "include": [{"class": "dining table", "count": 1, "color": "brown"}, {"class": "suitcase", "count": 1, "color": "white"}], "prompt": "a photo of a brown dining table and a white suitcase"}
-{"tag": "color_attr", "include": [{"class": "donut", "count": 1, "color": "orange"}, {"class": "stop sign", "count": 1, "color": "yellow"}], "prompt": "a photo of an orange donut and a yellow stop sign"}
-{"tag": "color_attr", "include": [{"class": "suitcase", "count": 1, "color": "green"}, {"class": "boat", "count": 1, "color": "blue"}], "prompt": "a photo of a green suitcase and a blue boat"}
-{"tag": "color_attr", "include": [{"class": "tennis racket", "count": 1, "color": "orange"}, {"class": "sports ball", "count": 1, "color": "yellow"}], "prompt": "a photo of an orange tennis racket and a yellow sports ball"}
-{"tag": "color_attr", "include": [{"class": "computer keyboard", "count": 1, "color": "purple"}, {"class": "chair", "count": 1, "color": "red"}], "prompt": "a photo of a purple computer keyboard and a red chair"}
-{"tag": "color_attr", "include": [{"class": "suitcase", "count": 1, "color": "purple"}, {"class": "pizza", "count": 1, "color": "orange"}], "prompt": "a photo of a purple suitcase and an orange pizza"}
-{"tag": "color_attr", "include": [{"class": "bottle", "count": 1, "color": "white"}, {"class": "sheep", "count": 1, "color": "blue"}], "prompt": "a photo of a white bottle and a blue sheep"}
-{"tag": "color_attr", "include": [{"class": "backpack", "count": 1, "color": "purple"}, {"class": "umbrella", "count": 1, "color": "white"}], "prompt": "a photo of a purple backpack and a white umbrella"}
-{"tag": "color_attr", "include": [{"class": "potted plant", "count": 1, "color": "orange"}, {"class": "spoon", "count": 1, "color": "black"}], "prompt": "a photo of an orange potted plant and a black spoon"}
-{"tag": "color_attr", "include": [{"class": "tennis racket", "count": 1, "color": "green"}, {"class": "dog", "count": 1, "color": "black"}], "prompt": "a photo of a green tennis racket and a black dog"}
-{"tag": "color_attr", "include": [{"class": "handbag", "count": 1, "color": "yellow"}, {"class": "refrigerator", "count": 1, "color": "blue"}], "prompt": "a photo of a yellow handbag and a blue refrigerator"}
-{"tag": "color_attr", "include": [{"class": "broccoli", "count": 1, "color": "pink"}, {"class": "sink", "count": 1, "color": "red"}], "prompt": "a photo of a pink broccoli and a red sink"}
-{"tag": "color_attr", "include": [{"class": "bowl", "count": 1, "color": "red"}, {"class": "sink", "count": 1, "color": "pink"}], "prompt": "a photo of a red bowl and a pink sink"}
-{"tag": "color_attr", "include": [{"class": "toilet", "count": 1, "color": "white"}, {"class": "apple", "count": 1, "color": "red"}], "prompt": "a photo of a white toilet and a red apple"}
-{"tag": "color_attr", "include": [{"class": "dining table", "count": 1, "color": "pink"}, {"class": "sandwich", "count": 1, "color": "black"}], "prompt": "a photo of a pink dining table and a black sandwich"}
-{"tag": "color_attr", "include": [{"class": "car", "count": 1, "color": "black"}, {"class": "parking meter", "count": 1, "color": "green"}], "prompt": "a photo of a black car and a green parking meter"}
-{"tag": "color_attr", "include": [{"class": "bird", "count": 1, "color": "yellow"}, {"class": "motorcycle", "count": 1, "color": "black"}], "prompt": "a photo of a yellow bird and a black motorcycle"}
-{"tag": "color_attr", "include": [{"class": "giraffe", "count": 1, "color": "brown"}, {"class": "stop sign", "count": 1, "color": "white"}], "prompt": "a photo of a brown giraffe and a white stop sign"}
-{"tag": "color_attr", "include": [{"class": "banana", "count": 1, "color": "white"}, {"class": "elephant", "count": 1, "color": "black"}], "prompt": "a photo of a white banana and a black elephant"}
-{"tag": "color_attr", "include": [{"class": "cow", "count": 1, "color": "orange"}, {"class": "sandwich", "count": 1, "color": "purple"}], "prompt": "a photo of an orange cow and a purple sandwich"}
-{"tag": "color_attr", "include": [{"class": "clock", "count": 1, "color": "red"}, {"class": "cell phone", "count": 1, "color": "black"}], "prompt": "a photo of a red clock and a black cell phone"}
-{"tag": "color_attr", "include": [{"class": "knife", "count": 1, "color": "brown"}, {"class": "donut", "count": 1, "color": "blue"}], "prompt": "a photo of a brown knife and a blue donut"}
-{"tag": "color_attr", "include": [{"class": "cup", "count": 1, "color": "red"}, {"class": "handbag", "count": 1, "color": "pink"}], "prompt": "a photo of a red cup and a pink handbag"}
-{"tag": "color_attr", "include": [{"class": "bicycle", "count": 1, "color": "yellow"}, {"class": "motorcycle", "count": 1, "color": "red"}], "prompt": "a photo of a yellow bicycle and a red motorcycle"}
-{"tag": "color_attr", "include": [{"class": "orange", "count": 1, "color": "red"}, {"class": "broccoli", "count": 1, "color": "purple"}], "prompt": "a photo of a red orange and a purple broccoli"}
-{"tag": "color_attr", "include": [{"class": "traffic light", "count": 1, "color": "orange"}, {"class": "toilet", "count": 1, "color": "white"}], "prompt": "a photo of an orange traffic light and a white toilet"}
-{"tag": "color_attr", "include": [{"class": "cup", "count": 1, "color": "green"}, {"class": "pizza", "count": 1, "color": "red"}], "prompt": "a photo of a green cup and a red pizza"}
-{"tag": "color_attr", "include": [{"class": "pizza", "count": 1, "color": "blue"}, {"class": "baseball glove", "count": 1, "color": "yellow"}], "prompt": "a photo of a blue pizza and a yellow baseball glove"}

eval/gen/geneval/prompts/generation_prompts.txt

@@ -1,553 +0,0 @@
-a photo of a bench
-a photo of a cow
-a photo of a bicycle
-a photo of a clock
-a photo of a carrot
-a photo of a suitcase
-a photo of a fork
-a photo of a surfboard
-a photo of a refrigerator
-a photo of a cup
-a photo of a microwave
-a photo of a potted plant
-a photo of a snowboard
-a photo of a zebra
-a photo of a parking meter
-a photo of a spoon
-a photo of a skateboard
-a photo of a car
-a photo of a motorcycle
-a photo of a traffic light
-a photo of a book
-a photo of a couch
-a photo of a backpack
-a photo of a computer keyboard
-a photo of a toaster
-a photo of a bird
-a photo of a bowl
-a photo of a dog
-a photo of a tie
-a photo of a laptop
-a photo of a computer mouse
-a photo of a sandwich
-a photo of a baseball bat
-a photo of a train
-a photo of a cell phone
-a photo of a chair
-a photo of a tv
-a photo of a broccoli
-a photo of a bed
-a photo of a skis
-a photo of a handbag
-a photo of a pizza
-a photo of a frisbee
-a photo of a scissors
-a photo of a bottle
-a photo of an elephant
-a photo of a toilet
-a photo of an oven
-a photo of an orange
-a photo of a person
-a photo of a teddy bear
-a photo of a vase
-a photo of a banana
-a photo of a toothbrush
-a photo of a tv remote
-a photo of a dining table
-a photo of a stop sign
-a photo of a sheep
-a photo of a fire hydrant
-a photo of an airplane
-a photo of a giraffe
-a photo of a horse
-a photo of a cat
-a photo of a donut
-a photo of a boat
-a photo of a baseball glove
-a photo of a hair drier
-a photo of a sink
-a photo of a cake
-a photo of a wine glass
-a photo of an apple
-a photo of a bus
-a photo of a tennis racket
-a photo of a knife
-a photo of a hot dog
-a photo of a truck
-a photo of an umbrella
-a photo of a sports ball
-a photo of a bear
-a photo of a kite
-a photo of a bench and a sports ball
-a photo of a toothbrush and a snowboard
-a photo of a toaster and an oven
-a photo of a broccoli and a vase
-a photo of a tennis racket and a wine glass
-a photo of a fork and a knife
-a photo of a hair drier and a cake
-a photo of a horse and a giraffe
-a photo of a horse and a computer keyboard
-a photo of a toothbrush and a carrot
-a photo of a cake and a zebra
-a photo of a hair drier and a bear
-a photo of a knife and a zebra
-a photo of a couch and a wine glass
-a photo of a frisbee and a vase
-a photo of a book and a laptop
-a photo of a dining table and a bear
-a photo of a frisbee and a couch
-a photo of a couch and a horse
-a photo of a toilet and a computer mouse
-a photo of a bottle and a refrigerator
-a photo of a potted plant and a backpack
-a photo of a skateboard and a cake
-a photo of a broccoli and a parking meter
-a photo of a zebra and a bed
-a photo of an oven and a bed
-a photo of a baseball bat and a fork
-a photo of a vase and a spoon
-a photo of a skateboard and a sink
-a photo of a pizza and a bench
-a photo of a bowl and a pizza
-a photo of a tennis racket and a bird
-a photo of a wine glass and a bear
-a photo of a fork and a book
-a photo of a scissors and a bowl
-a photo of a laptop and a carrot
-a photo of a stop sign and a bottle
-a photo of a microwave and a truck
-a photo of a person and a bear
-a photo of a frisbee and a cell phone
-a photo of a parking meter and a teddy bear
-a photo of a tennis racket and a bicycle
-a photo of a stop sign and a motorcycle
-a photo of a fire hydrant and a tennis racket
-a photo of a scissors and a sandwich
-a photo of a pizza and a book
-a photo of a giraffe and a computer mouse
-a photo of a stop sign and a toaster
-a photo of a computer mouse and a zebra
-a photo of a chair and a bench
-a photo of a tv and a carrot
-a photo of a surfboard and a suitcase
-a photo of a computer keyboard and a laptop
-a photo of a computer keyboard and a microwave
-a photo of a scissors and a bird
-a photo of a person and a snowboard
-a photo of a cow and a horse
-a photo of a handbag and a refrigerator
-a photo of a chair and a laptop
-a photo of a toothbrush and a bench
-a photo of a book and a baseball bat
-a photo of a horse and a train
-a photo of a bench and a vase
-a photo of a traffic light and a backpack
-a photo of a sports ball and a cow
-a photo of a computer mouse and a spoon
-a photo of a tv and a bicycle
-a photo of a bench and a snowboard
-a photo of a toothbrush and a toilet
-a photo of a person and an apple
-a photo of a sink and a sports ball
-a photo of a stop sign and a dog
-a photo of a knife and a stop sign
-a photo of a wine glass and a handbag
-a photo of a bowl and a skis
-a photo of a frisbee and an apple
-a photo of a computer keyboard and a cell phone
-a photo of a stop sign and a fork
-a photo of a potted plant and a boat
-a photo of a tv and a cell phone
-a photo of a tie and a broccoli
-a photo of a potted plant and a donut
-a photo of a person and a sink
-a photo of a couch and a snowboard
-a photo of a fork and a baseball glove
-a photo of an apple and a toothbrush
-a photo of a bus and a baseball glove
-a photo of a person and a stop sign
-a photo of a carrot and a couch
-a photo of a baseball bat and a bear
-a photo of a fire hydrant and a train
-a photo of a baseball glove and a carrot
-a photo of a microwave and a bench
-a photo of a cake and a stop sign
-a photo of a car and a computer mouse
-a photo of a suitcase and a dining table
-a photo of a person and a traffic light
-a photo of a cell phone and a horse
-a photo of a baseball bat and a giraffe
-a photo of two clocks
-a photo of two backpacks
-a photo of four handbags
-a photo of two frisbees
-a photo of three sports balls
-a photo of two bears
-a photo of two ties
-a photo of four sinks
-a photo of two toothbrushs
-a photo of three persons
-a photo of three tennis rackets
-a photo of four bowls
-a photo of four vases
-a photo of three cups
-a photo of four computer keyboards
-a photo of three sinks
-a photo of two ovens
-a photo of two toilets
-a photo of two bicycles
-a photo of two trains
-a photo of three oranges
-a photo of three buses
-a photo of three handbags
-a photo of three snowboards
-a photo of two snowboards
-a photo of four dogs
-a photo of three apples
-a photo of two sheeps
-a photo of three hot dogs
-a photo of three zebras
-a photo of three kites
-a photo of four apples
-a photo of three cell phones
-a photo of four baseball gloves
-a photo of three computer keyboards
-a photo of two beds
-a photo of two tv remotes
-a photo of three fire hydrants
-a photo of three books
-a photo of four giraffes
-a photo of two vases
-a photo of four donuts
-a photo of four chairs
-a photo of three baseball bats
-a photo of four stop signs
-a photo of two pizzas
-a photo of three refrigerators
-a photo of two fire hydrants
-a photo of three giraffes
-a photo of four tvs
-a photo of three wine glasses
-a photo of four broccolis
-a photo of three trucks
-a photo of two trucks
-a photo of two carrots
-a photo of two sandwichs
-a photo of four traffic lights
-a photo of four clocks
-a photo of two cars
-a photo of two bananas
-a photo of two wine glasses
-a photo of three pizzas
-a photo of four knifes
-a photo of three suitcases
-a photo of four zebras
-a photo of two teddy bears
-a photo of four skateboards
-a photo of four hot dogs
-a photo of three birds
-a photo of four boats
-a photo of four microwaves
-a photo of two hair driers
-a photo of three laptops
-a photo of three cows
-a photo of two parking meters
-a photo of four benchs
-a photo of three benchs
-a photo of four frisbees
-a photo of four books
-a photo of four buses
-a photo of a blue fire hydrant
-a photo of a pink car
-a photo of a purple cup
-a photo of a blue cow
-a photo of a yellow boat
-a photo of a blue umbrella
-a photo of a blue elephant
-a photo of a yellow elephant
-a photo of a red bicycle
-a photo of a purple suitcase
-a photo of a purple hair drier
-a photo of a white sandwich
-a photo of a purple elephant
-a photo of a green microwave
-a photo of a red zebra
-a photo of a red apple
-a photo of a yellow tv remote
-a photo of a blue toilet
-a photo of an orange orange
-a photo of a black donut
-a photo of a red vase
-a photo of a purple pizza
-a photo of a pink skateboard
-a photo of a green skateboard
-a photo of a purple bear
-a photo of a brown chair
-a photo of a brown computer keyboard
-a photo of an orange cow
-a photo of a brown skis
-a photo of a white kite
-a photo of a red dog
-a photo of a green couch
-a photo of a yellow airplane
-a photo of an orange tv
-a photo of a white scissors
-a photo of a pink cell phone
-a photo of a green surfboard
-a photo of a white fire hydrant
-a photo of a black bicycle
-a photo of a purple carrot
-a photo of a black dining table
-a photo of a purple potted plant
-a photo of a purple backpack
-a photo of a yellow train
-a photo of a pink potted plant
-a photo of a red giraffe
-a photo of a brown bear
-a photo of a black train
-a photo of an orange laptop
-a photo of a green hot dog
-a photo of a yellow parking meter
-a photo of a red potted plant
-a photo of a green traffic light
-a photo of a blue tv
-a photo of a brown refrigerator
-a photo of a black tv remote
-a photo of a purple scissors
-a photo of a yellow orange
-a photo of a brown toaster
-a photo of a red parking meter
-a photo of a brown orange
-a photo of a green clock
-a photo of a white sheep
-a photo of a yellow oven
-a photo of a green vase
-a photo of a black teddy bear
-a photo of a yellow carrot
-a photo of a black hot dog
-a photo of a red scissors
-a photo of a white teddy bear
-a photo of a black skis
-a photo of a blue dining table
-a photo of a black refrigerator
-a photo of a white dog
-a photo of an orange scissors
-a photo of a red cell phone
-a photo of a white orange
-a photo of a blue clock
-a photo of a blue carrot
-a photo of a green motorcycle
-a photo of a pink stop sign
-a photo of a black vase
-a photo of a black backpack
-a photo of a red car
-a photo of a green computer mouse
-a photo of a red backpack
-a photo of a green bus
-a photo of an orange toaster
-a photo of a yellow fork
-a photo of a pink parking meter
-a photo of a blue book
-a photo of a yellow broccoli
-a photo of an orange computer mouse
-a photo of a red cake
-a photo of a dog right of a teddy bear
-a photo of a wine glass above a kite
-a photo of a couch below a cup
-a photo of a laptop left of a cow
-a photo of a fork above a hair drier
-a photo of a tie right of a baseball bat
-a photo of a stop sign above a fork
-a photo of a bird below a skateboard
-a photo of an apple above a tv
-a photo of a train above a potted plant
-a photo of a truck left of a refrigerator
-a photo of a tv remote below a cow
-a photo of a bottle right of a train
-a photo of a dog above a cow
-a photo of a skateboard above a person
-a photo of a baseball glove below an umbrella
-a photo of a dining table right of an oven
-a photo of a hot dog left of a suitcase
-a photo of a bus below a toothbrush
-a photo of a backpack right of a sandwich
-a photo of a cake below a baseball bat
-a photo of a dog right of a tie
-a photo of a suitcase right of a boat
-a photo of a bear above a clock
-a photo of a tv remote left of an umbrella
-a photo of a sports ball left of an umbrella
-a photo of a train right of a dining table
-a photo of a hair drier below an elephant
-a photo of a tennis racket right of a spoon
-a photo of a wine glass right of a hot dog
-a photo of a computer mouse left of a bench
-a photo of a carrot left of an orange
-a photo of a kite above a toothbrush
-a photo of a toaster below a traffic light
-a photo of a cat below a baseball glove
-a photo of a skis right of a zebra
-a photo of a stop sign above a chair
-a photo of a stop sign above a parking meter
-a photo of a hot dog right of a skateboard
-a photo of a pizza below a computer keyboard
-a photo of a hair drier left of a toilet
-a photo of a cow left of a stop sign
-a photo of a suitcase above a skis
-a photo of a book above a laptop
-a photo of a toothbrush below a pizza
-a photo of a toilet left of a kite
-a photo of a tie above a sink
-a photo of a bird left of a couch
-a photo of a bed right of a sports ball
-a photo of an elephant below a surfboard
-a photo of a frisbee right of a motorcycle
-a photo of a vase above a fire hydrant
-a photo of a zebra left of an elephant
-a photo of a bench left of a bear
-a photo of a donut right of a bench
-a photo of a frisbee below a horse
-a photo of a computer keyboard above a snowboard
-a photo of a tv below a cow
-a photo of an elephant below a horse
-a photo of a suitcase left of a banana
-a photo of a train below an airplane
-a photo of a cat below a backpack
-a photo of a backpack below a cake
-a photo of a sandwich below a knife
-a photo of a bicycle above a parking meter
-a photo of a knife right of a suitcase
-a photo of a hot dog above a knife
-a photo of a zebra right of a parking meter
-a photo of a chair left of a zebra
-a photo of a cow below an airplane
-a photo of a cup left of an umbrella
-a photo of a zebra below a computer keyboard
-a photo of a zebra below a broccoli
-a photo of a laptop below a sports ball
-a photo of a truck left of a baseball bat
-a photo of a refrigerator above a baseball bat
-a photo of a tv above a baseball bat
-a photo of a baseball glove right of a bear
-a photo of a refrigerator below a scissors
-a photo of a dining table above a suitcase
-a photo of a parking meter above a broccoli
-a photo of a frisbee above a truck
-a photo of a pizza right of a banana
-a photo of a bus above a boat
-a photo of a cell phone left of a tennis racket
-a photo of a horse right of a broccoli
-a photo of a broccoli above a bottle
-a photo of a vase right of a horse
-a photo of a bear above a spoon
-a photo of a zebra right of a bed
-a photo of a cow right of a laptop
-a photo of a bed right of a frisbee
-a photo of a tie right of a motorcycle
-a photo of a laptop right of a tv
-a photo of a cell phone right of a chair
-a photo of a couch below a potted plant
-a photo of a clock below a tv
-a photo of a couch below a vase
-a photo of a donut below a cat
-a photo of a couch left of a toaster
-a photo of a purple wine glass and a black apple
-a photo of a green bus and a purple microwave
-a photo of a green skis and a brown airplane
-a photo of a yellow computer keyboard and a black sink
-a photo of a pink oven and a green motorcycle
-a photo of a purple parking meter and a red laptop
-a photo of a yellow skateboard and an orange computer mouse
-a photo of a red skis and a brown tie
-a photo of a pink skateboard and a black train
-a photo of a white handbag and a purple bed
-a photo of a purple elephant and a brown sports ball
-a photo of a purple dog and a black dining table
-a photo of a white dining table and a red car
-a photo of a blue cell phone and a green apple
-a photo of a red car and an orange potted plant
-a photo of a brown carrot and a white potted plant
-a photo of a black kite and a green bear
-a photo of a blue laptop and a brown bear
-a photo of a green teddy bear and a brown kite
-a photo of a yellow stop sign and a blue potted plant
-a photo of an orange snowboard and a green cat
-a photo of an orange truck and a pink sink
-a photo of a brown hot dog and a purple pizza
-a photo of a green couch and an orange umbrella
-a photo of a brown bed and a pink cell phone
-a photo of a black broccoli and a yellow cake
-a photo of a red train and a purple bear
-a photo of a purple tennis racket and a black sink
-a photo of a blue vase and a black banana
-a photo of a blue clock and a white cup
-a photo of a red umbrella and a blue couch
-a photo of a white handbag and a red giraffe
-a photo of a pink tv remote and a blue airplane
-a photo of a pink handbag and a black scissors
-a photo of a brown car and a pink hair drier
-a photo of a black bus and a brown cell phone
-a photo of a purple sheep and a pink banana
-a photo of a blue handbag and a white cell phone
-a photo of a white pizza and a green umbrella
-a photo of a white tie and a purple skateboard
-a photo of a yellow sports ball and a green boat
-a photo of a white wine glass and a brown giraffe
-a photo of a yellow bowl and a white baseball glove
-a photo of an orange microwave and a black spoon
-a photo of an orange skateboard and a pink bowl
-a photo of a blue toilet and a white suitcase
-a photo of a white boat and an orange hot dog
-a photo of a yellow dining table and a pink dog
-a photo of a red cake and a purple chair
-a photo of a blue tie and a pink dining table
-a photo of a blue cow and a black computer keyboard
-a photo of a yellow pizza and a green oven
-a photo of a red laptop and a brown car
-a photo of a purple computer keyboard and a blue scissors
-a photo of a green surfboard and an orange oven
-a photo of a yellow parking meter and a pink refrigerator
-a photo of a brown computer mouse and a purple bottle
-a photo of a red umbrella and a green cow
-a photo of a red giraffe and a black cell phone
-a photo of a brown oven and a purple train
-a photo of a blue baseball bat and a pink book
-a photo of a green cup and a yellow bowl
-a photo of a yellow suitcase and a brown bus
-a photo of an orange motorcycle and a pink donut
-a photo of an orange giraffe and a white baseball glove
-a photo of an orange handbag and a green carrot
-a photo of a black bottle and a white refrigerator
-a photo of a white dog and a blue potted plant
-a photo of an orange handbag and a red car
-a photo of a red stop sign and a blue book
-a photo of a yellow car and an orange toothbrush
-a photo of a black potted plant and a yellow toilet
-a photo of a brown dining table and a white suitcase
-a photo of an orange donut and a yellow stop sign
-a photo of a green suitcase and a blue boat
-a photo of an orange tennis racket and a yellow sports ball
-a photo of a purple computer keyboard and a red chair
-a photo of a purple suitcase and an orange pizza
-a photo of a white bottle and a blue sheep
-a photo of a purple backpack and a white umbrella
-a photo of an orange potted plant and a black spoon
-a photo of a green tennis racket and a black dog
-a photo of a yellow handbag and a blue refrigerator
-a photo of a pink broccoli and a red sink
-a photo of a red bowl and a pink sink
-a photo of a white toilet and a red apple
-a photo of a pink dining table and a black sandwich
-a photo of a black car and a green parking meter
-a photo of a yellow bird and a black motorcycle
-a photo of a brown giraffe and a white stop sign
-a photo of a white banana and a black elephant
-a photo of an orange cow and a purple sandwich
-a photo of a red clock and a black cell phone
-a photo of a brown knife and a blue donut
-a photo of a red cup and a pink handbag
-a photo of a yellow bicycle and a red motorcycle
-a photo of a red orange and a purple broccoli
-a photo of an orange traffic light and a white toilet
-a photo of a green cup and a red pizza
-a photo of a blue pizza and a yellow baseball glove

eval/gen/geneval/prompts/object_names.txt

@@ -1,80 +0,0 @@
-person
-bicycle
-car
-motorcycle
-airplane
-bus
-train
-truck
-boat
-traffic light
-fire hydrant
-stop sign
-parking meter
-bench
-bird
-cat
-dog
-horse
-sheep
-cow
-elephant
-bear
-zebra
-giraffe
-backpack
-umbrella
-handbag
-tie
-suitcase
-frisbee
-skis
-snowboard
-sports ball
-kite
-baseball bat
-baseball glove
-skateboard
-surfboard
-tennis racket
-bottle
-wine glass
-cup
-fork
-knife
-spoon
-bowl
-banana
-apple
-sandwich
-orange
-broccoli
-carrot
-hot dog
-pizza
-donut
-cake
-chair
-couch
-potted plant
-bed
-dining table
-toilet
-tv
-laptop
-computer mouse
-tv remote
-computer keyboard
-cell phone
-microwave
-oven
-toaster
-sink
-refrigerator
-book
-clock
-vase
-scissors
-teddy bear
-hair drier
-toothbrush

eval/gen/wise/cal_score.py

@@ -1,162 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-
-import json
-import os
-import argparse
-from collections import defaultdict
-
-
-def calculate_wiscore(consistency, realism, aesthetic_quality):
-    return 0.7 * consistency + 0.2 * realism + 0.1 * aesthetic_quality
-
-
-def cal_culture(file_path):
-    all_scores = []
-    total_objects = 0
-    has_9_9 = False
-    
-    with open(file_path, 'r') as file:
-        for line in file:
-            total_objects += 1
-            data = json.loads(line)
-            if 9.9 in [data['consistency'], data['realism'], data['aesthetic_quality']]:
-                has_9_9 = True
-            wiscore = calculate_wiscore(data['consistency'], data['realism'], data['aesthetic_quality'])
-            all_scores.append(wiscore)
-    
-    if has_9_9 or total_objects < 400:
-        print(f"Skipping file {file_path}: Contains 9.9 or has less than 400 objects.")
-        return None
-    
-    total_score = sum(all_scores)
-    avg_score = total_score / (len(all_scores)*2) if len(all_scores) > 0 else 0
-    
-    score = {
-        'total': total_score,
-        'average': avg_score
-    }
-
-    print(f"  Cultural - Total: {score['total']:.2f}, Average: {score['average']:.2f}")
-
-    return avg_score
-
-
-def cal_space_time(file_path):
-    categories = defaultdict(list)
-    total_objects = 0
-    has_9_9 = False
-    
-    with open(file_path, 'r') as file:
-        for line in file:
-            total_objects += 1
-            data = json.loads(line)
-            if 9.9 in [data['consistency'], data['realism'], data['aesthetic_quality']]:
-                has_9_9 = True
-            subcategory = data['Subcategory']
-            wiscore = calculate_wiscore(data['consistency'], data['realism'], data['aesthetic_quality'])
-            if subcategory in ['Longitudinal time', 'Horizontal time']:
-                categories['Time'].append(wiscore)
-            else:
-                categories['Space'].append(wiscore)
-    
-    if has_9_9 or total_objects < 300:
-        print(f"Skipping file {file_path}: Contains 9.9 or has less than 400 objects.")
-        return None
-    
-    total_scores = {category: sum(scores) for category, scores in categories.items()}
-    avg_scores = {category: sum(scores) / (len(scores) * 2 )if len(scores) > 0 else 0 for category, scores in categories.items()}
-    
-    scores = {
-        'total': total_scores,
-        'average': avg_scores
-    }
-
-    print(f"  Time - Total: {scores['total'].get('Time', 0):.2f}, Average: {scores['average'].get('Time', 0):.2f}")
-    print(f"  Space - Total: {scores['total'].get('Space', 0):.2f}, Average: {scores['average'].get('Space', 0):.2f}")
-
-    return avg_scores
-
-
-def cal_science(file_path):
-    categories = defaultdict(list)
-    total_objects = 0
-    has_9_9 = False
-    
-    with open(file_path, 'r') as file:
-        for line in file:
-            total_objects += 1
-            data = json.loads(line)
-            if 9.9 in [data['consistency'], data['realism'], data['aesthetic_quality']]:
-                has_9_9 = True
-            
-            prompt_id = data.get('prompt_id', 0)
-            if 701 <= prompt_id <= 800:
-                category = 'Biology'
-            elif 801 <= prompt_id <= 900:
-                category = 'Physics'
-            elif 901 <= prompt_id <= 1000:
-                category = 'Chemistry'
-            else:
-                category = "?"
-            
-            wiscore = calculate_wiscore(data['consistency'], data['realism'], data['aesthetic_quality'])
-            categories[category].append(wiscore)
-    
-    if has_9_9 or total_objects < 300: 
-        print(f"Skipping file {file_path}: Contains 9.9 or has less than 300 objects.")
-        return None
-    
-    total_scores = {category: sum(scores) for category, scores in categories.items()}
-    avg_scores = {category: sum(scores) / (len(scores)*2) if len(scores) > 0 else 0 for category, scores in categories.items()}
-
-    scores = {
-        'total': total_scores,
-        'average': avg_scores
-    }
-
-    for category in ['Biology', 'Physics', 'Chemistry']:
-        print(f"  {category} - Total: {scores['total'].get(category, 0):.2f}, Average: {scores['average'].get(category, 0):.2f}")
-    
-    return avg_scores
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Image Quality Assessment Tool')
-    parser.add_argument('--output_dir', required=True,
-                        help='Path to the output directory')
-    args = parser.parse_args()
-
-    avg_score = dict()
-
-    score = cal_culture(
-        os.path.join(args.output_dir, "cultural_common_sense_scores.jsonl")
-    )
-    avg_score['Cultural'] = score
-
-    scores = cal_space_time(
-        os.path.join(args.output_dir, "spatio-temporal_reasoning_scores.jsonl")
-    )
-    avg_score.update(scores)
-
-    scores = cal_science(
-        os.path.join(args.output_dir, "natural_science_scores.jsonl")
-    )
-    avg_score.update(scores)
-
-    avg_all = sum(avg_score.values()) / len(avg_score)
-
-    avg_score['Overall'] = avg_all
-    keys = ""
-    values = ""
-    for k, v in avg_score.items():
-        keys += f"{k} "
-        values += f"{v:.2f} "
-    print(keys)
-    print(values)
-
-    writer = open(os.path.join(args.output_dir, "results.txt"), 'w')
-    print(f"write results to file {os.path.join(args.output_dir, 'results.txt')}")
-    writer.write(keys + "\n")
-    writer.write(values + "\n")
-    writer.close()

eval/gen/wise/gpt_eval_mp.py

@@ -1,268 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-
-import json
-import os
-import base64
-import re
-import argparse
-import openai
-from pathlib import Path
-from typing import Dict, Any, List
-import concurrent.futures
-
-openai.api_key = os.getenv('OPENAI_API_KEY')
-print(openai.api_key)
-
-
-def parse_arguments():
-    parser = argparse.ArgumentParser(description='Image Quality Assessment Tool')
-
-    parser.add_argument('--json_path', required=True,
-                        help='Path to the prompts JSON file')
-    parser.add_argument('--image_dir', required=True,
-                        help='Path to the image directory')
-    parser.add_argument('--output_dir', required=True,
-                        help='Path to the output directory')
-
-    return parser.parse_args()
-
-
-def get_config(args):
-    filename = args.json_path.split("/")[-1].split(".")[0]
-    return {
-        "json_path": args.json_path,
-        "image_dir": args.image_dir,
-        "output_dir": args.output_dir,
-        "result_files": {
-            "full": f"{filename}_full.jsonl",
-            "scores": f"{filename}_scores.jsonl",
-        }
-    }
-
-
-def extract_scores(evaluation_text: str) -> Dict[str, float]:
-    score_pattern = r"\*{0,2}(Consistency|Realism|Aesthetic Quality)\*{0,2}\s*[:：]?\s*(\d)"
-    matches = re.findall(score_pattern, evaluation_text, re.IGNORECASE)
-
-    scores = {
-        "consistency": 9.9,
-        "realism": 9.9,
-        "aesthetic_quality": 9.9
-    }
-
-    for key, value in matches:
-        key = key.lower().replace(" ", "_")
-        if key in scores:
-            scores[key] = float(value)
-
-    return scores
-
-
-def encode_image(image_path: str) -> str:
-    with open(image_path, "rb") as image_file:
-        return base64.b64encode(image_file.read()).decode('utf-8')
-
-
-def load_prompts(json_path: str) -> Dict[int, Dict[str, Any]]:
-    with open(json_path, 'r') as f:
-        data = json.load(f)
-    return {item["prompt_id"]: item for item in data}
-
-
-def build_evaluation_messages(prompt_data: Dict, image_base64: str) -> list:
-    return [
-        {
-            "role": "system",
-            "content": [
-                {
-                    "type": "text",
-                    "text": "You are a professional Vincennes image quality audit expert, please evaluate the image quality strictly according to the protocol."
-                }
-            ]
-        },
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "text",
-                    "text": f"""Please evaluate strictly and return ONLY the three scores as requested.
-
-# Text-to-Image Quality Evaluation Protocol
-
-## System Instruction
-You are an AI quality auditor for text-to-image generation. Apply these rules with ABSOLUTE RUTHLESSNESS. Only images meeting the HIGHEST standards should receive top scores.
-
-**Input Parameters**  
-- PROMPT: [User's original prompt to]  
-- EXPLANATION: [Further explanation of the original prompt] 
----
-
-## Scoring Criteria
-
-**Consistency (0-2):**  How accurately and completely the image reflects the PROMPT.
-* **0 (Rejected):**  Fails to capture key elements of the prompt, or contradicts the prompt.
-* **1 (Conditional):** Partially captures the prompt. Some elements are present, but not all, or not accurately.  Noticeable deviations from the prompt's intent.
-* **2 (Exemplary):**  Perfectly and completely aligns with the PROMPT.  Every single element and nuance of the prompt is flawlessly represented in the image. The image is an ideal, unambiguous visual realization of the given prompt.
-
-**Realism (0-2):**  How realistically the image is rendered.
-* **0 (Rejected):**  Physically implausible and clearly artificial. Breaks fundamental laws of physics or visual realism.
-* **1 (Conditional):** Contains minor inconsistencies or unrealistic elements.  While somewhat believable, noticeable flaws detract from realism.
-* **2 (Exemplary):**  Achieves photorealistic quality, indistinguishable from a real photograph.  Flawless adherence to physical laws, accurate material representation, and coherent spatial relationships. No visual cues betraying AI generation.
-
-**Aesthetic Quality (0-2):**  The overall artistic appeal and visual quality of the image.
-* **0 (Rejected):**  Poor aesthetic composition, visually unappealing, and lacks artistic merit.
-* **1 (Conditional):**  Demonstrates basic visual appeal, acceptable composition, and color harmony, but lacks distinction or artistic flair.
-* **2 (Exemplary):**  Possesses exceptional aesthetic quality, comparable to a masterpiece.  Strikingly beautiful, with perfect composition, a harmonious color palette, and a captivating artistic style. Demonstrates a high degree of artistic vision and execution.
-
----
-
-## Output Format
-
-**Do not include any other text, explanations, or labels.** You must return only three lines of text, each containing a metric and the corresponding score, for example:
-
-**Example Output:**
-Consistency: 2
-Realism: 1
-Aesthetic Quality: 0
-
----
-
-**IMPORTANT Enforcement:**
-
-Be EXTREMELY strict in your evaluation. A score of '2' should be exceedingly rare and reserved only for images that truly excel and meet the highest possible standards in each metric. If there is any doubt, downgrade the score.
-
-For **Consistency**, a score of '2' requires complete and flawless adherence to every aspect of the prompt, leaving no room for misinterpretation or omission.
-
-For **Realism**, a score of '2' means the image is virtually indistinguishable from a real photograph in terms of detail, lighting, physics, and material properties.
-
-For **Aesthetic Quality**, a score of '2' demands exceptional artistic merit, not just pleasant visuals.
-
---- 
-Here are the Prompt and EXPLANATION for this evaluation:
-PROMPT: "{prompt_data['Prompt']}"
-EXPLANATION: "{prompt_data['Explanation']}"
-Please strictly adhere to the scoring criteria and follow the template format when providing your results."""
-                },
-                {
-                    "type": "image_url",
-                    "image_url": {
-                        "url": f"data:image/png;base64,{image_base64}"
-                    }
-                }
-            ]
-        }
-    ]
-
-
-def evaluate_image(prompt_data: Dict, image_path: str, config: Dict) -> Dict[str, Any]:
-    try:
-        base64_image = encode_image(image_path)
-        messages = build_evaluation_messages(prompt_data, base64_image)
-
-        response = openai_client.chat.completions.create(
-            model=model,
-            messages=messages,
-            temperature=0.0,
-            max_tokens=2000,
-            n=1,
-        )
-        response = response.to_dict()
-
-        evaluation_text = response['choices'][0]['message']['content'].strip()
-        scores = extract_scores(evaluation_text)
-
-        return {
-            "evaluation": evaluation_text,
-            **scores
-        }
-    except Exception as e:
-        return {
-            "evaluation": f"Evaluation failed: {str(e)}",
-            "consistency": 9.9,
-            "realism": 9.9,
-            "aesthetic_quality": 9.9
-        }
-
-
-def save_results(data, filename, config):
-    path = os.path.join(config["output_dir"], filename)
-
-    assert filename.endswith('.jsonl')
-    with open(path, 'a', encoding='utf-8') as f:
-        json_line = json.dumps(data, ensure_ascii=False)
-        f.write(json_line + '\n')
-
-
-def process_prompt(prompt_id, prompt_data, config):
-    image_path = os.path.join(config["image_dir"], f"{prompt_id}.png")
-
-    if not os.path.exists(image_path):
-        print(f"Warning: Image not found {image_path}")
-        return None
-
-    print(f"Evaluating prompt_id: {prompt_id}...")
-    evaluation_result = evaluate_image(prompt_data, image_path, config)
-
-    full_record = {
-        "prompt_id": prompt_id,
-        "prompt": prompt_data["Prompt"],
-        "key": prompt_data["Explanation"],
-        "image_path": image_path,
-        "evaluation": evaluation_result["evaluation"]
-    }
-
-    score_record = {
-        "prompt_id": prompt_id,
-        "Subcategory": prompt_data["Subcategory"],
-        "consistency": evaluation_result["consistency"],
-        "realism": evaluation_result["realism"],
-        "aesthetic_quality": evaluation_result["aesthetic_quality"]
-    }
-
-    return full_record, score_record
-
-
-if __name__ == "__main__":
-    api_key = openai.api_key
-    base_url = "your_api_url",
-    api_version = "2024-03-01-preview"
-    model = "gpt-4o-2024-11-20"
-
-    openai_client = openai.AzureOpenAI(
-        azure_endpoint=base_url,
-        api_version=api_version,
-        api_key=api_key,
-    )
-
-    args = parse_arguments()
-    config = get_config(args)
-    Path(config["output_dir"]).mkdir(parents=True, exist_ok=True)
-
-    prompts = load_prompts(config["json_path"])
-    
-    processed_ids = set()
-    if os.path.exists(os.path.join(config["output_dir"], config["result_files"]["full"])):
-        with open(os.path.join(config["output_dir"], config["result_files"]["full"]), 'r', encoding='utf-8') as f:
-            for line in f:
-                data = json.loads(line)
-                processed_ids.add(data["prompt_id"])
-    left_prompts = {k: v for k, v in prompts.items() if k not in processed_ids}
-    print(f"Process {len(left_prompts)} prompts...")
-
-    MAX_THREADS = 30
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
-        futures = [executor.submit(process_prompt, prompt_id, prompt_data, config)
-                   for prompt_id, prompt_data in left_prompts.items()]
-        for future in concurrent.futures.as_completed(futures):
-            try:
-                result = future.result()
-                if result:
-                    full_record, score_record = result
-                    print(full_record)
-                    save_results(full_record, config["result_files"]["full"], config)
-                    save_results(score_record, config["result_files"]["scores"], config)
-
-            except Exception as e:
-                print(f"An error occurred: {e}")

eval/vlm/__init__.py

@@ -1,2 +0,0 @@
-# Copyright 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0

eval/vlm/eval/mathvista/calculate_score.py

@@ -1,271 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-
-import pandas as pd
-# !pip install python-Levenshtein
-from Levenshtein import distance
-from utilities import *
-
-
-def get_most_similar(prediction, choices):
-    """
-    Use the Levenshtein distance (or edit distance) to determine which of the choices is most similar to the given prediction
-    """
-    distances = [distance(prediction, choice) for choice in choices]
-    ind = distances.index(min(distances))
-    return choices[ind]
-    # return min(choices, key=lambda choice: distance(prediction, choice))
-
-
-def normalize_extracted_answer(extraction, choices, question_type, answer_type, precision):
-    """
-    Normalize the extracted answer to match the answer type
-    """
-    if question_type == 'multi_choice':
-        # make sure the extraction is a string
-        if isinstance(extraction, str):
-            extraction = extraction.strip()
-        else:
-            try:
-                extraction = str(extraction)
-            except:
-                extraction = ''
-
-        # extract "A" from "(A) text"
-        letter = re.findall(r'\(([a-zA-Z])\)', extraction)
-        if len(letter) > 0:
-            extraction = letter[0].upper()
-
-        options = [chr(ord('A') + i) for i in range(len(choices))]
-
-        if extraction in options:
-            # convert option letter to text, e.g. "A" -> "text"
-            ind = options.index(extraction)
-            extraction = choices[ind]
-        else:
-            # select the most similar option
-            extraction = get_most_similar(extraction, choices)
-        assert extraction in choices
-
-    elif answer_type == 'integer':
-        try:
-            extraction = str(int(float(extraction)))
-        except:
-            extraction = None
-
-    elif answer_type == 'float':
-        try:
-            extraction = str(round(float(extraction), int(precision)))
-        except:
-            extraction = None
-
-    elif answer_type == 'list':
-        try:
-            extraction = str(extraction)
-        except:
-            extraction = None
-
-    return extraction
-
-
-def safe_equal(prediction, answer):
-    """
-    Check if the prediction is equal to the answer, even if they are of different types
-    """
-    try:
-        if prediction == answer:
-            return True
-        return False
-    except Exception as e:
-        print(e)
-        return False
-
-
-def get_acc_with_contion(res_pd, key, value):
-    if key == 'skills':
-        # if value in res_pd[key]:
-        total_pd = res_pd[res_pd[key].apply(lambda x: value in x)]
-    else:
-        total_pd = res_pd[res_pd[key] == value]
-
-    correct_pd = total_pd[total_pd['true_false'] == True] # noqa: E712
-    acc = '{:.2f}'.format(len(correct_pd) / len(total_pd) * 100)
-    return len(correct_pd), len(total_pd), acc
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--output_dir', type=str, default='./results')
-    parser.add_argument('--output_file', type=str, default='output.json')
-    parser.add_argument('--score_file', type=str, default='scores.json')
-    parser.add_argument('--gt_file', type=str, default='./eval/vlm/data/MathVista/annot_testmini.json', help='ground truth file')
-    parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
-    parser.add_argument('--rerun', action='store_true', help='rerun the evaluation')
-    parser.add_argument('--caculate_gain', action='store_true', help='caculate the score gains over random guess')
-    parser.add_argument('--random_file', type=str, default='score_random_guess.json')
-    args = parser.parse_args()
-
-    # args
-    output_file = os.path.join(args.output_dir, args.output_file)
-
-    # # quick test
-    # output_file = '../results/llava-llama-2-13b/output_llava_llama_2_13b.json'
-
-    # read json
-    print(f'Reading {output_file}...')
-    results = read_json(output_file)
-
-    # read ground truth
-    print(f'Reading {args.gt_file}...')
-    gts = read_json(args.gt_file)
-
-    # full pids
-    full_pids = list(results.keys())
-    if args.number > 0:
-        full_pids = full_pids[:min(args.number, len(full_pids))]
-    print('Number of testing problems:', len(full_pids))
-
-    ## [1] Evaluate if the prediction is true or false
-    print('\nEvaluating the predictions...')
-    update_json_flag = False
-    for pid in full_pids:
-        problem = results[pid]
-        # print(problem)
-
-        if args.rerun:
-            if 'prediction' in problem:
-                del problem['prediction']
-            if 'true_false' in problem:
-                del problem['true_false']
-
-        choices = problem['choices']
-        question_type = problem['question_type']
-        answer_type = problem['answer_type']
-        precision = problem['precision']
-        extraction = problem['extraction']
-
-        if 'answer' in problem:
-            answer = problem['answer']
-        else:
-            if pid in gts:
-                answer = gts[pid]['answer']
-            else:
-                answer = ''
-            problem['answer'] = answer
-
-        # normalize the extracted answer to match the answer type
-        prediction = normalize_extracted_answer(extraction, choices, question_type, answer_type, precision)
-
-        # verify the prediction is true or false
-        true_false = safe_equal(prediction, answer)
-
-        # update the problem
-        if 'true_false' not in problem:
-            update_json_flag = True
-
-        elif true_false != problem['true_false']:
-            update_json_flag = True
-
-        if 'prediction' not in problem:
-            update_json_flag = True
-
-        elif prediction != problem['prediction']:
-            update_json_flag = True
-
-        problem['prediction'] = prediction
-        problem['true_false'] = true_false
-
-    # save the updated json
-    if update_json_flag:
-        print('\n!!!Some problems are updated.!!!')
-        print(f'\nSaving {output_file}...')
-        save_json(results, output_file)
-
-    ## [2] Calculate the average accuracy
-    total = len(full_pids)
-    correct = 0
-    for pid in full_pids:
-        if results[pid]['true_false']:
-            correct += 1
-    accuracy = str(round(correct / total * 100, 2))
-    print(f'\nCorrect: {correct}, Total: {total}, Accuracy: {accuracy}%')
-
-    scores = {'average': {'accuracy': accuracy, 'correct': correct, 'total': total}}
-
-    ## [3] Calculate the fine-grained accuracy scores
-
-    # merge the 'metadata' attribute into the data
-    for pid in results:
-        results[pid].update(results[pid].pop('metadata'))
-
-    # convert the data to a pandas DataFrame
-    df = pd.DataFrame(results).T
-
-    print(len(df))
-    print('Number of test problems:', len(df))
-    # assert len(df) == 1000 # Important!!!
-
-    # asign the target keys for evaluation
-    target_keys = ['question_type', 'answer_type', 'language', 'source', 'category', 'task', 'context', 'grade',
-                   'skills']
-
-    for key in target_keys:
-        print(f'\nType: [{key}]')
-        # get the unique values of the key
-        if key == 'skills':
-            # the value is a list
-            values = []
-            for i in range(len(df)):
-                values += df[key][i]
-            values = list(set(values))
-        else:
-            values = df[key].unique()
-        # print(values)
-
-        # calculate the accuracy for each value
-        scores[key] = {}
-        for value in values:
-            correct, total, acc = get_acc_with_contion(df, key, value)
-            if total > 0:
-                print(f'[{value}]: {acc}% ({correct}/{total})')
-                scores[key][value] = {'accuracy': acc, 'correct': correct, 'total': total}
-
-        # sort the scores by accuracy
-        scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]['accuracy']), reverse=True))
-
-    # save the scores
-    scores_file = os.path.join(args.output_dir, args.score_file)
-    print(f'\nSaving {scores_file}...')
-    save_json(scores, scores_file)
-    print('\nDone!')
-
-    # [4] Calculate the score gains over random guess
-    if args.caculate_gain:
-        random_file = os.path.join(args.output_dir, args.random_file)
-        random_scores = json.load(open(random_file))
-
-        print('\nCalculating the score gains...')
-        for key in scores:
-            if key == 'average':
-                gain = round(float(scores[key]['accuracy']) - float(random_scores[key]['accuracy']), 2)
-                scores[key]['acc_gain'] = gain
-            else:
-                for sub_key in scores[key]:
-                    gain = round(
-                        float(scores[key][sub_key]['accuracy']) - float(random_scores[key][sub_key]['accuracy']), 2)
-                    scores[key][sub_key]['acc_gain'] = str(gain)
-
-        # save the score gains
-        print(f'\nSaving {scores_file}...')
-        save_json(scores, scores_file)
-        print('\nDone!')

eval/vlm/eval/mathvista/evaluate_mathvista.py

@@ -1,210 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import itertools
-import json
-import os
-import random
-
-import torch
-from datasets import concatenate_datasets, load_dataset
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from tqdm import tqdm
-
-ds_collections = {
-    'MathVista_testmini': {
-        'root': 'AI4Math/MathVista',
-        'max_new_tokens': 4096,
-        'min_new_tokens': 1,
-        'split': 'testmini'
-    },
-    'MathVista_test': {
-        'root': 'AI4Math/MathVista',
-        'max_new_tokens': 4096,
-        'min_new_tokens': 1,
-        'split': 'test'
-    },
-}
-
-
-COT_INSTRUCTION = (
-    'Your task is to answer the question below. '
-    "Give step by step reasoning before you answer, and when you're ready to answer, "
-    "please use the format \"Final answer: ..\""
-    '\n\n'
-    'Question:'
-    '\n\n'
-    '{question}'
-)
-
-
-def collate_fn(batches):
-    images = [_['images'] for _ in batches]
-    data_items = [_['data_item'] for _ in batches]
-    return images, data_items
-
-
-class MathVistaDataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, split):
-        dataset = load_dataset(root, cache_dir=os.path.join(os.getcwd(), 'eval/vlm/data/MathVista/'))
-        self.data = dataset[split]
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        data_item = self.data[idx]
-        image = data_item['decoded_image']
-        del data_item['decoded_image']
-
-        images = [image.convert('RGB') if image.mode != 'RGB' else image]
-
-        return {
-            'images': images,
-            'data_item': data_item,
-        }
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def evaluate_chat_model():
-    random.seed(args.seed)
-
-    for ds_name in args.datasets:
-        dataset = MathVistaDataset(
-            root=ds_collections[ds_name]['root'],
-            split=ds_collections[ds_name]['split'],
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        outputs = []
-        for _, (images, data_items) in tqdm(enumerate(dataloader)):
-            if args.cot:
-                question = COT_INSTRUCTION.format(question=data_items[0]['query'])
-            else:
-                question = data_items[0]['query']
-
-            images = images[0]
-            images, conversation = process_conversation(images, question)
-
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images,
-                prompt=conversation,
-                max_length=ds_collections[ds_name]['max_new_tokens'] if not args.cot else 4096, # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-
-            data_item = data_items[0]
-            data_item['response'] = pred
-            outputs.append(data_item)
-
-        torch.distributed.barrier()
-
-        world_size = torch.distributed.get_world_size()
-        merged_outputs = [None for _ in range(world_size)]
-        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
-
-        merged_outputs = [json.loads(_) for _ in merged_outputs]
-        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
-
-        if torch.distributed.get_rank() == 0:
-            temp = {}
-            for data_item in merged_outputs:
-                pid = data_item['pid']
-                temp[pid] = data_item
-
-            print(f'Evaluating {ds_name} ...')
-            results_file = 'results.json'
-            output_path = os.path.join(args.out_dir, 'results.json')
-            json.dump(temp, open(output_path, 'w'), indent=4)
-            print('Results saved to {}'.format(output_path))
-
-            if args.cot:
-                cmd = f'python eval/vlm/eval/mathvista/extract_answer_mp.py --output_file {results_file} --output_dir {args.out_dir}'
-            else:
-                cmd = f'python eval/vlm/eval/mathvista/extract_answer_mp.py --output_file {results_file} --output_dir {args.out_dir}'
-            print(cmd)
-            os.system(cmd)
-
-            cmd = f'python eval/vlm/eval/mathvista/calculate_score.py --output_file {results_file} --output_dir {args.out_dir} --score_file score.json'
-            print(cmd)
-            os.system(cmd)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='MathVista_testmini')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--cot', action='store_true')
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()

eval/vlm/eval/mathvista/extract_answer.py

@@ -1,160 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-
-from tqdm import tqdm
-from utilities import *
-
-openai.api_key = os.getenv('OPENAI_API_KEY')
-print(openai.api_key)
-
-# load demo prompt
-from prompts.ext_ans import demo_prompt
-
-
-def verify_extraction(extraction):
-    extraction = extraction.strip()
-    if extraction == '' or extraction is None:
-        return False
-    return True
-
-
-def create_test_prompt(demo_prompt, query, response):
-    demo_prompt = demo_prompt.strip()
-    test_prompt = f'{query}\n\n{response}'
-    full_prompt = f'{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: '
-    return full_prompt
-
-
-def _extract_answer(text):
-    match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE)
-    if match:
-        return match.group(2).strip()
-    return text
-
-
-def extract_answer(response, problem, quick_extract=False):
-    question_type = problem['question_type']
-    answer_type = problem['answer_type']
-    choices = problem['choices']
-    query = problem['query']
-
-    if response == '':
-        return ''
-
-    if question_type == 'multi_choice' and response in choices:
-        return response
-
-    if answer_type == 'integer':
-        try:
-            extraction = int(response)
-            return str(extraction)
-        except:
-            pass
-
-    if answer_type == 'float':
-        try:
-            extraction = str(float(response))
-            return extraction
-        except:
-            pass
-
-    # quick extraction
-    if quick_extract:
-        print('Quickly extracting answer...')
-        # The answer is "text". -> "text"
-        try:
-            result = _extract_answer(response)
-            return result
-            # result = re.search(r'The answer is "(.*)"\.', response)
-            # if result:
-            #     extraction = result.group(1)
-            #     return extraction
-        except:
-            pass
-
-    # general extraction
-    try:
-        full_prompt = create_test_prompt(demo_prompt, query, response)
-        extraction = get_chat_response(full_prompt, openai.api_key, patience=5)
-        return extraction
-    except Exception as e:
-        print(e)
-        print(f'Error in extracting answer for {pid}')
-
-    return ''
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    # input
-    parser.add_argument('--output_dir', type=str, default='./results')
-    parser.add_argument('--output_file', type=str, default='mathvista_answer.json')
-    parser.add_argument('--response_label', type=str, default='response', help='response label for the input file')
-    # model
-    parser.add_argument('--llm_engine', type=str, default='gpt-4-0613', help='llm engine',
-                        choices=['gpt-3.5-turbo', 'gpt-3.5', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613'])
-    parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
-    parser.add_argument('--quick_extract', action='store_true', help='use rules to extract answer for some problems')
-    parser.add_argument('--rerun', action='store_true', help='rerun the answer extraction')
-    # output
-    parser.add_argument('--save_every', type=int, default=10, help='save every n problems')
-    parser.add_argument('--output_label', type=str, default='', help='label for the output file')
-    args = parser.parse_args()
-
-    # args
-    label = args.response_label
-    result_file = os.path.join(args.output_dir, args.output_file)
-
-    if args.output_label != '':
-        output_file = result_file.replace('.json', f'_{args.output_label}.json')
-    else:
-        output_file = result_file
-
-    # read results
-    print(f'Reading {result_file}...')
-    results = read_json(result_file)
-
-    # full pids
-    full_pids = list(results.keys())
-    if args.number > 0:
-        full_pids = full_pids[:min(args.number, len(full_pids))]
-    print('Number of testing problems:', len(full_pids))
-
-    # test pids
-    if args.rerun:
-        test_pids = full_pids
-    else:
-        test_pids = []
-        for pid in full_pids:
-            # print(pid)
-            if 'extraction' not in results[pid] or not verify_extraction(results[pid]['extraction']):
-                test_pids.append(pid)
-
-    test_num = len(test_pids)
-    print('Number of problems to run:', test_num)
-    # print(test_pids)
-
-    # tqdm, enumerate results
-    for i, pid in enumerate(tqdm(test_pids)):
-        problem = results[pid]
-
-        assert label in problem
-        response = problem[label]
-
-        extraction = extract_answer(response, problem, args.quick_extract)
-        results[pid]['extraction'] = extraction
-
-        if i % args.save_every == 0 or i == test_num - 1:
-            print(f'Saving results to {output_file}...')
-            save_json(results, output_file)
-            print(f'Results saved.')

eval/vlm/eval/mathvista/extract_answer_mp.py

@@ -1,161 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-
-import argparse
-import os
-import re
-import json
-import openai
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from tqdm import tqdm
-from utilities import *
-from prompts.ext_ans import demo_prompt
-
-openai.api_key = os.getenv('OPENAI_API_KEY')
-print(openai.api_key)
-
-def verify_extraction(extraction):
-    extraction = extraction.strip()
-    if extraction == '' or extraction is None:
-        return False
-    return True
-
-def create_test_prompt(demo_prompt, query, response):
-    demo_prompt = demo_prompt.strip()
-    test_prompt = f'{query}\n\n{response}'
-    full_prompt = f'{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: '
-    return full_prompt
-
-def _extract_answer(text):
-    match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE)
-    if match:
-        return match.group(2).strip()
-    return text
-
-def extract_answer(response, problem, quick_extract=False):
-    question_type = problem['question_type']
-    answer_type = problem['answer_type']
-    choices = problem['choices']
-    query = problem['query']
-
-    if response == '':
-        return ''
-
-    if question_type == 'multi_choice' and response in choices:
-        return response
-
-    if answer_type == 'integer':
-        try:
-            extraction = int(response)
-            return str(extraction)
-        except:
-            pass
-
-    if answer_type == 'float':
-        try:
-            extraction = str(float(response))
-            return extraction
-        except:
-            pass
-
-    # quick extraction
-    if quick_extract:
-        print('Quickly extracting answer...')
-        try:
-            result = _extract_answer(response)
-            return result
-        except:
-            pass
-
-    try:
-        full_prompt = create_test_prompt(demo_prompt, query, response)
-        extraction = get_chat_response(full_prompt, openai.api_key, patience=5, model=args.llm_engine)
-        return extraction
-    except Exception as e:
-        print(e)
-
-    return ''
-
-def process_problem(pid, results, label, args):
-    problem = results[pid]
-    response = problem[label]
-    extraction = extract_answer(response, problem, args.quick_extract)
-    return pid, extraction
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    # input
-    parser.add_argument('--output_dir', type=str, default='./results')
-    parser.add_argument('--output_file', type=str, default='mathvista_answer.json')
-    parser.add_argument('--response_label', type=str, default='response', help='response label for the input file')
-    # model
-    parser.add_argument('--llm_engine', type=str, default='gpt-4o-2024-11-20', help='llm engine',
-                        choices=['gpt-3.5-turbo', 'gpt-3.5', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613',
-                                 'gpt-4o-2024-08-06', 'gpt-4o-2024-11-20'])
-    parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
-    parser.add_argument('--quick_extract', action='store_true', help='use rules to extract answer for some problems')
-    parser.add_argument('--rerun', action='store_true', help='rerun the answer extraction')
-    # output
-    parser.add_argument('--save_every', type=int, default=100, help='save every n problems')
-    parser.add_argument('--output_label', type=str, default='', help='label for the output file')
-    parser.add_argument('--max_workers', type=int, default=40, help='max workers for ThreadPoolExecutor')
-    args = parser.parse_args()
-
-    label = args.response_label
-    result_file = os.path.join(args.output_dir, args.output_file)
-
-    if args.output_label != '':
-        output_file = result_file.replace('.json', f'_{args.output_label}.json')
-    else:
-        output_file = result_file
-
-    print(f'Reading {result_file}...')
-    results = read_json(result_file)
-
-    full_pids = list(results.keys())
-    if args.number > 0:
-        full_pids = full_pids[:min(args.number, len(full_pids))]
-    print('Number of total problems:', len(full_pids))
-
-    if args.rerun:
-        test_pids = full_pids
-    else:
-        test_pids = []
-        for pid in full_pids:
-            if 'extraction' not in results[pid] or not verify_extraction(results[pid]['extraction']):
-                test_pids.append(pid)
-
-    test_num = len(test_pids)
-    print('Number of problems to run:', test_num)
-
-    with ThreadPoolExecutor(max_workers=args.max_workers) as executor:
-        future_to_pid = {}
-        for pid in test_pids:
-            future = executor.submit(process_problem, pid, results, label, args)
-            future_to_pid[future] = pid
-
-        completed_count = 0
-        for future in tqdm(as_completed(future_to_pid), total=test_num):
-            pid = future_to_pid[future]
-            try:
-                pid_result, extraction = future.result()
-                results[pid_result]['extraction'] = extraction
-            except Exception as e:
-                print(f'Error processing pid={pid}: {e}')
-
-            completed_count += 1
-            if (completed_count % args.save_every == 0) or (completed_count == test_num):
-                print(f'Saving results to {output_file}... [{completed_count}/{test_num}]')
-                save_json(results, output_file)
-                print('Results saved.')
-
-    print('All done!')

eval/vlm/eval/mathvista/prompts/ext_ans.py

@@ -1,51 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-# pids = 852,  104,  824,  506,  540
-
-demo_prompt = """
-Please read the following example. Then extract the answer from the model response and type it at the end of the prompt.
-
-Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end.
-Question: Which number is missing?
-
-Model response: The number missing in the sequence is 14.
-
-Extracted answer: 14
-
-Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end.
-Question: What is the fraction of females facing the camera?
-
-Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera.
-
-Extracted answer: 0.6
-
-Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end.
-Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $)
-
-Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy.
-
-Extracted answer: 1.45
-
-Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end.
-Question: Between which two years does the line  graph saw its maximum peak?
-
-Model response: The line graph saw its maximum peak between 2007 and 2008.
-
-Extracted answer: [2007, 2008]
-
-Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end.
-Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5
-
-Model response: The correct answer is (B) 8/11.
-
-Extracted answer: B
-"""

eval/vlm/eval/mathvista/utilities.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import json
-import os
-import pickle
-import re
-import time
-
-import cv2
-import openai
-from word2number import w2n
-
-openai_client = None
-
-
-def create_dir(output_dir):
-    if not os.path.exists(output_dir):
-        os.makedirs(output_dir)
-
-
-def read_csv(file):
-    data = []
-    with open(file, 'r') as f:
-        for line in f:
-            data.append(line.strip())
-    return data
-
-
-def read_pandas_csv(csv_path):
-    # read a pandas csv sheet
-    import pandas as pd
-    df = pd.read_csv(csv_path)
-    return df
-
-
-def read_json(path):
-    with open(path, 'r', encoding='utf-8') as f:
-        return json.load(f)
-
-
-def read_jsonl(file):
-    with open(file, 'r') as f:
-        data = [json.loads(line) for line in f]
-    return data
-
-
-def read_pickle(path):
-    with open(path, 'rb') as f:
-        return pickle.load(f)
-
-
-def save_json(data, path):
-    with open(path, 'w') as f:
-        json.dump(data, f, indent=4)
-
-
-def save_array_img(path, image):
-    cv2.imwrite(path, image)
-
-
-def contains_digit(text):
-    # check if text contains a digit
-    if any(char.isdigit() for char in text):
-        return True
-    return False
-
-
-def contains_number_word(text):
-    # check if text contains a number word
-    ignore_words = ['a', 'an', 'point']
-    words = re.findall(r'\b\w+\b', text)  # This regex pattern matches any word in the text
-    for word in words:
-        if word in ignore_words:
-            continue
-        try:
-            w2n.word_to_num(word)
-            return True  # If the word can be converted to a number, return True
-        except ValueError:
-            continue  # If the word can't be converted to a number, continue with the next word
-
-    # check if text contains a digit
-    if any(char.isdigit() for char in text):
-        return True
-
-    return False  # If none of the words could be converted to a number, return False
-
-
-def contains_quantity_word(text, special_keep_words=[]):
-    # check if text contains a quantity word
-    quantity_words = ['most', 'least', 'fewest'
-                                       'more', 'less', 'fewer',
-                      'largest', 'smallest', 'greatest',
-                      'larger', 'smaller', 'greater',
-                      'highest', 'lowest', 'higher', 'lower',
-                      'increase', 'decrease',
-                      'minimum', 'maximum', 'max', 'min',
-                      'mean', 'average', 'median',
-                      'total', 'sum', 'add', 'subtract',
-                      'difference', 'quotient', 'gap',
-                      'half', 'double', 'twice', 'triple',
-                      'square', 'cube', 'root',
-                      'approximate', 'approximation',
-                      'triangle', 'rectangle', 'circle', 'square', 'cube', 'sphere', 'cylinder', 'cone', 'pyramid',
-                      'multiply', 'divide',
-                      'percentage', 'percent', 'ratio', 'proportion', 'fraction', 'rate',
-                      ]
-
-    quantity_words += special_keep_words  # dataset specific words
-
-    words = re.findall(r'\b\w+\b', text)  # This regex pattern matches any word in the text
-    if any(word in quantity_words for word in words):
-        return True
-
-    return False  # If none of the words could be converted to a number, return False
-
-
-def is_bool_word(text):
-    if text in ['Yes', 'No', 'True', 'False',
-                'yes', 'no', 'true', 'false',
-                'YES', 'NO', 'TRUE', 'FALSE']:
-        return True
-    return False
-
-
-def is_digit_string(text):
-    # remove ".0000"
-    text = text.strip()
-    text = re.sub(r'\.0+$', '', text)
-    try:
-        int(text)
-        return True
-    except ValueError:
-        return False
-
-
-def is_float_string(text):
-    # text is a float string if it contains a "." and can be converted to a float
-    if '.' in text:
-        try:
-            float(text)
-            return True
-        except ValueError:
-            return False
-    return False
-
-
-def copy_image(image_path, output_image_path):
-    from shutil import copyfile
-    copyfile(image_path, output_image_path)
-
-
-def copy_dir(src_dir, dst_dir):
-    from shutil import copytree
-
-    # copy the source directory to the target directory
-    copytree(src_dir, dst_dir)
-
-
-import PIL.Image as Image
-
-
-def get_image_size(img_path):
-    img = Image.open(img_path)
-    width, height = img.size
-    return width, height
-
-
-def get_chat_response(
-        promot="", api_key="", 
-        base_url="your_api_url",
-        api_version="2024-03-01-preview", model="gpt-4-0613", 
-        temperature=0, max_tokens=256, n=1, patience=10000000, sleep_time=0
-    ):
-    openai_client = openai.AzureOpenAI(
-        azure_endpoint=base_url,
-        api_version=api_version,
-        api_key=api_key,
-    )
-
-    messages = [
-        {'role': 'user', 'content': promot},
-    ]
-    while patience > 0:
-        patience -= 1
-        try:
-            response = openai_client.chat.completions.create(
-                model=model,
-                messages=messages,
-                # api_key=api_key,
-                temperature=temperature,
-                max_tokens=max_tokens,
-                n=n,
-            )
-            response = response.to_dict()
-            if n == 1:
-                prediction = response['choices'][0]['message']['content'].strip()
-                if prediction != '' and prediction is not None:
-                    return prediction
-            else:
-                prediction = [choice['message']['content'].strip() for choice in response['choices']]
-                if prediction[0] != '' and prediction[0] is not None:
-                    return prediction
-
-        except Exception as e:
-            if 'Rate limit' not in str(e):
-                print(e)
-
-            if 'Please reduce the length of the messages' in str(e):
-                print('!!Reduce promot size')
-                # reduce input prompt and keep the tail
-                new_size = int(len(promot) * 0.9)
-                new_start = len(promot) - new_size
-                promot = promot[new_start:]
-                messages = [
-                    {'role': 'user', 'content': promot},
-                ]
-
-            if sleep_time > 0:
-                time.sleep(sleep_time)
-    return ''

eval/vlm/eval/mmbench/evaluate_mmbench.py

@@ -1,283 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import base64
-import itertools
-import json
-import os
-import random
-from io import BytesIO
-
-import pandas as pd
-import torch
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'mmbench_dev_20230712': {
-        'root': 'eval/vlm/data/mmbench/mmbench_dev_20230712.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'dev',
-        'language': 'en'
-    },
-    'mmbench_dev_cn_20231003': {
-        'root': 'eval/vlm/data/mmbench/mmbench_dev_cn_20231003.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'dev',
-        'language': 'cn'
-    },
-    'mmbench_dev_en_20231003': {
-        'root': 'eval/vlm/data/mmbench/mmbench_dev_en_20231003.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'dev',
-        'language': 'en'
-    },
-    'mmbench_test_cn_20231003': {
-        'root': 'eval/vlm/data/mmbench/mmbench_test_cn_20231003.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'test',
-        'language': 'cn'
-    },
-    'mmbench_test_en_20231003': {
-        'root': 'eval/vlm/data/mmbench/mmbench_test_en_20231003.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'test',
-        'language': 'en'
-    },
-    'ccbench_dev_cn': {
-        'root': 'eval/vlm/data/mmbench/CCBench_legacy.tsv',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-        'type': 'dev',
-        'language': 'cn'
-    }
-}
-
-
-def collate_fn(batches):
-    questions = [_['question'] for _ in batches]
-    images = [_['images'] for _ in batches]
-    conversation = [_['conversation'] for _ in batches]
-    answers = [_['answer'] for _ in batches]
-    indexes = [_['index'] for _ in batches]
-    options = [_['option'] for _ in batches]
-    return questions, images, conversation, answers, indexes, options
-
-
-class MMBenchDataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, prompt, language):
-        self.df = pd.read_csv(root, sep='\t')
-        self.prompt = prompt
-        self.language = language
-
-    def __len__(self):
-        return len(self.df)
-
-    def __getitem__(self, idx):
-        index = self.df.iloc[idx]['index']
-        image = self.df.iloc[idx]['image']
-        question = self.df.iloc[idx]['question']
-        answer = self.df.iloc[idx]['answer'] if 'answer' in self.df.iloc[0].keys() else None
-        # catetory = self.df.iloc[idx]['category']
-        # l2_catetory = self.df.iloc[idx]['l2-category']
-
-        image = Image.open(BytesIO(base64.b64decode(image))).convert('RGB')
-        images = [image]
-
-        option_candidate = ['A', 'B', 'C', 'D', 'E']
-        options = {
-            cand: self.load_from_df(idx, cand)
-            for cand in option_candidate
-            if self.load_from_df(idx, cand) is not None
-        }
-
-        hint = self.load_from_df(idx, 'hint')
-        if hint is not None:
-            question = hint + '\n' + question
-        for key, item in options.items():
-            question += f'\n{key}. {item}'
-        if self.language == 'cn':
-            question = question + '\n' + self.prompt['cn']
-        else:
-            question = question + '\n' + self.prompt['en']
-
-        images, conversation = process_conversation(images, question)
-
-        return {
-            'question': question,
-            'images': images,
-            'conversation': conversation,
-            'answer': answer,
-            'index': index,
-            'option': options
-        }
-
-    def load_from_df(self, idx, key):
-        if key in self.df.iloc[idx] and not pd.isna(self.df.iloc[idx][key]):
-            return self.df.iloc[idx][key]
-        else:
-            return None
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def post_process(pred, option):
-    pred = pred.strip()
-    option_candidate = list(option.keys())
-    if len(pred) == 1:
-        return pred
-    if len(pred) == 0:
-        pred = "C"
-    elif len(pred) != 1 and pred[0] in option_candidate:
-        return pred[0]
-    elif len(pred) != 1 and pred[0] not in option_candidate:
-        for k, v in option.items():
-            if v in pred:
-                return k
-
-    return pred
-
-
-def evaluate_chat_model():
-    random.seed(args.seed)
-
-    for ds_name in args.datasets:
-        dataset = MMBenchDataset(
-            root=ds_collections[ds_name]['root'],
-            prompt=prompt,
-            language=ds_collections[ds_name]['language'],
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        outputs = []
-        for _, (questions, images, conversation, answers, indexes, options) in tqdm(enumerate(dataloader)):
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images[0], # batch=1
-                prompt=conversation[0], # batch=1
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-            preds = [post_process(pred, options[0])]
-
-            for question, pred, answer, index in zip(questions, preds, answers, indexes):
-                outputs.append({
-                    'question': question,
-                    'answer': pred,
-                    'gt_answers': answer,
-                    'index': int(index)
-                })
-
-        torch.distributed.barrier()
-
-        world_size = torch.distributed.get_world_size()
-        merged_outputs = [None for _ in range(world_size)]
-        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
-
-        merged_outputs = [json.loads(_) for _ in merged_outputs]
-        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
-
-        if torch.distributed.get_rank() == 0:
-            print(f'Evaluating {ds_name} ...')
-            results_file = 'results.xlsx'
-            output_path = os.path.join(args.out_dir, results_file)
-            df = pd.read_table(ds_collections[ds_name]['root'])
-            cur_df = df.copy()
-            if 'mmbench' in ds_name:
-                cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
-                cur_df.insert(6, 'prediction', None)
-            else:
-                cur_df = cur_df.drop(columns=['category', 'image'])
-                cur_df.insert(8, 'prediction', None)
-            for item in merged_outputs:
-                cur_df.loc[df['index'] == item['index'], 'prediction'] = item['answer']
-
-            cur_df.to_excel(output_path, index=False, engine='openpyxl')
-            print('Results saved to {}'.format(output_path))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='mmbench_dev_20230712')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    prompt = {
-        'en': "Answer with the option's letter from the given choices directly.",
-        'cn': '请直接回答选项字母。'
-    }
-    evaluate_chat_model()

eval/vlm/eval/mme/Your_Results/OCR.txt

@@ -1,40 +0,0 @@
-0001.jpg	Is the word in the logo "angie's"? Please answer yes or no.	Yes
-0001.jpg	Is the word in the logo "angle's"? Please answer yes or no.	No
-0002.jpg	Is the word in the logo "c'est cheese"? Please answer yes or no.	Yes
-0002.jpg	Is the word in the logo "crest cheese"? Please answer yes or no.	No
-0003.jpg	Is the word in the logo "beavertails pastry"? Please answer yes or no.	Yes
-0003.jpg	Is the word in the logo "beavertalls pastry"? Please answer yes or no.	No
-0004.jpg	Is the word in the logo "old market sundries"? Please answer yes or no.	Yes
-0004.jpg	Is the word in the logo "old market hundreds"? Please answer yes or no.	No
-0005.jpg	Is the word in the logo "kress"? Please answer yes or no.	Yes
-0005.jpg	Is the word in the logo "dress"? Please answer yes or no.	No
-0006.jpg	Is the word in the logo "the beatles story liver pool"? Please answer yes or no.	Yes
-0006.jpg	Is the word in the logo "the beats story liver pool"? Please answer yes or no.	No
-0007.jpg	Is the phone number in the picture "0131 555 6363"? Please answer yes or no.	Yes
-0007.jpg	Is the phone number in the picture "0137 556 6363"? Please answer yes or no.	No
-0008.jpg	Is the word in the logo "phil's market"? Please answer yes or no.	Yes
-0008.jpg	Is the word in the logo "phll's market"? Please answer yes or no.	No
-0009.jpg	Is the word in the logo "fenders diner"? Please answer yes or no.	Yes
-0009.jpg	Is the word in the logo "finders diner"? Please answer yes or no.	No
-0010.jpg	Is the word in the logo "high time coffee shop"? Please answer yes or no.	Yes
-0010.jpg	Is the word in the logo "high tite cofeee shop"? Please answer yes or no.	No
-0011.jpg	Is the word in the logo "ihop restaurant"? Please answer yes or no.	Yes
-0011.jpg	Is the word in the logo "lhop restaurant"? Please answer yes or no.	No
-0012.jpg	Is the word in the logo "casa grecque restaurants"? Please answer yes or no.	Yes
-0012.jpg	Is the word in the logo "case grecque restaurants"? Please answer yes or no.	No
-0013.jpg	Is the word in the picture "seabreeze motel"? Please answer yes or no.	Yes
-0013.jpg	Is the word in the picture "seebreeze model"? Please answer yes or no.	No
-0014.jpg	Is the word in the logo "penarth pier built 1894"? Please answer yes or no.	Yes
-0014.jpg	Is the word in the logo "penarth pies buid 1894"? Please answer yes or no.	No
-0015.jpg	Is the text in the picture "hollywood"? Please answer yes or no.	Yes
-0015.jpg	Is the text in the picture "holly word"? Please answer yes or no.	No
-0016.jpg	Is the word in the logo "shop rite"? Please answer yes or no.	Yes
-0016.jpg	Is the word in the logo "stop rite"? Please answer yes or no.	No
-0017.jpg	Is the word in the logo "hardco industrial construction"? Please answer yes or no.	Yes
-0017.jpg	Is the word in the logo "hardto industal construction"? Please answer yes or no.	No
-0018.jpg	Is the word in the logo "oldsmobile service"? Please answer yes or no.	Yes
-0018.jpg	Is the word in the logo "old mobile service"? Please answer yes or no.	No
-0019.jpg	Is the word in the logo "exchange hotel"? Please answer yes or no.	Yes
-0019.jpg	Is the word in the logo "excharge hotel"? Please answer yes or no.	No
-0020.jpg	Is the word in the logo "cold drinks"? Please answer yes or no.	Yes
-0020.jpg	Is the word in the logo "cold rinks"? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/artwork.txt

@@ -1,400 +0,0 @@
-10002.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-10002.jpg	Does this artwork exist in the form of glassware? Please answer yes or no.	No
-10049.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-10049.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	No
-10256.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-10256.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	No
-10358.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-10358.jpg	Does this artwork exist in the form of glassware? Please answer yes or no.	No
-10543.jpg	Is this artwork displayed in fogg art museum, harvard university, cambridge? Please answer yes or no.	Yes
-10543.jpg	Is this artwork displayed in museo civico, pistoia? Please answer yes or no.	No
-10581.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	Yes
-10581.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	No
-1060.jpg	Is this artwork created by antoniazzo romano? Please answer yes or no.	Yes
-1060.jpg	Is this artwork created by gentile da fabriano? Please answer yes or no.	No
-10881.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-10881.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-10970.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-10970.jpg	Does this artwork belong to the type of study? Please answer yes or no.	No
-11276.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	Yes
-11276.jpg	Does this artwork exist in the form of graphics? Please answer yes or no.	No
-11331.jpg	Is this artwork created by donatello? Please answer yes or no.	Yes
-11331.jpg	Is this artwork created by zichy, mihály? Please answer yes or no.	No
-11488.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	Yes
-11488.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	No
-11724.jpg	Is this artwork created by duccio di buoninsegna? Please answer yes or no.	Yes
-11724.jpg	Is this artwork created by giani, felice? Please answer yes or no.	No
-11726.jpg	Is this artwork titled temptation on the mountain (detail)? Please answer yes or no.	Yes
-11726.jpg	Is this artwork titled in the forest of fontainebleau? Please answer yes or no.	No
-12133.jpg	Is this artwork titled hand study with bible? Please answer yes or no.	Yes
-12133.jpg	Is this artwork titled self-portrait aged 78? Please answer yes or no.	No
-12439.jpg	Is this artwork created by dürer, albrecht? Please answer yes or no.	Yes
-12439.jpg	Is this artwork created by koekkoek, barend cornelis? Please answer yes or no.	No
-12561.jpg	Is this artwork created by eberlein, gustav heinrich? Please answer yes or no.	Yes
-12561.jpg	Is this artwork created by gillemans, jan pauwel the younger? Please answer yes or no.	No
-12652.jpg	Is this artwork displayed in stedelijk museum de lakenhal, leiden? Please answer yes or no.	Yes
-12652.jpg	Is this artwork displayed in palazzo ducale, mantua? Please answer yes or no.	No
-12736.jpg	Is this artwork displayed in cannon hall museum, barnsley? Please answer yes or no.	Yes
-12736.jpg	Is this artwork displayed in protestant parish church, gelnhausen? Please answer yes or no.	No
-12902.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-12902.jpg	Is this artwork displayed in musée national gustave-moreau, paris? Please answer yes or no.	No
-12908.jpg	Is this artwork titled ruth and boaz? Please answer yes or no.	Yes
-12908.jpg	Is this artwork titled view of dresden from the right bank of the elbe with the augustus bridge? Please answer yes or no.	No
-13091.jpg	Is this artwork titled italianate landscape with figures by classical ruins? Please answer yes or no.	Yes
-13091.jpg	Is this artwork titled two boys singing? Please answer yes or no.	No
-13174.jpg	Is this artwork titled nobility? Please answer yes or no.	Yes
-13174.jpg	Is this artwork titled aretino in the studio of tintoretto? Please answer yes or no.	No
-13239.jpg	Is this artwork titled doge ziani receiving the benediction of pope alexander iii? Please answer yes or no.	Yes
-13239.jpg	Is this artwork titled the adoration of the shepherds? Please answer yes or no.	No
-13288.jpg	Does this artwork exist in the form of architecture? Please answer yes or no.	Yes
-13288.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-13696.jpg	Is this artwork displayed in pinacoteca nazionale, siena? Please answer yes or no.	Yes
-13696.jpg	Is this artwork displayed in british embassy, paris? Please answer yes or no.	No
-13760.jpg	Is this artwork titled noli me tangere? Please answer yes or no.	Yes
-13760.jpg	Is this artwork titled profile study of a bearded man? Please answer yes or no.	No
-13821.jpg	Is this artwork created by frangipane, niccolò? Please answer yes or no.	Yes
-13821.jpg	Is this artwork created by drevet, pierre? Please answer yes or no.	No
-13901.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-13901.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-14283.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-14283.jpg	Does this artwork exist in the form of mosaic? Please answer yes or no.	No
-14499.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-14499.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	No
-14777.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-14777.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	No
-15028.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	Yes
-15028.jpg	Does this artwork belong to the type of study? Please answer yes or no.	No
-15232.jpg	Is this artwork created by giordano, luca? Please answer yes or no.	Yes
-15232.jpg	Is this artwork created by heyerdahl, hans olaf? Please answer yes or no.	No
-15246.jpg	Is this artwork displayed in palazzo medici riccardi, florence? Please answer yes or no.	Yes
-15246.jpg	Is this artwork displayed in abbey church of sainte-foy, conques (aveyron)? Please answer yes or no.	No
-15311.jpg	Is this artwork created by giorgione? Please answer yes or no.	Yes
-15311.jpg	Is this artwork created by marilhat, prosper? Please answer yes or no.	No
-15989.jpg	Is this artwork displayed in pinacoteca, vatican? Please answer yes or no.	Yes
-15989.jpg	Is this artwork displayed in cathedral museum, zamora? Please answer yes or no.	No
-16006.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-16006.jpg	Is this artwork displayed in cathedral of san geminiano, modena? Please answer yes or no.	No
-16249.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	Yes
-16249.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	No
-16538.jpg	Is this artwork created by gogh, vincent van? Please answer yes or no.	Yes
-16538.jpg	Is this artwork created by altdorfer, albrecht? Please answer yes or no.	No
-16835.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-16835.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	No
-16911.jpg	Is this artwork created by gossart, jan? Please answer yes or no.	Yes
-16911.jpg	Is this artwork created by stanzione, massimo? Please answer yes or no.	No
-17311.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-17311.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	No
-17317.jpg	Is this artwork created by gozzoli, benozzo? Please answer yes or no.	Yes
-17317.jpg	Is this artwork created by coriolano, cristoforo? Please answer yes or no.	No
-17535.jpg	Is this artwork created by grebber, pieter de? Please answer yes or no.	Yes
-17535.jpg	Is this artwork created by massys, quentin? Please answer yes or no.	No
-17823.jpg	Is this artwork created by greuze, jean-baptiste? Please answer yes or no.	Yes
-17823.jpg	Is this artwork created by landseer, sir edwin henry? Please answer yes or no.	No
-17838.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-17838.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-17998.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-17998.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	No
-18566.jpg	Is this artwork created by hamen, juan van der? Please answer yes or no.	Yes
-18566.jpg	Is this artwork created by starnina, gherardo di jacopo? Please answer yes or no.	No
-18604.jpg	Is this artwork created by hardouin-mansart, jules? Please answer yes or no.	Yes
-18604.jpg	Is this artwork created by kerseboom, friedrich? Please answer yes or no.	No
-18722.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-18722.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	No
-1873.jpg	Does this artwork exist in the form of architecture? Please answer yes or no.	Yes
-1873.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	No
-18902.jpg	Is this artwork created by herrera, francisco de, the elder? Please answer yes or no.	Yes
-18902.jpg	Is this artwork created by ingres, jean-auguste-dominique? Please answer yes or no.	No
-18926.jpg	Is this artwork created by herring, john frederick the younger? Please answer yes or no.	Yes
-18926.jpg	Is this artwork created by cozens, john robert? Please answer yes or no.	No
-19087.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-19087.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-19154.jpg	Is this artwork titled portrait of the merchant georg gisze (detail)? Please answer yes or no.	Yes
-19154.jpg	Is this artwork titled pair of table candlesticks? Please answer yes or no.	No
-19417.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-19417.jpg	Does this artwork exist in the form of mosaic? Please answer yes or no.	No
-19452.jpg	Is this artwork titled the artist and his model? Please answer yes or no.	Yes
-19452.jpg	Is this artwork titled the lovesick maiden (detail)? Please answer yes or no.	No
-19839.jpg	Is this artwork created by janneck, franz christoph? Please answer yes or no.	Yes
-19839.jpg	Is this artwork created by goupil, jules-adolphe? Please answer yes or no.	No
-19863.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-19863.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	No
-19993.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-19993.jpg	Is this artwork displayed in cathedral of st paul, liège? Please answer yes or no.	No
-20176.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-20176.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-20437.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-20437.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-20442.jpg	Is this artwork created by kucharski, aleksander? Please answer yes or no.	Yes
-20442.jpg	Is this artwork created by pourbus, frans the elder? Please answer yes or no.	No
-20455.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-20455.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-20483.jpg	Is this artwork titled allegory of the regency? Please answer yes or no.	Yes
-20483.jpg	Is this artwork titled breton woman bathing? Please answer yes or no.	No
-20490.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-20490.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	No
-20551.jpg	Is this artwork created by lagrenée, jean-jacques? Please answer yes or no.	Yes
-20551.jpg	Is this artwork created by scultori, diana? Please answer yes or no.	No
-20651.jpg	Is this artwork titled a highland landscape? Please answer yes or no.	Yes
-20651.jpg	Is this artwork titled a dog and a cat fighting in a kitchen interior? Please answer yes or no.	No
-20724.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	Yes
-20724.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	No
-21048.jpg	Is this artwork created by lemoyne, jean-baptiste ii? Please answer yes or no.	Yes
-21048.jpg	Is this artwork created by kneller, sir godfrey? Please answer yes or no.	No
-21097.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-21097.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	No
-21244.jpg	Does this artwork belong to the type of study? Please answer yes or no.	Yes
-21244.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	No
-21469.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	Yes
-21469.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-21580.jpg	Is this artwork created by linard, jacques? Please answer yes or no.	Yes
-21580.jpg	Is this artwork created by bonino da campione? Please answer yes or no.	No
-21712.jpg	Is this artwork titled st john the evangelist resuscitating drusiana? Please answer yes or no.	Yes
-21712.jpg	Is this artwork titled la finette? Please answer yes or no.	No
-22329.jpg	Is this artwork titled marriage of the virgin? Please answer yes or no.	Yes
-22329.jpg	Is this artwork titled landscape with river and figures (detail)? Please answer yes or no.	No
-22366.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-22366.jpg	Does this artwork exist in the form of glassware? Please answer yes or no.	No
-22667.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-22667.jpg	Is this artwork displayed in san francesco d'assisi, pavia? Please answer yes or no.	No
-22760.jpg	Is this artwork titled madonna and child (detail)? Please answer yes or no.	Yes
-22760.jpg	Is this artwork titled view of the south and east walls? Please answer yes or no.	No
-22842.jpg	Is this artwork titled ukrainian peasant girl? Please answer yes or no.	Yes
-22842.jpg	Is this artwork titled virtue crowning merit? Please answer yes or no.	No
-23229.jpg	Is this artwork displayed in national gallery, london? Please answer yes or no.	Yes
-23229.jpg	Is this artwork displayed in notre-dame-la-riche, tours? Please answer yes or no.	No
-23427.jpg	Is this artwork displayed in the hermitage, st. petersburg? Please answer yes or no.	Yes
-23427.jpg	Is this artwork displayed in national gallery of victoria, melbourne? Please answer yes or no.	No
-23465.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-23465.jpg	Is this artwork displayed in cistertian church, zirc? Please answer yes or no.	No
-23824.jpg	Is this artwork titled christ walking on the water? Please answer yes or no.	Yes
-23824.jpg	Is this artwork titled mademoiselle romaine lacaux? Please answer yes or no.	No
-24122.jpg	Is this artwork displayed in museo correr, venice? Please answer yes or no.	Yes
-24122.jpg	Is this artwork displayed in church of brou, bourg-en-bresse? Please answer yes or no.	No
-24260.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-24260.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	No
-24291.jpg	Is this artwork titled virgin and child with sts catherine, cecilia, barbara, and ursula? Please answer yes or no.	Yes
-24291.jpg	Is this artwork titled sorrow? Please answer yes or no.	No
-24723.jpg	Is this artwork titled tomb of henry the lion and his wife matilda? Please answer yes or no.	Yes
-24723.jpg	Is this artwork titled god the father? Please answer yes or no.	No
-2490.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	Yes
-2490.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	No
-2507.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-2507.jpg	Is this artwork displayed in st. vitus's cathedral, prague? Please answer yes or no.	No
-25312.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-25312.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-25476.jpg	Is this artwork created by michelangelo buonarroti? Please answer yes or no.	Yes
-25476.jpg	Is this artwork created by beuckelaer, joachim? Please answer yes or no.	No
-25492.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	Yes
-25492.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	No
-25513.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-25513.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	No
-26521.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	Yes
-26521.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-26973.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-26973.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	No
-27021.jpg	Is this artwork created by miniaturist, german? Please answer yes or no.	Yes
-27021.jpg	Is this artwork created by trinquesse, louis-rolland? Please answer yes or no.	No
-27662.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	Yes
-27662.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	No
-27936.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	Yes
-27936.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	No
-28039.jpg	Is this artwork displayed in cappella palatina, palermo? Please answer yes or no.	Yes
-28039.jpg	Is this artwork displayed in musée des beaux-arts, chambéry? Please answer yes or no.	No
-28345.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-28345.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-28400.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-28400.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	No
-28698.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-28698.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-28758.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-28758.jpg	Does this artwork exist in the form of graphics? Please answer yes or no.	No
-28974.jpg	Is this artwork titled prayer before the meal? Please answer yes or no.	Yes
-28974.jpg	Is this artwork titled rest in the mountains? Please answer yes or no.	No
-29266.jpg	Is this artwork created by palma vecchio? Please answer yes or no.	Yes
-29266.jpg	Is this artwork created by maris, jacobus hendricus? Please answer yes or no.	No
-30443.jpg	Is this artwork titled the crucifixion with sts jerome and christopher? Please answer yes or no.	Yes
-30443.jpg	Is this artwork titled tomb of michelangelo (detail)? Please answer yes or no.	No
-3085.jpg	Is this artwork created by bartsius, willem? Please answer yes or no.	Yes
-3085.jpg	Is this artwork created by oehme, ernst ferdinand? Please answer yes or no.	No
-30875.jpg	Is this artwork created by pomarancio? Please answer yes or no.	Yes
-30875.jpg	Is this artwork created by steen, jan? Please answer yes or no.	No
-3114.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-3114.jpg	Does this artwork belong to the type of study? Please answer yes or no.	No
-31808.jpg	Is this artwork created by raffaello sanzio? Please answer yes or no.	Yes
-31808.jpg	Is this artwork created by simon von taisten? Please answer yes or no.	No
-32147.jpg	Is this artwork titled lucretia? Please answer yes or no.	Yes
-32147.jpg	Is this artwork titled rinaldo abandoning armida (detail)? Please answer yes or no.	No
-3241.jpg	Is this artwork titled holy family? Please answer yes or no.	Yes
-3241.jpg	Is this artwork titled friedrich iii, the wise, and johann i, the constant, electors of saxony? Please answer yes or no.	No
-33017.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-33017.jpg	Does this artwork exist in the form of glassware? Please answer yes or no.	No
-33069.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	Yes
-33069.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	No
-33173.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-33173.jpg	Does this artwork exist in the form of graphics? Please answer yes or no.	No
-33753.jpg	Is this artwork titled vanitas? Please answer yes or no.	Yes
-33753.jpg	Is this artwork titled legend of st francis: 18. apparition at arles (detail)? Please answer yes or no.	No
-33854.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-33854.jpg	Does this artwork belong to the type of study? Please answer yes or no.	No
-339.jpg	Is this artwork displayed in staatliche museen, berlin? Please answer yes or no.	Yes
-339.jpg	Is this artwork displayed in national museum of religious carvings, valladolid? Please answer yes or no.	No
-33933.jpg	Is this artwork titled madonna and child? Please answer yes or no.	Yes
-33933.jpg	Is this artwork titled the bacino di san marco? Please answer yes or no.	No
-3404.jpg	Is this artwork displayed in szépmûvészeti múzeum, budapest? Please answer yes or no.	Yes
-3404.jpg	Is this artwork displayed in s. eustorgio, milan? Please answer yes or no.	No
-34109.jpg	Is this artwork displayed in national gallery of art, washington? Please answer yes or no.	Yes
-34109.jpg	Is this artwork displayed in abbey church of sainte-foy, conques? Please answer yes or no.	No
-34363.jpg	Is this artwork displayed in museo del prado, madrid? Please answer yes or no.	Yes
-34363.jpg	Is this artwork displayed in state tretyakov gallery, moscow? Please answer yes or no.	No
-34539.jpg	Is this artwork titled the victory of eucharistic truth over heresy? Please answer yes or no.	Yes
-34539.jpg	Is this artwork titled a sunday afternoon on the ile de la grande jatte? Please answer yes or no.	No
-34627.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	Yes
-34627.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	No
-34638.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-34638.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-34669.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	Yes
-34669.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	No
-35345.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-35345.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	No
-35439.jpg	Is this artwork titled madonna and child with a host of musical angels? Please answer yes or no.	Yes
-35439.jpg	Is this artwork titled garden in fontenay? Please answer yes or no.	No
-35460.jpg	Is this artwork created by schinkel, karl friedrich? Please answer yes or no.	Yes
-35460.jpg	Is this artwork created by giolfino, bartolomeo? Please answer yes or no.	No
-35486.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-35486.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-35513.jpg	Is this artwork created by schongauer, martin? Please answer yes or no.	Yes
-35513.jpg	Is this artwork created by cassioli, amos? Please answer yes or no.	No
-3552.jpg	Is this artwork titled madonna degli alberetti? Please answer yes or no.	Yes
-3552.jpg	Is this artwork titled peter gillis? Please answer yes or no.	No
-35658.jpg	Is this artwork created by sebastiano del piombo? Please answer yes or no.	Yes
-35658.jpg	Is this artwork created by jacobsz., dirck? Please answer yes or no.	No
-35736.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-35736.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-35861.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	Yes
-35861.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-36805.jpg	Is this artwork titled weir? Please answer yes or no.	Yes
-36805.jpg	Is this artwork titled view of the window wall? Please answer yes or no.	No
-36966.jpg	Does this artwork belong to the type of portrait? Please answer yes or no.	Yes
-36966.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	No
-37010.jpg	Is this artwork titled madonna and child with the young st john? Please answer yes or no.	Yes
-37010.jpg	Is this artwork titled sketch for attila? Please answer yes or no.	No
-37077.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-37077.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-37439.jpg	Is this artwork titled the message? Please answer yes or no.	Yes
-37439.jpg	Is this artwork titled the descent from the cross? Please answer yes or no.	No
-37819.jpg	Is this artwork created by tiepolo, giovanni battista? Please answer yes or no.	Yes
-37819.jpg	Is this artwork created by kerricx, willem ignatius? Please answer yes or no.	No
-37866.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	Yes
-37866.jpg	Does this artwork belong to the type of still-life? Please answer yes or no.	No
-381.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-381.jpg	Does this artwork exist in the form of architecture? Please answer yes or no.	No
-38178.jpg	Is this artwork created by tintoretto? Please answer yes or no.	Yes
-38178.jpg	Is this artwork created by morel, jean-baptiste? Please answer yes or no.	No
-38536.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-38536.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-38546.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-38546.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-38694.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-38694.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-38740.jpg	Is this artwork displayed in musée toulouse-lautrec, albi? Please answer yes or no.	Yes
-38740.jpg	Is this artwork displayed in kupferstichkabinett, gotha? Please answer yes or no.	No
-38881.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	Yes
-38881.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	No
-38993.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-38993.jpg	Does this artwork exist in the form of illumination? Please answer yes or no.	No
-39026.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-39026.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	No
-39124.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-39124.jpg	Does this artwork exist in the form of graphics? Please answer yes or no.	No
-39188.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-39188.jpg	Does this artwork exist in the form of architecture? Please answer yes or no.	No
-39482.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-39482.jpg	Does this artwork exist in the form of metalwork? Please answer yes or no.	No
-39556.jpg	Is this artwork created by unknown master, dutch? Please answer yes or no.	Yes
-39556.jpg	Is this artwork created by cuyp, benjamin gerritsz.? Please answer yes or no.	No
-41036.jpg	Is this artwork displayed in kunsthistorisches museum, vienna? Please answer yes or no.	Yes
-41036.jpg	Is this artwork displayed in national museum of art, minsk? Please answer yes or no.	No
-41371.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-41371.jpg	Does this artwork exist in the form of architecture? Please answer yes or no.	No
-41484.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-41484.jpg	Does this artwork belong to the type of historical? Please answer yes or no.	No
-41594.jpg	Is this artwork created by veronese, paolo? Please answer yes or no.	Yes
-41594.jpg	Is this artwork created by jeaurat, etienne? Please answer yes or no.	No
-416.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	Yes
-416.jpg	Does this artwork exist in the form of others? Please answer yes or no.	No
-41653.jpg	Is this artwork titled view of the sala del collegio? Please answer yes or no.	Yes
-41653.jpg	Is this artwork titled reine lefebvre and margot? Please answer yes or no.	No
-41944.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-41944.jpg	Does this artwork exist in the form of mosaic? Please answer yes or no.	No
-42152.jpg	Is this artwork titled the pieterskerk in leiden? Please answer yes or no.	Yes
-42152.jpg	Is this artwork titled portrait of cardinal reginald pole? Please answer yes or no.	No
-42288.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-42288.jpg	Does this artwork exist in the form of stained-glass? Please answer yes or no.	No
-42303.jpg	Is this artwork displayed in art museum, cincinnati? Please answer yes or no.	Yes
-42303.jpg	Is this artwork displayed in banca del monte di bologna e ravenna, bologna? Please answer yes or no.	No
-42401.jpg	Is this artwork created by waldmüller, fedinand georg? Please answer yes or no.	Yes
-42401.jpg	Is this artwork created by seeman, enoch? Please answer yes or no.	No
-42447.jpg	Is this artwork displayed in musée du louvre, paris? Please answer yes or no.	Yes
-42447.jpg	Is this artwork displayed in santa catarina, pisa? Please answer yes or no.	No
-42585.jpg	Is this artwork created by werff, pieter van der? Please answer yes or no.	Yes
-42585.jpg	Is this artwork created by domenichini, apollonio? Please answer yes or no.	No
-42706.jpg	Is this artwork displayed in musée du louvre, paris? Please answer yes or no.	Yes
-42706.jpg	Is this artwork displayed in galleria nazionale d'arte moderna e contemporanea, rome? Please answer yes or no.	No
-42796.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-42796.jpg	Is this artwork displayed in museo di san salvi, florence? Please answer yes or no.	No
-42857.jpg	Does this artwork belong to the type of landscape? Please answer yes or no.	Yes
-42857.jpg	Does this artwork belong to the type of study? Please answer yes or no.	No
-42905.jpg	Is this artwork created by wit, jacob de? Please answer yes or no.	Yes
-42905.jpg	Is this artwork created by vittone, bernardo antonio? Please answer yes or no.	No
-42941.jpg	Is this artwork created by witte, emanuel de? Please answer yes or no.	Yes
-42941.jpg	Is this artwork created by bicci di neri? Please answer yes or no.	No
-42956.jpg	Is this artwork titled view of rome with the tiberand castel sant'angelo? Please answer yes or no.	Yes
-42956.jpg	Is this artwork titled st bonaventure enters the franciscan order? Please answer yes or no.	No
-42987.jpg	Is this artwork created by witz, konrad? Please answer yes or no.	Yes
-42987.jpg	Is this artwork created by christus, petrus? Please answer yes or no.	No
-43142.jpg	Does this artwork belong to the type of mythological? Please answer yes or no.	Yes
-43142.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	No
-43175.jpg	Is this artwork displayed in private collection? Please answer yes or no.	Yes
-43175.jpg	Is this artwork displayed in smith college museum of art, northampton? Please answer yes or no.	No
-43349.jpg	Is this artwork created by zuccarelli, francesco? Please answer yes or no.	Yes
-43349.jpg	Is this artwork created by baccanelli, giovanni antonio di giulio? Please answer yes or no.	No
-43445.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-43445.jpg	Does this artwork belong to the type of interior? Please answer yes or no.	No
-4836.jpg	Is this artwork displayed in villa cornaro, piombino dese? Please answer yes or no.	Yes
-4836.jpg	Is this artwork displayed in palais saint-vaast, arras? Please answer yes or no.	No
-5227.jpg	Is this artwork created by botticelli, sandro? Please answer yes or no.	Yes
-5227.jpg	Is this artwork created by vigri, caterina? Please answer yes or no.	No
-526.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-526.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-5906.jpg	Is this artwork created by bronzino, agnolo? Please answer yes or no.	Yes
-5906.jpg	Is this artwork created by pellegrino da san daniele? Please answer yes or no.	No
-6168.jpg	Does this artwork exist in the form of graphics? Please answer yes or no.	Yes
-6168.jpg	Does this artwork exist in the form of tapestry? Please answer yes or no.	No
-6297.jpg	Is this artwork titled peasants making merry outside a tavern 'the swan'? Please answer yes or no.	Yes
-6297.jpg	Is this artwork titled allegory of quietude? Please answer yes or no.	No
-6478.jpg	Does this artwork belong to the type of religious? Please answer yes or no.	Yes
-6478.jpg	Does this artwork belong to the type of genre? Please answer yes or no.	No
-6969.jpg	Is this artwork titled letizia ramolino bonaparte? Please answer yes or no.	Yes
-6969.jpg	Is this artwork titled job and his daughters? Please answer yes or no.	No
-701.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-701.jpg	Does this artwork exist in the form of others? Please answer yes or no.	No
-7702.jpg	Is this artwork titled reine lefebvre and margot? Please answer yes or no.	Yes
-7702.jpg	Is this artwork titled fire in the oil depot at san marcuola? Please answer yes or no.	No
-8101.jpg	Is this artwork displayed in museu de arte, são paulo? Please answer yes or no.	Yes
-8101.jpg	Is this artwork displayed in national széchényi library, budapest? Please answer yes or no.	No
-815.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-815.jpg	Does this artwork exist in the form of furniture? Please answer yes or no.	No
-8797.jpg	Is this artwork created by coecke van aelst, pieter? Please answer yes or no.	Yes
-8797.jpg	Is this artwork created by abaquesne, masséot? Please answer yes or no.	No
-8885.jpg	Is this artwork displayed in art museum, saint louis? Please answer yes or no.	Yes
-8885.jpg	Is this artwork displayed in museo civico d'arte antica, turin? Please answer yes or no.	No
-9153.jpg	Is this artwork displayed in galleria nazionale, parma? Please answer yes or no.	Yes
-9153.jpg	Is this artwork displayed in hospital de san bernardo, seville? Please answer yes or no.	No
-9395.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-9395.jpg	Does this artwork exist in the form of stained-glass? Please answer yes or no.	No
-9405.jpg	Is this artwork created by courbet, gustave? Please answer yes or no.	Yes
-9405.jpg	Is this artwork created by milani, aureliano? Please answer yes or no.	No
-9599.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	Yes
-9599.jpg	Does this artwork exist in the form of ceramics? Please answer yes or no.	No
-995.jpg	Does this artwork exist in the form of sculpture? Please answer yes or no.	Yes
-995.jpg	Does this artwork exist in the form of painting? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/celebrity.txt

@@ -1,340 +0,0 @@
-tt0032138_shot_0395_img_0.jpg	Is the actor inside the red bounding box named Frank Morgan? Please answer yes or no.	Yes
-tt0032138_shot_0395_img_0.jpg	Is the actor inside the red bounding box named Eric Schniewind? Please answer yes or no.	No
-tt0035423_shot_0464_img_0.jpg	Is the actor inside the red bounding box called Hugh Jackman? Please answer yes or no.	Yes
-tt0035423_shot_0464_img_0.jpg	Is the actor inside the red bounding box called Lizzie Hopley? Please answer yes or no.	No
-tt0038650_shot_0737_img_1.jpg	Is the person inside the red bounding box called James Stewart? Please answer yes or no.	Yes
-tt0038650_shot_0737_img_1.jpg	Is the person inside the red bounding box called Phil Selway? Please answer yes or no.	No
-tt0047396_shot_0333_img_0.jpg	Is the actor inside the red bounding box named James Stewart? Please answer yes or no.	Yes
-tt0047396_shot_0333_img_0.jpg	Is the actor inside the red bounding box named Ron Blair? Please answer yes or no.	No
-tt0048545_shot_0124_img_0.jpg	Is the actor inside the red bounding box called Natalie Wood? Please answer yes or no.	Yes
-tt0048545_shot_0124_img_0.jpg	Is the actor inside the red bounding box called Rebecca Jackson Mendoza? Please answer yes or no.	No
-tt0049470_shot_0279_img_0.jpg	Is the person inside the red bounding box called James Stewart? Please answer yes or no.	Yes
-tt0049470_shot_0279_img_0.jpg	Is the person inside the red bounding box called Matt Pashkow? Please answer yes or no.	No
-tt0049730_shot_0273_img_0.jpg	Is the person inside the red bounding box called Vera Miles? Please answer yes or no.	Yes
-tt0049730_shot_0273_img_0.jpg	Is the person inside the red bounding box called Addie Yungmee? Please answer yes or no.	No
-tt0052357_shot_0511_img_0.jpg	Is the actor inside the red bounding box called Kim Novak? Please answer yes or no.	Yes
-tt0052357_shot_0511_img_0.jpg	Is the actor inside the red bounding box called Abigail Van Alyn? Please answer yes or no.	No
-tt0053221_shot_0197_img_0.jpg	Is the actor inside the red bounding box named John Wayne? Please answer yes or no.	Yes
-tt0053221_shot_0197_img_0.jpg	Is the actor inside the red bounding box named Claude-Oliver Rudolph? Please answer yes or no.	No
-tt0054167_shot_0122_img_0.jpg	Is the person inside the red bounding box called Anna Massey? Please answer yes or no.	Yes
-tt0054167_shot_0122_img_0.jpg	Is the person inside the red bounding box called Eddie Tagoe? Please answer yes or no.	No
-tt0056869_shot_0320_img_0.jpg	Is the person inside the red bounding box called Tippi Hedren? Please answer yes or no.	Yes
-tt0056869_shot_0320_img_0.jpg	Is the person inside the red bounding box called Denise Mack? Please answer yes or no.	No
-tt0056923_shot_0835_img_0.jpg	Is the actor inside the red bounding box called Audrey Hepburn? Please answer yes or no.	Yes
-tt0056923_shot_0835_img_0.jpg	Is the actor inside the red bounding box called Chris April? Please answer yes or no.	No
-tt0057115_shot_0686_img_0.jpg	Is the person inside the red bounding box named James Garner? Please answer yes or no.	Yes
-tt0057115_shot_0686_img_0.jpg	Is the person inside the red bounding box named Chutimon Chuengcharoensukying? Please answer yes or no.	No
-tt0058331_shot_0353_img_0.jpg	Is the actor inside the red bounding box named Julie Andrews? Please answer yes or no.	Yes
-tt0058331_shot_0353_img_0.jpg	Is the actor inside the red bounding box named Ed Geldart? Please answer yes or no.	No
-tt0058461_shot_0901_img_0.jpg	Is the actor inside the red bounding box called Gian Maria Volontè? Please answer yes or no.	Yes
-tt0058461_shot_0901_img_0.jpg	Is the actor inside the red bounding box called Jennifer Connelly? Please answer yes or no.	No
-tt0061418_shot_0148_img_0.jpg	Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no.	Yes
-tt0061418_shot_0148_img_0.jpg	Is the actor inside the red bounding box named Warona Seane? Please answer yes or no.	No
-tt0061722_shot_0259_img_0.jpg	Is the actor inside the red bounding box called Dustin Hoffman? Please answer yes or no.	Yes
-tt0061722_shot_0259_img_0.jpg	Is the actor inside the red bounding box called Christopher Olsen? Please answer yes or no.	No
-tt0062622_shot_0291_img_0.jpg	Is the actor inside the red bounding box named Keir Dullea? Please answer yes or no.	Yes
-tt0062622_shot_0291_img_0.jpg	Is the actor inside the red bounding box named Frank Albanese? Please answer yes or no.	No
-tt0063442_shot_0702_img_0.jpg	Is the actor inside the red bounding box called Linda Harrison? Please answer yes or no.	Yes
-tt0063442_shot_0702_img_0.jpg	Is the actor inside the red bounding box called Michael McKean? Please answer yes or no.	No
-tt0064115_shot_0367_img_0.jpg	Is the actor inside the red bounding box named Robert Redford? Please answer yes or no.	Yes
-tt0064115_shot_0367_img_0.jpg	Is the actor inside the red bounding box named Cooper Murray? Please answer yes or no.	No
-tt0064665_shot_0300_img_0.jpg	Is the actor inside the red bounding box called Jon Voight? Please answer yes or no.	Yes
-tt0064665_shot_0300_img_0.jpg	Is the actor inside the red bounding box called Harvey Meyer? Please answer yes or no.	No
-tt0065214_shot_0366_img_0.jpg	Is the person inside the red bounding box called Robert Ryan? Please answer yes or no.	Yes
-tt0065214_shot_0366_img_0.jpg	Is the person inside the red bounding box called Victor Verhaeghe? Please answer yes or no.	No
-tt0065724_shot_0320_img_1.jpg	Is the person inside the red bounding box named Karen Black? Please answer yes or no.	Yes
-tt0065724_shot_0320_img_1.jpg	Is the person inside the red bounding box named Nick Discenza? Please answer yes or no.	No
-tt0066026_shot_0085_img_0.jpg	Is the person inside the red bounding box called Donald Sutherland? Please answer yes or no.	Yes
-tt0066026_shot_0085_img_0.jpg	Is the person inside the red bounding box called Michael Wollet? Please answer yes or no.	No
-tt0066921_shot_0631_img_0.jpg	Is the actor inside the red bounding box called Malcolm McDowell? Please answer yes or no.	Yes
-tt0066921_shot_0631_img_0.jpg	Is the actor inside the red bounding box called Darling Légitimus? Please answer yes or no.	No
-tt0067116_shot_0122_img_0.jpg	Is the actor inside the red bounding box called Gene Hackman? Please answer yes or no.	Yes
-tt0067116_shot_0122_img_0.jpg	Is the actor inside the red bounding box called Russell G. Jones? Please answer yes or no.	No
-tt0068646_shot_0166_img_0.jpg	Is the actor inside the red bounding box called Marlon Brando? Please answer yes or no.	Yes
-tt0068646_shot_0166_img_0.jpg	Is the actor inside the red bounding box called Voltaire Sterling? Please answer yes or no.	No
-tt0069762_shot_0723_img_0.jpg	Is the person inside the red bounding box named Sissy Spacek? Please answer yes or no.	Yes
-tt0069762_shot_0723_img_0.jpg	Is the person inside the red bounding box named Monica Giordano? Please answer yes or no.	No
-tt0070047_shot_0255_img_0.jpg	Is the actor inside the red bounding box called Ellen Burstyn? Please answer yes or no.	Yes
-tt0070047_shot_0255_img_0.jpg	Is the actor inside the red bounding box called Shawnee Smith? Please answer yes or no.	No
-tt0070379_shot_0569_img_0.jpg	Is the actor inside the red bounding box named Richard Romanus? Please answer yes or no.	Yes
-tt0070379_shot_0569_img_0.jpg	Is the actor inside the red bounding box named Valerie Colgan? Please answer yes or no.	No
-tt0070511_shot_0639_img_0.jpg	Is the person inside the red bounding box called Dustin Hoffman? Please answer yes or no.	Yes
-tt0070511_shot_0639_img_0.jpg	Is the person inside the red bounding box called Fernando Lueches? Please answer yes or no.	No
-tt0070735_shot_0818_img_0.jpg	Is the person inside the red bounding box named Robert Redford? Please answer yes or no.	Yes
-tt0070735_shot_0818_img_0.jpg	Is the person inside the red bounding box named Ellin Dennis? Please answer yes or no.	No
-tt0070849_shot_0021_img_1.jpg	Is the person inside the red bounding box named Maria Schneider? Please answer yes or no.	Yes
-tt0070849_shot_0021_img_1.jpg	Is the person inside the red bounding box named Mary Kellogg? Please answer yes or no.	No
-tt0071315_shot_0153_img_0.jpg	Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no.	Yes
-tt0071315_shot_0153_img_0.jpg	Is the actor inside the red bounding box named Kelly Hitman? Please answer yes or no.	No
-tt0071562_shot_0684_img_0.jpg	Is the actor inside the red bounding box named Al Pacino? Please answer yes or no.	Yes
-tt0071562_shot_0684_img_0.jpg	Is the actor inside the red bounding box named Debie Jarczewski? Please answer yes or no.	No
-tt0072684_shot_0512_img_1.jpg	Is the person inside the red bounding box named Marisa Berenson? Please answer yes or no.	Yes
-tt0072684_shot_0512_img_1.jpg	Is the person inside the red bounding box named Graham Bohea? Please answer yes or no.	No
-tt0073195_shot_0280_img_0.jpg	Is the actor inside the red bounding box named Roy Scheider? Please answer yes or no.	Yes
-tt0073195_shot_0280_img_0.jpg	Is the actor inside the red bounding box named Abdul Qadir Farookh? Please answer yes or no.	No
-tt0073629_shot_0700_img_0.jpg	Is the person inside the red bounding box named Barry Bostwick? Please answer yes or no.	Yes
-tt0073629_shot_0700_img_0.jpg	Is the person inside the red bounding box named Johnny Galecki? Please answer yes or no.	No
-tt0074119_shot_0814_img_0.jpg	Is the actor inside the red bounding box called Robert Redford? Please answer yes or no.	Yes
-tt0074119_shot_0814_img_0.jpg	Is the actor inside the red bounding box called Delroy Lindo? Please answer yes or no.	No
-tt0074285_shot_0535_img_1.jpg	Is the person inside the red bounding box named William Katt? Please answer yes or no.	Yes
-tt0074285_shot_0535_img_1.jpg	Is the person inside the red bounding box named Stephen Rider? Please answer yes or no.	No
-tt0075148_shot_0618_img_0.jpg	Is the actor inside the red bounding box called Sylvester Stallone? Please answer yes or no.	Yes
-tt0075148_shot_0618_img_0.jpg	Is the actor inside the red bounding box called Eric Hatch? Please answer yes or no.	No
-tt0075686_shot_0373_img_0.jpg	Is the actor inside the red bounding box called Woody Allen? Please answer yes or no.	Yes
-tt0075686_shot_0373_img_0.jpg	Is the actor inside the red bounding box called Penny Wallace? Please answer yes or no.	No
-tt0076729_shot_0451_img_0.jpg	Is the actor inside the red bounding box called Sally Field? Please answer yes or no.	Yes
-tt0076729_shot_0451_img_0.jpg	Is the actor inside the red bounding box called Giorgio Libassi? Please answer yes or no.	No
-tt0076759_shot_0930_img_0.jpg	Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no.	Yes
-tt0076759_shot_0930_img_0.jpg	Is the actor inside the red bounding box called Ryoko Sadoshima? Please answer yes or no.	No
-tt0077402_shot_1220_img_0.jpg	Is the person inside the red bounding box named Scott H. Reiniger? Please answer yes or no.	Yes
-tt0077402_shot_1220_img_0.jpg	Is the person inside the red bounding box named Chris Delaney? Please answer yes or no.	No
-tt0077405_shot_0150_img_0.jpg	Is the actor inside the red bounding box named Sam Shepard? Please answer yes or no.	Yes
-tt0077405_shot_0150_img_0.jpg	Is the actor inside the red bounding box named Bijou Phillips? Please answer yes or no.	No
-tt0077416_shot_1442_img_0.jpg	Is the person inside the red bounding box named Robert De Niro? Please answer yes or no.	Yes
-tt0077416_shot_1442_img_0.jpg	Is the person inside the red bounding box named Stu Smith? Please answer yes or no.	No
-tt0077651_shot_0133_img_0.jpg	Is the person inside the red bounding box called Jamie Lee Curtis? Please answer yes or no.	Yes
-tt0077651_shot_0133_img_0.jpg	Is the person inside the red bounding box called Paris Arrowsmith? Please answer yes or no.	No
-tt0078788_shot_1434_img_0.jpg	Is the person inside the red bounding box called Martin Sheen? Please answer yes or no.	Yes
-tt0078788_shot_1434_img_0.jpg	Is the person inside the red bounding box called Le Capriccio Français? Please answer yes or no.	No
-tt0078841_shot_0692_img_0.jpg	Is the actor inside the red bounding box named Shirley MacLaine? Please answer yes or no.	Yes
-tt0078841_shot_0692_img_0.jpg	Is the actor inside the red bounding box named Tomas Choy? Please answer yes or no.	No
-tt0079417_shot_0735_img_0.jpg	Is the actor inside the red bounding box called Meryl Streep? Please answer yes or no.	Yes
-tt0079417_shot_0735_img_0.jpg	Is the actor inside the red bounding box called Ross Lacy? Please answer yes or no.	No
-tt0079470_shot_0798_img_0.jpg	Is the person inside the red bounding box named Eric Idle? Please answer yes or no.	Yes
-tt0079470_shot_0798_img_0.jpg	Is the person inside the red bounding box named Quincy Taylor? Please answer yes or no.	No
-tt0079945_shot_1411_img_0.jpg	Is the person inside the red bounding box named Persis Khambatta? Please answer yes or no.	Yes
-tt0079945_shot_1411_img_0.jpg	Is the person inside the red bounding box named Alison Waddell? Please answer yes or no.	No
-tt0080339_shot_0711_img_0.jpg	Is the actor inside the red bounding box named Robert Hays? Please answer yes or no.	Yes
-tt0080339_shot_0711_img_0.jpg	Is the actor inside the red bounding box named Grace Sullivan? Please answer yes or no.	No
-tt0080684_shot_1574_img_2.jpg	Is the actor inside the red bounding box called Mark Hamill? Please answer yes or no.	Yes
-tt0080684_shot_1574_img_2.jpg	Is the actor inside the red bounding box called Rodion Salnikov? Please answer yes or no.	No
-tt0081505_shot_0449_img_0.jpg	Is the actor inside the red bounding box called Shelley Duvall? Please answer yes or no.	Yes
-tt0081505_shot_0449_img_0.jpg	Is the actor inside the red bounding box called Antony Carrick? Please answer yes or no.	No
-tt0082089_shot_0046_img_0.jpg	Is the actor inside the red bounding box named Kathleen Turner? Please answer yes or no.	Yes
-tt0082089_shot_0046_img_0.jpg	Is the actor inside the red bounding box named Aaron Henderson? Please answer yes or no.	No
-tt0082198_shot_1353_img_0.jpg	Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no.	Yes
-tt0082198_shot_1353_img_0.jpg	Is the person inside the red bounding box named Tim Herlihy? Please answer yes or no.	No
-tt0082971_shot_0831_img_0.jpg	Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no.	Yes
-tt0082971_shot_0831_img_0.jpg	Is the actor inside the red bounding box called Richard Angarola? Please answer yes or no.	No
-tt0083658_shot_0963_img_0.jpg	Is the person inside the red bounding box called Rutger Hauer? Please answer yes or no.	Yes
-tt0083658_shot_0963_img_0.jpg	Is the person inside the red bounding box called Stéphane Julien? Please answer yes or no.	No
-tt0083866_shot_0364_img_0.jpg	Is the actor inside the red bounding box called Robert MacNaughton? Please answer yes or no.	Yes
-tt0083866_shot_0364_img_0.jpg	Is the actor inside the red bounding box called Seam Turay? Please answer yes or no.	No
-tt0083907_shot_0633_img_0.jpg	Is the actor inside the red bounding box named Bruce Campbell? Please answer yes or no.	Yes
-tt0083907_shot_0633_img_0.jpg	Is the actor inside the red bounding box named Kaden Leos? Please answer yes or no.	No
-tt0083929_shot_0405_img_0.jpg	Is the actor inside the red bounding box named Jennifer Jason Leigh? Please answer yes or no.	Yes
-tt0083929_shot_0405_img_0.jpg	Is the actor inside the red bounding box named Eric D. Sandgren? Please answer yes or no.	No
-tt0084726_shot_0283_img_0.jpg	Is the actor inside the red bounding box named Leonard Nimoy? Please answer yes or no.	Yes
-tt0084726_shot_0283_img_0.jpg	Is the actor inside the red bounding box named John Cusack? Please answer yes or no.	No
-tt0086190_shot_0815_img_0.jpg	Is the actor inside the red bounding box named Carrie Fisher? Please answer yes or no.	Yes
-tt0086190_shot_0815_img_0.jpg	Is the actor inside the red bounding box named Ernie Adams? Please answer yes or no.	No
-tt0086250_shot_1079_img_0.jpg	Is the actor inside the red bounding box called Steven Bauer? Please answer yes or no.	Yes
-tt0086250_shot_1079_img_0.jpg	Is the actor inside the red bounding box called Bill Nunn? Please answer yes or no.	No
-tt0086856_shot_0929_img_0.jpg	Is the actor inside the red bounding box called Peter Weller? Please answer yes or no.	Yes
-tt0086856_shot_0929_img_0.jpg	Is the actor inside the red bounding box called Tracee Cocco? Please answer yes or no.	No
-tt0086879_shot_0158_img_0.jpg	Is the person inside the red bounding box called Elizabeth Berridge? Please answer yes or no.	Yes
-tt0086879_shot_0158_img_0.jpg	Is the person inside the red bounding box called Ralph Ineson? Please answer yes or no.	No
-tt0087332_shot_0798_img_0.jpg	Is the person inside the red bounding box called Bill Murray? Please answer yes or no.	Yes
-tt0087332_shot_0798_img_0.jpg	Is the person inside the red bounding box called Jiao Xu? Please answer yes or no.	No
-tt0087469_shot_0049_img_2.jpg	Is the person inside the red bounding box named Harrison Ford? Please answer yes or no.	Yes
-tt0087469_shot_0049_img_2.jpg	Is the person inside the red bounding box named Paulo Benedeti? Please answer yes or no.	No
-tt0088847_shot_0109_img_0.jpg	Is the actor inside the red bounding box named Anthony Michael Hall? Please answer yes or no.	Yes
-tt0088847_shot_0109_img_0.jpg	Is the actor inside the red bounding box named Luis Javier? Please answer yes or no.	No
-tt0088944_shot_0634_img_0.jpg	Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no.	Yes
-tt0088944_shot_0634_img_0.jpg	Is the actor inside the red bounding box named Shaine Jones? Please answer yes or no.	No
-tt0088993_shot_0569_img_0.jpg	Is the actor inside the red bounding box called George A. Romero? Please answer yes or no.	Yes
-tt0088993_shot_0569_img_0.jpg	Is the actor inside the red bounding box called James Eckhouse? Please answer yes or no.	No
-tt0089218_shot_0327_img_0.jpg	Is the person inside the red bounding box named Sean Astin? Please answer yes or no.	Yes
-tt0089218_shot_0327_img_0.jpg	Is the person inside the red bounding box named Dan Hunter? Please answer yes or no.	No
-tt0089881_shot_0034_img_0.jpg	Is the actor inside the red bounding box called Tatsuya Nakadai? Please answer yes or no.	Yes
-tt0089881_shot_0034_img_0.jpg	Is the actor inside the red bounding box called Nancy Vee? Please answer yes or no.	No
-tt0090022_shot_0464_img_0.jpg	Is the actor inside the red bounding box called Scott Glenn? Please answer yes or no.	Yes
-tt0090022_shot_0464_img_0.jpg	Is the actor inside the red bounding box called Robert Ryan? Please answer yes or no.	No
-tt0090605_shot_0344_img_0.jpg	Is the person inside the red bounding box called Sigourney Weaver? Please answer yes or no.	Yes
-tt0090605_shot_0344_img_0.jpg	Is the person inside the red bounding box called Lia Beldam? Please answer yes or no.	No
-tt0090756_shot_0135_img_0.jpg	Is the person inside the red bounding box named Laura Dern? Please answer yes or no.	Yes
-tt0090756_shot_0135_img_0.jpg	Is the person inside the red bounding box named Keith Frost? Please answer yes or no.	No
-tt0091042_shot_0098_img_0.jpg	Is the person inside the red bounding box called Matthew Broderick? Please answer yes or no.	Yes
-tt0091042_shot_0098_img_0.jpg	Is the person inside the red bounding box called Mina E. Mina? Please answer yes or no.	No
-tt0091738_shot_0073_img_1.jpg	Is the actor inside the red bounding box called Kathleen Turner? Please answer yes or no.	Yes
-tt0091738_shot_0073_img_1.jpg	Is the actor inside the red bounding box called Pat Kiernan? Please answer yes or no.	No
-tt0091867_shot_0422_img_2.jpg	Is the person inside the red bounding box named Simon Callow? Please answer yes or no.	Yes
-tt0091867_shot_0422_img_2.jpg	Is the person inside the red bounding box named Rusty Goffe? Please answer yes or no.	No
-tt0092099_shot_0455_img_1.jpg	Is the person inside the red bounding box called Tom Cruise? Please answer yes or no.	Yes
-tt0092099_shot_0455_img_1.jpg	Is the person inside the red bounding box called Carol Krolick? Please answer yes or no.	No
-tt0092699_shot_0208_img_0.jpg	Is the actor inside the red bounding box called William Hurt? Please answer yes or no.	Yes
-tt0092699_shot_0208_img_0.jpg	Is the actor inside the red bounding box called Hildur Ruriks? Please answer yes or no.	No
-tt0093565_shot_0409_img_0.jpg	Is the actor inside the red bounding box named Cher? Please answer yes or no.	Yes
-tt0093565_shot_0409_img_0.jpg	Is the actor inside the red bounding box named Mark Brady? Please answer yes or no.	No
-tt0093748_shot_0346_img_0.jpg	Is the actor inside the red bounding box called John Candy? Please answer yes or no.	Yes
-tt0093748_shot_0346_img_0.jpg	Is the actor inside the red bounding box called Sarah Heller? Please answer yes or no.	No
-tt0093773_shot_0212_img_0.jpg	Is the person inside the red bounding box named Jesse Ventura? Please answer yes or no.	Yes
-tt0093773_shot_0212_img_0.jpg	Is the person inside the red bounding box named Akio Mitamura? Please answer yes or no.	No
-tt0093779_shot_1047_img_0.jpg	Is the person inside the red bounding box named Peter Falk? Please answer yes or no.	Yes
-tt0093779_shot_1047_img_0.jpg	Is the person inside the red bounding box named Lisa Ann Walter? Please answer yes or no.	No
-tt0094226_shot_0237_img_2.jpg	Is the actor inside the red bounding box called Kevin Costner? Please answer yes or no.	Yes
-tt0094226_shot_0237_img_2.jpg	Is the actor inside the red bounding box called Colin Hill? Please answer yes or no.	No
-tt0094737_shot_0567_img_0.jpg	Is the person inside the red bounding box called Tom Hanks? Please answer yes or no.	Yes
-tt0094737_shot_0567_img_0.jpg	Is the person inside the red bounding box called Chris McHallem? Please answer yes or no.	No
-tt0095016_shot_1170_img_0.jpg	Is the actor inside the red bounding box called Paul Gleason? Please answer yes or no.	Yes
-tt0095016_shot_1170_img_0.jpg	Is the actor inside the red bounding box called Carl Palmer? Please answer yes or no.	No
-tt0095250_shot_0509_img_0.jpg	Is the actor inside the red bounding box named Jean Reno? Please answer yes or no.	Yes
-tt0095250_shot_0509_img_0.jpg	Is the actor inside the red bounding box named Ralph Meyering Jr.? Please answer yes or no.	No
-tt0095765_shot_0008_img_0.jpg	Is the actor inside the red bounding box called Antonella Attili? Please answer yes or no.	Yes
-tt0095765_shot_0008_img_0.jpg	Is the actor inside the red bounding box called Amber Estrada? Please answer yes or no.	No
-tt0095953_shot_0412_img_0.jpg	Is the person inside the red bounding box named Tom Cruise? Please answer yes or no.	Yes
-tt0095953_shot_0412_img_0.jpg	Is the person inside the red bounding box named Lara Mulcahy? Please answer yes or no.	No
-tt0096320_shot_0085_img_0.jpg	Is the actor inside the red bounding box called Arnold Schwarzenegger? Please answer yes or no.	Yes
-tt0096320_shot_0085_img_0.jpg	Is the actor inside the red bounding box called Dan Duran? Please answer yes or no.	No
-tt0096754_shot_0570_img_1.jpg	Is the person inside the red bounding box named Todd Graff? Please answer yes or no.	Yes
-tt0096754_shot_0570_img_1.jpg	Is the person inside the red bounding box named Guy Carleton? Please answer yes or no.	No
-tt0096874_shot_0647_img_0.jpg	Is the actor inside the red bounding box named Michael J. Fox? Please answer yes or no.	Yes
-tt0096874_shot_0647_img_0.jpg	Is the actor inside the red bounding box named Momoko Komatsu? Please answer yes or no.	No
-tt0096895_shot_0819_img_1.jpg	Is the person inside the red bounding box called Michael Keaton? Please answer yes or no.	Yes
-tt0096895_shot_0819_img_1.jpg	Is the person inside the red bounding box called Ben Foster? Please answer yes or no.	No
-tt0097216_shot_0381_img_0.jpg	Is the actor inside the red bounding box named Danny Aiello? Please answer yes or no.	Yes
-tt0097216_shot_0381_img_0.jpg	Is the actor inside the red bounding box named Taissa Farmiga? Please answer yes or no.	No
-tt0097428_shot_0106_img_0.jpg	Is the actor inside the red bounding box named Bill Murray? Please answer yes or no.	Yes
-tt0097428_shot_0106_img_0.jpg	Is the actor inside the red bounding box named Michael Fawcett? Please answer yes or no.	No
-tt0097576_shot_1010_img_2.jpg	Is the actor inside the red bounding box named Harrison Ford? Please answer yes or no.	Yes
-tt0097576_shot_1010_img_2.jpg	Is the actor inside the red bounding box named M. Emmet Walsh? Please answer yes or no.	No
-tt0098635_shot_0556_img_0.jpg	Is the actor inside the red bounding box named Meg Ryan? Please answer yes or no.	Yes
-tt0098635_shot_0556_img_0.jpg	Is the actor inside the red bounding box named Tom Branch? Please answer yes or no.	No
-tt0098724_shot_0474_img_0.jpg	Is the person inside the red bounding box named Andie MacDowell? Please answer yes or no.	Yes
-tt0098724_shot_0474_img_0.jpg	Is the person inside the red bounding box named Linda Taylor? Please answer yes or no.	No
-tt0099423_shot_1010_img_0.jpg	Is the person inside the red bounding box called Bruce Willis? Please answer yes or no.	Yes
-tt0099423_shot_1010_img_0.jpg	Is the person inside the red bounding box called Trevor Eve? Please answer yes or no.	No
-tt0099487_shot_0123_img_0.jpg	Is the actor inside the red bounding box named Johnny Depp? Please answer yes or no.	Yes
-tt0099487_shot_0123_img_0.jpg	Is the actor inside the red bounding box named Farrah Forke? Please answer yes or no.	No
-tt0099674_shot_1356_img_0.jpg	Is the person inside the red bounding box named Al Pacino? Please answer yes or no.	Yes
-tt0099674_shot_1356_img_0.jpg	Is the person inside the red bounding box named Nick Porrazzo? Please answer yes or no.	No
-tt0099685_shot_1132_img_0.jpg	Is the actor inside the red bounding box called Ray Liotta? Please answer yes or no.	Yes
-tt0099685_shot_1132_img_0.jpg	Is the actor inside the red bounding box called Chick Allan? Please answer yes or no.	No
-tt0099810_shot_0285_img_0.jpg	Is the person inside the red bounding box called Alec Baldwin? Please answer yes or no.	Yes
-tt0099810_shot_0285_img_0.jpg	Is the person inside the red bounding box called Jennifer Anglin? Please answer yes or no.	No
-tt0100157_shot_0365_img_0.jpg	Is the actor inside the red bounding box named James Caan? Please answer yes or no.	Yes
-tt0100157_shot_0365_img_0.jpg	Is the actor inside the red bounding box named Bryan Johnson? Please answer yes or no.	No
-tt0100403_shot_0517_img_0.jpg	Is the person inside the red bounding box called Gary Busey? Please answer yes or no.	Yes
-tt0100403_shot_0517_img_0.jpg	Is the person inside the red bounding box called Alfred Tiaki Hotu? Please answer yes or no.	No
-tt0100405_shot_0786_img_0.jpg	Is the actor inside the red bounding box named Jason Alexander? Please answer yes or no.	Yes
-tt0100405_shot_0786_img_0.jpg	Is the actor inside the red bounding box named Alexandra Bastedo? Please answer yes or no.	No
-tt0101410_shot_0105_img_0.jpg	Is the person inside the red bounding box named John Turturro? Please answer yes or no.	Yes
-tt0101410_shot_0105_img_0.jpg	Is the person inside the red bounding box named David Gore? Please answer yes or no.	No
-tt0102492_shot_0086_img_0.jpg	Is the actor inside the red bounding box called Jamie Lee Curtis? Please answer yes or no.	Yes
-tt0102492_shot_0086_img_0.jpg	Is the actor inside the red bounding box called Heidi Fischer? Please answer yes or no.	No
-tt0103064_shot_1206_img_0.jpg	Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no.	Yes
-tt0103064_shot_1206_img_0.jpg	Is the actor inside the red bounding box named Gigi Lee? Please answer yes or no.	No
-tt0103064_shot_2602_img_1.jpg	Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no.	Yes
-tt0103064_shot_2602_img_1.jpg	Is the person inside the red bounding box named Candice Azzara? Please answer yes or no.	No
-tt0103776_shot_0719_img_0.jpg	Is the person inside the red bounding box called Michael Keaton? Please answer yes or no.	Yes
-tt0103776_shot_0719_img_0.jpg	Is the person inside the red bounding box called Nicholas Rice? Please answer yes or no.	No
-tt0104036_shot_0336_img_1.jpg	Is the person inside the red bounding box named Stephen Rea? Please answer yes or no.	Yes
-tt0104036_shot_0336_img_1.jpg	Is the person inside the red bounding box named Mimi Lizio? Please answer yes or no.	No
-tt0104257_shot_0477_img_0.jpg	Is the person inside the red bounding box named Jack Nicholson? Please answer yes or no.	Yes
-tt0104257_shot_0477_img_0.jpg	Is the person inside the red bounding box named Emma Julia Jacobs? Please answer yes or no.	No
-tt0104348_shot_0340_img_0.jpg	Is the person inside the red bounding box called Ed Harris? Please answer yes or no.	Yes
-tt0104348_shot_0340_img_0.jpg	Is the person inside the red bounding box called Carla Lizzette Mejia? Please answer yes or no.	No
-tt0105236_shot_0193_img_0.jpg	Is the actor inside the red bounding box named Harvey Keitel? Please answer yes or no.	Yes
-tt0105236_shot_0193_img_0.jpg	Is the actor inside the red bounding box named Terence Yin? Please answer yes or no.	No
-tt0105665_shot_0351_img_0.jpg	Is the actor inside the red bounding box named Kyle MacLachlan? Please answer yes or no.	Yes
-tt0105665_shot_0351_img_0.jpg	Is the actor inside the red bounding box named Julia Hsu? Please answer yes or no.	No
-tt0105695_shot_1436_img_1.jpg	Is the person inside the red bounding box called Jaimz Woolvett? Please answer yes or no.	Yes
-tt0105695_shot_1436_img_1.jpg	Is the person inside the red bounding box called Hermione Baddeley? Please answer yes or no.	No
-tt0106977_shot_1604_img_0.jpg	Is the person inside the red bounding box named Tommy Lee Jones? Please answer yes or no.	Yes
-tt0106977_shot_1604_img_0.jpg	Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no.	No
-tt0107614_shot_0116_img_0.jpg	Is the person inside the red bounding box called Sally Field? Please answer yes or no.	Yes
-tt0107614_shot_0116_img_0.jpg	Is the person inside the red bounding box called Arthur Senzy? Please answer yes or no.	No
-tt0108399_shot_0778_img_0.jpg	Is the actor inside the red bounding box called Christopher Walken? Please answer yes or no.	Yes
-tt0108399_shot_0778_img_0.jpg	Is the actor inside the red bounding box called Fiona Sit? Please answer yes or no.	No
-tt0109831_shot_0298_img_0.jpg	Is the person inside the red bounding box called Hugh Grant? Please answer yes or no.	Yes
-tt0109831_shot_0298_img_0.jpg	Is the person inside the red bounding box called Renée Zellweger? Please answer yes or no.	No
-tt0111280_shot_0258_img_0.jpg	Is the actor inside the red bounding box named Gates McFadden? Please answer yes or no.	Yes
-tt0111280_shot_0258_img_0.jpg	Is the actor inside the red bounding box named Michael Angarano? Please answer yes or no.	No
-tt0111280_shot_1479_img_2.jpg	Is the actor inside the red bounding box called William Shatner? Please answer yes or no.	Yes
-tt0111280_shot_1479_img_2.jpg	Is the actor inside the red bounding box called Richard Rohrbough? Please answer yes or no.	No
-tt0112384_shot_0878_img_0.jpg	Is the person inside the red bounding box called Kathleen Quinlan? Please answer yes or no.	Yes
-tt0112384_shot_0878_img_0.jpg	Is the person inside the red bounding box called Veronica Diaz Carranza? Please answer yes or no.	No
-tt0112641_shot_0412_img_1.jpg	Is the actor inside the red bounding box called Robert De Niro? Please answer yes or no.	Yes
-tt0112641_shot_0412_img_1.jpg	Is the actor inside the red bounding box called Pierre Malherbe? Please answer yes or no.	No
-tt0112740_shot_1056_img_0.jpg	Is the person inside the red bounding box named Denzel Washington? Please answer yes or no.	Yes
-tt0112740_shot_1056_img_0.jpg	Is the person inside the red bounding box named Bill Pullman? Please answer yes or no.	No
-tt0113101_shot_0547_img_0.jpg	Is the person inside the red bounding box named Tim Roth? Please answer yes or no.	Yes
-tt0113101_shot_0547_img_0.jpg	Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no.	No
-tt0114369_shot_1138_img_0.jpg	Is the person inside the red bounding box named Brad Pitt? Please answer yes or no.	Yes
-tt0114369_shot_1138_img_0.jpg	Is the person inside the red bounding box named Benjamin Nitze? Please answer yes or no.	No
-tt0114388_shot_0162_img_0.jpg	Is the actor inside the red bounding box called Emma Thompson? Please answer yes or no.	Yes
-tt0114388_shot_0162_img_0.jpg	Is the actor inside the red bounding box called Francis P. Hughes? Please answer yes or no.	No
-tt0114388_shot_1207_img_1.jpg	Is the person inside the red bounding box called Hugh Grant? Please answer yes or no.	Yes
-tt0114388_shot_1207_img_1.jpg	Is the person inside the red bounding box called Zach Hopkins? Please answer yes or no.	No
-tt0115798_shot_0844_img_1.jpg	Is the person inside the red bounding box named Jim Carrey? Please answer yes or no.	Yes
-tt0115798_shot_0844_img_1.jpg	Is the person inside the red bounding box named Renee Herlocker? Please answer yes or no.	No
-tt0116367_shot_0755_img_0.jpg	Is the actor inside the red bounding box named George Clooney? Please answer yes or no.	Yes
-tt0116367_shot_0755_img_0.jpg	Is the actor inside the red bounding box named Ben Crowley? Please answer yes or no.	No
-tt0116629_shot_1570_img_2.jpg	Is the person inside the red bounding box called Will Smith? Please answer yes or no.	Yes
-tt0116629_shot_1570_img_2.jpg	Is the person inside the red bounding box called E. Katherine Kerr? Please answer yes or no.	No
-tt0116695_shot_0343_img_0.jpg	Is the person inside the red bounding box named Tom Cruise? Please answer yes or no.	Yes
-tt0116695_shot_0343_img_0.jpg	Is the person inside the red bounding box named Billy Dee? Please answer yes or no.	No
-tt0117060_shot_0412_img_0.jpg	Is the actor inside the red bounding box called Tom Cruise? Please answer yes or no.	Yes
-tt0117060_shot_0412_img_0.jpg	Is the actor inside the red bounding box called Carrie Lazar? Please answer yes or no.	No
-tt0117060_shot_1401_img_0.jpg	Is the actor inside the red bounding box called Jean Reno? Please answer yes or no.	Yes
-tt0117060_shot_1401_img_0.jpg	Is the actor inside the red bounding box called Jill Teed? Please answer yes or no.	No
-tt0117381_shot_0798_img_1.jpg	Is the person inside the red bounding box called Edward Norton? Please answer yes or no.	Yes
-tt0117381_shot_0798_img_1.jpg	Is the person inside the red bounding box called Michael Tezla? Please answer yes or no.	No
-tt0117500_shot_2467_img_0.jpg	Is the actor inside the red bounding box called Ed Harris? Please answer yes or no.	Yes
-tt0117500_shot_2467_img_0.jpg	Is the actor inside the red bounding box called Paul J.Q. Lee? Please answer yes or no.	No
-tt0117509_shot_0041_img_0.jpg	Is the actor inside the red bounding box named Paul Rudd? Please answer yes or no.	Yes
-tt0117509_shot_0041_img_0.jpg	Is the actor inside the red bounding box named Max Martini? Please answer yes or no.	No
-tt0117571_shot_0475_img_0.jpg	Is the person inside the red bounding box named Neve Campbell? Please answer yes or no.	Yes
-tt0117571_shot_0475_img_0.jpg	Is the person inside the red bounding box named Frank Hoyt Taylor? Please answer yes or no.	No
-tt0117731_shot_0300_img_0.jpg	Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no.	Yes
-tt0117731_shot_0300_img_0.jpg	Is the actor inside the red bounding box called Debra Montague? Please answer yes or no.	No
-tt0117731_shot_1067_img_0.jpg	Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no.	Yes
-tt0117731_shot_1067_img_0.jpg	Is the actor inside the red bounding box called Jenny Wilson? Please answer yes or no.	No
-tt0118548_shot_1296_img_0.jpg	Is the actor inside the red bounding box called Clint Eastwood? Please answer yes or no.	Yes
-tt0118548_shot_1296_img_0.jpg	Is the actor inside the red bounding box called Kate Winslet? Please answer yes or no.	No
-tt0118571_shot_0627_img_0.jpg	Is the actor inside the red bounding box called Glenn Close? Please answer yes or no.	Yes
-tt0118571_shot_0627_img_0.jpg	Is the actor inside the red bounding box called Arlene Farber? Please answer yes or no.	No
-tt0118636_shot_0007_img_1.jpg	Is the person inside the red bounding box called Brad Renfro? Please answer yes or no.	Yes
-tt0118636_shot_0007_img_1.jpg	Is the person inside the red bounding box called Sandra Park? Please answer yes or no.	No
-tt0118636_shot_0344_img_0.jpg	Is the actor inside the red bounding box called Brad Renfro? Please answer yes or no.	Yes
-tt0118636_shot_0344_img_0.jpg	Is the actor inside the red bounding box called Karen Strassman? Please answer yes or no.	No
-tt0118655_shot_0279_img_0.jpg	Is the person inside the red bounding box called Robert Wagner? Please answer yes or no.	Yes
-tt0118655_shot_0279_img_0.jpg	Is the person inside the red bounding box called Arthur Birnbaum? Please answer yes or no.	No
-tt0118655_shot_1152_img_2.jpg	Is the actor inside the red bounding box called Seth Green? Please answer yes or no.	Yes
-tt0118655_shot_1152_img_2.jpg	Is the actor inside the red bounding box called Sue Doucette? Please answer yes or no.	No
-tt0118689_shot_0706_img_0.jpg	Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no.	Yes
-tt0118689_shot_0706_img_0.jpg	Is the actor inside the red bounding box called Hugo Perez? Please answer yes or no.	No
-tt0118689_shot_0969_img_2.jpg	Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no.	Yes
-tt0118689_shot_0969_img_2.jpg	Is the actor inside the red bounding box called Jack Shields? Please answer yes or no.	No
-tt0118715_shot_0079_img_0.jpg	Is the actor inside the red bounding box called Jeff Bridges? Please answer yes or no.	Yes
-tt0118715_shot_0079_img_0.jpg	Is the actor inside the red bounding box called Scott Adkins? Please answer yes or no.	No
-tt0118749_shot_0795_img_0.jpg	Is the person inside the red bounding box called John C. Reilly? Please answer yes or no.	Yes
-tt0118749_shot_0795_img_0.jpg	Is the person inside the red bounding box called Chris Lowell? Please answer yes or no.	No
-tt0118883_shot_0691_img_1.jpg	Is the actor inside the red bounding box called Julia Roberts? Please answer yes or no.	Yes
-tt0118883_shot_0691_img_1.jpg	Is the actor inside the red bounding box called Roger Bart? Please answer yes or no.	No
-tt0118971_shot_0679_img_0.jpg	Is the actor inside the red bounding box called Charlize Theron? Please answer yes or no.	Yes
-tt0118971_shot_0679_img_0.jpg	Is the actor inside the red bounding box called Young-min Kim? Please answer yes or no.	No
-tt0119008_shot_0979_img_0.jpg	Is the actor inside the red bounding box named Al Pacino? Please answer yes or no.	Yes
-tt0119008_shot_0979_img_0.jpg	Is the actor inside the red bounding box named Neil Tweddle? Please answer yes or no.	No
-tt0119094_shot_0446_img_2.jpg	Is the actor inside the red bounding box called Nicolas Cage? Please answer yes or no.	Yes
-tt0119094_shot_0446_img_2.jpg	Is the actor inside the red bounding box called Juan Gabriel Pareja? Please answer yes or no.	No
-tt0119116_shot_0721_img_0.jpg	Is the actor inside the red bounding box called Bruce Willis? Please answer yes or no.	Yes
-tt0119116_shot_0721_img_0.jpg	Is the actor inside the red bounding box called Troye Sivan? Please answer yes or no.	No
-tt0119174_shot_0439_img_0.jpg	Is the actor inside the red bounding box named Michael Douglas? Please answer yes or no.	Yes
-tt0119174_shot_0439_img_0.jpg	Is the actor inside the red bounding box named Carola McGuinness? Please answer yes or no.	No
-tt0119314_shot_0572_img_0.jpg	Is the actor inside the red bounding box called Scarlett Johansson? Please answer yes or no.	Yes
-tt0119314_shot_0572_img_0.jpg	Is the actor inside the red bounding box called Daisy Beaumont? Please answer yes or no.	No
-tt0119528_shot_0171_img_0.jpg	Is the person inside the red bounding box called Jim Carrey? Please answer yes or no.	Yes
-tt0119528_shot_0171_img_0.jpg	Is the person inside the red bounding box called Eliot Paton? Please answer yes or no.	No
-tt0119528_shot_0761_img_1.jpg	Is the actor inside the red bounding box named Jim Carrey? Please answer yes or no.	Yes
-tt0119528_shot_0761_img_1.jpg	Is the actor inside the red bounding box named Jari Kinnunen? Please answer yes or no.	No
-tt0119643_shot_0330_img_0.jpg	Is the actor inside the red bounding box named Brad Pitt? Please answer yes or no.	Yes
-tt0119643_shot_0330_img_0.jpg	Is the actor inside the red bounding box named Anthony Hopkins? Please answer yes or no.	No
-tt0119738_shot_0201_img_0.jpg	Is the person inside the red bounding box named Christopher Masterson? Please answer yes or no.	Yes
-tt0119738_shot_0201_img_0.jpg	Is the person inside the red bounding box named Edwin Craig? Please answer yes or no.	No
-tt0119822_shot_0878_img_0.jpg	Is the person inside the red bounding box named Greg Kinnear? Please answer yes or no.	Yes
-tt0119822_shot_0878_img_0.jpg	Is the person inside the red bounding box named Aleksandr Dubina? Please answer yes or no.	No
-tt0120338_shot_0444_img_2.jpg	Is the actor inside the red bounding box named Kate Winslet? Please answer yes or no.	Yes
-tt0120338_shot_0444_img_2.jpg	Is the actor inside the red bounding box named Donald Gibb? Please answer yes or no.	No
-tt0120338_shot_1130_img_2.jpg	Is the person inside the red bounding box called Leonardo DiCaprio? Please answer yes or no.	Yes
-tt0120338_shot_1130_img_2.jpg	Is the person inside the red bounding box called Anne Betancourt? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/code_reasoning.txt

@@ -1,40 +0,0 @@
-0001.png	The image shows a python code. Is the output of the code 'Hello'? Please answer yes or no.	Yes
-0001.png	The image shows a python code. Is the output of the code 'World'? Please answer yes or no.	No
-0002.png	The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no.	Yes
-0002.png	The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no.	No
-0003.png	The image shows a python code. Is the output of the code '12'? Please answer yes or no.	Yes
-0003.png	The image shows a python code. Is the output of the code '5'? Please answer yes or no.	No
-0004.png	The image shows a python code. Is the output of the code '3'? Please answer yes or no.	Yes
-0004.png	The image shows a python code. Is the output of the code '2'? Please answer yes or no.	No
-0005.png	The image shows a python code. Is the output of the code '12'? Please answer yes or no.	Yes
-0005.png	The image shows a python code. Is the output of the code '5'? Please answer yes or no.	No
-0006.png	The image shows a python code. Is the output of the code '0'? Please answer yes or no.	Yes
-0006.png	The image shows a python code. Is the output of the code '1'? Please answer yes or no.	No
-0007.png	Is a c++ code shown in the picture? Please answer yes or no.	Yes
-0007.png	Is a python code shown in the picture? Please answer yes or no.	No
-0008.png	The image shows a python code. Is the output of the code '1234'? Please answer yes or no.	Yes
-0008.png	The image shows a python code. Is the output of the code '12345'? Please answer yes or no.	No
-0009.png	The image shows a python code. Is the output of the code '36'? Please answer yes or no.	Yes
-0009.png	The image shows a python code. Is the output of the code '6'? Please answer yes or no.	No
-0010.png	The image shows a python code. Is the output of the code '1'? Please answer yes or no.	Yes
-0010.png	The image shows a python code. Is the output of the code '5'? Please answer yes or no.	No
-0011.png	The image shows a python code. Is the output of the code '0'? Please answer yes or no.	Yes
-0011.png	The image shows a python code. Is the output of the code '1'? Please answer yes or no.	No
-0012.png	The image shows a python code. Is the output of the code 'working hard'? Please answer yes or no.	Yes
-0012.png	The image shows a python code. Is the output of the code 'playing hard'? Please answer yes or no.	No
-0013.png	The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no.	Yes
-0013.png	The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no.	No
-0014.png	The image shows a python code. Is the output of the code '7'? Please answer yes or no.	Yes
-0014.png	The image shows a python code. Is the output of the code '1'? Please answer yes or no.	No
-0015.png	The image shows a python code. Is the output of the code '11'? Please answer yes or no.	Yes
-0015.png	The image shows a python code. Is the output of the code '9'? Please answer yes or no.	No
-0016.png	The image shows a python code. Is the output of the code 'x is smaller than 10'? Please answer yes or no.	Yes
-0016.png	The image shows a python code. Is the output of the code 'x is larger than 10'? Please answer yes or no.	No
-0017.png	The image shows a python code. Will the number 3 appear in the output of the code? Please answer yes or no.	Yes
-0017.png	The image shows a python code. Will the number 6 appear in the output of the code? Please answer yes or no.	No
-0018.png	The image shows a python code. Is the output of the code '11'? Please answer yes or no.	Yes
-0018.png	The image shows a python code. Is the output of the code '12'? Please answer yes or no.	No
-0019.png	The image shows a python code. Is the output of the code 'the list has more than 2 numbers'? Please answer yes or no.	Yes
-0019.png	The image shows a python code. Is the output of the code 'the list has less than 2 numbers'? Please answer yes or no.	No
-0020.png	Is a python code shown in the picture? Please answer yes or no.	Yes
-0020.png	Is a c++ code shown in the picture? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/color.txt

@@ -1,60 +0,0 @@
-000000006723.jpg	Is there a red brick building in the image? Please answer yes or no.	Yes
-000000006723.jpg	Is there a yellow brick building in the image? Please answer yes or no.	No
-000000008277.jpg	Is there a white plate in the image? Please answer yes or no.	Yes
-000000008277.jpg	Is there a yellow plate in the image? Please answer yes or no.	No
-000000012120.jpg	Is there a blue court in the image? Please answer yes or no.	Yes
-000000012120.jpg	Is there a purple court in the image? Please answer yes or no.	No
-000000014831.jpg	Is there a brown and white animal in the image? Please answer yes or no.	Yes
-000000014831.jpg	Is there a green and red animal in the image? Please answer yes or no.	No
-000000028993.jpg	Are there yellow poles in the image? Please answer yes or no.	Yes
-000000028993.jpg	Are there blue poles in the image? Please answer yes or no.	No
-000000029393.jpg	Is there a brown dog in the image? Please answer yes or no.	Yes
-000000029393.jpg	Is there a black dog in the image? Please answer yes or no.	No
-000000035770.jpg	Is there a black and white toilet in the image? Please answer yes or no.	Yes
-000000035770.jpg	Is there a red and white toilet in the image? Please answer yes or no.	No
-000000038118.jpg	Is there a red coat in the image? Please answer yes or no.	Yes
-000000038118.jpg	Is there a yellow coat in the image? Please answer yes or no.	No
-000000047112.jpg	Is there a white plate in the image? Please answer yes or no.	Yes
-000000047112.jpg	Is there a yellow plate in the image? Please answer yes or no.	No
-000000047121.jpg	Is there a black cat in the image? Please answer yes or no.	Yes
-000000047121.jpg	Is there a brown cat in the image? Please answer yes or no.	No
-000000053529.jpg	Is there a green hat in the image? Please answer yes or no.	Yes
-000000053529.jpg	Is there a red hat in the image? Please answer yes or no.	No
-000000053994.jpg	Is there a gray wall in the image? Please answer yes or no.	Yes
-000000053994.jpg	Is there a red wall in the image? Please answer yes or no.	No
-000000055072.jpg	Is there a brown giraffe in the image?  Please answer yes or no.	Yes
-000000055072.jpg	Is there a black giraffe in the image? Please answer yes or no.	No
-000000057597.jpg	Are there any red shoes in the image? Please answer yes or no.	Yes
-000000057597.jpg	Are there any yellow shoes in the image? Please answer yes or no.	No
-000000061658.jpg	Are there a white dish in the image? Please answer yes or no.	Yes
-000000061658.jpg	Are there a green dish in the image? Please answer yes or no.	No
-000000338560.jpg	Is there a blue and yellow fire hydrant in the image? Please answer yes or no.	Yes
-000000338560.jpg	Is there a blue and orange fire hydrant in the image? Please answer yes or no.	No
-000000370208.jpg	Is there a red bicycle with white handlebars in the image? Please answer yes or no.	Yes
-000000370208.jpg	Is there a red bicycle with black handlebars in the image? Please answer yes or no.	No
-000000377723.jpg	Is there a blue bus in the image? Please answer yes or no.	Yes
-000000377723.jpg	Is there a orange bus in the image? Please answer yes or no.	No
-000000405205.jpg	Is there a white bus in the image?  Please answer yes or no.	Yes
-000000405205.jpg	Is there a red bus in the image?  Please answer yes or no.	No
-000000410612.jpg	Is there a red boat in the image? Please answer yes or no.	Yes
-000000410612.jpg	Is there a gray boat in the image? Please answer yes or no.	No
-000000427034.jpg	Is there a brown and black dog in the image? Please answer yes or no.	Yes
-000000427034.jpg	Is there a brown and white dog in the image? Please answer yes or no.	No
-000000442456.jpg	Is there a man wearing a red shirt in the image? Please answer yes or no.	Yes
-000000442456.jpg	Is there a man wearing a white shirt in the image? Please answer yes or no.	No
-000000492362.jpg	Is there a skateboard with red wheels in the image? Please answer yes or no.	Yes
-000000492362.jpg	Is there a skateboard with black wheels in the image? Please answer yes or no.	No
-000000492992.jpg	Is there a white bird in the image? Please answer yes or no.	Yes
-000000492992.jpg	Is there a yellow bird in the image? Please answer yes or no.	No
-000000512929.jpg	Are there any green beans in the image? Please answer yes or no.	Yes
-000000512929.jpg	Are there any orange beans in the image? Please answer yes or no.	No
-000000530457.jpg	Are there any red flowers in the image? Please answer yes or no.	Yes
-000000530457.jpg	Are there any green flowers in the image? Please answer yes or no.	No
-000000532761.jpg	Is there a living room painted yellow in the image? Please answer yes or no.	Yes
-000000532761.jpg	Is there a living room painted black in the image? Please answer yes or no.	No
-000000534041.jpg	Is there a purple bottle in the image? Please answer yes or no.	Yes
-000000534041.jpg	Is there a white bottle in the image? Please answer yes or no.	No
-000000563758.jpg	Is there a red scarf in the image?  Please answer yes or no.	Yes
-000000563758.jpg	Is there a brown scarf in the image?  Please answer yes or no.	No
-000000564280.jpg	Is there a red couch in the image? Please answer yes or no.	Yes
-000000564280.jpg	Is there a black couch in the image? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/commonsense_reasoning.txt

@@ -1,140 +0,0 @@
-0001.png	Here are the order details for my taxi ride. Should I actually pay $29.42? Please answer yes or no.	Yes
-0001.png	Here are the order details for my taxi ride. Should I actually pay $32.42? Please answer yes or no.	No
-0002.png	Should I stop when I'm about to cross the street and see the sign in the picture? Please answer yes or no.	Yes
-0002.png	When I see the sign in the picture, can I cross the street? Please answer yes or no.	No
-0003.png	May I ask if in the game of finger-guessing game, did the right side of the picture win? Please answer yes or no.	Yes
-0003.png	May I ask if in the game of finger-guessing game, did the left side of the picture win? Please answer yes or no.	No
-0004.png	Does the fruit in the picture look stale? Please answer yes or no.	Yes
-0004.png	Does the fruit in the picture look very fresh? Please answer yes or no.	No
-0005.png	The office's normal closing time is 5 p.m. Now is afternoon. Should I continue to work at the time shown in the picture? Please answer yes or no.	Yes
-0005.png	The office's normal closing time is 5 p.m. Now is afternoon. Could I leave work at the time shown in the picture? Please answer yes or no.	No
-0006.png	I recently want to go on vacation to relax and go to a place full of fresh air. Is the venue in the picture appropriate? Please answer yes or no.	Yes
-0006.png	I want to go where there are a lot of people. Is the venue in the picture appropriate? Please answer yes or no.	No
-0007.png	I want to clean the house and I want to choose a tool. Is the tool in the picture an appropriate choice? Please answer yes or no.	Yes
-0007.png	I want to transport something and I want to choose a tool to help me. Is the tool in the picture an appropriate choice? Please answer yes or no.	No
-0008.png	Can I smoke where the picture is? Please answer yes or no.	Yes
-0008.png	Is smoking prohibited in the location of the picture? Please answer yes or no.	No
-0009.png	Will green be obtained by mixing the above two colors? Please answer yes or no.	Yes
-0009.png	Will red be obtained by mixing the above two colors? Please answer yes or no.	No
-0010.png	I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no.	Yes
-0010.png	I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no.	No
-0011.png	If I am allergic to durian, can I finish the fruit in the picture? Please answer yes or no.	Yes
-0011.png	If I am allergic to banana, can I finish the fruit in the picture? Please answer yes or no.	No
-0012.png	I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no.	Yes
-0012.png	I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no.	No
-0013.png	I am going to a formal dinner party. Is the shoe in the picture an appropriate choice? Please answer yes or no.	Yes
-0013.png	I am going to play basketball. Is the shoe in the picture an appropriate choice? Please answer yes or no.	No
-0014.png	In this line chart, the vertical axis is height and the horizontal axis is age. Does Maria's height exceed Jane's height in the end? Please answer yes or no.	Yes
-0014.png	In this line chart, the vertical axis is height and the horizontal axis is age. Does Jane's height exceed Kangkang's height in the end? Please answer yes or no.	No
-0015.png	Is the ball usually played with hands? Please answer yes or no.	Yes
-0015.png	Is the ball usually played with feet? Please answer yes or no.	No
-0016.png	Is the place in the picture a good place to enjoy the cool in a sunny day? Please answer yes or no.	Yes
-0016.png	Is the place in the picture a good shelter from the rain when it thunders outside? Please answer yes or no.	No
-0017.png	Are the vehicles in the pictures usually environmentally friendly? Please answer yes or no.	Yes
-0017.png	Does the vehicle in the picture usually run faster than the car? Please answer yes or no.	No
-0018.png	This is a picture of some kind of animal. Does it eat leaves? Please answer yes or no.	Yes
-0018.png	This is a picture of some kind of animal. Does it eat meat? Please answer yes or no.	No
-0019.png	Is the water flow in the picture from the top to the bottom? Please answer yes or no.	Yes
-0019.png	Is the water flow in the picture from the bottom to the top? Please answer yes or no.	No
-0020.png	Can the item in the picture be used to measure length? Please answer yes or no.	Yes
-0020.png	Can the item in the picture be used to measure angles? Please answer yes or no.	No
-0021.png	This is a toilet guide sign. I am a man. Should I go to the toilet on the left? Please answer yes or no.	Yes
-0021.png	This is a toilet guide sign. I am a man. Should I go to the toilet on the right? Please answer yes or no.	No
-0022.png	Does the animal in the picture usually catch mice? Please answer yes or no.	Yes
-0022.png	Is the animal in the picture usually used in search and rescue? Please answer yes or no.	No
-0023.png	If you want to keep your fruit fresh in summer, should you put it in the appliance in the picture? Please answer yes or no.	Yes
-0023.png	Is the appliance in the picture more suitable for winter than summer? Please answer yes or no.	No
-0024.png	I want to go skating. Is the shoe in the picture usually appropriate? Please answer yes or no.	Yes
-0024.png	I want to go roller skating. Is the shoe in the picture usually appropriate? Please answer yes or no.	No
-0025.png	I feel very thirsty in the desert now. Can the thing in the picture help me? Please answer yes or no.	Yes
-0025.png	I don't like clear cups. Is the cup in the picture my type? Please answer yes or no.	No
-0026.png	I want to go for a run and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no.	Yes
-0026.png	I want to practice ballet and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no.	No
-0027.png	Are the pants in the picture usually suitable for casual wear? Please answer yes or no.	Yes
-0027.png	Are the pants in the picture usually suitable for playing basketball? Please answer yes or no.	No
-0028.png	This is a picture from a real scene. Is there only one real cat in this picture? Please answer yes or no.	Yes
-0028.png	This is a picture from a real scene. Is there only two real cats in this picture? Please answer yes or no.	No
-0029.png	The three cats in the picture, the one without a beard, is the middle one? Please answer yes or no.	Yes
-0029.png	The three cats in the picture, the one without a beard, is the right one? Please answer yes or no.	No
-0030.png	I'm going to 501. Do I need to turn left at the intersection? Please answer yes or no.	Yes
-0030.png	I'm going to 502. Do I need to turn left at the intersection? Please answer yes or no.	No
-0031.png	Is the drink in the picture usually suitable for a party? Please answer yes or no.	Yes
-0031.png	Is the drink in the picture usually suitable for drinking together with cephalosporin? Please answer yes or no.	No
-0032.png	Here is a picture of the cake I cut. Did I cut it at least twice? Please answer yes or no.	Yes
-0032.png	Here is a picture of the cake I cut. Did I cut it at least once? Please answer yes or no.	No
-0033.png	This is a picture from a real scene. Is there only one real apple in this picture? Please answer yes or no.	Yes
-0033.png	This is a picture from a real scene. Is there only two real apples in this picture? Please answer yes or no.	No
-0034.png	Here is a pie chart counting the favorite fruits of all employees in our company. Is the durian the most popular fruit? Please answer yes or no.	Yes
-0034.png	Here is a pie chart counting the favorite fruits of all employees in our company. Is the mango the most popular fruit? Please answer yes or no.	No
-0035.png	This is the sales chart of this month. Is Tina the runner-up in sales this month? Please answer yes or no.	Yes
-0035.png	This is the sales chart of this month. Is John the runner-up in sales this month? Please answer yes or no.	No
-0036.png	Is it a good time to walk through the road in the picture? Please answer yes or no.	Yes
-0036.png	Is it a good time to drive a car through the road in the picture? Please answer yes or no.	No
-0037.png	Here is a photo of the sun's position at a certain time. Could it be dusk now? Please answer yes or no.	Yes
-0037.png	Here is a photo of the sun's position at a certain time. Could it be noon now? Please answer yes or no.	No
-0038.png	All apples are shown in the picture. If I eat an apple every day, can I eat it for four days? Please answer yes or no.	Yes
-0038.png	All apples are shown in the picture. If I eat an apple every day, can I eat it for three days? Please answer yes or no.	No
-0039.png	This line chart is used to count the sales of two types of burgers. Are chicken burgers more popular? Please answer yes or no.	Yes
-0039.png	This line chart is used to count the sales of two types of burgers. Are beef burgers more popular? Please answer yes or no.	No
-0040.png	I want to supplement protein. Is it appropriate to eat the food in the picture? Please answer yes or no.	Yes
-0040.png	I don't like to eat any food related to chicken. Is the food in the picture my type? Please answer yes or no.	No
-0041.png	Is the fruit in the picture usually sweet? Please answer yes or no.	Yes
-0041.png	Is the fruit in the picture usually spicy? Please answer yes or no.	No
-0042.png	Are there usually cars in the area shown in the picture? Please answer yes or no.	Yes
-0042.png	Is it appropriate to cross the road directly from the place shown in the picture? Please answer yes or no.	No
-0043.png	Is the animal in the picture usually not seen in winter? Please answer yes or no.	Yes
-0043.png	Is the animal in the picture usually seen in winter? Please answer yes or no.	No
-0044.png	This is a flowchart of a program. I enter 3 and 6. Is the output 'No'? Please answer yes or no.	Yes
-0044.png	This is a flowchart of a program. I enter 3 and 6. Is the output 'Yes'? Please answer yes or no.	No
-0045.png	There is a sign at the intersection, can I turn left? Please answer yes or no.	Yes
-0045.png	There is a sign at the intersection, can I turn right? Please answer yes or no.	No
-0046.png	Vitamin C is very helpful for human health. Does the food on in the picture usually contain Vitamin C? Please answer yes or no.	Yes
-0046.png	Is the food in the picture commonly used to build muscle? Please answer yes or no.	No
-0047.png	All apples are shown in the picture. My brother and I divide the apples equally. May I have one apple? Please answer yes or no.	Yes
-0047.png	All apples are shown in the picture. My brother and I divide the apples equally. May I have two apples? Please answer yes or no.	No
-0048.png	Here is a picture of eating fruit. Am I eating a strawberry? Please answer yes or no.	Yes
-0048.png	Here is a picture of eating fruit. Am I eating a cherry tomato? Please answer yes or no.	No
-0049.png	Does the vehicle in the picture usually have its Windows closed during fast driving? Please answer yes or no.	Yes
-0049.png	Does the vehicle in the picture usually have its Windows opened during fast driving? Please answer yes or no.	No
-0050.png	Do people commonly use the item in the picture for makeup in their daily lives? Please answer yes or no.	Yes
-0050.png	Do people commonly use the item in the picture to write in their daily lives? Please answer yes or no.	No
-0051.png	This is a flowchart of a program. When the input is 5, is the output 6? Please answer yes or no.	Yes
-0051.png	This is a flowchart of a program. When the input is 6, is the output 5? Please answer yes or no.	No
-0052.png	I want to lose weight. Is the food in the picture an appropriate choice? Please answer yes or no.	Yes
-0052.png	I want to gain weight. Is the food in the picture an appropriate choice? Please answer yes or no.	No
-0053.png	Is the car in the picture going to make a right turn after going through a straight road section? Please answer yes or no.	Yes
-0053.png	Is the car in the picture going to make a left turn after going through a straight road section? Please answer yes or no.	No
-0054.png	May I ask if the plants in the picture can survive in the water? Please answer yes or no.	Yes
-0054.png	May I ask if the plants in the picture can survive in the soil? Please answer yes or no.	No
-0055.png	The man in the picture is eating. Does he eat noodles? Please answer yes or no.	Yes
-0055.png	The man in the picture is eating. Does he eat rice? Please answer yes or no.	No
-0056.png	Can the item in the picture output water? Please answer yes or no.	Yes
-0056.png	Can the item in picture be used for blowing air? Please answer yes or no.	No
-0057.png	Does the vehicle in the picture usually run faster than a horse? Please answer yes or no.	Yes
-0057.png	Does the vehicle in the picture usually fly? Please answer yes or no.	No
-0058.png	Can't I smoke here? Please answer yes or no.	Yes
-0058.png	May I smoke here? Please answer yes or no.	No
-0059.png	This pie chart is the age distribution of our company. Is the proportion of people aged 30-50 more than 40%? Please answer yes or no.	Yes
-0059.png	This pie chart is the age distribution of our company. Is the proportion of people aged 40-50 more than 30%? Please answer yes or no.	No
-0060.png	This is the histogram of fruit sales today. Do more men buy watermelons than women buy bananas? Please answer yes or no.	Yes
-0060.png	This is the histogram of fruit sales today. Do more men buy peach than women buy apple? Please answer yes or no.	No
-0061.png	Is the tool in the picture common in tall buildings? Please answer yes or no.	Yes
-0061.png	In case of fire, is it appropriate to choose the tool in the picture to go downstairs? Please answer yes or no.	No
-0062.png	It's snowing outside the window now. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no.	Yes
-0062.png	It's very hot outside. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no.	No
-0063.png	Is the animal in the picture suitable as a pet? Please answer yes or no.	Yes
-0063.png	Is the animal in the pictures usually stronger than adult tigers? Please answer yes or no.	No
-0064.png	I want to play basketball. Is the venue in the picture a good choice? Please answer yes or no.	Yes
-0064.png	I want to play football. Is the venue in the picture a good choice? Please answer yes or no.	No
-0065.png	Is it appropriate to wear a down jacket during the season in the picture? Please answer yes or no.	Yes
-0065.png	Is it appropriate to only wear short sleeves during the season in the picture? Please answer yes or no.	No
-0066.png	I want to carry one thing with me on a rainy day. Is the thing in the image an appropriate choice? Please answer yes or no.	Yes
-0066.png	It is raining outside. I am in a house and I don't need to go out. Is this thing in the picture necessary for me to use? Please answer yes or no.	No
-0067.png	I feel very hot. Is the tool in the picture suitable for use? Please answer yes or no.	Yes
-0067.png	I feel very cold. Is the tool in the picture suitable for use? Please answer yes or no.	No
-0068.png	Is it unhealthy to eat the food in the picture too often? Please answer yes or no.	Yes
-0068.png	Is the food in the picture usually low in calories? Please answer yes or no.	No
-0069.png	Is the phone in the photo connected to a charger? Please answer yes or no.	Yes
-0069.png	Is the phone in the photo charging? Please answer yes or no.	No
-0070.png	I want to turn the screw. Is the tool in the picture usually appropriate? Please answer yes or no.	Yes
-0070.png	Is the tool in the picture usually suitable for smashing walnuts? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/count.txt

@@ -1,60 +0,0 @@
-000000006040.jpg	Is there a train in the picture? Please answer yes or no.	Yes
-000000006040.jpg	Are there a total of two trains in the picture? Please answer yes or no.	No
-000000044279.jpg	Is there a total of two people in the image? Please answer yes or no.	Yes
-000000044279.jpg	Is there only one people in the image? Please answer yes or no.	No
-000000067213.jpg	Is there only one dog in the image? Please answer yes or no.	Yes
-000000067213.jpg	Is there two dogs in the image? Please answer yes or no.	No
-000000071226.jpg	Is there a total of two dogs in the image? Please answer yes or no.	Yes
-000000071226.jpg	Is there only one dogs in the image? Please answer yes or no.	No
-000000097994.jpg	Are there three laptops in the picture? Please answer yes or no.	Yes
-000000097994.jpg	Are there four laptops in the picture? Please answer yes or no.	No
-000000195918.jpg	Is there a total of two display devices in the image? Please answer yes or no.	Yes
-000000195918.jpg	Is there only one display device in the image?  Please answer yes or no.	No
-000000236721.jpg	Are there two bananas in the image? Please answer yes or no.	Yes
-000000236721.jpg	Are there three bananas in the image? Please answer yes or no.	No
-000000261712.jpg	Are there two giraffes in this image? Please answer yes or no.	Yes
-000000261712.jpg	Are there three giraffes in this picture? Please answer yes or no.	No
-000000274066.jpg	Are there four people appear in this image? Please answer yes or no.	Yes
-000000274066.jpg	Are there only three people appear in this image? Please answer yes or no.	No
-000000276434.jpg	Is there a total of three cakes in this image? Please answer yes or no.	Yes
-000000276434.jpg	Are there only two cakes in this image? Please answer yes or no.	No
-000000289059.jpg	Is there a total of two person appear in the image? Please answer yes or no.	Yes
-000000289059.jpg	Is there only one person appear in the image? Please answer yes or no.	No
-000000290081.jpg	Is there only one bowl in this image? Please answer yes or no.	Yes
-000000290081.jpg	Are there two bowls in this image? Please answer yes or no.	No
-000000301867.jpg	Are there three people appear in this image? Please answer yes or no.	Yes
-000000301867.jpg	Are there only two people appear in this image? Please answer yes or no.	No
-000000335954.jpg	Are there two bowls in this image? Please answer yes or no.	Yes
-000000335954.jpg	Are there three bowls in this image? Please answer yes or no.	No
-000000357816.jpg	Are there four people in this image? Please answer yes or no.	Yes
-000000357816.jpg	Are there five people in this image? Please answer yes or no.	No
-000000372819.jpg	Are there four dogs appear in this image? Please answer yes or no.	Yes
-000000372819.jpg	Are there only three dogs appear in this image? Please answer yes or no.	No
-000000410612.jpg	Is there only one ship in the picture? Please answer yes or no.	Yes
-000000410612.jpg	Is there a total of two ships in the picture? Please answer yes or no.	No
-000000423944.jpg	Is there no person in this picture? Please answer yes or no.	Yes
-000000423944.jpg	Are there two people appear in this image? Please answer yes or no.	No
-000000427034.jpg	Is there a dog in the picture? Please answer yes or no.	Yes
-000000427034.jpg	Are there a total of two dogs in the picture? Please answer yes or no.	No
-000000430286.jpg	Are there three remotes in this image? Please answer yes or no.	Yes
-000000430286.jpg	Are there only two remotes in this image? Please answer yes or no.	No
-000000432468.jpg	Are there three zippers in the picture? Please answer yes or no.	Yes
-000000432468.jpg	Is there a zipper in the picture? Please answer yes or no.	No
-000000434479.jpg	Are there two pieces of pizza in this image? Please answer yes or no.	Yes
-000000434479.jpg	Is there only one piece of pizza in this image? Please answer yes or no.	No
-000000438304.jpg	Are there two tennis rackets in the picture? Please answer yes or no.	Yes
-000000438304.jpg	Are there only one tennis racket in the picture? Please answer yes or no.	No
-000000450303.jpg	Are there six people appear in this image? Please answer yes or no.	Yes
-000000450303.jpg	Are there seven people appear in this image? Please answer yes or no.	No
-000000470121.jpg	Is there only one bottle in the image? Please answer yes or no.	Yes
-000000470121.jpg	Is there two bottles in the image? Please answer yes or no.	No
-000000476215.jpg	Are there two horses in this image? Please answer yes or no.	Yes
-000000476215.jpg	Is there only one horse in this image? Please answer yes or no.	No
-000000482100.jpg	Are there two toilets in the picture? Please answer yes or no.	Yes
-000000482100.jpg	Is there only one toilet in the picture? Please answer yes or no.	No
-000000491867.jpg	Is there only one necktie in the image? Please answer yes or no.	Yes
-000000491867.jpg	Is there three neckties in the image? Please answer yes or no.	No
-000000556000.jpg	Are there four people in the image? Please answer yes or no.	Yes
-000000556000.jpg	Are there only three people in the image? Please answer yes or no.	No
-000000565045.jpg	Are there two bath towels in the picture? Please answer yes or no.	Yes
-000000565045.jpg	Is there only one bath towel in the picture? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/existence.txt

@@ -1,60 +0,0 @@
-000000006040.jpg	Is there a train in this image? Please answer yes or no.	Yes
-000000006040.jpg	Is there a bed in this image? Please answer yes or no.	No
-000000006471.jpg	Is there a baseball bat in this image? Please answer yes or no.	Yes
-000000006471.jpg	Is there a giraffe in this image? Please answer yes or no.	No
-000000007108.jpg	Is there a elephant in this image? Please answer yes or no.	Yes
-000000007108.jpg	Is there a hair drier in this image? Please answer yes or no.	No
-000000007816.jpg	Is there a motorcycle in this image? Please answer yes or no.	Yes
-000000007816.jpg	Is there a airplane in this image? Please answer yes or no.	No
-000000007977.jpg	Is there a skateboard in this image? Please answer yes or no.	Yes
-000000007977.jpg	Is there a spoon in this image? Please answer yes or no.	No
-000000008844.jpg	Is there a person in this image? Please answer yes or no.	Yes
-000000008844.jpg	Is there a sink in this image? Please answer yes or no.	No
-000000009590.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
-000000009590.jpg	Is there a scissors in this image? Please answer yes or no.	No
-000000010363.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
-000000010363.jpg	Is there a apple in this image? Please answer yes or no.	No
-000000011197.jpg	Is there a car in this image? Please answer yes or no.	Yes
-000000011197.jpg	Is there a fork in this image? Please answer yes or no.	No
-000000015254.jpg	Is there a spoon in this image? Please answer yes or no.	Yes
-000000015254.jpg	Is there a donut in this image? Please answer yes or no.	No
-000000015517.jpg	Is there a bus in this image? Please answer yes or no.	Yes
-000000015517.jpg	Is there a cow in this image? Please answer yes or no.	No
-000000015746.jpg	Is there a fire hydrant in this image? Please answer yes or no.	Yes
-000000015746.jpg	Is there a person in this image? Please answer yes or no.	No
-000000037751.jpg	Is there a backpack in this image? Please answer yes or no.	Yes
-000000037751.jpg	Is there a microwave in this image? Please answer yes or no.	No
-000000050145.jpg	Is there a bicycle in this image? Please answer yes or no.	Yes
-000000050145.jpg	Is there a apple in this image? Please answer yes or no.	No
-000000061418.jpg	Is there a chair in this image? Please answer yes or no.	Yes
-000000061418.jpg	Is there a airplane in this image? Please answer yes or no.	No
-000000417779.jpg	Is there a car in this image? Please answer yes or no.	Yes
-000000417779.jpg	Is there a kite in this image? Please answer yes or no.	No
-000000424521.jpg	Is there a skateboard in this image? Please answer yes or no.	Yes
-000000424521.jpg	Is there a banana in this image? Please answer yes or no.	No
-000000438304.jpg	Is there a sports ball in this image? Please answer yes or no.	Yes
-000000438304.jpg	Is there a horse in this image? Please answer yes or no.	No
-000000494427.jpg	Is there a laptop in this image? Please answer yes or no.	Yes
-000000494427.jpg	Is there a potted plant in this image? Please answer yes or no.	No
-000000495448.jpg	Is there a cake in this image? Please answer yes or no.	Yes
-000000495448.jpg	Is there a tie in this image? Please answer yes or no.	No
-000000498463.jpg	Is there a refrigerator in this image? Please answer yes or no.	Yes
-000000498463.jpg	Is there a donut in this image? Please answer yes or no.	No
-000000519039.jpg	Is there a truck in this image? Please answer yes or no.	Yes
-000000519039.jpg	Is there a book in this image? Please answer yes or no.	No
-000000523241.jpg	Is there a car in this image? Please answer yes or no.	Yes
-000000523241.jpg	Is there a cell phone in this image? Please answer yes or no.	No
-000000530162.jpg	Is there a umbrella in this image? Please answer yes or no.	Yes
-000000530162.jpg	Is there a horse in this image? Please answer yes or no.	No
-000000537812.jpg	Is there a chair in this image? Please answer yes or no.	Yes
-000000537812.jpg	Is there a baseball bat in this image? Please answer yes or no.	No
-000000541952.jpg	Is there a clock in this image? Please answer yes or no.	Yes
-000000541952.jpg	Is there a bottle in this image? Please answer yes or no.	No
-000000546626.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
-000000546626.jpg	Is there a mouse in this image? Please answer yes or no.	No
-000000556000.jpg	Is there a chair in this image? Please answer yes or no.	Yes
-000000556000.jpg	Is there a dog in this image? Please answer yes or no.	No
-000000557258.jpg	Is there a toilet in this image? Please answer yes or no.	Yes
-000000557258.jpg	Is there a pizza in this image? Please answer yes or no.	No
-000000572956.jpg	Is there a motorcycle in this image? Please answer yes or no.	Yes
-000000572956.jpg	Is there a bus in this image? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/landmark.txt

@@ -1,400 +0,0 @@
-0006adf999ccc899.jpg	Is this a photo of Muktinath? Please answer yes or no.	Yes
-0006adf999ccc899.jpg	Is this a photo of Lohikoski Church? Please answer yes or no.	No
-0015309638e5a9b1.jpg	Is this a picture of Teufelsmauer (Harz)? Please answer yes or no.	Yes
-0015309638e5a9b1.jpg	Is this a picture of Romfo kirke? Please answer yes or no.	No
-001f0bfe8464254a.jpg	Is this a picture of Saint Catherine of Alexandria church in Warsaw? Please answer yes or no.	Yes
-001f0bfe8464254a.jpg	Is this a picture of Kaap Skil? Please answer yes or no.	No
-0022666a36b80ea5.jpg	Is this an image of Hall of Supreme Harmony (Forbidden City)? Please answer yes or no.	Yes
-0022666a36b80ea5.jpg	Is this an image of Brickegickel? Please answer yes or no.	No
-0023bd687cbc0a44.jpg	Is this a photo of Utrecht Dom Tower? Please answer yes or no.	Yes
-0023bd687cbc0a44.jpg	Is this a photo of Great Palace Mosaic Museum? Please answer yes or no.	No
-00261324328490fc.jpg	Is this an image of Loch Ossian? Please answer yes or no.	Yes
-00261324328490fc.jpg	Is this an image of Tanemakidaishi? Please answer yes or no.	No
-002ae473a76362a6.jpg	Is this an image of Gerlache Strait? Please answer yes or no.	Yes
-002ae473a76362a6.jpg	Is this an image of Schmiedebruch (Naturschutzgebiet)? Please answer yes or no.	No
-003f8af9477a8eb8.jpg	Is this a photo of Le Destroit? Please answer yes or no.	Yes
-003f8af9477a8eb8.jpg	Is this a photo of Kharkov Mosque? Please answer yes or no.	No
-004543cec064cd0b.jpg	Is this a picture of Wollaton Park? Please answer yes or no.	Yes
-004543cec064cd0b.jpg	Is this a picture of St John's Church, Blackpool? Please answer yes or no.	No
-004beea6937b3a8c.jpg	Is this a photo of Brussels Park? Please answer yes or no.	Yes
-004beea6937b3a8c.jpg	Is this a photo of Forest Hills Cemetery (Boston, Massachusetts)? Please answer yes or no.	No
-005150f80de6f487.jpg	Is this a photo of Schloss Burgpreppach? Please answer yes or no.	Yes
-005150f80de6f487.jpg	Is this a photo of Pinkas Synagogue? Please answer yes or no.	No
-0053894534d1e259.jpg	Is this a picture of Goethe- und Schiller-Denkmal (Weimar)? Please answer yes or no.	Yes
-0053894534d1e259.jpg	Is this a picture of Grey River? Please answer yes or no.	No
-0059fc817c5fd5b7.jpg	Is this a photo of KLCC Park? Please answer yes or no.	Yes
-0059fc817c5fd5b7.jpg	Is this a photo of Neath Castle? Please answer yes or no.	No
-005c68530bb6ed7c.jpg	Is this a photo of Balvenie Castle? Please answer yes or no.	Yes
-005c68530bb6ed7c.jpg	Is this a photo of Draper Utah Temple? Please answer yes or no.	No
-006e3342b57aa3b4.jpg	Is this a photo of Larvs kyrka? Please answer yes or no.	Yes
-006e3342b57aa3b4.jpg	Is this a photo of Geitgallien? Please answer yes or no.	No
-0075a4d212539625.jpg	Is this a photo of Cardiff Castle? Please answer yes or no.	Yes
-0075a4d212539625.jpg	Is this a photo of Boudhanath? Please answer yes or no.	No
-008053e6bb5afa33.jpg	Is this a photo of Rochers des Mourres? Please answer yes or no.	Yes
-008053e6bb5afa33.jpg	Is this a photo of Schloss Weinfelden? Please answer yes or no.	No
-00c1349a9622390c.jpg	Is this a photo of Amar Jawan Jyoti (Tomb of the Unknown Soldier)? Please answer yes or no.	Yes
-00c1349a9622390c.jpg	Is this a photo of Porta Garibaldi (Marsala)? Please answer yes or no.	No
-00cae9f165d8d2cc.jpg	Is this an image of Fisherman's Bastion? Please answer yes or no.	Yes
-00cae9f165d8d2cc.jpg	Is this an image of Equestrian statue of George IV, Trafalgar Square, London? Please answer yes or no.	No
-00f4fb71575efb0b.jpg	Is this a picture of 1965)? Please answer yes or no.	Yes
-00f4fb71575efb0b.jpg	Is this a picture of Plaza Monumental de Toros de Pueblo Nuevo? Please answer yes or no.	No
-00fa476690a260a8.jpg	Is this an image of Bathampton Toll Bridge? Please answer yes or no.	Yes
-00fa476690a260a8.jpg	Is this an image of Moose River Plains Wild Forest? Please answer yes or no.	No
-010260e62e7d847d.jpg	Is this a photo of Visingsborg? Please answer yes or no.	Yes
-010260e62e7d847d.jpg	Is this a photo of Teutsche Schule (Schleusingen)? Please answer yes or no.	No
-010b5b5c9c461f65.jpg	Is this an image of Het Steen? Please answer yes or no.	Yes
-010b5b5c9c461f65.jpg	Is this an image of Castello di Montalto Dora? Please answer yes or no.	No
-01114538f95f1ce8.jpg	Is this a picture of Svirzh Castle? Please answer yes or no.	Yes
-01114538f95f1ce8.jpg	Is this a picture of Bistrica, Ionian Sea? Please answer yes or no.	No
-01121b9b3c79afc4.jpg	Is this a photo of Alamannenmuseum Ellwangen? Please answer yes or no.	Yes
-01121b9b3c79afc4.jpg	Is this a photo of Sint-Jozefkerk, Enschede? Please answer yes or no.	No
-011a19afda5df3c7.jpg	Is this a picture of San Prospero (Reggio Emilia)? Please answer yes or no.	Yes
-011a19afda5df3c7.jpg	Is this a picture of Lake DeFuniak? Please answer yes or no.	No
-012856462a95ada9.jpg	Is this a picture of Chingshui Cliff? Please answer yes or no.	Yes
-012856462a95ada9.jpg	Is this a picture of St. John's Church, Portland? Please answer yes or no.	No
-012ea376a16f17bf.jpg	Is this a picture of Church of Saint Giles in Prague? Please answer yes or no.	Yes
-012ea376a16f17bf.jpg	Is this a picture of Pfarrkirche St. Martin an der Raab? Please answer yes or no.	No
-013e1a1e887cb915.jpg	Is this a picture of Real Casa de Correos, Madrid? Please answer yes or no.	Yes
-013e1a1e887cb915.jpg	Is this a picture of Ayton Castle, Scottish Borders? Please answer yes or no.	No
-0140a7d5446719c3.jpg	Is this a photo of Friday Mosque, Herat? Please answer yes or no.	Yes
-0140a7d5446719c3.jpg	Is this a photo of Hexenturm (Burg (bei Magdeburg))? Please answer yes or no.	No
-01416124a7ab211a.jpg	Is this a photo of Ajuntament de Mollerussa? Please answer yes or no.	Yes
-01416124a7ab211a.jpg	Is this a photo of Episcopal Diocese of Northern Indiana? Please answer yes or no.	No
-01425c4d35880aad.jpg	Is this an image of Museu de Sant Boi de Llobregat? Please answer yes or no.	Yes
-01425c4d35880aad.jpg	Is this an image of Bezdonnoye Lake, Moscow Oblast? Please answer yes or no.	No
-014c50e91cf799c7.jpg	Is this a photo of Clearwater Beach, Florida? Please answer yes or no.	Yes
-014c50e91cf799c7.jpg	Is this a photo of Church of Holy Trinity in Vratislavice nad Nisou? Please answer yes or no.	No
-015543ddea466c54.jpg	Is this a picture of Galician market square in Skansen, Sanok? Please answer yes or no.	Yes
-015543ddea466c54.jpg	Is this a picture of Walworth Castle? Please answer yes or no.	No
-0169c4235270f65b.jpg	Is this a photo of St. Ricarius Church, Aberford? Please answer yes or no.	Yes
-0169c4235270f65b.jpg	Is this a photo of Schloss Prebberede? Please answer yes or no.	No
-016e68628ea6c7ee.jpg	Is this an image of Jewish Ceremonial Hall, Prague-Josefov? Please answer yes or no.	Yes
-016e68628ea6c7ee.jpg	Is this an image of Our Lady Church (Ettlingen)? Please answer yes or no.	No
-01862535ce8961ee.jpg	Is this an image of Mont Tendre? Please answer yes or no.	Yes
-01862535ce8961ee.jpg	Is this an image of Ogden Utah Temple? Please answer yes or no.	No
-0186742e7963fb3c.jpg	Is this a photo of Ozegahara? Please answer yes or no.	Yes
-0186742e7963fb3c.jpg	Is this a photo of San Giovenale (Orvieto)? Please answer yes or no.	No
-01928acd6dfe6cdd.jpg	Is this a picture of Watergate Bay? Please answer yes or no.	Yes
-01928acd6dfe6cdd.jpg	Is this a picture of Cairnpapple Hill? Please answer yes or no.	No
-01a6842099d62f22.jpg	Is this an image of Clark Veterans Cemetery? Please answer yes or no.	Yes
-01a6842099d62f22.jpg	Is this an image of Mirdif City Centre? Please answer yes or no.	No
-01aca2d30b43d05f.jpg	Is this a picture of Monte Minerva (Villanova Monteleone)? Please answer yes or no.	Yes
-01aca2d30b43d05f.jpg	Is this a picture of Dorfkirche Gnandstein? Please answer yes or no.	No
-01b12048bab7439f.jpg	Is this a photo of Kaptai Lake? Please answer yes or no.	Yes
-01b12048bab7439f.jpg	Is this a photo of Castle of the Conception? Please answer yes or no.	No
-01b715ceb7050bce.jpg	Is this a picture of Kalamita fortress? Please answer yes or no.	Yes
-01b715ceb7050bce.jpg	Is this a picture of Augsburger Hotelturm? Please answer yes or no.	No
-01b9b58a573c45f0.jpg	Is this a picture of St Mary Redcliffe? Please answer yes or no.	Yes
-01b9b58a573c45f0.jpg	Is this a picture of Kirche zur Heiligsten Dreifaltigkeit, Vienna? Please answer yes or no.	No
-01bc5557ed2616ad.jpg	Is this a photo of Intercession of the Theotokos church in Lutsk? Please answer yes or no.	Yes
-01bc5557ed2616ad.jpg	Is this a photo of Cap 110? Please answer yes or no.	No
-01c0e0ff84631905.jpg	Is this a photo of Ribblehead Viaduct? Please answer yes or no.	Yes
-01c0e0ff84631905.jpg	Is this a photo of Church of Saint George in Koptevo? Please answer yes or no.	No
-01d6e386c4a01ab3.jpg	Is this a picture of Rotes Rathaus? Please answer yes or no.	Yes
-01d6e386c4a01ab3.jpg	Is this a picture of Mahipatgad? Please answer yes or no.	No
-01d9bef0120557a5.jpg	Is this a photo of Andronikov Monastery? Please answer yes or no.	Yes
-01d9bef0120557a5.jpg	Is this a photo of Ghatkopar? Please answer yes or no.	No
-01e198bea8285816.jpg	Is this an image of Saint Basil church, Volodymyr-Volynskyi? Please answer yes or no.	Yes
-01e198bea8285816.jpg	Is this an image of Hoher Markt, Vienna? Please answer yes or no.	No
-01f23b4c5253e9c6.jpg	Is this a picture of Goethes Gartenhaus? Please answer yes or no.	Yes
-01f23b4c5253e9c6.jpg	Is this a picture of Lamba, Shetland Islands? Please answer yes or no.	No
-01ffc77bdf7fe084.jpg	Is this a picture of Oldemorstoft? Please answer yes or no.	Yes
-01ffc77bdf7fe084.jpg	Is this a picture of Roman Catholic Diocese of Antwerp? Please answer yes or no.	No
-0200cce957e9c4a6.jpg	Is this a photo of Malmuerta Tower? Please answer yes or no.	Yes
-0200cce957e9c4a6.jpg	Is this a photo of Monte Lema? Please answer yes or no.	No
-0202aac6b0bfbcd8.jpg	Is this a picture of Gettysburg National Military Park? Please answer yes or no.	Yes
-0202aac6b0bfbcd8.jpg	Is this a picture of Auron? Please answer yes or no.	No
-0205343362e60daf.jpg	Is this a picture of Church of Our Lady of Sorrows in Riga? Please answer yes or no.	Yes
-0205343362e60daf.jpg	Is this a picture of Simon en Judaskerk (Ootmarsum)? Please answer yes or no.	No
-020ae4dd487a8ab0.jpg	Is this a photo of Kashueti Church? Please answer yes or no.	Yes
-020ae4dd487a8ab0.jpg	Is this a photo of Colorado State Capitol? Please answer yes or no.	No
-02259144c3f094f0.jpg	Is this a picture of The American Adventure? Please answer yes or no.	Yes
-02259144c3f094f0.jpg	Is this a picture of Langenwaldschanze? Please answer yes or no.	No
-0237fc92f6b31aeb.jpg	Is this a photo of Abbaye de Mortemer? Please answer yes or no.	Yes
-0237fc92f6b31aeb.jpg	Is this a photo of St Thomas Rest Park? Please answer yes or no.	No
-0240727dbd2b8531.jpg	Is this a photo of Nicolaikerk Appingedam? Please answer yes or no.	Yes
-0240727dbd2b8531.jpg	Is this a photo of Cutimbo? Please answer yes or no.	No
-02545ca051c6c480.jpg	Is this a photo of Podilski Tovtry National Nature Park? Please answer yes or no.	Yes
-02545ca051c6c480.jpg	Is this a photo of Lyndonville, Vermont? Please answer yes or no.	No
-02598153523880aa.jpg	Is this a photo of Marilao Church? Please answer yes or no.	Yes
-02598153523880aa.jpg	Is this a photo of Madliena, Malta? Please answer yes or no.	No
-02737bb70957827d.jpg	Is this an image of White Pagoda in Zhakou? Please answer yes or no.	Yes
-02737bb70957827d.jpg	Is this an image of Norrstrandskyrkan? Please answer yes or no.	No
-0273f127895337be.jpg	Is this a photo of Ringkirche? Please answer yes or no.	Yes
-0273f127895337be.jpg	Is this a photo of Huxi Mosque? Please answer yes or no.	No
-02785530b6ac4f56.jpg	Is this a picture of Church of Nicholas the Wet? Please answer yes or no.	Yes
-02785530b6ac4f56.jpg	Is this a picture of Razula? Please answer yes or no.	No
-028348883685db69.jpg	Is this an image of Saint Andrew the Apostle Church? Please answer yes or no.	Yes
-028348883685db69.jpg	Is this an image of EYE Film Institute Netherlands? Please answer yes or no.	No
-029431b10472c228.jpg	Is this a picture of Santa Caterina (Sassari)? Please answer yes or no.	Yes
-029431b10472c228.jpg	Is this a picture of Former Tainan Assembly Hall? Please answer yes or no.	No
-0295877d5cd76190.jpg	Is this a picture of Groothoofdspoort? Please answer yes or no.	Yes
-0295877d5cd76190.jpg	Is this a picture of 707 17th Street? Please answer yes or no.	No
-02b16429304b00b7.jpg	Is this a picture of De Haag? Please answer yes or no.	Yes
-02b16429304b00b7.jpg	Is this a picture of Villa Melchett? Please answer yes or no.	No
-02b7f3ae549582f1.jpg	Is this a photo of Pont-canal de Digoin? Please answer yes or no.	Yes
-02b7f3ae549582f1.jpg	Is this a photo of Trenton Battle Monument? Please answer yes or no.	No
-02d11e392d3ab93f.jpg	Is this an image of Santa Monica Parish Church in Angat, Bulacan? Please answer yes or no.	Yes
-02d11e392d3ab93f.jpg	Is this an image of Bangkok City Pillar Shrine? Please answer yes or no.	No
-02e38cd5a3850f60.jpg	Is this a photo of Palazzo Cavalli Franchetti (Venice)? Please answer yes or no.	Yes
-02e38cd5a3850f60.jpg	Is this a photo of Isbister Chambered Cairn? Please answer yes or no.	No
-02e3e418198427e7.jpg	Is this an image of Goldener Reiter? Please answer yes or no.	Yes
-02e3e418198427e7.jpg	Is this an image of Palacio de la Mosquera? Please answer yes or no.	No
-030d9a26b7d5755e.jpg	Is this a photo of Medvedgrad? Please answer yes or no.	Yes
-030d9a26b7d5755e.jpg	Is this a photo of Tammisto? Please answer yes or no.	No
-0317376043fb00eb.jpg	Is this a photo of Butler's Wharf? Please answer yes or no.	Yes
-0317376043fb00eb.jpg	Is this a photo of Church of Saints Philip and James (Lelekovice)? Please answer yes or no.	No
-03280db41fd4e4f5.jpg	Is this a picture of Oxford Street? Please answer yes or no.	Yes
-03280db41fd4e4f5.jpg	Is this a picture of Blake Fell? Please answer yes or no.	No
-034733eb61d663cd.jpg	Is this a picture of Colonna dell'Immacolata (Rome)? Please answer yes or no.	Yes
-034733eb61d663cd.jpg	Is this a picture of Brookmill Park? Please answer yes or no.	No
-034d066523bb20d1.jpg	Is this a photo of Ibrahim-al-Ibrahim Mosque? Please answer yes or no.	Yes
-034d066523bb20d1.jpg	Is this a photo of Atami Castle? Please answer yes or no.	No
-036b60d84c6ebc63.jpg	Is this a photo of Frederik V statue in Amalienborg Palace? Please answer yes or no.	Yes
-036b60d84c6ebc63.jpg	Is this a photo of Karl-Bittel-Park? Please answer yes or no.	No
-03729394d18f7e9a.jpg	Is this an image of Sapporo Clock Tower? Please answer yes or no.	Yes
-03729394d18f7e9a.jpg	Is this an image of Saint Nikolaus Church (Beuster)? Please answer yes or no.	No
-0384fd0664c0ca62.jpg	Is this an image of St. Martin (Sontheim)? Please answer yes or no.	Yes
-0384fd0664c0ca62.jpg	Is this an image of Hot Creek (Mono County, California)? Please answer yes or no.	No
-038bdff81be029a6.jpg	Is this an image of Fosso Reale (Livorno)? Please answer yes or no.	Yes
-038bdff81be029a6.jpg	Is this an image of Farlington Marshes? Please answer yes or no.	No
-039bbdf71cd17b3f.jpg	Is this a picture of Qiantang River Bridge? Please answer yes or no.	Yes
-039bbdf71cd17b3f.jpg	Is this a picture of Casa Museo Boschi Di Stefano (Milan)? Please answer yes or no.	No
-03aa5dd2e3aab548.jpg	Is this a photo of Central Police Station? Please answer yes or no.	Yes
-03aa5dd2e3aab548.jpg	Is this a photo of Kontai-ji? Please answer yes or no.	No
-03c4b1901e4195f4.jpg	Is this an image of St. Johanniskirche Eppendorf? Please answer yes or no.	Yes
-03c4b1901e4195f4.jpg	Is this an image of Maidenhead Bridge? Please answer yes or no.	No
-03c8b2c1306c0038.jpg	Is this an image of Ermita de Sant Joan (Montserrat)? Please answer yes or no.	Yes
-03c8b2c1306c0038.jpg	Is this an image of Hamilton Grange National Memorial? Please answer yes or no.	No
-03d5e3bfc958be38.jpg	Is this an image of Pietrarsa railway museum? Please answer yes or no.	Yes
-03d5e3bfc958be38.jpg	Is this an image of Washita Battlefield National Historic Site? Please answer yes or no.	No
-03dc7df1f1df7b8f.jpg	Is this a picture of Yevhen Paton Bridge? Please answer yes or no.	Yes
-03dc7df1f1df7b8f.jpg	Is this a picture of Minoan Villa of Makrygialos? Please answer yes or no.	No
-03f65f52c0077b6c.jpg	Is this a photo of Reading Lighthouse? Please answer yes or no.	Yes
-03f65f52c0077b6c.jpg	Is this a photo of St Nicholas Kirk, Aberdeen? Please answer yes or no.	No
-03f873280d814ae7.jpg	Is this a photo of Clarence Dock (Leeds)? Please answer yes or no.	Yes
-03f873280d814ae7.jpg	Is this a photo of Madonna delle Grazie (Roccastrada)? Please answer yes or no.	No
-03f8abf9f8473a6f.jpg	Is this a picture of Parque nacional del Teide? Please answer yes or no.	Yes
-03f8abf9f8473a6f.jpg	Is this a picture of Church of Saint Mary the Virgin, Skiemonys? Please answer yes or no.	No
-03fb778d388ad9a0.jpg	Is this an image of Mercado Central de Santiago? Please answer yes or no.	Yes
-03fb778d388ad9a0.jpg	Is this an image of Castelo de Alegrete? Please answer yes or no.	No
-041da60e053c5906.jpg	Is this a picture of Schloss Weesenstein? Please answer yes or no.	Yes
-041da60e053c5906.jpg	Is this a picture of Mas Cabrafiga? Please answer yes or no.	No
-042ca5291cefe0b2.jpg	Is this a picture of Pfarrkirche hl. Augustinus, Perchtoldsdorf? Please answer yes or no.	Yes
-042ca5291cefe0b2.jpg	Is this a picture of Mark Twain Riverboat? Please answer yes or no.	No
-0443da4f4fcdea52.jpg	Is this an image of Port of Funchal? Please answer yes or no.	Yes
-0443da4f4fcdea52.jpg	Is this an image of Rauscherpark? Please answer yes or no.	No
-04531b37dfe5c991.jpg	Is this a picture of Roebling Suspension Bridge? Please answer yes or no.	Yes
-04531b37dfe5c991.jpg	Is this a picture of St. James Episcopal Church (Wilmington, North Carolina)? Please answer yes or no.	No
-04534ba775aa38ad.jpg	Is this an image of Cork City Gaol? Please answer yes or no.	Yes
-04534ba775aa38ad.jpg	Is this an image of Taramati Baradari? Please answer yes or no.	No
-04590c1648dd25a0.jpg	Is this an image of Zhejiang Sci-Tech University? Please answer yes or no.	Yes
-04590c1648dd25a0.jpg	Is this an image of Khalti Lake? Please answer yes or no.	No
-045b65ea196dac3f.jpg	Is this a photo of Republican architecture in Manizales? Please answer yes or no.	Yes
-045b65ea196dac3f.jpg	Is this a photo of Kayauchi-banta Cliff? Please answer yes or no.	No
-045ec5631638d290.jpg	Is this a picture of Baldachin of Saint Peter's Basilica? Please answer yes or no.	Yes
-045ec5631638d290.jpg	Is this a picture of Jean Baptiste Point Du Sable Homesite? Please answer yes or no.	No
-04a5226570dce6c3.jpg	Is this a picture of Church of the Assumption of the Virgin Mary in Chrudim? Please answer yes or no.	Yes
-04a5226570dce6c3.jpg	Is this a picture of Queen Elizabeth Country Park? Please answer yes or no.	No
-04b6c84ce89461ec.jpg	Is this an image of Tomb of Xerxes I? Please answer yes or no.	Yes
-04b6c84ce89461ec.jpg	Is this an image of Park HaMesila? Please answer yes or no.	No
-04d67117549e07a8.jpg	Is this a photo of Laguna de Salinas? Please answer yes or no.	Yes
-04d67117549e07a8.jpg	Is this a photo of Raspyatsky Monastery (Serpukhov)? Please answer yes or no.	No
-04ebe2885134597f.jpg	Is this a picture of The South Africa (Delville Wood) National Memorial? Please answer yes or no.	Yes
-04ebe2885134597f.jpg	Is this a picture of Imogiri? Please answer yes or no.	No
-050464e4ebc0b36d.jpg	Is this an image of Tai Wong Ye Temple, Wong Chuk Hang? Please answer yes or no.	Yes
-050464e4ebc0b36d.jpg	Is this an image of Hierve el Agua? Please answer yes or no.	No
-05095dd2b6715fc3.jpg	Is this an image of Sampsonievsky Cathedral? Please answer yes or no.	Yes
-05095dd2b6715fc3.jpg	Is this an image of I. and E. Greenwald Steam Engine No. 1058? Please answer yes or no.	No
-05685b1070b828f5.jpg	Is this a photo of Wurstelprater? Please answer yes or no.	Yes
-05685b1070b828f5.jpg	Is this a photo of Nanmyaebonthar Sannandawya Sandamuni Pagoda? Please answer yes or no.	No
-05b01b255cec2bfb.jpg	Is this a photo of Puthoorppilly Sree Krishnaswamy Temple? Please answer yes or no.	Yes
-05b01b255cec2bfb.jpg	Is this a photo of Villa Reale (Marlia)? Please answer yes or no.	No
-05bcf05b4460d2c3.jpg	Is this an image of Museo archeologico nazionale dell'Umbria? Please answer yes or no.	Yes
-05bcf05b4460d2c3.jpg	Is this an image of Mode Gakuen Cocoon Tower? Please answer yes or no.	No
-05c3aa404351f407.jpg	Is this an image of Pasaje Gutierrez (Valladolid)? Please answer yes or no.	Yes
-05c3aa404351f407.jpg	Is this an image of Pieve di Santa Maria a Carli? Please answer yes or no.	No
-05d56e249af839e7.jpg	Is this a photo of Serbian Orthodox Cathedral in Sarajevo? Please answer yes or no.	Yes
-05d56e249af839e7.jpg	Is this a photo of Seewaldsee (Sankt Koloman)? Please answer yes or no.	No
-06134984e2aa9bcc.jpg	Is this a picture of Grote Kerk (Groede)? Please answer yes or no.	Yes
-06134984e2aa9bcc.jpg	Is this a picture of Goldeck? Please answer yes or no.	No
-0613d7e8f292ee15.jpg	Is this a photo of Museo Archeologico G. Rambotti? Please answer yes or no.	Yes
-0613d7e8f292ee15.jpg	Is this a photo of Sanrei? Please answer yes or no.	No
-062298815e9197e7.jpg	Is this a photo of Liuyang Confucius Temple? Please answer yes or no.	Yes
-062298815e9197e7.jpg	Is this a photo of Astley Castle? Please answer yes or no.	No
-06483ba2bff074bc.jpg	Is this an image of Museo Universitario del Chopo? Please answer yes or no.	Yes
-06483ba2bff074bc.jpg	Is this an image of Cedar Park, Seattle, Washington? Please answer yes or no.	No
-06740b93ad1f6c02.jpg	Is this a picture of Borisoglebsky Monastery (Borisoglebsky)? Please answer yes or no.	Yes
-06740b93ad1f6c02.jpg	Is this a picture of Aletsch Arena? Please answer yes or no.	No
-067ea27ec08c9354.jpg	Is this a picture of Schlosskirche St. Aegidien (Bernburg)? Please answer yes or no.	Yes
-067ea27ec08c9354.jpg	Is this a picture of The Haining? Please answer yes or no.	No
-06a85d0b5d70881d.jpg	Is this an image of Silwan? Please answer yes or no.	Yes
-06a85d0b5d70881d.jpg	Is this an image of Jagdschloss Platte? Please answer yes or no.	No
-06ab45292ecd8bf7.jpg	Is this a picture of Maribor Regional Museum? Please answer yes or no.	Yes
-06ab45292ecd8bf7.jpg	Is this a picture of Myres Castle? Please answer yes or no.	No
-06b6531dab42ccab.jpg	Is this a picture of Roland (Riga)? Please answer yes or no.	Yes
-06b6531dab42ccab.jpg	Is this a picture of Wudangshan Shangdi Temple? Please answer yes or no.	No
-06d039ff97afef04.jpg	Is this a picture of Arch of Galerius (Thessaloniki)? Please answer yes or no.	Yes
-06d039ff97afef04.jpg	Is this a picture of Catholic Church of Wanchin? Please answer yes or no.	No
-06e453b06a3e4054.jpg	Is this an image of Pendeen Lighthouse? Please answer yes or no.	Yes
-06e453b06a3e4054.jpg	Is this an image of Silberbach (Heubach)? Please answer yes or no.	No
-06e927f784642ce4.jpg	Is this a picture of Museo Delta Antico? Please answer yes or no.	Yes
-06e927f784642ce4.jpg	Is this a picture of OSTEC Exhibition Hall? Please answer yes or no.	No
-070c8d6326a87397.jpg	Is this an image of Lanxmeerpoort? Please answer yes or no.	Yes
-070c8d6326a87397.jpg	Is this an image of Tramway du Cap-Ferret? Please answer yes or no.	No
-0713451824a443d4.jpg	Is this a picture of Castillo de Sax? Please answer yes or no.	Yes
-0713451824a443d4.jpg	Is this a picture of Roman Catholic Diocese of Orlando? Please answer yes or no.	No
-0743e58605497887.jpg	Is this an image of Capitoline temple (Brescia)? Please answer yes or no.	Yes
-0743e58605497887.jpg	Is this an image of Burg Horn? Please answer yes or no.	No
-075264ec41057049.jpg	Is this a picture of Imatrankoski? Please answer yes or no.	Yes
-075264ec41057049.jpg	Is this a picture of Jarlsberg travbane? Please answer yes or no.	No
-07551cf96af77299.jpg	Is this a photo of Friedhof Wilmersdorf? Please answer yes or no.	Yes
-07551cf96af77299.jpg	Is this a photo of Partisan cemetery in Mostar? Please answer yes or no.	No
-075f8d86303a020e.jpg	Is this a photo of Church of the Nativity (Bethlehem)? Please answer yes or no.	Yes
-075f8d86303a020e.jpg	Is this a photo of Pennsylvania Railroad Office Building? Please answer yes or no.	No
-07ac52cf4620519f.jpg	Is this a photo of San Pietro (Reggio Emilia)? Please answer yes or no.	Yes
-07ac52cf4620519f.jpg	Is this a photo of Plaza de toros de La Condomina? Please answer yes or no.	No
-07b04584e67cb39d.jpg	Is this a picture of Smithfield, Dublin? Please answer yes or no.	Yes
-07b04584e67cb39d.jpg	Is this a picture of Novyi Svit Sanctuary? Please answer yes or no.	No
-07ed43dd266b669b.jpg	Is this a photo of Rocky Mountain National Park? Please answer yes or no.	Yes
-07ed43dd266b669b.jpg	Is this a photo of Col-des-Roches? Please answer yes or no.	No
-083dd5a3ba3d96dc.jpg	Is this a picture of Teddington Cemetery? Please answer yes or no.	Yes
-083dd5a3ba3d96dc.jpg	Is this a picture of Khutir Nadia? Please answer yes or no.	No
-08442f24f3c25565.jpg	Is this a photo of Abdijkerk Loosduinen? Please answer yes or no.	Yes
-08442f24f3c25565.jpg	Is this a photo of Rinsey Head? Please answer yes or no.	No
-08a0d2fd3fe6dc95.jpg	Is this a picture of Dreifaltigkeitskirche (Ulm)? Please answer yes or no.	Yes
-08a0d2fd3fe6dc95.jpg	Is this a picture of Wat Pa Mamuang? Please answer yes or no.	No
-08c43cd8f2be6709.jpg	Is this a picture of Parchi di Nervi? Please answer yes or no.	Yes
-08c43cd8f2be6709.jpg	Is this a picture of Altlayer Schweiz? Please answer yes or no.	No
-09232aeaf46dec64.jpg	Is this a picture of Canadian Museum of Nature? Please answer yes or no.	Yes
-09232aeaf46dec64.jpg	Is this a picture of Gwangtonggwan? Please answer yes or no.	No
-092d0859ff6424de.jpg	Is this a photo of Kasteel Lunenburg? Please answer yes or no.	Yes
-092d0859ff6424de.jpg	Is this a photo of Thomas Paine Cottage? Please answer yes or no.	No
-092e075e8e44c228.jpg	Is this a picture of ChaoTianGong? Please answer yes or no.	Yes
-092e075e8e44c228.jpg	Is this a picture of Pygmalion Theater Wien? Please answer yes or no.	No
-093ad5564a004653.jpg	Is this a picture of Crescent Lake (Dunhuang)? Please answer yes or no.	Yes
-093ad5564a004653.jpg	Is this a picture of Kirjurinluoto Arena? Please answer yes or no.	No
-0962855b67159733.jpg	Is this a photo of Monastery of Monsalud? Please answer yes or no.	Yes
-0962855b67159733.jpg	Is this a photo of ToonSeum? Please answer yes or no.	No
-09a51d517f1a72d5.jpg	Is this a picture of Ford's Theatre? Please answer yes or no.	Yes
-09a51d517f1a72d5.jpg	Is this a picture of Burg Hachen? Please answer yes or no.	No
-09ae0628a75e4fcc.jpg	Is this a picture of Onze-Lieve-Vrouw Hemelvaartkerk (Zottegem)? Please answer yes or no.	Yes
-09ae0628a75e4fcc.jpg	Is this a picture of Salvation Army in Canada? Please answer yes or no.	No
-09b7acf87c8046ec.jpg	Is this an image of Church of the Epiphany (Yaroslavl)? Please answer yes or no.	Yes
-09b7acf87c8046ec.jpg	Is this an image of Szlenkier Palace? Please answer yes or no.	No
-09b94a2f304a540c.jpg	Is this a photo of Abbaye Notre-Dame du Val? Please answer yes or no.	Yes
-09b94a2f304a540c.jpg	Is this a photo of Hotarumachi, Osaka? Please answer yes or no.	No
-09ce89cbf237da03.jpg	Is this a picture of Church of Saint Mary Magdalene (Brno)? Please answer yes or no.	Yes
-09ce89cbf237da03.jpg	Is this a picture of Wain Wath Force? Please answer yes or no.	No
-0a3035bfca2ab920.jpg	Is this an image of Ortigia? Please answer yes or no.	Yes
-0a3035bfca2ab920.jpg	Is this an image of Montmayeur castle? Please answer yes or no.	No
-0ab2ed007db301d5.jpg	Is this a picture of Highgate Cemetery? Please answer yes or no.	Yes
-0ab2ed007db301d5.jpg	Is this a picture of Isola Maggiore? Please answer yes or no.	No
-0acc12d1261d99ca.jpg	Is this an image of Wat Chedi Si Hong? Please answer yes or no.	Yes
-0acc12d1261d99ca.jpg	Is this an image of Leschi Park? Please answer yes or no.	No
-0adb088e24d88123.jpg	Is this a picture of Iglesia de Santo Domingo (La Serena)? Please answer yes or no.	Yes
-0adb088e24d88123.jpg	Is this a picture of Canelles de Baix (la Vall de Bianya)? Please answer yes or no.	No
-0ae3355e0a54db21.jpg	Is this a photo of Grimspound? Please answer yes or no.	Yes
-0ae3355e0a54db21.jpg	Is this a photo of Lough Owel? Please answer yes or no.	No
-0b4240b5ef69362e.jpg	Is this a photo of Saint Anne church in Nowa Ruda? Please answer yes or no.	Yes
-0b4240b5ef69362e.jpg	Is this a photo of Bray Wick? Please answer yes or no.	No
-0b64944086932c90.jpg	Is this a picture of Alte Pfarrkirche St. Martin (Moosach)? Please answer yes or no.	Yes
-0b64944086932c90.jpg	Is this a picture of Maya Devi Temple, Lumbini? Please answer yes or no.	No
-0b8fa4152f1bc3ed.jpg	Is this a picture of Lac du Salagou? Please answer yes or no.	Yes
-0b8fa4152f1bc3ed.jpg	Is this a picture of Holsljunga kyrka? Please answer yes or no.	No
-0ba6fa7ade050d49.jpg	Is this a photo of Churches in Zemaiciu Kalvarija? Please answer yes or no.	Yes
-0ba6fa7ade050d49.jpg	Is this a photo of Waiola Church? Please answer yes or no.	No
-0bd72080836eac97.jpg	Is this a picture of Castelo de Sesimbra? Please answer yes or no.	Yes
-0bd72080836eac97.jpg	Is this a picture of Gouffre Berger? Please answer yes or no.	No
-0bdd954254310e73.jpg	Is this a photo of Melrose House? Please answer yes or no.	Yes
-0bdd954254310e73.jpg	Is this a photo of Former water tower (Tsubame, Niigata)? Please answer yes or no.	No
-0c06fe97f38f6f88.jpg	Is this a photo of Belvoir Castle? Please answer yes or no.	Yes
-0c06fe97f38f6f88.jpg	Is this a photo of Su'ao Cold Spring? Please answer yes or no.	No
-0c6b0c5d2a401285.jpg	Is this a photo of Pelham Crescent? Please answer yes or no.	Yes
-0c6b0c5d2a401285.jpg	Is this a photo of Manila Ocean Park? Please answer yes or no.	No
-0cc5201e37961a1d.jpg	Is this a photo of Adare Manor? Please answer yes or no.	Yes
-0cc5201e37961a1d.jpg	Is this a photo of Leopoldspark? Please answer yes or no.	No
-0cd8fda9668b3ed2.jpg	Is this a picture of Church of the Holy Mandylion (Klyazma)? Please answer yes or no.	Yes
-0cd8fda9668b3ed2.jpg	Is this a picture of Kalayar Kovil? Please answer yes or no.	No
-0cf2e28266a0f1a1.jpg	Is this an image of Piazza Bra (Verona)? Please answer yes or no.	Yes
-0cf2e28266a0f1a1.jpg	Is this an image of Keeneland? Please answer yes or no.	No
-0cf48b827c9c99de.jpg	Is this a photo of Beatus Rhenanus Bridge? Please answer yes or no.	Yes
-0cf48b827c9c99de.jpg	Is this a photo of Ossian Hall? Please answer yes or no.	No
-0d23f6c345b68c05.jpg	Is this an image of Monte Olivia? Please answer yes or no.	Yes
-0d23f6c345b68c05.jpg	Is this an image of Crackstate? Please answer yes or no.	No
-0d70bceb2557d1f4.jpg	Is this an image of Spaso-Preobrazhensky Cathedral (Pereslavl-Zalessky)? Please answer yes or no.	Yes
-0d70bceb2557d1f4.jpg	Is this an image of Manta Rota? Please answer yes or no.	No
-0d7716f4aed0c9fa.jpg	Is this a photo of Willemsbrug in Rotterdam? Please answer yes or no.	Yes
-0d7716f4aed0c9fa.jpg	Is this a photo of Grumentum, Parco archeologico? Please answer yes or no.	No
-0d89a8a5e8c269e4.jpg	Is this an image of Burg Normannstein? Please answer yes or no.	Yes
-0d89a8a5e8c269e4.jpg	Is this an image of Cape Borda Lighthouse? Please answer yes or no.	No
-0e04b4f7f9801ea9.jpg	Is this a picture of Vase sculpture, Bremen? Please answer yes or no.	Yes
-0e04b4f7f9801ea9.jpg	Is this a picture of Mount Wilhelm? Please answer yes or no.	No
-0e0511c6e33636ca.jpg	Is this a picture of Victoria Stadium? Please answer yes or no.	Yes
-0e0511c6e33636ca.jpg	Is this a picture of Villa Belvedere (San Giuliano Terme)? Please answer yes or no.	No
-0e553d5d99e21c32.jpg	Is this a picture of Mustafa Pasha Bridge? Please answer yes or no.	Yes
-0e553d5d99e21c32.jpg	Is this a picture of Scrattons Eco Park? Please answer yes or no.	No
-0eb857c972ddc4c8.jpg	Is this an image of Nabq Protected Area? Please answer yes or no.	Yes
-0eb857c972ddc4c8.jpg	Is this an image of Maletsunyane Falls? Please answer yes or no.	No
-0f92862c0366ab60.jpg	Is this a photo of Cotehele? Please answer yes or no.	Yes
-0f92862c0366ab60.jpg	Is this a photo of Mozirje Grove? Please answer yes or no.	No
-0fc48f210ea7d363.jpg	Is this a photo of Schauenburg (Oberkirch)? Please answer yes or no.	Yes
-0fc48f210ea7d363.jpg	Is this a photo of Dartford Library? Please answer yes or no.	No
-100e5402cfb2dfcf.jpg	Is this a picture of Nanzenji? Please answer yes or no.	Yes
-100e5402cfb2dfcf.jpg	Is this a picture of Bagakain Lake? Please answer yes or no.	No
-101786bd2ec5800e.jpg	Is this a picture of Church of the Merciful Saviour in Kuskovo (Moscow)? Please answer yes or no.	Yes
-101786bd2ec5800e.jpg	Is this a picture of Central Board of Secondary Education? Please answer yes or no.	No
-1023b7ec812bb4c6.jpg	Is this a photo of St. Joseph's Cathedral, Dunedin? Please answer yes or no.	Yes
-1023b7ec812bb4c6.jpg	Is this a photo of Fagervik Church? Please answer yes or no.	No
-1053f4cebfceabd8.jpg	Is this an image of Woolwich foot tunnel? Please answer yes or no.	Yes
-1053f4cebfceabd8.jpg	Is this an image of Kirchenbibliothek St. Marien Barth? Please answer yes or no.	No
-10a944858a4283e0.jpg	Is this a picture of Harding Tomb? Please answer yes or no.	Yes
-10a944858a4283e0.jpg	Is this a picture of Kartause Ittingen? Please answer yes or no.	No
-10b2b8741f283a3c.jpg	Is this a photo of De Bataaf, Winterswijk? Please answer yes or no.	Yes
-10b2b8741f283a3c.jpg	Is this a photo of Porte Guillaume-Lion? Please answer yes or no.	No
-11b6bb2c35f4ee22.jpg	Is this an image of Ku-ring-gai Chase National Park? Please answer yes or no.	Yes
-11b6bb2c35f4ee22.jpg	Is this an image of Saint Mark's Coptic Orthodox Cathedral, Cairo? Please answer yes or no.	No
-127be7d34a21e5ed.jpg	Is this a picture of Beachy Head? Please answer yes or no.	Yes
-127be7d34a21e5ed.jpg	Is this a picture of Jindo Bridge? Please answer yes or no.	No
-130846b4f29fe4ba.jpg	Is this a photo of Duomo (Viterbo)? Please answer yes or no.	Yes
-130846b4f29fe4ba.jpg	Is this a photo of Raffles Praslin? Please answer yes or no.	No
-13a563a1a8ef38ae.jpg	Is this a photo of Torre del Rellotge d'Olesa de Montserrat? Please answer yes or no.	Yes
-13a563a1a8ef38ae.jpg	Is this a photo of Salzburg Museum? Please answer yes or no.	No
-14d3aba340d2fba9.jpg	Is this an image of Regard Saint-Martin? Please answer yes or no.	Yes
-14d3aba340d2fba9.jpg	Is this an image of Mount Donna Buang? Please answer yes or no.	No
-15b75eb6fca4aeb6.jpg	Is this a photo of City Duma building (Saint Petersburg)? Please answer yes or no.	Yes
-15b75eb6fca4aeb6.jpg	Is this a photo of Komadome-Hachiman-jinja? Please answer yes or no.	No
-15ec5d2c0898a56e.jpg	Is this a photo of Kids Marina Rotterdam? Please answer yes or no.	Yes
-15ec5d2c0898a56e.jpg	Is this a photo of Kirkwood Observatory? Please answer yes or no.	No
-15f326a030396dba.jpg	Is this a picture of Macquarie Centre? Please answer yes or no.	Yes
-15f326a030396dba.jpg	Is this a picture of California Historical Landmarks in San Mateo County, California? Please answer yes or no.	No
-162a77504f4e686e.jpg	Is this an image of Wieliczka Castle? Please answer yes or no.	Yes
-162a77504f4e686e.jpg	Is this an image of Naval Station Guantanamo Bay? Please answer yes or no.	No
-16b76bdd7eaaef1b.jpg	Is this a picture of Evangelische Kirche Weinfelden? Please answer yes or no.	Yes
-16b76bdd7eaaef1b.jpg	Is this a picture of Gorg Blau? Please answer yes or no.	No
-16d55614b78b6b8b.jpg	Is this an image of Church of the Assumption in Bishche? Please answer yes or no.	Yes
-16d55614b78b6b8b.jpg	Is this an image of Water of Girvan? Please answer yes or no.	No
-17e0cd732e35d4aa.jpg	Is this a photo of Millennium Tower (San Francisco, California)? Please answer yes or no.	Yes
-17e0cd732e35d4aa.jpg	Is this a photo of New Orleans Botanical Garden? Please answer yes or no.	No
-18608ca3b785abf8.jpg	Is this a picture of Solvognen? Please answer yes or no.	Yes
-18608ca3b785abf8.jpg	Is this a picture of Schenley Farms National Historic District? Please answer yes or no.	No
-197e440391d309f8.jpg	Is this a picture of Skansin? Please answer yes or no.	Yes
-197e440391d309f8.jpg	Is this a picture of Bardstown Historic District? Please answer yes or no.	No
-19fe04377265454e.jpg	Is this a picture of Alexander Nevsky Church, Belgrade? Please answer yes or no.	Yes
-19fe04377265454e.jpg	Is this a picture of Seppiko-Mineyama Hyogo Prefectural Nature Park? Please answer yes or no.	No
-1a32959819038291.jpg	Is this an image of Id Kah Mosque? Please answer yes or no.	Yes
-1a32959819038291.jpg	Is this an image of Katharinenkirche (Zwickau)? Please answer yes or no.	No
-1ac0e3c493073995.jpg	Is this an image of Church of Saint John the Baptist (Jihlava)? Please answer yes or no.	Yes
-1ac0e3c493073995.jpg	Is this an image of Silte kyrka? Please answer yes or no.	No
-1d4be36153741972.jpg	Is this a picture of Marie Guyart Building? Please answer yes or no.	Yes
-1d4be36153741972.jpg	Is this a picture of Motol? Please answer yes or no.	No
-1f233b807b660d25.jpg	Is this a photo of Aussichtsturm Pyramidenkogel? Please answer yes or no.	Yes
-1f233b807b660d25.jpg	Is this a photo of Vvedenskoe cemetery? Please answer yes or no.	No
-27a4267b9ea26cd0.jpg	Is this an image of Oude Salviuskerk (Limbricht)? Please answer yes or no.	Yes
-27a4267b9ea26cd0.jpg	Is this an image of Festung Heldrungen? Please answer yes or no.	No
-2a895eb889d6fd99.jpg	Is this an image of Beijing Guozijian? Please answer yes or no.	Yes
-2a895eb889d6fd99.jpg	Is this an image of Klinikkirche (Pfafferode)? Please answer yes or no.	No
-2f25654bcb445452.jpg	Is this a photo of Rinku Gate Tower Building? Please answer yes or no.	Yes
-2f25654bcb445452.jpg	Is this a photo of Bigorski Monastery? Please answer yes or no.	No
-3c89eb4043bcefac.jpg	Is this a picture of Begijnhofkerk (Mechelen)? Please answer yes or no.	Yes
-3c89eb4043bcefac.jpg	Is this a picture of Panther Hollow Lake? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/numerical_calculation.txt

@@ -1,40 +0,0 @@
-0001.png	Is the answer to the arithmetic question in the image 225? Please answer yes or no.	Yes
-0001.png	Is the answer to the arithmetic question in the image 1515? Please answer yes or no.	No
-0002.png	Is the answer to the arithmetic question in the image 340? Please answer yes or no.	Yes
-0002.png	Is the answer to the arithmetic question in the image 17? Please answer yes or no.	No
-0003.png	Is the answer to the arithmetic question in the image 65? Please answer yes or no.	Yes
-0003.png	Is the answer to the arithmetic question in the image 56? Please answer yes or no.	No
-0004.png	Is the answer to the arithmetic question in the image 33? Please answer yes or no.	Yes
-0004.png	Is the answer to the arithmetic question in the image 32? Please answer yes or no.	No
-0005.png	Is the area of the square in the picture equal to 40? Please answer yes or no.	Yes
-0005.png	Is the area of the square in the picture equal to 8? Please answer yes or no.	No
-0006.png	Is the area of the square in the picture equal to 9? Please answer yes or no.	Yes
-0006.png	Is the area of the square in the picture equal to 3? Please answer yes or no.	No
-0007.png	Is the answer to the arithmetic question in the image 49? Please answer yes or no.	Yes
-0007.png	Is the answer to the arithmetic question in the image 39? Please answer yes or no.	No
-0008.png	Should the value of "a" in the picture equal 7? Please answer yes or no.	Yes
-0008.png	Should the value of "a" in the picture equal 14? Please answer yes or no.	No
-0009.png	Should the value of "a" in the picture equal 2? Please answer yes or no.	Yes
-0009.png	Should the value of "a" in the picture equal 3? Please answer yes or no.	No
-0010.png	Is the answer to the arithmetic question in the image 13? Please answer yes or no.	Yes
-0010.png	Is the answer to the arithmetic question in the image 12? Please answer yes or no.	No
-0011.png	Is the area of the parallelogram in the picture equal to 24? Please answer yes or no.	Yes
-0011.png	Is the area of the parallelogram in the picture equal to 6? Please answer yes or no.	No
-0012.png	Should the value of "a" in the picture equal 9? Please answer yes or no.	Yes
-0012.png	Should the value of "a" in the picture equal 1? Please answer yes or no.	No
-0013.png	Is the area of the right triangle in the picture equal to 24? Please answer yes or no.	Yes
-0013.png	Is the area of the right triangle in the picture equal to 8? Please answer yes or no.	No
-0014.png	Is the answer to the arithmetic question in the image 200? Please answer yes or no.	Yes
-0014.png	Is the answer to the arithmetic question in the image 400? Please answer yes or no.	No
-0015.png	Is the answer to the arithmetic question in the image 11? Please answer yes or no.	Yes
-0015.png	Is the answer to the arithmetic question in the image 111? Please answer yes or no.	No
-0016.png	Is the answer to the arithmetic question in the image 9? Please answer yes or no.	Yes
-0016.png	Is the answer to the arithmetic question in the image 16? Please answer yes or no.	No
-0017.png	Is the answer to the arithmetic question in the image 14? Please answer yes or no.	Yes
-0017.png	Is the answer to the arithmetic question in the image 83? Please answer yes or no.	No
-0018.png	Should the value of "a" in the picture equal 3? Please answer yes or no.	Yes
-0018.png	Should the value of "a" in the picture equal 2? Please answer yes or no.	No
-0019.png	Is the answer to the arithmetic question in the image 18? Please answer yes or no.	Yes
-0019.png	Is the answer to the arithmetic question in the image 36? Please answer yes or no.	No
-0020.png	Is the answer to the arithmetic question in the image 9? Please answer yes or no.	Yes
-0020.png	Is the answer to the arithmetic question in the image 45? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/position.txt

@@ -1,60 +0,0 @@
-000000006471.jpg	Is the cricket bat above the batter's body? Please answer yes or no.	Yes
-000000006471.jpg	Is the cricket bat under the batter's body Please answer yes or no.	No
-000000007281.jpg	Is the sea behind people in the image? Please answer yes or no.	Yes
-000000007281.jpg	Is the sea in front of people in the image? Please answer yes or no.	No
-000000014038.jpg	Is the refrigerator on the left side of the picture? Please answer yes or no.	Yes
-000000014038.jpg	Is the refrigerator on the right side of the picture Please answer yes or no.	No
-000000031248.jpg	Is there a sofa in the middle of potted plants in the image? Please answer yes or no.	Yes
-000000031248.jpg	Is there a sofa in the right side of potted plants in the image? Please answer yes or no.	No
-000000048504.jpg	Is the gray elephant in front of the brown elephant? Please answer yes or no.	Yes
-000000048504.jpg	Is the brown elephant in front of the gray elephant? Please answer yes or no.	No
-000000052007.jpg	Are the pedestrians on the right of the bus? Please answer yes or no.	Yes
-000000052007.jpg	Are the pedestrians on the left of the bus? Please answer yes or no.	No
-000000056127.jpg	Is the light above the fire hydrant in the image? Please answer yes or no.	Yes
-000000056127.jpg	Is the light under the fire hydrant in the image?  Please answer yes or no.	No
-000000062025.jpg	Is the trash can under the cup in the image？ Please answer yes or no.	Yes
-000000062025.jpg	Is the trash can above the cup in the image？ Please answer yes or no.	No
-000000062808.jpg	Is the phone above the pizza in the image? Please answer yes or no.	Yes
-000000062808.jpg	Is the phone under the pizza in the image? Please answer yes or no.	No
-000000067213.jpg	Is the dog above the pool in the image? Please answer yes or no.	Yes
-000000067213.jpg	Is the dog under the pool in the image? Please answer yes or no.	No
-000000097994.jpg	Is the light above the computer in the image? Please answer yes or no.	Yes
-000000097994.jpg	Is the light under the computer in the image? Please answer yes or no.	No
-000000204871.jpg	Is the car on the right side of the fire hydrant in the picture? Please answer yes or no.	Yes
-000000204871.jpg	Is the car on the left side of the fire hydrant in the picture? Please answer yes or no.	No
-000000206487.jpg	Is the motorcycle on the right side of the bus? Please answer yes or no.	Yes
-000000206487.jpg	Is the motorcycle on the left side of the bus Please answer yes or no.	No
-000000211825.jpg	Is the cake on the left side of the camera? Please answer yes or no.	Yes
-000000211825.jpg	Is the cake on the right side of the camera? Please answer yes or no.	No
-000000212800.jpg	Is the blue umbrella under the black umbrella? Please answer yes or no.	Yes
-000000212800.jpg	Is the blue umbrella above the black umbrella? Please answer yes or no.	No
-000000395701.jpg	Is the TV on the left of the bookshelf? Please answer yes or no.	Yes
-000000395701.jpg	Is the TV on the right of the bookshelf? Please answer yes or no.	No
-000000395801.jpg	Is the clock above people? Please answer yes or no.	Yes
-000000395801.jpg	Is the clock under people? Please answer yes or no.	No
-000000405970.jpg	Is the grey sofa on the right of the TV? Please answer yes or no.	Yes
-000000405970.jpg	Is the grey sofa on the left of the TV? Please answer yes or no.	No
-000000426241.jpg	Is the white mouse on the right of the black keyboard? Please answer yes or no.	Yes
-000000426241.jpg	Is the white mouse on the left of the black keyboard? Please answer yes or no.	No
-000000450303.jpg	Is the monitor on top of a person? Please answer yes or no.	Yes
-000000450303.jpg	Is the monitor under the person? Please answer yes or no.	No
-000000458410.jpg	Is the TV on the left of the lamp? Please answer yes or no.	Yes
-000000458410.jpg	Is the TV on the right of the lamp? Please answer yes or no.	No
-000000472046.jpg	Is the pineapple on the left of the pot in the image? Please answer yes or no.	Yes
-000000472046.jpg	Is the pineapple on the right of the pot in the image? Please answer yes or no.	No
-000000477955.jpg	Is the person under the kite? Please answer yes or no.	Yes
-000000477955.jpg	Is the person above the kite? Please answer yes or no.	No
-000000482585.jpg	Is the person on the right of the train? Please answer yes or no.	Yes
-000000482585.jpg	Is the person on the left of the train? Please answer yes or no.	No
-000000494869.jpg	Is the baby on the right of the dog in the image? Please answer yes or no.	Yes
-000000494869.jpg	Is the baby on the left of the dog in the image? Please answer yes or no.	No
-000000509699.jpg	Is the mirror above the TV? Please answer yes or no.	Yes
-000000509699.jpg	Is the mirror under the TV? Please answer yes or no.	No
-000000519569.jpg	Is the vase on the left of the bottle? Please answer yes or no.	Yes
-000000519569.jpg	Is the vase on the right of the bottle? Please answer yes or no.	No
-000000530162.jpg	Is the big red and black umbrella on the top of people? Please answer yes or no.	Yes
-000000530162.jpg	Is the big red and black umbrella under people? Please answer yes or no.	No
-000000551660.jpg	Is the spoon in the bowl? Please answer yes or no.	Yes
-000000551660.jpg	Is the spoon out of the bowl? Please answer yes or no.	No
-000000578922.jpg	Is the vase on the left of the toothbrush? Please answer yes or no.	Yes
-000000578922.jpg	Is the vase on the right of the toothbrush? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/posters.txt

@@ -1,294 +0,0 @@
-tt0048028.jpg	Is this movie directed by elia kazan? Please answer yes or no.	Yes
-tt0048028.jpg	Is this movie directed by maneesh sharma? Please answer yes or no.	No
-tt0053221.jpg	Is this movie titled rio bravo (1959)? Please answer yes or no.	Yes
-tt0053221.jpg	Is this movie titled the matrix (1999)? Please answer yes or no.	No
-tt0062622.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	Yes
-tt0062622.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
-tt0063442.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0063442.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
-tt0065724.jpg	Is this movie directed by bob rafelson? Please answer yes or no.	Yes
-tt0065724.jpg	Is this movie directed by anthony minghella? Please answer yes or no.	No
-tt0067116.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0067116.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
-tt0067140.jpg	Is this movie titled duck, you sucker (1971)? Please answer yes or no.	Yes
-tt0067140.jpg	Is this movie titled crimson tide (1995)? Please answer yes or no.	No
-tt0067185.jpg	Is this movie directed by hal ashby? Please answer yes or no.	Yes
-tt0067185.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	No
-tt0068646.jpg	Is this movie titled the godfather (1972)? Please answer yes or no.	Yes
-tt0068646.jpg	Is this movie titled bring on the melody (2017)? Please answer yes or no.	No
-tt0070291.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0070291.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
-tt0071315.jpg	Is this movie titled chinatown (1974)? Please answer yes or no.	Yes
-tt0071315.jpg	Is this movie titled planet of the apes (1968)? Please answer yes or no.	No
-tt0074119.jpg	Is this movie directed by alan j. pakula? Please answer yes or no.	Yes
-tt0074119.jpg	Is this movie directed by stanley kubrick? Please answer yes or no.	No
-tt0074749.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0074749.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
-tt0082186.jpg	Is this movie directed by desmond davis? Please answer yes or no.	Yes
-tt0082186.jpg	Is this movie directed by edward zwick? Please answer yes or no.	No
-tt0083907.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0083907.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	No
-tt0083946.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	Yes
-tt0083946.jpg	Is this movie originated from the country or region of denmark? Please answer yes or no.	No
-tt0084787.jpg	Is this movie titled the thing (1982)? Please answer yes or no.	Yes
-tt0084787.jpg	Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no.	No
-tt0086250.jpg	Is this movie directed by brian de palma? Please answer yes or no.	Yes
-tt0086250.jpg	Is this movie directed by patrice leconte? Please answer yes or no.	No
-tt0093565.jpg	Is this movie titled moonstruck (1987)? Please answer yes or no.	Yes
-tt0093565.jpg	Is this movie titled now you see me (2013)? Please answer yes or no.	No
-tt0093773.jpg	Is this movie titled predator (1987)? Please answer yes or no.	Yes
-tt0093773.jpg	Is this movie titled the cell (2000)? Please answer yes or no.	No
-tt0094291.jpg	Is this movie directed by oliver stone? Please answer yes or no.	Yes
-tt0094291.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
-tt0096446.jpg	Is this movie directed by ron howard? Please answer yes or no.	Yes
-tt0096446.jpg	Is this movie directed by peter jackson? Please answer yes or no.	No
-tt0096463.jpg	Is this movie directed by mike nichols? Please answer yes or no.	Yes
-tt0096463.jpg	Is this movie directed by cary joji fukunaga? Please answer yes or no.	No
-tt0096895.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0096895.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
-tt0097576.jpg	Is this movie directed by steven spielberg? Please answer yes or no.	Yes
-tt0097576.jpg	Is this movie directed by francis ford coppola? Please answer yes or no.	No
-tt0098635.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0098635.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
-tt0098724.jpg	Is this movie titled sex, lies, and videotape (1989)? Please answer yes or no.	Yes
-tt0098724.jpg	Is this movie titled final destination (2000)? Please answer yes or no.	No
-tt0099653.jpg	Is this movie directed by jerry zucker? Please answer yes or no.	Yes
-tt0099653.jpg	Is this movie directed by felix chong,alan mak? Please answer yes or no.	No
-tt0100112.jpg	Is this movie originated from the country or region of france? Please answer yes or no.	Yes
-tt0100112.jpg	Is this movie originated from the country or region of new zealand? Please answer yes or no.	No
-tt0100150.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0100150.jpg	Is this movie originated from the country or region of hong kong? Please answer yes or no.	No
-tt0100935.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0100935.jpg	Is this movie originated from the country or region of canada? Please answer yes or no.	No
-tt0101889.jpg	Is this movie titled the fisher king (1991)? Please answer yes or no.	Yes
-tt0101889.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
-tt0101921.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0101921.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
-tt0107653.jpg	Is this movie titled naked (1993)? Please answer yes or no.	Yes
-tt0107653.jpg	Is this movie titled captain america: civil war (2016)? Please answer yes or no.	No
-tt0107808.jpg	Is this movie titled a perfect world (1993)? Please answer yes or no.	Yes
-tt0107808.jpg	Is this movie titled harold and maude (1971)? Please answer yes or no.	No
-tt0110201.jpg	Is this movie titled hail the judge (1994)? Please answer yes or no.	Yes
-tt0110201.jpg	Is this movie titled batman (1989)? Please answer yes or no.	No
-tt0112740.jpg	Is this movie titled crimson tide (1995)? Please answer yes or no.	Yes
-tt0112740.jpg	Is this movie titled insidious: the last key (2018)? Please answer yes or no.	No
-tt0113497.jpg	Is this movie titled jumanji (1995)? Please answer yes or no.	Yes
-tt0113497.jpg	Is this movie titled morgan (2016)? Please answer yes or no.	No
-tt0114369.jpg	Is this movie titled se7en (1995)? Please answer yes or no.	Yes
-tt0114369.jpg	Is this movie titled a perfect world (1993)? Please answer yes or no.	No
-tt0114388.jpg	Is this movie directed by ang lee? Please answer yes or no.	Yes
-tt0114388.jpg	Is this movie directed by david lowery? Please answer yes or no.	No
-tt0116209.jpg	Is this movie directed by anthony minghella? Please answer yes or no.	Yes
-tt0116209.jpg	Is this movie directed by john cassavetes? Please answer yes or no.	No
-tt0116629.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0116629.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
-tt0118636.jpg	Is this movie titled apt pupil (1998)? Please answer yes or no.	Yes
-tt0118636.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
-tt0118655.jpg	Is this movie directed by jay roach? Please answer yes or no.	Yes
-tt0118655.jpg	Is this movie directed by peter jackson? Please answer yes or no.	No
-tt0118715.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0118715.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
-tt0119303.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0119303.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
-tt0120738.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0120738.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
-tt0129387.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0129387.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
-tt0133093.jpg	Is this movie directed by lana wachowski,lilly wachowski? Please answer yes or no.	Yes
-tt0133093.jpg	Is this movie directed by siu-tung ching? Please answer yes or no.	No
-tt0163025.jpg	Is this movie directed by joe johnston? Please answer yes or no.	Yes
-tt0163025.jpg	Is this movie directed by adam robitel? Please answer yes or no.	No
-tt0166924.jpg	Is this movie titled mulholland drive (2001)? Please answer yes or no.	Yes
-tt0166924.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	No
-tt0167261.jpg	Is this movie directed by peter jackson? Please answer yes or no.	Yes
-tt0167261.jpg	Is this movie directed by gabe ibáñez? Please answer yes or no.	No
-tt0182789.jpg	Is this movie titled bicentennial man (1999)? Please answer yes or no.	Yes
-tt0182789.jpg	Is this movie titled ocean's eleven (2001)? Please answer yes or no.	No
-tt0187078.jpg	Is this movie titled gone in 60 seconds (2000)? Please answer yes or no.	Yes
-tt0187078.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
-tt0195714.jpg	Is this movie directed by james wong? Please answer yes or no.	Yes
-tt0195714.jpg	Is this movie directed by bob rafelson? Please answer yes or no.	No
-tt0209958.jpg	Is this movie titled the cell (2000)? Please answer yes or no.	Yes
-tt0209958.jpg	Is this movie titled the matrix revolutions (2003)? Please answer yes or no.	No
-tt0232500.jpg	Is this movie titled the fast and the furious (2001)? Please answer yes or no.	Yes
-tt0232500.jpg	Is this movie titled east of eden (1955)? Please answer yes or no.	No
-tt0240772.jpg	Is this movie directed by steven soderbergh? Please answer yes or no.	Yes
-tt0240772.jpg	Is this movie directed by paul mcguigan? Please answer yes or no.	No
-tt0242653.jpg	Is this movie titled the matrix revolutions (2003)? Please answer yes or no.	Yes
-tt0242653.jpg	Is this movie titled sense and sensibility (1995)? Please answer yes or no.	No
-tt0244244.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0244244.jpg	Is this movie originated from the country or region of argentina? Please answer yes or no.	No
-tt0258463.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0258463.jpg	Is this movie originated from the country or region of italy? Please answer yes or no.	No
-tt0261392.jpg	Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no.	Yes
-tt0261392.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
-tt0264395.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	Yes
-tt0264395.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
-tt0266697.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	Yes
-tt0266697.jpg	Is this movie directed by peyton reed? Please answer yes or no.	No
-tt0268978.jpg	Is this movie titled a beautiful mind (2001)? Please answer yes or no.	Yes
-tt0268978.jpg	Is this movie titled blue jasmine (2013)? Please answer yes or no.	No
-tt0318627.jpg	Is this movie titled resident evil: apocalypse (2004)? Please answer yes or no.	Yes
-tt0318627.jpg	Is this movie titled the meddler (2015)? Please answer yes or no.	No
-tt0328107.jpg	Is this movie titled man on fire (2004)? Please answer yes or no.	Yes
-tt0328107.jpg	Is this movie titled real steel (2011)? Please answer yes or no.	No
-tt0338751.jpg	Is this movie directed by martin scorsese? Please answer yes or no.	Yes
-tt0338751.jpg	Is this movie directed by chris columbus? Please answer yes or no.	No
-tt0341495.jpg	Is this movie directed by siu-tung ching? Please answer yes or no.	Yes
-tt0341495.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	No
-tt0351977.jpg	Is this movie directed by kevin bray? Please answer yes or no.	Yes
-tt0351977.jpg	Is this movie directed by joe johnston? Please answer yes or no.	No
-tt0369702.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	Yes
-tt0369702.jpg	Is this movie originated from the country or region of france? Please answer yes or no.	No
-tt0373469.jpg	Is this movie directed by shane black? Please answer yes or no.	Yes
-tt0373469.jpg	Is this movie directed by dennis dugan? Please answer yes or no.	No
-tt0378194.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	Yes
-tt0378194.jpg	Is this movie directed by todd phillips? Please answer yes or no.	No
-tt0379786.jpg	Is this movie titled serenity (2005)? Please answer yes or no.	Yes
-tt0379786.jpg	Is this movie titled bad genius (2017)? Please answer yes or no.	No
-tt0393109.jpg	Is this movie titled brick (2005)? Please answer yes or no.	Yes
-tt0393109.jpg	Is this movie titled sense and sensibility (1995)? Please answer yes or no.	No
-tt0399201.jpg	Is this movie directed by michael bay? Please answer yes or no.	Yes
-tt0399201.jpg	Is this movie directed by angelina jolie? Please answer yes or no.	No
-tt0425210.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	Yes
-tt0425210.jpg	Is this movie titled the godfather (1972)? Please answer yes or no.	No
-tt0433035.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0433035.jpg	Is this movie originated from the country or region of new zealand? Please answer yes or no.	No
-tt0443680.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0443680.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
-tt0449088.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0449088.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
-tt0450259.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt0450259.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
-tt0462499.jpg	Is this movie titled rambo (2008)? Please answer yes or no.	Yes
-tt0462499.jpg	Is this movie titled walking tall (2004)? Please answer yes or no.	No
-tt0499549.jpg	Is this movie directed by james cameron? Please answer yes or no.	Yes
-tt0499549.jpg	Is this movie directed by jerry zucker? Please answer yes or no.	No
-tt0780653.jpg	Is this movie titled the wolfman (2010)? Please answer yes or no.	Yes
-tt0780653.jpg	Is this movie titled wild at heart (1990)? Please answer yes or no.	No
-tt0829482.jpg	Is this movie directed by greg mottola? Please answer yes or no.	Yes
-tt0829482.jpg	Is this movie directed by jing wong? Please answer yes or no.	No
-tt0848228.jpg	Is this movie titled the avengers (2012)? Please answer yes or no.	Yes
-tt0848228.jpg	Is this movie titled wolf warrior 2 (2017)? Please answer yes or no.	No
-tt0898367.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	Yes
-tt0898367.jpg	Is this movie directed by franklin j. schaffner? Please answer yes or no.	No
-tt0942385.jpg	Is this movie titled tropic thunder (2008)? Please answer yes or no.	Yes
-tt0942385.jpg	Is this movie titled bad teacher (2011)? Please answer yes or no.	No
-tt0993846.jpg	Is this movie titled the wolf of wall street (2013)? Please answer yes or no.	Yes
-tt0993846.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	No
-tt1017460.jpg	Is this movie directed by vincenzo natali? Please answer yes or no.	Yes
-tt1017460.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
-tt1057500.jpg	Is this movie directed by clint eastwood? Please answer yes or no.	Yes
-tt1057500.jpg	Is this movie directed by john cassavetes? Please answer yes or no.	No
-tt1068680.jpg	Is this movie titled yes man (2008)? Please answer yes or no.	Yes
-tt1068680.jpg	Is this movie titled rio bravo (1959)? Please answer yes or no.	No
-tt1099212.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1099212.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
-tt1119646.jpg	Is this movie directed by todd phillips? Please answer yes or no.	Yes
-tt1119646.jpg	Is this movie directed by franklin j. schaffner? Please answer yes or no.	No
-tt1124037.jpg	Is this movie directed by gary ross? Please answer yes or no.	Yes
-tt1124037.jpg	Is this movie directed by francis ford coppola? Please answer yes or no.	No
-tt1229822.jpg	Is this movie titled jane eyre (2011)? Please answer yes or no.	Yes
-tt1229822.jpg	Is this movie titled kiss kiss bang bang (2005)? Please answer yes or no.	No
-tt1284575.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1284575.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	No
-tt1291150.jpg	Is this movie directed by jonathan liebesman? Please answer yes or no.	Yes
-tt1291150.jpg	Is this movie directed by masahide ichii? Please answer yes or no.	No
-tt1300851.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1300851.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
-tt1305806.jpg	Is this movie titled the secret in their eyes (2009)? Please answer yes or no.	Yes
-tt1305806.jpg	Is this movie titled ocean's eleven (2001)? Please answer yes or no.	No
-tt1343092.jpg	Is this movie directed by baz luhrmann? Please answer yes or no.	Yes
-tt1343092.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	No
-tt1385826.jpg	Is this movie titled the adjustment bureau (2011)? Please answer yes or no.	Yes
-tt1385826.jpg	Is this movie titled the last of sheila (1973)? Please answer yes or no.	No
-tt1403865.jpg	Is this movie directed by ethan coen,joel coen? Please answer yes or no.	Yes
-tt1403865.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	No
-tt1438176.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1438176.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	No
-tt1454029.jpg	Is this movie titled the help (2011)? Please answer yes or no.	Yes
-tt1454029.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
-tt1560747.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1560747.jpg	Is this movie originated from the country or region of canada? Please answer yes or no.	No
-tt1564367.jpg	Is this movie titled just go with it (2011)? Please answer yes or no.	Yes
-tt1564367.jpg	Is this movie titled journey to the west (2013)? Please answer yes or no.	No
-tt1606378.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt1606378.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
-tt1637725.jpg	Is this movie titled ted (2012)? Please answer yes or no.	Yes
-tt1637725.jpg	Is this movie titled lost in space (1998)? Please answer yes or no.	No
-tt1670345.jpg	Is this movie titled now you see me (2013)? Please answer yes or no.	Yes
-tt1670345.jpg	Is this movie titled bicentennial man (1999)? Please answer yes or no.	No
-tt1703957.jpg	Is this movie titled genius (2016)? Please answer yes or no.	Yes
-tt1703957.jpg	Is this movie titled fan (2016)? Please answer yes or no.	No
-tt1722484.jpg	Is this movie directed by hermine huntgeburth? Please answer yes or no.	Yes
-tt1722484.jpg	Is this movie directed by todd phillips? Please answer yes or no.	No
-tt1800241.jpg	Is this movie directed by david o. russell? Please answer yes or no.	Yes
-tt1800241.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
-tt1809398.jpg	Is this movie titled unbroken (2014)? Please answer yes or no.	Yes
-tt1809398.jpg	Is this movie titled home alone 3 (1997)? Please answer yes or no.	No
-tt1971325.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	Yes
-tt1971325.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
-tt1974419.jpg	Is this movie titled the neon demon (2016)? Please answer yes or no.	Yes
-tt1974419.jpg	Is this movie titled man on fire (2004)? Please answer yes or no.	No
-tt2017561.jpg	Is this movie directed by stephen chow,chi-kin kwok? Please answer yes or no.	Yes
-tt2017561.jpg	Is this movie directed by craig gillespie? Please answer yes or no.	No
-tt2078768.jpg	Is this movie originated from the country or region of hong kong? Please answer yes or no.	Yes
-tt2078768.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
-tt2132285.jpg	Is this movie titled the bling ring (2013)? Please answer yes or no.	Yes
-tt2132285.jpg	Is this movie titled tropic thunder (2008)? Please answer yes or no.	No
-tt2238032.jpg	Is this movie titled skiptrace (2016)? Please answer yes or no.	Yes
-tt2238032.jpg	Is this movie titled the bourne identity (2002)? Please answer yes or no.	No
-tt2334873.jpg	Is this movie titled blue jasmine (2013)? Please answer yes or no.	Yes
-tt2334873.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
-tt2788732.jpg	Is this movie titled pete's dragon (2016)? Please answer yes or no.	Yes
-tt2788732.jpg	Is this movie titled willow (1988)? Please answer yes or no.	No
-tt2802144.jpg	Is this movie directed by matthew vaughn? Please answer yes or no.	Yes
-tt2802144.jpg	Is this movie directed by dietrich brüggemann? Please answer yes or no.	No
-tt2908856.jpg	Is this movie directed by israel horovitz? Please answer yes or no.	Yes
-tt2908856.jpg	Is this movie directed by mike nichols? Please answer yes or no.	No
-tt2980516.jpg	Is this movie titled the theory of everything (2014)? Please answer yes or no.	Yes
-tt2980516.jpg	Is this movie titled stations of the cross (2014)? Please answer yes or no.	No
-tt3077214.jpg	Is this movie titled suffragette (2015)? Please answer yes or no.	Yes
-tt3077214.jpg	Is this movie titled east of eden (1955)? Please answer yes or no.	No
-tt3316960.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt3316960.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	No
-tt3465916.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	Yes
-tt3465916.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
-tt3495026.jpg	Is this movie titled fan (2016)? Please answer yes or no.	Yes
-tt3495026.jpg	Is this movie titled from afar (2015)? Please answer yes or no.	No
-tt3498820.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt3498820.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
-tt3700804.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt3700804.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
-tt3824458.jpg	Is this movie titled tangerine (2015)? Please answer yes or no.	Yes
-tt3824458.jpg	Is this movie titled miller's crossing (1990)? Please answer yes or no.	No
-tt3860916.jpg	Is this movie directed by ben howling? Please answer yes or no.	Yes
-tt3860916.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	No
-tt4046784.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt4046784.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
-tt4242158.jpg	Is this movie titled lost in hong kong (2015)? Please answer yes or no.	Yes
-tt4242158.jpg	Is this movie titled harold and maude (1971)? Please answer yes or no.	No
-tt4273292.jpg	Is this movie directed by babak anvari? Please answer yes or no.	Yes
-tt4273292.jpg	Is this movie directed by ethan coen,joel coen? Please answer yes or no.	No
-tt4501454.jpg	Is this movie directed by lorene scafaria? Please answer yes or no.	Yes
-tt4501454.jpg	Is this movie directed by herbert ross? Please answer yes or no.	No
-tt4520364.jpg	Is this movie directed by luke scott? Please answer yes or no.	Yes
-tt4520364.jpg	Is this movie directed by dominic sena? Please answer yes or no.	No
-tt4651520.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
-tt4651520.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
-tt4721400.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	Yes
-tt4721400.jpg	Is this movie directed by jonathan liebesman? Please answer yes or no.	No
-tt4972062.jpg	Is this movie directed by mike birbiglia? Please answer yes or no.	Yes
-tt4972062.jpg	Is this movie directed by david fincher? Please answer yes or no.	No
-tt5564148.jpg	Is this movie directed by masahide ichii? Please answer yes or no.	Yes
-tt5564148.jpg	Is this movie directed by dietrich brüggemann? Please answer yes or no.	No
-tt5688868.jpg	Is this movie titled primal rage: the legend of konga (2018)? Please answer yes or no.	Yes
-tt5688868.jpg	Is this movie titled the neon demon (2016)? Please answer yes or no.	No
-tt5726086.jpg	Is this movie directed by adam robitel? Please answer yes or no.	Yes
-tt5726086.jpg	Is this movie directed by gore verbinski? Please answer yes or no.	No
-tt6788942.jpg	Is this movie originated from the country or region of thailand? Please answer yes or no.	Yes
-tt6788942.jpg	Is this movie originated from the country or region of argentina? Please answer yes or no.	No
-tt7055592.jpg	Is this movie titled brotherhood of blades ii: the infernal battlefield (2017)? Please answer yes or no.	Yes
-tt7055592.jpg	Is this movie titled stations of the cross (2014)? Please answer yes or no.	No
-tt7131870.jpg	Is this movie directed by jing wu? Please answer yes or no.	Yes
-tt7131870.jpg	Is this movie directed by david lynch? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/scene.txt

@@ -1,400 +0,0 @@
-Places365_val_00000001.jpg	Is this picture captured in a place of greenhouse indoor? Please answer yes or no.	Yes
-Places365_val_00000001.jpg	Is this picture captured in a place of waiting room? Please answer yes or no.	No
-Places365_val_00000002.jpg	Is this photo taken in a place of wet bar? Please answer yes or no.	Yes
-Places365_val_00000002.jpg	Is this photo taken in a place of stage indoor? Please answer yes or no.	No
-Places365_val_00000003.jpg	Is this picture captured in a place of clean room? Please answer yes or no.	Yes
-Places365_val_00000003.jpg	Is this picture captured in a place of bookstore? Please answer yes or no.	No
-Places365_val_00000004.jpg	Is this picture captured in a place of golf course? Please answer yes or no.	Yes
-Places365_val_00000004.jpg	Is this picture captured in a place of computer room? Please answer yes or no.	No
-Places365_val_00000005.jpg	Is this picture captured in a place of rock arch? Please answer yes or no.	Yes
-Places365_val_00000005.jpg	Is this picture captured in a place of desert sand? Please answer yes or no.	No
-Places365_val_00000006.jpg	Is this photo taken in a place of corridor? Please answer yes or no.	Yes
-Places365_val_00000006.jpg	Is this photo taken in a place of jail cell? Please answer yes or no.	No
-Places365_val_00000007.jpg	Does this image describe a place of canyon? Please answer yes or no.	Yes
-Places365_val_00000007.jpg	Does this image describe a place of basement? Please answer yes or no.	No
-Places365_val_00000008.jpg	Is this picture captured in a place of dining room? Please answer yes or no.	Yes
-Places365_val_00000008.jpg	Is this picture captured in a place of ball pit? Please answer yes or no.	No
-Places365_val_00000009.jpg	Is this picture captured in a place of forest broadleaf? Please answer yes or no.	Yes
-Places365_val_00000009.jpg	Is this picture captured in a place of swamp? Please answer yes or no.	No
-Places365_val_00000010.jpg	Does this image describe a place of shopping mall indoor? Please answer yes or no.	Yes
-Places365_val_00000010.jpg	Does this image describe a place of mosque outdoor? Please answer yes or no.	No
-Places365_val_00000011.jpg	Is this photo taken in a place of baseball field? Please answer yes or no.	Yes
-Places365_val_00000011.jpg	Is this photo taken in a place of bus station indoor? Please answer yes or no.	No
-Places365_val_00000012.jpg	Does this image describe a place of campus? Please answer yes or no.	Yes
-Places365_val_00000012.jpg	Does this image describe a place of sandbox? Please answer yes or no.	No
-Places365_val_00000013.jpg	Is this picture captured in a place of beach house? Please answer yes or no.	Yes
-Places365_val_00000013.jpg	Is this picture captured in a place of alcove? Please answer yes or no.	No
-Places365_val_00000014.jpg	Does this image describe a place of art gallery? Please answer yes or no.	Yes
-Places365_val_00000014.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	No
-Places365_val_00000015.jpg	Does this image describe a place of bus interior? Please answer yes or no.	Yes
-Places365_val_00000015.jpg	Does this image describe a place of butchers shop? Please answer yes or no.	No
-Places365_val_00000016.jpg	Does this image describe a place of gymnasium indoor? Please answer yes or no.	Yes
-Places365_val_00000016.jpg	Does this image describe a place of crevasse? Please answer yes or no.	No
-Places365_val_00000017.jpg	Is this picture captured in a place of glacier? Please answer yes or no.	Yes
-Places365_val_00000017.jpg	Is this picture captured in a place of fishpond? Please answer yes or no.	No
-Places365_val_00000018.jpg	Is this photo taken in a place of nursing home? Please answer yes or no.	Yes
-Places365_val_00000018.jpg	Is this photo taken in a place of server room? Please answer yes or no.	No
-Places365_val_00000019.jpg	Does this image describe a place of storage room? Please answer yes or no.	Yes
-Places365_val_00000019.jpg	Does this image describe a place of aquarium? Please answer yes or no.	No
-Places365_val_00000020.jpg	Is this picture captured in a place of florist shop indoor? Please answer yes or no.	Yes
-Places365_val_00000020.jpg	Is this picture captured in a place of office? Please answer yes or no.	No
-Places365_val_00000021.jpg	Is this picture captured in a place of restaurant kitchen? Please answer yes or no.	Yes
-Places365_val_00000021.jpg	Is this picture captured in a place of garage outdoor? Please answer yes or no.	No
-Places365_val_00000022.jpg	Is this photo taken in a place of recreation room? Please answer yes or no.	Yes
-Places365_val_00000022.jpg	Is this photo taken in a place of runway? Please answer yes or no.	No
-Places365_val_00000023.jpg	Does this image describe a place of schoolhouse? Please answer yes or no.	Yes
-Places365_val_00000023.jpg	Does this image describe a place of raft? Please answer yes or no.	No
-Places365_val_00000024.jpg	Does this image describe a place of amusement arcade? Please answer yes or no.	Yes
-Places365_val_00000024.jpg	Does this image describe a place of medina? Please answer yes or no.	No
-Places365_val_00000025.jpg	Is this photo taken in a place of ocean? Please answer yes or no.	Yes
-Places365_val_00000025.jpg	Is this photo taken in a place of sauna? Please answer yes or no.	No
-Places365_val_00000026.jpg	Does this image describe a place of river? Please answer yes or no.	Yes
-Places365_val_00000026.jpg	Does this image describe a place of corral? Please answer yes or no.	No
-Places365_val_00000027.jpg	Is this photo taken in a place of pub indoor? Please answer yes or no.	Yes
-Places365_val_00000027.jpg	Is this photo taken in a place of basement? Please answer yes or no.	No
-Places365_val_00000028.jpg	Is this picture captured in a place of lecture room? Please answer yes or no.	Yes
-Places365_val_00000028.jpg	Is this picture captured in a place of aqueduct? Please answer yes or no.	No
-Places365_val_00000029.jpg	Is this photo taken in a place of japanese garden? Please answer yes or no.	Yes
-Places365_val_00000029.jpg	Is this photo taken in a place of pharmacy? Please answer yes or no.	No
-Places365_val_00000030.jpg	Does this image describe a place of motel? Please answer yes or no.	Yes
-Places365_val_00000030.jpg	Does this image describe a place of toyshop? Please answer yes or no.	No
-Places365_val_00000031.jpg	Does this image describe a place of assembly line? Please answer yes or no.	Yes
-Places365_val_00000031.jpg	Does this image describe a place of beer garden? Please answer yes or no.	No
-Places365_val_00000032.jpg	Does this image describe a place of campsite? Please answer yes or no.	Yes
-Places365_val_00000032.jpg	Does this image describe a place of elevator shaft? Please answer yes or no.	No
-Places365_val_00000033.jpg	Is this photo taken in a place of florist shop indoor? Please answer yes or no.	Yes
-Places365_val_00000033.jpg	Is this photo taken in a place of sushi bar? Please answer yes or no.	No
-Places365_val_00000034.jpg	Does this image describe a place of racecourse? Please answer yes or no.	Yes
-Places365_val_00000034.jpg	Does this image describe a place of galley? Please answer yes or no.	No
-Places365_val_00000035.jpg	Is this photo taken in a place of kennel outdoor? Please answer yes or no.	Yes
-Places365_val_00000035.jpg	Is this photo taken in a place of bathroom? Please answer yes or no.	No
-Places365_val_00000036.jpg	Is this picture captured in a place of bus interior? Please answer yes or no.	Yes
-Places365_val_00000036.jpg	Is this picture captured in a place of mansion? Please answer yes or no.	No
-Places365_val_00000037.jpg	Does this image describe a place of dressing room? Please answer yes or no.	Yes
-Places365_val_00000037.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	No
-Places365_val_00000038.jpg	Does this image describe a place of wind farm? Please answer yes or no.	Yes
-Places365_val_00000038.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	No
-Places365_val_00000039.jpg	Is this picture captured in a place of auto factory? Please answer yes or no.	Yes
-Places365_val_00000039.jpg	Is this picture captured in a place of tree farm? Please answer yes or no.	No
-Places365_val_00000040.jpg	Is this picture captured in a place of beach? Please answer yes or no.	Yes
-Places365_val_00000040.jpg	Is this picture captured in a place of library outdoor? Please answer yes or no.	No
-Places365_val_00000041.jpg	Is this photo taken in a place of mountain? Please answer yes or no.	Yes
-Places365_val_00000041.jpg	Is this photo taken in a place of formal garden? Please answer yes or no.	No
-Places365_val_00000042.jpg	Does this image describe a place of pavilion? Please answer yes or no.	Yes
-Places365_val_00000042.jpg	Does this image describe a place of stable? Please answer yes or no.	No
-Places365_val_00000043.jpg	Does this image describe a place of tree house? Please answer yes or no.	Yes
-Places365_val_00000043.jpg	Does this image describe a place of patio? Please answer yes or no.	No
-Places365_val_00000044.jpg	Is this photo taken in a place of coffee shop? Please answer yes or no.	Yes
-Places365_val_00000044.jpg	Is this photo taken in a place of elevator lobby? Please answer yes or no.	No
-Places365_val_00000045.jpg	Is this photo taken in a place of skyscraper? Please answer yes or no.	Yes
-Places365_val_00000045.jpg	Is this photo taken in a place of forest broadleaf? Please answer yes or no.	No
-Places365_val_00000046.jpg	Does this image describe a place of street? Please answer yes or no.	Yes
-Places365_val_00000046.jpg	Does this image describe a place of golf course? Please answer yes or no.	No
-Places365_val_00000047.jpg	Is this picture captured in a place of alley? Please answer yes or no.	Yes
-Places365_val_00000047.jpg	Is this picture captured in a place of boardwalk? Please answer yes or no.	No
-Places365_val_00000048.jpg	Is this photo taken in a place of supermarket? Please answer yes or no.	Yes
-Places365_val_00000048.jpg	Is this photo taken in a place of creek? Please answer yes or no.	No
-Places365_val_00000049.jpg	Is this picture captured in a place of arena hockey? Please answer yes or no.	Yes
-Places365_val_00000049.jpg	Is this picture captured in a place of ski slope? Please answer yes or no.	No
-Places365_val_00000050.jpg	Is this picture captured in a place of zen garden? Please answer yes or no.	Yes
-Places365_val_00000050.jpg	Is this picture captured in a place of bazaar indoor? Please answer yes or no.	No
-Places365_val_00000051.jpg	Is this picture captured in a place of library indoor? Please answer yes or no.	Yes
-Places365_val_00000051.jpg	Is this picture captured in a place of orchard? Please answer yes or no.	No
-Places365_val_00000052.jpg	Is this photo taken in a place of ice skating rink indoor? Please answer yes or no.	Yes
-Places365_val_00000052.jpg	Is this photo taken in a place of wind farm? Please answer yes or no.	No
-Places365_val_00000053.jpg	Is this picture captured in a place of arena hockey? Please answer yes or no.	Yes
-Places365_val_00000053.jpg	Is this picture captured in a place of loading dock? Please answer yes or no.	No
-Places365_val_00000054.jpg	Does this image describe a place of chalet? Please answer yes or no.	Yes
-Places365_val_00000054.jpg	Does this image describe a place of tower? Please answer yes or no.	No
-Places365_val_00000055.jpg	Does this image describe a place of bedchamber? Please answer yes or no.	Yes
-Places365_val_00000055.jpg	Does this image describe a place of music studio? Please answer yes or no.	No
-Places365_val_00000056.jpg	Is this photo taken in a place of botanical garden? Please answer yes or no.	Yes
-Places365_val_00000056.jpg	Is this photo taken in a place of legislative chamber? Please answer yes or no.	No
-Places365_val_00000057.jpg	Is this picture captured in a place of galley? Please answer yes or no.	Yes
-Places365_val_00000057.jpg	Is this picture captured in a place of physics laboratory? Please answer yes or no.	No
-Places365_val_00000058.jpg	Does this image describe a place of department store? Please answer yes or no.	Yes
-Places365_val_00000058.jpg	Does this image describe a place of rope bridge? Please answer yes or no.	No
-Places365_val_00000059.jpg	Is this photo taken in a place of schoolhouse? Please answer yes or no.	Yes
-Places365_val_00000059.jpg	Is this photo taken in a place of hangar outdoor? Please answer yes or no.	No
-Places365_val_00000060.jpg	Is this photo taken in a place of atrium public? Please answer yes or no.	Yes
-Places365_val_00000060.jpg	Is this photo taken in a place of subway station platform? Please answer yes or no.	No
-Places365_val_00000061.jpg	Is this photo taken in a place of dam? Please answer yes or no.	Yes
-Places365_val_00000061.jpg	Is this photo taken in a place of ice skating rink indoor? Please answer yes or no.	No
-Places365_val_00000062.jpg	Does this image describe a place of army base? Please answer yes or no.	Yes
-Places365_val_00000062.jpg	Does this image describe a place of train station platform? Please answer yes or no.	No
-Places365_val_00000063.jpg	Does this image describe a place of gas station? Please answer yes or no.	Yes
-Places365_val_00000063.jpg	Does this image describe a place of doorway outdoor? Please answer yes or no.	No
-Places365_val_00000064.jpg	Does this image describe a place of ballroom? Please answer yes or no.	Yes
-Places365_val_00000064.jpg	Does this image describe a place of corn field? Please answer yes or no.	No
-Places365_val_00000065.jpg	Does this image describe a place of biology laboratory? Please answer yes or no.	Yes
-Places365_val_00000065.jpg	Does this image describe a place of candy store? Please answer yes or no.	No
-Places365_val_00000066.jpg	Does this image describe a place of pantry? Please answer yes or no.	Yes
-Places365_val_00000066.jpg	Does this image describe a place of lighthouse? Please answer yes or no.	No
-Places365_val_00000067.jpg	Is this picture captured in a place of greenhouse indoor? Please answer yes or no.	Yes
-Places365_val_00000067.jpg	Is this picture captured in a place of pasture? Please answer yes or no.	No
-Places365_val_00000068.jpg	Is this photo taken in a place of kasbah? Please answer yes or no.	Yes
-Places365_val_00000068.jpg	Is this photo taken in a place of swimming hole? Please answer yes or no.	No
-Places365_val_00000069.jpg	Is this picture captured in a place of mausoleum? Please answer yes or no.	Yes
-Places365_val_00000069.jpg	Is this picture captured in a place of alcove? Please answer yes or no.	No
-Places365_val_00000070.jpg	Is this picture captured in a place of booth indoor? Please answer yes or no.	Yes
-Places365_val_00000070.jpg	Is this picture captured in a place of bazaar indoor? Please answer yes or no.	No
-Places365_val_00000071.jpg	Is this picture captured in a place of youth hostel? Please answer yes or no.	Yes
-Places365_val_00000071.jpg	Is this picture captured in a place of landfill? Please answer yes or no.	No
-Places365_val_00000072.jpg	Does this image describe a place of fire escape? Please answer yes or no.	Yes
-Places365_val_00000072.jpg	Does this image describe a place of playroom? Please answer yes or no.	No
-Places365_val_00000073.jpg	Is this photo taken in a place of rice paddy? Please answer yes or no.	Yes
-Places365_val_00000073.jpg	Is this photo taken in a place of bus station indoor? Please answer yes or no.	No
-Places365_val_00000074.jpg	Is this photo taken in a place of mausoleum? Please answer yes or no.	Yes
-Places365_val_00000074.jpg	Is this photo taken in a place of train station platform? Please answer yes or no.	No
-Places365_val_00000075.jpg	Is this photo taken in a place of entrance hall? Please answer yes or no.	Yes
-Places365_val_00000075.jpg	Is this photo taken in a place of residential neighborhood? Please answer yes or no.	No
-Places365_val_00000076.jpg	Is this photo taken in a place of bakery shop? Please answer yes or no.	Yes
-Places365_val_00000076.jpg	Is this photo taken in a place of doorway outdoor? Please answer yes or no.	No
-Places365_val_00000077.jpg	Does this image describe a place of escalator indoor? Please answer yes or no.	Yes
-Places365_val_00000077.jpg	Does this image describe a place of ice shelf? Please answer yes or no.	No
-Places365_val_00000078.jpg	Is this picture captured in a place of elevator shaft? Please answer yes or no.	Yes
-Places365_val_00000078.jpg	Is this picture captured in a place of fire escape? Please answer yes or no.	No
-Places365_val_00000079.jpg	Is this picture captured in a place of industrial area? Please answer yes or no.	Yes
-Places365_val_00000079.jpg	Is this picture captured in a place of botanical garden? Please answer yes or no.	No
-Places365_val_00000080.jpg	Is this photo taken in a place of escalator indoor? Please answer yes or no.	Yes
-Places365_val_00000080.jpg	Is this photo taken in a place of coffee shop? Please answer yes or no.	No
-Places365_val_00000081.jpg	Does this image describe a place of operating room? Please answer yes or no.	Yes
-Places365_val_00000081.jpg	Does this image describe a place of house? Please answer yes or no.	No
-Places365_val_00000082.jpg	Is this photo taken in a place of mosque outdoor? Please answer yes or no.	Yes
-Places365_val_00000082.jpg	Is this photo taken in a place of recreation room? Please answer yes or no.	No
-Places365_val_00000083.jpg	Is this photo taken in a place of cottage? Please answer yes or no.	Yes
-Places365_val_00000083.jpg	Is this photo taken in a place of assembly line? Please answer yes or no.	No
-Places365_val_00000084.jpg	Does this image describe a place of courtyard? Please answer yes or no.	Yes
-Places365_val_00000084.jpg	Does this image describe a place of greenhouse outdoor? Please answer yes or no.	No
-Places365_val_00000085.jpg	Is this photo taken in a place of flea market indoor? Please answer yes or no.	Yes
-Places365_val_00000085.jpg	Is this photo taken in a place of pavilion? Please answer yes or no.	No
-Places365_val_00000086.jpg	Does this image describe a place of banquet hall? Please answer yes or no.	Yes
-Places365_val_00000086.jpg	Does this image describe a place of storage room? Please answer yes or no.	No
-Places365_val_00000087.jpg	Is this photo taken in a place of auto factory? Please answer yes or no.	Yes
-Places365_val_00000087.jpg	Is this photo taken in a place of motel? Please answer yes or no.	No
-Places365_val_00000088.jpg	Is this photo taken in a place of cockpit? Please answer yes or no.	Yes
-Places365_val_00000088.jpg	Is this photo taken in a place of gift shop? Please answer yes or no.	No
-Places365_val_00000089.jpg	Is this photo taken in a place of martial arts gym? Please answer yes or no.	Yes
-Places365_val_00000089.jpg	Is this photo taken in a place of atrium public? Please answer yes or no.	No
-Places365_val_00000090.jpg	Is this picture captured in a place of general store outdoor? Please answer yes or no.	Yes
-Places365_val_00000090.jpg	Is this picture captured in a place of bazaar outdoor? Please answer yes or no.	No
-Places365_val_00000091.jpg	Is this photo taken in a place of motel? Please answer yes or no.	Yes
-Places365_val_00000091.jpg	Is this photo taken in a place of chemistry lab? Please answer yes or no.	No
-Places365_val_00000092.jpg	Is this photo taken in a place of playground? Please answer yes or no.	Yes
-Places365_val_00000092.jpg	Is this photo taken in a place of snowfield? Please answer yes or no.	No
-Places365_val_00000093.jpg	Is this picture captured in a place of basement? Please answer yes or no.	Yes
-Places365_val_00000093.jpg	Is this picture captured in a place of jewelry shop? Please answer yes or no.	No
-Places365_val_00000094.jpg	Is this picture captured in a place of playroom? Please answer yes or no.	Yes
-Places365_val_00000094.jpg	Is this picture captured in a place of pier? Please answer yes or no.	No
-Places365_val_00000095.jpg	Is this photo taken in a place of fabric store? Please answer yes or no.	Yes
-Places365_val_00000095.jpg	Is this photo taken in a place of beauty salon? Please answer yes or no.	No
-Places365_val_00000096.jpg	Is this picture captured in a place of forest path? Please answer yes or no.	Yes
-Places365_val_00000096.jpg	Is this picture captured in a place of balcony interior? Please answer yes or no.	No
-Places365_val_00000097.jpg	Does this image describe a place of hangar indoor? Please answer yes or no.	Yes
-Places365_val_00000097.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	No
-Places365_val_00000098.jpg	Does this image describe a place of crosswalk? Please answer yes or no.	Yes
-Places365_val_00000098.jpg	Does this image describe a place of snowfield? Please answer yes or no.	No
-Places365_val_00000099.jpg	Does this image describe a place of physics laboratory? Please answer yes or no.	Yes
-Places365_val_00000099.jpg	Does this image describe a place of stage indoor? Please answer yes or no.	No
-Places365_val_00000100.jpg	Is this picture captured in a place of pub indoor? Please answer yes or no.	Yes
-Places365_val_00000100.jpg	Is this picture captured in a place of pier? Please answer yes or no.	No
-Places365_val_00000101.jpg	Is this photo taken in a place of schoolhouse? Please answer yes or no.	Yes
-Places365_val_00000101.jpg	Is this photo taken in a place of vineyard? Please answer yes or no.	No
-Places365_val_00000102.jpg	Is this photo taken in a place of industrial area? Please answer yes or no.	Yes
-Places365_val_00000102.jpg	Is this photo taken in a place of street? Please answer yes or no.	No
-Places365_val_00000103.jpg	Is this picture captured in a place of baseball field? Please answer yes or no.	Yes
-Places365_val_00000103.jpg	Is this picture captured in a place of airfield? Please answer yes or no.	No
-Places365_val_00000104.jpg	Is this picture captured in a place of ice floe? Please answer yes or no.	Yes
-Places365_val_00000104.jpg	Is this picture captured in a place of kindergarden classroom? Please answer yes or no.	No
-Places365_val_00000105.jpg	Is this picture captured in a place of train interior? Please answer yes or no.	Yes
-Places365_val_00000105.jpg	Is this picture captured in a place of movie theater indoor? Please answer yes or no.	No
-Places365_val_00000106.jpg	Is this picture captured in a place of desert sand? Please answer yes or no.	Yes
-Places365_val_00000106.jpg	Is this picture captured in a place of burial chamber? Please answer yes or no.	No
-Places365_val_00000107.jpg	Is this photo taken in a place of castle? Please answer yes or no.	Yes
-Places365_val_00000107.jpg	Is this photo taken in a place of computer room? Please answer yes or no.	No
-Places365_val_00000108.jpg	Does this image describe a place of burial chamber? Please answer yes or no.	Yes
-Places365_val_00000108.jpg	Does this image describe a place of oilrig? Please answer yes or no.	No
-Places365_val_00000109.jpg	Is this photo taken in a place of nursing home? Please answer yes or no.	Yes
-Places365_val_00000109.jpg	Is this photo taken in a place of amusement park? Please answer yes or no.	No
-Places365_val_00000110.jpg	Does this image describe a place of church indoor? Please answer yes or no.	Yes
-Places365_val_00000110.jpg	Does this image describe a place of general store outdoor? Please answer yes or no.	No
-Places365_val_00000111.jpg	Is this picture captured in a place of palace? Please answer yes or no.	Yes
-Places365_val_00000111.jpg	Is this picture captured in a place of tree house? Please answer yes or no.	No
-Places365_val_00000112.jpg	Does this image describe a place of bus station indoor? Please answer yes or no.	Yes
-Places365_val_00000112.jpg	Does this image describe a place of hardware store? Please answer yes or no.	No
-Places365_val_00000113.jpg	Is this picture captured in a place of atrium public? Please answer yes or no.	Yes
-Places365_val_00000113.jpg	Is this picture captured in a place of music studio? Please answer yes or no.	No
-Places365_val_00000114.jpg	Is this photo taken in a place of butte? Please answer yes or no.	Yes
-Places365_val_00000114.jpg	Is this photo taken in a place of valley? Please answer yes or no.	No
-Places365_val_00000115.jpg	Is this picture captured in a place of zen garden? Please answer yes or no.	Yes
-Places365_val_00000115.jpg	Is this picture captured in a place of banquet hall? Please answer yes or no.	No
-Places365_val_00000116.jpg	Is this picture captured in a place of racecourse? Please answer yes or no.	Yes
-Places365_val_00000116.jpg	Is this picture captured in a place of basketball court indoor? Please answer yes or no.	No
-Places365_val_00000117.jpg	Is this picture captured in a place of cliff? Please answer yes or no.	Yes
-Places365_val_00000117.jpg	Is this picture captured in a place of sauna? Please answer yes or no.	No
-Places365_val_00000118.jpg	Is this photo taken in a place of laundromat? Please answer yes or no.	Yes
-Places365_val_00000118.jpg	Is this photo taken in a place of archive? Please answer yes or no.	No
-Places365_val_00000119.jpg	Does this image describe a place of wave? Please answer yes or no.	Yes
-Places365_val_00000119.jpg	Does this image describe a place of shed? Please answer yes or no.	No
-Places365_val_00000120.jpg	Is this photo taken in a place of legislative chamber? Please answer yes or no.	Yes
-Places365_val_00000120.jpg	Is this photo taken in a place of nursery? Please answer yes or no.	No
-Places365_val_00000121.jpg	Is this photo taken in a place of elevator shaft? Please answer yes or no.	Yes
-Places365_val_00000121.jpg	Is this photo taken in a place of berth? Please answer yes or no.	No
-Places365_val_00000122.jpg	Is this photo taken in a place of hayfield? Please answer yes or no.	Yes
-Places365_val_00000122.jpg	Is this photo taken in a place of skyscraper? Please answer yes or no.	No
-Places365_val_00000123.jpg	Is this photo taken in a place of hospital room? Please answer yes or no.	Yes
-Places365_val_00000123.jpg	Is this photo taken in a place of mosque outdoor? Please answer yes or no.	No
-Places365_val_00000124.jpg	Does this image describe a place of swimming hole? Please answer yes or no.	Yes
-Places365_val_00000124.jpg	Does this image describe a place of television room? Please answer yes or no.	No
-Places365_val_00000125.jpg	Is this picture captured in a place of biology laboratory? Please answer yes or no.	Yes
-Places365_val_00000125.jpg	Is this picture captured in a place of lake natural? Please answer yes or no.	No
-Places365_val_00000126.jpg	Is this picture captured in a place of church indoor? Please answer yes or no.	Yes
-Places365_val_00000126.jpg	Is this picture captured in a place of server room? Please answer yes or no.	No
-Places365_val_00000127.jpg	Is this picture captured in a place of temple asia? Please answer yes or no.	Yes
-Places365_val_00000127.jpg	Is this picture captured in a place of ball pit? Please answer yes or no.	No
-Places365_val_00000128.jpg	Is this photo taken in a place of landfill? Please answer yes or no.	Yes
-Places365_val_00000128.jpg	Is this photo taken in a place of viaduct? Please answer yes or no.	No
-Places365_val_00000129.jpg	Does this image describe a place of arena hockey? Please answer yes or no.	Yes
-Places365_val_00000129.jpg	Does this image describe a place of clothing store? Please answer yes or no.	No
-Places365_val_00000130.jpg	Is this photo taken in a place of baseball field? Please answer yes or no.	Yes
-Places365_val_00000130.jpg	Is this photo taken in a place of art gallery? Please answer yes or no.	No
-Places365_val_00000131.jpg	Is this picture captured in a place of gazebo exterior? Please answer yes or no.	Yes
-Places365_val_00000131.jpg	Is this picture captured in a place of driveway? Please answer yes or no.	No
-Places365_val_00000132.jpg	Does this image describe a place of greenhouse outdoor? Please answer yes or no.	Yes
-Places365_val_00000132.jpg	Does this image describe a place of stage indoor? Please answer yes or no.	No
-Places365_val_00000133.jpg	Is this picture captured in a place of beauty salon? Please answer yes or no.	Yes
-Places365_val_00000133.jpg	Is this picture captured in a place of junkyard? Please answer yes or no.	No
-Places365_val_00000134.jpg	Does this image describe a place of ice shelf? Please answer yes or no.	Yes
-Places365_val_00000134.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	No
-Places365_val_00000135.jpg	Is this picture captured in a place of islet? Please answer yes or no.	Yes
-Places365_val_00000135.jpg	Is this picture captured in a place of recreation room? Please answer yes or no.	No
-Places365_val_00000136.jpg	Is this picture captured in a place of railroad track? Please answer yes or no.	Yes
-Places365_val_00000136.jpg	Is this picture captured in a place of department store? Please answer yes or no.	No
-Places365_val_00000137.jpg	Is this photo taken in a place of bazaar outdoor? Please answer yes or no.	Yes
-Places365_val_00000137.jpg	Is this photo taken in a place of office? Please answer yes or no.	No
-Places365_val_00000138.jpg	Is this photo taken in a place of basement? Please answer yes or no.	Yes
-Places365_val_00000138.jpg	Is this photo taken in a place of banquet hall? Please answer yes or no.	No
-Places365_val_00000139.jpg	Is this picture captured in a place of television studio? Please answer yes or no.	Yes
-Places365_val_00000139.jpg	Is this picture captured in a place of general store indoor? Please answer yes or no.	No
-Places365_val_00000140.jpg	Is this picture captured in a place of mezzanine? Please answer yes or no.	Yes
-Places365_val_00000140.jpg	Is this picture captured in a place of bedroom? Please answer yes or no.	No
-Places365_val_00000141.jpg	Does this image describe a place of roof garden? Please answer yes or no.	Yes
-Places365_val_00000141.jpg	Does this image describe a place of television room? Please answer yes or no.	No
-Places365_val_00000142.jpg	Does this image describe a place of restaurant kitchen? Please answer yes or no.	Yes
-Places365_val_00000142.jpg	Does this image describe a place of fabric store? Please answer yes or no.	No
-Places365_val_00000143.jpg	Is this photo taken in a place of house? Please answer yes or no.	Yes
-Places365_val_00000143.jpg	Is this photo taken in a place of swimming hole? Please answer yes or no.	No
-Places365_val_00000144.jpg	Is this picture captured in a place of vineyard? Please answer yes or no.	Yes
-Places365_val_00000144.jpg	Is this picture captured in a place of castle? Please answer yes or no.	No
-Places365_val_00000145.jpg	Is this picture captured in a place of driveway? Please answer yes or no.	Yes
-Places365_val_00000145.jpg	Is this picture captured in a place of badlands? Please answer yes or no.	No
-Places365_val_00000146.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	Yes
-Places365_val_00000146.jpg	Does this image describe a place of industrial area? Please answer yes or no.	No
-Places365_val_00000147.jpg	Does this image describe a place of cabin outdoor? Please answer yes or no.	Yes
-Places365_val_00000147.jpg	Does this image describe a place of creek? Please answer yes or no.	No
-Places365_val_00000148.jpg	Is this photo taken in a place of restaurant? Please answer yes or no.	Yes
-Places365_val_00000148.jpg	Is this photo taken in a place of apartment building outdoor? Please answer yes or no.	No
-Places365_val_00000149.jpg	Is this picture captured in a place of auditorium? Please answer yes or no.	Yes
-Places365_val_00000149.jpg	Is this picture captured in a place of rock arch? Please answer yes or no.	No
-Places365_val_00000150.jpg	Is this photo taken in a place of street? Please answer yes or no.	Yes
-Places365_val_00000150.jpg	Is this photo taken in a place of toyshop? Please answer yes or no.	No
-Places365_val_00000151.jpg	Does this image describe a place of lagoon? Please answer yes or no.	Yes
-Places365_val_00000151.jpg	Does this image describe a place of aquarium? Please answer yes or no.	No
-Places365_val_00000152.jpg	Is this picture captured in a place of hangar indoor? Please answer yes or no.	Yes
-Places365_val_00000152.jpg	Is this picture captured in a place of street? Please answer yes or no.	No
-Places365_val_00000153.jpg	Does this image describe a place of bar? Please answer yes or no.	Yes
-Places365_val_00000153.jpg	Does this image describe a place of tree house? Please answer yes or no.	No
-Places365_val_00000154.jpg	Is this picture captured in a place of classroom? Please answer yes or no.	Yes
-Places365_val_00000154.jpg	Is this picture captured in a place of wheat field? Please answer yes or no.	No
-Places365_val_00000155.jpg	Does this image describe a place of creek? Please answer yes or no.	Yes
-Places365_val_00000155.jpg	Does this image describe a place of youth hostel? Please answer yes or no.	No
-Places365_val_00000156.jpg	Does this image describe a place of stable? Please answer yes or no.	Yes
-Places365_val_00000156.jpg	Does this image describe a place of junkyard? Please answer yes or no.	No
-Places365_val_00000157.jpg	Is this picture captured in a place of swamp? Please answer yes or no.	Yes
-Places365_val_00000157.jpg	Is this picture captured in a place of village? Please answer yes or no.	No
-Places365_val_00000158.jpg	Is this picture captured in a place of ice shelf? Please answer yes or no.	Yes
-Places365_val_00000158.jpg	Is this picture captured in a place of motel? Please answer yes or no.	No
-Places365_val_00000159.jpg	Is this picture captured in a place of train interior? Please answer yes or no.	Yes
-Places365_val_00000159.jpg	Is this picture captured in a place of jewelry shop? Please answer yes or no.	No
-Places365_val_00000160.jpg	Does this image describe a place of windmill? Please answer yes or no.	Yes
-Places365_val_00000160.jpg	Does this image describe a place of lake natural? Please answer yes or no.	No
-Places365_val_00000161.jpg	Is this photo taken in a place of archive? Please answer yes or no.	Yes
-Places365_val_00000161.jpg	Is this photo taken in a place of village? Please answer yes or no.	No
-Places365_val_00000162.jpg	Does this image describe a place of train station platform? Please answer yes or no.	Yes
-Places365_val_00000162.jpg	Does this image describe a place of manufactured home? Please answer yes or no.	No
-Places365_val_00000163.jpg	Is this photo taken in a place of chalet? Please answer yes or no.	Yes
-Places365_val_00000163.jpg	Is this photo taken in a place of throne room? Please answer yes or no.	No
-Places365_val_00000164.jpg	Does this image describe a place of lake natural? Please answer yes or no.	Yes
-Places365_val_00000164.jpg	Does this image describe a place of athletic field outdoor? Please answer yes or no.	No
-Places365_val_00000165.jpg	Is this picture captured in a place of playground? Please answer yes or no.	Yes
-Places365_val_00000165.jpg	Is this picture captured in a place of viaduct? Please answer yes or no.	No
-Places365_val_00000166.jpg	Is this photo taken in a place of dorm room? Please answer yes or no.	Yes
-Places365_val_00000166.jpg	Is this photo taken in a place of ski resort? Please answer yes or no.	No
-Places365_val_00000167.jpg	Is this picture captured in a place of flea market indoor? Please answer yes or no.	Yes
-Places365_val_00000167.jpg	Is this picture captured in a place of cliff? Please answer yes or no.	No
-Places365_val_00000168.jpg	Does this image describe a place of airplane cabin? Please answer yes or no.	Yes
-Places365_val_00000168.jpg	Does this image describe a place of rock arch? Please answer yes or no.	No
-Places365_val_00000169.jpg	Is this picture captured in a place of drugstore? Please answer yes or no.	Yes
-Places365_val_00000169.jpg	Is this picture captured in a place of canyon? Please answer yes or no.	No
-Places365_val_00000170.jpg	Is this picture captured in a place of pharmacy? Please answer yes or no.	Yes
-Places365_val_00000170.jpg	Is this picture captured in a place of orchestra pit? Please answer yes or no.	No
-Places365_val_00000171.jpg	Is this picture captured in a place of greenhouse outdoor? Please answer yes or no.	Yes
-Places365_val_00000171.jpg	Is this picture captured in a place of archaelogical excavation? Please answer yes or no.	No
-Places365_val_00000172.jpg	Is this photo taken in a place of bathroom? Please answer yes or no.	Yes
-Places365_val_00000172.jpg	Is this photo taken in a place of auto factory? Please answer yes or no.	No
-Places365_val_00000173.jpg	Is this photo taken in a place of shoe shop? Please answer yes or no.	Yes
-Places365_val_00000173.jpg	Is this photo taken in a place of car interior? Please answer yes or no.	No
-Places365_val_00000174.jpg	Does this image describe a place of moat water? Please answer yes or no.	Yes
-Places365_val_00000174.jpg	Does this image describe a place of marsh? Please answer yes or no.	No
-Places365_val_00000175.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	Yes
-Places365_val_00000175.jpg	Does this image describe a place of parking garage indoor? Please answer yes or no.	No
-Places365_val_00000176.jpg	Is this picture captured in a place of bowling alley? Please answer yes or no.	Yes
-Places365_val_00000176.jpg	Is this picture captured in a place of hotel outdoor? Please answer yes or no.	No
-Places365_val_00000177.jpg	Does this image describe a place of embassy? Please answer yes or no.	Yes
-Places365_val_00000177.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	No
-Places365_val_00000178.jpg	Is this photo taken in a place of youth hostel? Please answer yes or no.	Yes
-Places365_val_00000178.jpg	Is this photo taken in a place of hangar indoor? Please answer yes or no.	No
-Places365_val_00000179.jpg	Does this image describe a place of hot spring? Please answer yes or no.	Yes
-Places365_val_00000179.jpg	Does this image describe a place of mausoleum? Please answer yes or no.	No
-Places365_val_00000180.jpg	Does this image describe a place of cottage? Please answer yes or no.	Yes
-Places365_val_00000180.jpg	Does this image describe a place of parking garage indoor? Please answer yes or no.	No
-Places365_val_00000181.jpg	Does this image describe a place of general store outdoor? Please answer yes or no.	Yes
-Places365_val_00000181.jpg	Does this image describe a place of hotel room? Please answer yes or no.	No
-Places365_val_00000182.jpg	Is this photo taken in a place of chalet? Please answer yes or no.	Yes
-Places365_val_00000182.jpg	Is this photo taken in a place of dorm room? Please answer yes or no.	No
-Places365_val_00000183.jpg	Is this photo taken in a place of flea market indoor? Please answer yes or no.	Yes
-Places365_val_00000183.jpg	Is this photo taken in a place of tree house? Please answer yes or no.	No
-Places365_val_00000184.jpg	Is this photo taken in a place of shower? Please answer yes or no.	Yes
-Places365_val_00000184.jpg	Is this photo taken in a place of living room? Please answer yes or no.	No
-Places365_val_00000185.jpg	Is this picture captured in a place of supermarket? Please answer yes or no.	Yes
-Places365_val_00000185.jpg	Is this picture captured in a place of orchard? Please answer yes or no.	No
-Places365_val_00000186.jpg	Is this picture captured in a place of bowling alley? Please answer yes or no.	Yes
-Places365_val_00000186.jpg	Is this picture captured in a place of ice skating rink outdoor? Please answer yes or no.	No
-Places365_val_00000187.jpg	Is this picture captured in a place of barn? Please answer yes or no.	Yes
-Places365_val_00000187.jpg	Is this picture captured in a place of lobby? Please answer yes or no.	No
-Places365_val_00000188.jpg	Is this photo taken in a place of japanese garden? Please answer yes or no.	Yes
-Places365_val_00000188.jpg	Is this photo taken in a place of hangar indoor? Please answer yes or no.	No
-Places365_val_00000189.jpg	Is this picture captured in a place of swimming hole? Please answer yes or no.	Yes
-Places365_val_00000189.jpg	Is this picture captured in a place of bedroom? Please answer yes or no.	No
-Places365_val_00000190.jpg	Is this picture captured in a place of pizzeria? Please answer yes or no.	Yes
-Places365_val_00000190.jpg	Is this picture captured in a place of computer room? Please answer yes or no.	No
-Places365_val_00000191.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	Yes
-Places365_val_00000191.jpg	Does this image describe a place of church indoor? Please answer yes or no.	No
-Places365_val_00000192.jpg	Is this photo taken in a place of market indoor? Please answer yes or no.	Yes
-Places365_val_00000192.jpg	Is this photo taken in a place of home office? Please answer yes or no.	No
-Places365_val_00000193.jpg	Is this picture captured in a place of ice floe? Please answer yes or no.	Yes
-Places365_val_00000193.jpg	Is this picture captured in a place of closet? Please answer yes or no.	No
-Places365_val_00000194.jpg	Does this image describe a place of lake natural? Please answer yes or no.	Yes
-Places365_val_00000194.jpg	Does this image describe a place of beer hall? Please answer yes or no.	No
-Places365_val_00000195.jpg	Does this image describe a place of mountain path? Please answer yes or no.	Yes
-Places365_val_00000195.jpg	Does this image describe a place of construction site? Please answer yes or no.	No
-Places365_val_00000196.jpg	Is this photo taken in a place of orchestra pit? Please answer yes or no.	Yes
-Places365_val_00000196.jpg	Is this photo taken in a place of burial chamber? Please answer yes or no.	No
-Places365_val_00000197.jpg	Does this image describe a place of village? Please answer yes or no.	Yes
-Places365_val_00000197.jpg	Does this image describe a place of underwater ocean deep? Please answer yes or no.	No
-Places365_val_00000198.jpg	Does this image describe a place of waterfall? Please answer yes or no.	Yes
-Places365_val_00000198.jpg	Does this image describe a place of booth indoor? Please answer yes or no.	No
-Places365_val_00000199.jpg	Is this photo taken in a place of greenhouse indoor? Please answer yes or no.	Yes
-Places365_val_00000199.jpg	Is this photo taken in a place of aqueduct? Please answer yes or no.	No
-Places365_val_00000200.jpg	Is this photo taken in a place of television studio? Please answer yes or no.	Yes
-Places365_val_00000200.jpg	Is this photo taken in a place of hunting lodge outdoor? Please answer yes or no.	No

eval/vlm/eval/mme/Your_Results/text_translation.txt

@@ -1,40 +0,0 @@
-0001.png	Is it appropriate to translate the Chinese in the image into English 'classic taste' in the picture? Please answer yes or no.	Yes
-0001.png	Is it appropriate to translate the Chinese in the image into English 'classic strawberry flavor' in the picture? Please answer yes or no.	No
-0002.png	Is it appropriate to translate the Chinese in the image into English 'a delicious dinner' in the picture? Please answer yes or no.	Yes
-0002.png	Is it appropriate to translate the Chinese in the image into English 'hamburger and chips' in the picture? Please answer yes or no.	No
-0003.png	Is it appropriate to translate the Chinese in the image into English 'sunny weather' in the picture? Please answer yes or no.	Yes
-0003.png	Is it appropriate to translate the Chinese in the image into English 'cold weather' in the picture? Please answer yes or no.	No
-0004.png	Is it appropriate to translate the Chinese in the image into English 'run very fast' in the picture? Please answer yes or no.	Yes
-0004.png	Is it appropriate to translate the Chinese in the image into English 'run very slow' in the picture? Please answer yes or no.	No
-0005.png	Is it appropriate to translate the Chinese in the image into English 'feeling happy' in the picture? Please answer yes or no.	Yes
-0005.png	Is it appropriate to translate the Chinese in the image into English 'feeling bored' in the picture? Please answer yes or no.	No
-0006.png	Is it appropriate to translate the Chinese in the image into English 'work hard together' in the picture? Please answer yes or no.	Yes
-0006.png	Is it appropriate to translate the Chinese in the image into English 'be filled with intrigue' in the picture? Please answer yes or no.	No
-0007.png	Is it appropriate to translate the Chinese in the image into English 'walking very slowly' in the picture? Please answer yes or no.	Yes
-0007.png	Is it appropriate to translate the Chinese in the image into English 'runing very slowly' in the picture? Please answer yes or no.	No
-0008.png	Is it appropriate to translate the Chinese in the image into English 'very proud' in the picture? Please answer yes or no.	Yes
-0008.png	Is it appropriate to translate the Chinese in the image into English 'very thankful' in the picture? Please answer yes or no.	No
-0009.png	Is it appropriate to translate the Chinese in the image into English 'creative people' in the picture? Please answer yes or no.	Yes
-0009.png	Is it appropriate to translate the Chinese in the image into English 'leading people' in the picture? Please answer yes or no.	No
-0010.png	Is it appropriate to translate the Chinese in the image into English 'a beautiful garden' in the picture? Please answer yes or no.	Yes
-0010.png	Is it appropriate to translate the Chinese in the image into English 'a beautiful campus' in the picture? Please answer yes or no.	No
-0011.png	Is it appropriate to translate the Chinese in the image into English 'a difficult work' in the picture? Please answer yes or no.	Yes
-0011.png	Is it appropriate to translate the Chinese in the image into English 'a easy work' in the picture? Please answer yes or no.	No
-0012.png	Is it appropriate to translate the Chinese in the image into English 'a small amount' in the picture? Please answer yes or no.	Yes
-0012.png	Is it appropriate to translate the Chinese in the image into English 'difficult and dangerous' in the picture? Please answer yes or no.	No
-0013.png	Is it appropriate to translate the Chinese in the image into English 'feeling frustrated' in the picture? Please answer yes or no.	Yes
-0013.png	Is it appropriate to translate the Chinese in the image into English 'feeling relaxed' in the picture? Please answer yes or no.	No
-0014.png	Is it appropriate to translate the Chinese in the image into English 'waiting for a long time' in the picture? Please answer yes or no.	Yes
-0014.png	Is it appropriate to translate the Chinese in the image into English 'sleeping for a long time' in the picture? Please answer yes or no.	No
-0015.png	Is it appropriate to translate the Chinese in the image into English 'very powerful' in the picture? Please answer yes or no.	Yes
-0015.png	Is it appropriate to translate the Chinese in the image into English 'to be fragile throughout the world' in the picture? Please answer yes or no.	No
-0016.png	Is it appropriate to translate the Chinese in the image into English 'all talk and no action' in the picture? Please answer yes or no.	Yes
-0016.png	Is it appropriate to translate the Chinese in the image into English 'hands-on practice' in the picture? Please answer yes or no.	No
-0017.png	Is it appropriate to translate the Chinese in the image into English 'delicious fruit' in the picture? Please answer yes or no.	Yes
-0017.png	Is it appropriate to translate the Chinese in the image into English 'banana' in the picture? Please answer yes or no.	No
-0018.png	Is it appropriate to translate the Chinese in the image into English 'very unforgettable' in the picture? Please answer yes or no.	Yes
-0018.png	Is it appropriate to translate the Chinese in the image into English 'very happy' in the picture? Please answer yes or no.	No
-0019.png	Is it appropriate to translate the Chinese in the image into English 'get along well' in the picture? Please answer yes or no.	Yes
-0019.png	Is it appropriate to translate the Chinese in the image into English 'for own self-interest' in the picture? Please answer yes or no.	No
-0020.png	Is it appropriate to translate the Chinese in the image into English 'rank first' in the picture? Please answer yes or no.	Yes
-0020.png	Is it appropriate to translate the Chinese in the image into English 'to add the finishing touches' in the picture? Please answer yes or no.	No

eval/vlm/eval/mme/calculation.py

@@ -1,184 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import os
-
-from sklearn.metrics import (accuracy_score, confusion_matrix, precision_score,
-                             recall_score)
-
-parser = argparse.ArgumentParser()
-# parser.add_argument('--results_dir', default='./LaVIN', type=str)
-parser.add_argument('--out-dir', default='./', type=str)
-
-eval_type_dict = {
-    'Perception': ['existence', 'count', 'position', 'color', 'posters', 'celebrity', 'scene', 'landmark', 'artwork', 'OCR'],
-    'Cognition': ['commonsense_reasoning', 'numerical_calculation', 'text_translation', 'code_reasoning']
-}
-
-
-class calculate_metrics:
-    def divide_chunks(self, l, n=2):
-        # looping till length l
-        for i in range(0, len(l), n):
-            yield l[i:i + n]
-
-        return
-
-    def parse_pred_ans(self, pred_ans):
-        pred_label = None
-        if pred_ans in ['yes', 'no']:
-            pred_label = pred_ans
-        else:
-            prefix_pred_ans = pred_ans[:4]
-
-            if 'yes' in prefix_pred_ans:
-                pred_label = 'yes'
-            elif 'no' in prefix_pred_ans:
-                pred_label = 'no'
-            else:
-                pred_label = 'other'
-
-        return pred_label
-
-    def compute_metric(self, gts, preds):
-        assert len(gts) == len(preds)
-
-        label_map = {
-            'yes': 1,
-            'no': 0,
-            'other': -1,
-        }
-
-        gts = [label_map[x] for x in gts]
-        preds = [label_map[x] for x in preds]
-
-        acc = accuracy_score(gts, preds)
-
-        clean_gts = []
-        clean_preds = []
-        other_num = 0
-        for gt, pred in zip(gts, preds):
-            if pred == -1:
-                other_num += 1
-                continue
-            clean_gts.append(gt)
-            clean_preds.append(pred)
-
-        conf_mat = confusion_matrix(clean_gts, clean_preds, labels=[1,0])
-        precision = precision_score(clean_gts, clean_preds, average='binary')
-        recall = recall_score(clean_gts, clean_preds, average='binary')
-        tp, fn = conf_mat[0]
-        fp, tn = conf_mat[1]
-
-        metric_dict = dict()
-        metric_dict = {
-            'TP': tp,
-            'FN': fn,
-            'TN': tn,
-            'FP': fp,
-            'precision': precision,
-            'recall': recall,
-            'other_num': other_num,
-            'acc': acc,
-        }
-
-        return metric_dict
-
-    def process_result(self, results_dir):
-        ret_message = ""
-        model_score_dict = dict()
-        for eval_type, task_name_list in eval_type_dict.items():
-            print('===========', eval_type, '===========')
-            ret_message += f"=========== {eval_type} ===========\n"
-
-            scores = 0
-            task_score_dict = dict()
-
-            for task_name in task_name_list:
-
-                task_txt = os.path.join(results_dir, task_name + '.txt')
-                lines = open(task_txt, 'r').readlines()
-                chunk_lines = list(self.divide_chunks(lines)) # one image corresponds to two questions
-
-                img_num = len(chunk_lines)
-                task_other_ans_num = 0
-                task_score = 0
-                acc_plus_correct_num = 0
-                gts = []
-                preds = []
-
-                for img_items in chunk_lines:
-                    assert len(img_items) == 2
-                    img_correct_num = 0
-
-                    for img_item in img_items:
-                        try:
-                            img_name, question, gt_ans, pred_ans = img_item.split('\t')
-                        except:
-                            print(img_item)
-                            continue
-                        gt_ans = gt_ans.lower()
-                        pred_ans = pred_ans.lower()
-
-                        assert gt_ans in ['yes', 'no'] # gt can only be yes or no.
-
-                        pred_ans = self.parse_pred_ans(pred_ans)
-                        assert pred_ans in ['yes', 'no', 'other']
-
-                        gts.append(gt_ans)
-                        preds.append(pred_ans)
-
-                        if gt_ans == pred_ans:
-                            img_correct_num += 1
-
-                        if pred_ans not in ['yes', 'no']:
-                            task_other_ans_num += 1
-
-                    if img_correct_num == 2:
-                        acc_plus_correct_num += 1
-
-                # cal TP precision acc, etc.
-                metric_dict = self.compute_metric(gts, preds)
-                acc_plus = acc_plus_correct_num / img_num
-                metric_dict['acc_plus'] = acc_plus
-
-                for k, v in metric_dict.items():
-                    if k in ['acc', 'acc_plus']:
-                        task_score += v*100
-
-                task_score_dict[task_name] = task_score
-
-                scores += task_score
-
-            print('total score:', scores, '\n')
-            ret_message += f"total score: {scores} \n\n"
-            for task_name, score in task_score_dict.items():
-                print('\t', task_name, ' score:', score)
-                ret_message += f"\t {task_name} score: {score}\n"
-            print('\n')
-            ret_message += "\n\n"
-
-        return ret_message
-
-
-if __name__ == '__main__':
-    cal = calculate_metrics()
-
-    args = parser.parse_args()
-    # results_dir = args.results_dir
-    results_dir = args.out_dir
-    ret_message = cal.process_result(results_dir)
-
-    writer = open(os.path.join(args.out_dir, "results.txt"), 'w')
-    print(f"write results to file {os.path.join(args.out_dir, 'results.txt')}")
-    writer.write(ret_message)
-    writer.close()

eval/vlm/eval/mme/eval.py

@@ -1,74 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import os
-import re
-
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-
-def post_processing(response):
-    response = response.replace('\n', '').replace('不是', 'No').replace('是', 'Yes').replace('否', 'No')
-    response = response.lower().replace('true', 'yes').replace('false', 'no')
-    pattern = re.compile(r'[\u4e00-\u9fa5]')
-    response = re.sub(pattern, '', response)
-    return response
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--root', type=str, default='eval/vlm/eval/mme/Your_Results')
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    os.makedirs(args.out_dir, exist_ok=True)
-    prompt = 'Answer the question using a single word or phrase.'
-
-    for filename in os.listdir(args.root):
-        fin = open(os.path.join(args.root, filename), 'r', encoding='utf-8')
-        fout = open(os.path.join(args.out_dir, filename), 'w', encoding='utf-8')
-        lines = fin.readlines()
-        filename = filename.replace('.txt', '')
-        for line in tqdm(lines):
-            img, question, gt = line.strip().split('\t')
-            question = question + ' ' + prompt
-            img_path = os.path.join('eval/vlm/data/mme/MME_Benchmark_release_version', filename, img)
-            if not os.path.exists(img_path):
-                img_path = os.path.join('eval/vlm/data/mme/MME_Benchmark_release_version', filename, "images", img)
-            if not os.path.exists(img_path):
-                continue
-            images = [Image.open(img_path).convert('RGB')]
-            images, conversation = process_conversation(images, question)
-
-            response = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images,
-                prompt=conversation,
-                max_length=20,
-            )
-            response = post_processing(response)
-            print(img, question, gt, response, sep='\t', file=fout)
-        fin.close()
-        fout.close()
-
-    os.system(f"python -m eval.vlm.eval.mme.calculation --out-dir {args.out_dir}")

eval/vlm/eval/mmmu/data_utils.py

@@ -1,187 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-"""Utils for data load, save, and process (e.g., prompt construction)"""
-
-import json
-import os
-import re
-
-import yaml
-
-DOMAIN_CAT2SUB_CAT = {
-    'Art and Design': ['Art', 'Art_Theory', 'Design', 'Music'],
-    'Business': ['Accounting', 'Economics', 'Finance', 'Manage', 'Marketing'],
-    'Science': ['Biology', 'Chemistry', 'Geography', 'Math', 'Physics', ],
-    'Health and Medicine': ['Basic_Medical_Science', 'Clinical_Medicine', 'Diagnostics_and_Laboratory_Medicine',
-                            'Pharmacy', 'Public_Health'],
-    'Humanities and Social Science': ['History', 'Literature', 'Sociology', 'Psychology'],
-    'Tech and Engineering': ['Agriculture', 'Architecture_and_Engineering', 'Computer_Science', 'Electronics',
-                             'Energy_and_Power', 'Materials', 'Mechanical_Engineering'],
-}
-
-CAT_SHORT2LONG = {
-    'acc': 'Accounting',
-    'agri': 'Agriculture',
-    'arch': 'Architecture_and_Engineering',
-    'art': 'Art',
-    'art_theory': 'Art_Theory',
-    'bas_med': 'Basic_Medical_Science',
-    'bio': 'Biology',
-    'chem': 'Chemistry',
-    'cli_med': 'Clinical_Medicine',
-    'cs': 'Computer_Science',
-    'design': 'Design',
-    'diag_med': 'Diagnostics_and_Laboratory_Medicine',
-    'econ': 'Economics',
-    'elec': 'Electronics',
-    'ep': 'Energy_and_Power',
-    'fin': 'Finance',
-    'geo': 'Geography',
-    'his': 'History',
-    'liter': 'Literature',
-    'manage': 'Manage',
-    'mark': 'Marketing',
-    'mate': 'Materials',
-    'math': 'Math',
-    'mech': 'Mechanical_Engineering',
-    'music': 'Music',
-    'phar': 'Pharmacy',
-    'phys': 'Physics',
-    'psy': 'Psychology',
-    'pub_health': 'Public_Health',
-    'socio': 'Sociology'
-}
-
-
-# DATA SAVING
-def save_json(filename, ds):
-    with open(filename, 'w') as f:
-        json.dump(ds, f, indent=4)
-
-
-def get_multi_choice_info(options):
-    """
-    Given the list of options for multiple choice question
-    Return the index2ans and all_choices
-    """
-
-    start_chr = 'A'
-    all_choices = []
-    index2ans = {}
-    for i, option in enumerate(options):
-        index2ans[chr(ord(start_chr) + i)] = option
-        all_choices.append(chr(ord(start_chr) + i))
-
-    return index2ans, all_choices
-
-
-def load_yaml(file_path):
-    with open(file_path, 'r') as stream:
-        try:
-            yaml_dict = yaml.safe_load(stream)
-        except yaml.YAMLError as exc:
-            print(exc)
-
-    return yaml_dict
-
-
-def parse_img_path(text):
-    matches = re.findall("<img='(.*?)'>", text)
-    return matches
-
-
-def process_single_sample(data):
-    question = data['question']
-    o_imgs_paths = []
-    for option in data['options']:
-        current_o_imgs_paths = parse_img_path(option)
-        for img_path in current_o_imgs_paths:
-            o_imgs_paths.append(img_path)
-    images = [data['image_1'], data['image_2'], data['image_3'], data['image_4'],
-              data['image_5'], data['image_6'], data['image_7']]
-    return {'id': data['id'], 'question': question, 'options': data['options'], 'answer': data['answer'],
-            'image': images, 'question_type': data['question_type']}
-
-
-# DATA SAVING
-def save_json(filename, ds):
-    with open(filename, 'w') as f:
-        json.dump(ds, f, indent=4)
-
-
-def save_jsonl(filename, data):
-    """
-    Save a dictionary of data to a JSON Lines file with the filename as key and caption as value.
-
-    Args:
-        filename (str): The path to the file where the data should be saved.
-        data (dict): The dictionary containing the data to save where key is the image path and value is the caption.
-    """
-    with open(filename, 'w', encoding='utf-8') as f:
-        for img_path, caption in data.items():
-            # Extract the base filename without the extension
-            base_filename = os.path.basename(img_path)
-            # Create a JSON object with the filename as the key and caption as the value
-            json_record = json.dumps({base_filename: caption}, ensure_ascii=False)
-            # Write the JSON object to the file, one per line
-            f.write(json_record + '\n')
-
-
-def save_args(args, path_dir):
-    argsDict = args.__dict__
-    with open(path_dir + 'setting.txt', 'w') as f:
-        f.writelines('------------------ start ------------------' + '\n')
-        for eachArg, value in argsDict.items():
-            f.writelines(eachArg + ' : ' + str(value) + '\n')
-        f.writelines('------------------- end -------------------')
-
-
-# DATA PROCESSING
-def construct_prompt(sample, config):
-    question = sample['question']
-    options = eval(sample['options'])
-    example = ''
-    if sample['question_type'] == 'multiple-choice':
-        start_chr = 'A'
-        prediction_range = []
-        index2ans = {}
-        for option in options:
-            prediction_range.append(start_chr)
-            example += f'({start_chr}) {option}\n'
-            index2ans[start_chr] = option
-            start_chr = chr(ord(start_chr) + 1)
-        empty_prompt_sample_structure = config['multi_choice_example_format']
-        empty_prompt = empty_prompt_sample_structure.format(question, example)
-        res_dict = {}
-        res_dict['index2ans'] = index2ans
-        res_dict['correct_choice'] = sample['answer']
-        res_dict['all_choices'] = prediction_range
-        res_dict['empty_prompt'] = empty_prompt
-        if config['task_instructions']:
-            res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt
-        else:
-            res_dict['final_input_prompt'] = empty_prompt
-
-        res_dict['gt_content'] = options[ord(sample['answer'].upper()) - ord('A')]
-    else:
-        empty_prompt_sample_structure = config['short_ans_example_format']
-        empty_prompt = empty_prompt_sample_structure.format(question)
-        res_dict = {}
-        res_dict['empty_prompt'] = empty_prompt
-        if config['task_instructions']:
-            res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt
-        else:
-            res_dict['final_input_prompt'] = empty_prompt
-        res_dict['gt_content'] = sample['answer']
-
-    res_dict.update(sample)
-    return res_dict

eval/vlm/eval/mmmu/eval_utils.py

@@ -1,275 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-"""Response Parsing and Evaluation for various models"""
-import random
-import re
-from typing import Dict
-
-random.seed(42)
-import numpy as np
-
-
-# ----------- Process Multi-choice -------------
-def parse_multi_choice_response(response, all_choices, index2ans):
-    """
-    Parse the prediction from the generated response.
-    Return the predicted index e.g., A, B, C, D.
-    """
-    for char in [',', '.', '!', '?', ';', ':', "'"]:
-        response = response.strip(char)
-    response = ' ' + response + ' '  # add space to avoid partial match
-
-    index_ans = True
-    ans_with_brack = False
-    candidates = []
-    for choice in all_choices:  # e.g., (A) (B) (C) (D)
-        if f'({choice})' in response:
-            candidates.append(choice)
-            ans_with_brack = True
-
-    if len(candidates) == 0:
-        for choice in all_choices:  # e.g., A B C D
-            if f' {choice} ' in response:
-                candidates.append(choice)
-
-    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
-    if len(candidates) == 0 and len(response.split()) > 5:
-        for index, ans in index2ans.items():
-            if ans.lower() in response.lower():
-                candidates.append(index)
-                index_ans = False  # it's content ans.
-
-    if len(candidates) == 0:  # still not get answer, randomly choose one.
-        pred_index = random.choice(all_choices)
-    elif len(candidates) > 1:
-        start_indexes = []
-        if index_ans:
-            if ans_with_brack:
-                for can in candidates:
-                    index = response.rfind(f'({can})')
-                    start_indexes.append(index)  # -1 will be ignored anyway
-                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
-            else:
-                for can in candidates:
-                    index = response.rfind(f' {can} ')
-                    start_indexes.append(index)
-        else:
-            for can in candidates:
-                index = response.lower().rfind(index2ans[can].lower())
-                start_indexes.append(index)
-        # get the last one
-        pred_index = candidates[np.argmax(start_indexes)]
-    else:  # if only one candidate, use it.
-        pred_index = candidates[0]
-
-    return pred_index
-
-
-# ----------- Process Open -------------
-def check_is_number(string):
-    """
-    Check if the given string a number.
-    """
-    try:
-        float(string.replace(',', ''))
-        return True
-    except ValueError:
-        # check if there's comma inside
-        return False
-
-
-def normalize_str(string):
-    """
-    Normalize the str to lower case and make them float numbers if possible.
-    """
-    # check if characters in the string
-
-    # if number, numerize it.
-    string = string.strip()
-
-    is_number = check_is_number(string)
-
-    if is_number:
-        string = string.replace(',', '')
-        string = float(string)
-        # leave 2 decimal
-        string = round(string, 2)
-        return [string]
-    else:  # it's likely to be a string
-        # lower it
-        string = string.lower()
-        if len(string) == 1:
-            return [' ' + string, string + ' ']  # avoid trivial matches
-        return [string]
-
-
-def extract_numbers(string):
-    """
-    Exact all forms of numbers from a string with regex.
-    """
-    # Pattern for numbers with commas
-    pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b'
-    # Pattern for scientific notation
-    pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+'
-    # Pattern for simple numbers without commas
-    pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])'
-
-    # Extract numbers with commas
-    numbers_with_commas = re.findall(pattern_commas, string)
-    # Extract numbers in scientific notation
-    numbers_scientific = re.findall(pattern_scientific, string)
-    # Extract simple numbers without commas
-    numbers_simple = re.findall(pattern_simple, string)
-
-    # Combine all extracted numbers
-    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
-    return all_numbers
-
-
-def parse_open_response(response):
-    """
-    Parse the prediction from the generated response.
-    Return a list of predicted strings or numbers.
-    """
-
-    # content = content.strip("\n").strip(".").strip(" ")
-    def get_key_subresponses(response):
-        key_responses = []
-        response = response.strip().strip('.').lower()
-        sub_responses = re.split(r'\.\s(?=[A-Z])|\n', response)
-        indicators_of_keys = ['could be ', 'so ', 'is ',
-                              'thus ', 'therefore ', 'final ', 'answer ', 'result ']
-        key_responses = []
-        for index, resp in enumerate(sub_responses):
-            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
-            if index == len(sub_responses) - 1:
-                indicators_of_keys.extend(['='])
-            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
-            for indicator in indicators_of_keys:
-                if indicator in resp:
-                    if not shortest_key_response:
-                        shortest_key_response = resp.split(indicator)[-1].strip()
-                    else:
-                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
-                            shortest_key_response = resp.split(indicator)[-1].strip()
-                    # key_responses.append(resp.split(indicator)[1].strip())
-
-            if shortest_key_response:
-                # and it's not trivial
-                if shortest_key_response.strip() not in [':', ',', '.', '!', '?', ';', ':', "'"]:
-                    key_responses.append(shortest_key_response)
-        if len(key_responses) == 0:  # did not found any
-            return [response]
-        return key_responses
-
-    # pdb.set_trace()
-    key_responses = get_key_subresponses(response)
-
-    pred_list = key_responses.copy()  # keep the original string response
-    for resp in key_responses:
-        pred_list.extend(extract_numbers(resp))
-
-    tmp_pred_list = []
-    for i in range(len(pred_list)):
-        tmp_pred_list.extend(normalize_str(pred_list[i]))
-    pred_list = tmp_pred_list
-
-    # remove duplicates
-    pred_list = list(set(pred_list))
-
-    return pred_list
-
-
-# ----------- Evaluation -------------
-
-def eval_multi_choice(gold_i, pred_i):
-    """
-    Evaluate a multiple choice instance.
-    """
-    correct = False
-    # only they are exactly the same, we consider it as correct
-    if isinstance(gold_i, list):
-        for answer in gold_i:
-            if answer == pred_i:
-                correct = True
-                break
-    else:  # gold_i is a string
-        if gold_i == pred_i:
-            correct = True
-    return correct
-
-
-def eval_open(gold_i, pred_i):
-    """
-    Evaluate an open question instance
-    """
-    correct = False
-    if isinstance(gold_i, list):
-        # use float to avoid trivial matches
-        norm_answers = []
-        for answer in gold_i:
-            norm_answers.extend(normalize_str(answer))
-    else:
-        norm_answers = normalize_str(gold_i)
-    for pred in pred_i:  # pred is already normalized in parse response phase
-        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
-            for norm_ans in norm_answers:
-                # only see if the string answer in the string pred
-                if isinstance(norm_ans, str) and norm_ans in pred:
-                    if not correct:
-                        correct = True
-                    break
-        else:  # it's a float number
-            if pred in norm_answers:
-                if not correct:
-                    correct = True
-                break
-    return correct
-
-
-# ----------- Batch Evaluation -------------
-def evaluate(samples):
-    """
-    Batch evaluation for multiple choice and open questions.
-    """
-    pred_correct = 0
-    judge_dict = dict()
-    for sample in samples:
-        gold_i = sample['answer']
-        pred_i = sample['parsed_pred']
-        if sample['question_type'] == 'multiple-choice':
-            correct = eval_multi_choice(gold_i, pred_i)
-        else:  # open question
-            correct = eval_open(gold_i, pred_i)
-
-        if correct:
-            judge_dict[sample['id']] = 'Correct'
-            pred_correct += 1
-        else:
-            judge_dict[sample['id']] = 'Wrong'
-
-    if len(samples) == 0:
-        return {'acc': 0}
-    return judge_dict, {'acc': pred_correct / len(samples)}
-
-
-# ----------- Calculate Accuracy -------------
-def calculate_ins_level_acc(results: Dict):
-    """Calculate the instruction level accuracy for given Subject results"""
-    acc = 0
-    ins_num = 0
-    for cat_results in results.values():
-        acc += cat_results['acc'] * cat_results['num_example']
-        ins_num += cat_results['num_example']
-    if ins_num == 0:
-        return 0
-    return acc / ins_num

eval/vlm/eval/mmmu/evaluate_mmmu.py

@@ -1,272 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import itertools
-import json
-import os
-import random
-
-import torch
-from .data_utils import CAT_SHORT2LONG, process_single_sample
-from datasets import concatenate_datasets, load_dataset
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'MMMU_validation': {
-        'root': 'MMMU/MMMU',
-        'max_new_tokens': 10,
-        'min_new_tokens': 1,
-        'split': 'validation'
-    },
-    'MMMU_test': {
-        'root': 'MMMU/MMMU',
-        'max_new_tokens': 10,
-        'min_new_tokens': 1,
-        'split': 'test'
-    },
-    'MMMU_dev': {
-        'root': 'MMMU/MMMU',
-        'max_new_tokens': 10,
-        'min_new_tokens': 1,
-        'split': 'dev'
-    },
-}
-
-
-def collate_fn(batches):
-    questions = [_['question'] for _ in batches]
-    images = [_['images'] for _ in batches]
-    conversation = [_['conversation'] for _ in batches]
-    answers = [_['answer'] for _ in batches]
-    data_ids = [_['data_id'] for _ in batches]
-    options = [_['option'] for _ in batches]
-    return questions, images, conversation, answers, data_ids, options
-
-
-class MMMUDataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, split, prompt):
-        # run for each subject
-        sub_dataset_list = []
-        for subject in tqdm(CAT_SHORT2LONG.values()):
-            sub_dataset = load_dataset(root, subject, split=split, cache_dir=os.path.join(os.getcwd(), 'eval/vlm/data/MMMU/'))
-            sub_dataset_list.append(sub_dataset)
-
-        # merge all dataset
-        self.data = concatenate_datasets(sub_dataset_list)
-        self.prompt = prompt
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-
-        data = process_single_sample(self.data[idx])
-        data_id = data['id']
-        question = data['question'].strip()
-        pil_images = data['image']
-        question_type = data['question_type']
-
-        choices = eval(data['options'])
-        answer = data['answer'] if 'answer' in data else None
-
-        choice_list = []
-        options = {}
-        multiple_choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M']
-        for i, c in enumerate(choices):
-            choice_list.append('{}. {}'.format(multiple_choices[i], c.strip()))
-            options[multiple_choices[i]] = c.strip()
-        choice_txt = '\n'.join(choice_list)
-        images = []
-        for idx, pil_image in enumerate(pil_images):
-            if pil_image is not None:
-                if idx == 0:
-                    pil_image = pil_image.resize((pil_image.width * 2, pil_image.height * 2), Image.BILINEAR)
-                images.append(pil_image)
-
-        if len(choice_txt) > 0:
-            question += '\n' + choice_txt
-        question += '\n' + self.prompt[question_type]
-        question = question.strip()
-
-        # NOTE: Do not add <image> since <image 1> has been added
-        # question = "<image>" * len(images) + "\n" + question
-
-        images, conversation = process_conversation(images, question)
-
-        return {
-            'question': question,
-            'images': images,
-            'conversation': conversation,
-            'answer': answer,
-            'option': options,
-            'data_id': data_id
-        }
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def post_process(pred, option):
-    pred = pred.strip()
-    option_candidate = list(option.keys())
-    if len(pred) == 1:
-        return pred
-    elif len(pred) == 0:
-        pred = "C"
-    elif len(pred) != 1 and pred[0] in option_candidate:
-        return pred[0]
-    elif len(pred) != 1 and pred[0] not in option_candidate:
-        for k, v in option.items():
-            if v in pred:
-                return k
-
-    return pred
-
-
-def evaluate_chat_model():
-    prompt = {
-        'multiple-choice': "Answer with the option's letter from the given choices directly.",
-        'open': 'Answer the question using a single word or phrase.'
-    }
-    random.seed(args.seed)
-
-    for ds_name in args.datasets:
-        dataset = MMMUDataset(
-            root=ds_collections[ds_name]['root'],
-            split=ds_collections[ds_name]['split'],
-            prompt=prompt,
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        outputs = []
-        for _, (questions, images, conversation, answers, data_ids, options) in tqdm(enumerate(dataloader)):
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images[0], # batch=1
-                prompt=conversation[0], # batch=1
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-            if len(options[0]) == 0:
-                preds = [pred]
-            else:
-                preds = [post_process(pred, options[0])]
-
-            for question, pred, answer, data_id in zip(questions, preds, answers, data_ids):
-                outputs.append({
-                    'question': question,
-                    'answer': pred,
-                    'gt_answers': answer,
-                    'data_id': data_id
-                })
-
-        torch.distributed.barrier()
-
-        world_size = torch.distributed.get_world_size()
-        merged_outputs = [None for _ in range(world_size)]
-        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
-
-        merged_outputs = [json.loads(_) for _ in merged_outputs]
-        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
-
-        if torch.distributed.get_rank() == 0:
-            print(f'Evaluating {ds_name} ...')
-            output_path = os.path.join(args.out_dir, "prediction.json")
-            outputs = {}
-            for item in merged_outputs:
-                outputs[item['data_id']] = item['answer']
-            with open(output_path, 'w') as f:
-                json.dump(outputs, f, indent=4)
-            print('Results saved to {}'.format(output_path))
-            if ds_collections[ds_name]['split'] == 'validation':
-                print('Evaluating ...')
-                cmd = f'python -m eval.vlm.eval.mmmu.main_eval_only ' \
-                      f'--output_path {output_path} ' \
-                      f'--answer_path eval/vlm/eval/mmmu/answer_dict_val.json ' \
-                      f'--out-dir {args.out_dir}'
-                print(cmd)
-                os.system(cmd)
-            output_path = os.path.join(args.out_dir, "results.jsonl")
-            writer = open(output_path, 'w')
-            for item in merged_outputs:
-                writer.write(json.dumps(item) + '\n')
-            writer.close()
-            print('Results saved to {}'.format(output_path))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='MMMU_validation')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()

eval/vlm/eval/mmmu/evaluate_mmmu_cot.py

@@ -1,252 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import json
-import os
-import random
-
-import torch
-from .data_utils import CAT_SHORT2LONG, process_single_sample
-from datasets import concatenate_datasets, load_dataset
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'MMMU_validation_cot': {
-        'root': 'MMMU/MMMU',
-        'max_new_tokens': 16384,
-        'min_new_tokens': 1,
-        'split': 'validation'
-    },
-}
-
-COT_INSTRUCTION = (
-    'Your task is to answer the question below. '
-    "Give step by step reasoning before you answer, and when you're ready to answer, "
-    "please use the format \"Final answer: ..\""
-    '\n\n'
-    'Question:'
-    '\n\n'
-    '{question}'
-)
-
-COT_OPEN_INSTRUCTION = (
-    'Question: {question}'
-)
-
-COT_MC_INSTRUCTION = (
-    'Question: {question} Options: {options}\n'
-    'Try to reason about the question step by step to help you get the correct answer. '
-    'You might find that sometimes no reasoning is needed if the answer is straightforward. '
-    'Sometimes listing out a few reasoning steps will be helpful. '
-    'In any case, please keep the reasoning concise.\n\n'
-
-    'First, please describe what is included in the image. '
-    'Then, respond with your reason first and output the final answer in the format "Final Answer: <answer>" where <answer> is the single correct letter choice A, B, C, D, E, F, etc, when options are provided. '
-    'If you find that your reasoning lead to none of the choice, reject your reasoning and choose the most likely answer. '
-    'You have to answer with one of the choices. If no options are provided, <answer> is your answer. '
-    'If you would like to skip reasoning, just directly output the "Final Answer" part.'
-)
-
-COT_OPEN_INSTRUCTION_V2 = (
-    "You should first think about the reasoning process in the mind and then provide the user with the answer. The reasoning process is enclosed within <think> </think> tags, i.e. <think> reasoning process here </think> answer here." +\
-    "{question}" + "\nAnswer the question using a single word or phrase."
-)
-
-COT_MC_INSTRUCTION_V2 = (
-    "You should first think about the reasoning process in the mind and then provide the user with the answer. The reasoning process is enclosed within <think> </think> tags, i.e. <think> reasoning process here </think> answer here." +\
-    "Question: {question} Options: {options} " + "\nAnswer with the option's letter from the given choices directly."
-)
-
-
-def collate_fn(batches):
-    questions = [_['question'] for _ in batches]
-    images = [_['images'] for _ in batches]
-    conversation = [_['conversation'] for _ in batches]
-    answers = [_['answer'] for _ in batches]
-    data_ids = [_['data_id'] for _ in batches]
-    options = [_['option'] for _ in batches]
-    return questions, images, conversation, answers, data_ids, options
-
-
-class MMMUDataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, split):
-        # run for each subject
-        sub_dataset_list = []
-        for subject in tqdm(CAT_SHORT2LONG.values()):
-            sub_dataset = load_dataset(root, subject, split=split, cache_dir=os.path.join(os.getcwd(), 'eval/vlm/data/MMMU/'))
-            sub_dataset_list.append(sub_dataset)
-
-        # merge all dataset
-        self.data = concatenate_datasets(sub_dataset_list)
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-
-        data = process_single_sample(self.data[idx])
-        data_id = data['id']
-        question = data['question'].strip()
-        pil_images = data['image']
-        question_type = data['question_type'] # "open", "multiple-choice"
-
-        choices = eval(data['options'])
-        answer = data['answer'] if 'answer' in data else None
-
-        choice_list = []
-        options = {}
-        multiple_choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M']
-        for i, c in enumerate(choices):
-            choice_list.append('{}. {}'.format(multiple_choices[i], c.strip()))
-            options[multiple_choices[i]] = c.strip()
-        choice_txt = '\n'.join(choice_list)
-        images = []
-        for idx, pil_image in enumerate(pil_images):
-            if pil_image is not None:
-                if idx == 0:
-                    pil_image = pil_image.resize((pil_image.width * 2, pil_image.height * 2), Image.BILINEAR)
-                images.append(pil_image)
-
-        if len(choice_txt) > 0:
-            question = COT_MC_INSTRUCTION_V2.format(question=question.strip(), options=choice_txt.strip())
-        else:
-            question = COT_OPEN_INSTRUCTION_V2.format(question=question.strip())
-
-        # NOTE: Do not add <image> since <image 1> has been added
-        # question = "<image>" * len(images) + "\n" + question
-
-        images, conversation = process_conversation(images, question)
-
-        return {
-            'question': question,
-            'images': images,
-            'conversation': conversation,
-            'answer': answer,
-            'option': options,
-            'data_id': data_id
-        }
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def evaluate_chat_model():
-    random.seed(args.seed)
-    output_path = os.path.join(args.out_dir, "results.jsonl")
-
-    processed_ids = set()
-    if os.path.exists(output_path):
-        with open(output_path) as f:
-            lines = f.readlines()
-            for line in lines:
-                processed_ids.add(json.loads(line)["data_id"])
-
-    writer = open(output_path, 'a')
-
-    for ds_name in args.datasets:
-        dataset = MMMUDataset(
-            root=ds_collections[ds_name]['root'],
-            split=ds_collections[ds_name]['split'],
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        for _, (questions, images, conversation, answers, data_ids, options) in tqdm(enumerate(dataloader)):
-            if data_ids[0] in processed_ids:
-                continue
-
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images[0], # batch=1
-                prompt=conversation[0], # batch=1
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-            preds = [pred.strip()]
-
-            for question, pred, answer, data_id in zip(questions, preds, answers, data_ids):
-                cur_output = {
-                    'question': question,
-                    'answer': pred,
-                    'gt_answers': answer,
-                    'data_id': data_id
-                }
-                writer.write(json.dumps(cur_output) + '\n')
-
-    writer.close()
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='MMMU_validation')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()

eval/vlm/eval/mmmu/main_eval_only.py

@@ -1,113 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-"""Parse and Evalate"""
-import os
-import json
-from argparse import ArgumentParser
-
-from .data_utils import CAT_SHORT2LONG, DOMAIN_CAT2SUB_CAT, save_json
-from .eval_utils import calculate_ins_level_acc, evaluate, parse_open_response
-
-if __name__ == '__main__':
-
-    parser = ArgumentParser()
-    parser.add_argument('--output_path', type=str, default='./example_outputs/qwen_vl/total_val_output.json',
-                        help='The path to model output file.')
-    parser.add_argument('--answer_path', type=str, default='./answer_dict_val.json', help='Answer file path.')
-    parser.add_argument('--out-dir', type=str, default='results')
-    args = parser.parse_args()
-
-    output_dict = json.load(open(args.output_path))
-    answer_dict = json.load(open(args.answer_path))
-
-    # group by category
-    output_dict_w_cat = {}
-    for data_id, parsed_pred in output_dict.items():
-        category = '_'.join(data_id.split('_')[1:-1])
-        if category not in output_dict_w_cat:
-            output_dict_w_cat.update({category: {}})
-        output_dict_w_cat[category].update({data_id: parsed_pred})
-
-    # group by category
-    answer_dict_w_cat = {}
-    for data_id, parsed_pred in answer_dict.items():
-        category = '_'.join(data_id.split('_')[1:-1])
-        if category not in answer_dict_w_cat:
-            answer_dict_w_cat.update({category: {}})
-        answer_dict_w_cat[category].update({data_id: parsed_pred})
-
-    evaluation_result = {}
-
-    for category in CAT_SHORT2LONG.values():
-        print('Evaluating: {}'.format(category))
-        # get cat_outputs and cat_answers
-        try:
-            cat_outputs = output_dict_w_cat[category]
-            cat_answers = answer_dict_w_cat[category]
-        except KeyError:
-            print('Skipping {} for not found'.format(category))
-            continue
-
-        exampels_to_eval = []
-        for data_id, parsed_pred in cat_outputs.items():
-            question_type = cat_answers[data_id]['question_type']
-            if question_type != 'multiple-choice':
-                parsed_pred = parse_open_response(parsed_pred)  # mainly for type consistency (make it number, etc.)
-            else:
-                parsed_pred = parsed_pred
-
-            exampels_to_eval.append({
-                'id': data_id,
-                'question_type': question_type,
-                'answer': cat_answers[data_id]['ground_truth'],
-                'parsed_pred': parsed_pred
-            })
-
-        judge_dict, metric_dict = evaluate(exampels_to_eval)
-        metric_dict.update({'num_example': len(exampels_to_eval)})
-
-        evaluation_result[category] = metric_dict
-
-    printable_results = {}
-    # pdb.set_trace()
-    # add domain Subject
-    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
-        in_domain_cat_results = {}
-        for cat_name in in_domain_cats:  # use the order in DOMAIN_CAT2SUB_CAT
-            if cat_name in evaluation_result.keys():
-                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
-            else:
-                pass
-        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
-        in_domain_data_num = sum([cat_results['num_example'] for cat_results in in_domain_cat_results.values()])
-        printable_results['Overall-' + domain] = {'num': int(in_domain_data_num),
-                                                  'acc': round(in_domain_ins_acc, 3)
-                                                  }
-        # add sub category
-        for cat_name, cat_results in in_domain_cat_results.items():
-            printable_results[cat_name] = {'num': int(cat_results['num_example']),
-                                           'acc': round(cat_results['acc'], 3)
-                                           }
-
-    # table.append(["-----------------------------", "-----", "----"])
-    all_ins_acc = calculate_ins_level_acc(evaluation_result)
-    printable_results['Overall'] = {
-        'num': sum([cat_results['num_example'] for cat_results in evaluation_result.values()]),
-        'acc': round(all_ins_acc, 3)}
-
-    print(printable_results)
-    writer = open(os.path.join(args.out_dir, "results.txt"), 'w')
-    print(f"write results to file {os.path.join(args.out_dir, 'results.txt')}")
-    for key, value in printable_results.items():
-        line = f'{key}: num={value["num"]}, acc={value["acc"]}\n'
-        writer.write(line)
-    writer.close()

eval/vlm/eval/mmvet/evaluate_mmvet.py

@@ -1,112 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import json
-import os
-import random
-
-import torch
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'mmvet': {
-        'root': 'eval/vlm/data/mm-vet/images',
-        'question': 'eval/vlm/data/mm-vet/llava-mm-vet.jsonl',
-        'metric': None,
-        'max_new_tokens': 1000,
-        'min_new_tokens': 1,
-    }
-}
-
-
-class VQADataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, data, prompt):
-        self.root = root
-        self.data = open(data).readlines()
-        self.prompt = prompt
-        
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        data = json.loads(self.data[idx].strip())
-        image, question, question_id, annotation = data['image'], data[
-            'text'], data['question_id'], data.get('answer', None)
-
-        image = os.path.join(self.root, image)
-        image = Image.open(image).convert('RGB')
-        images = [image]
-        
-        question = question + ' ' + self.prompt
-
-        images, conversation = process_conversation(images, question)
-
-        return question_id, question, images, conversation, annotation
-
-
-def evaluate_chat_model():
-    random.seed(args.seed)
-    prompt = ''
-
-    for ds_name in args.datasets:
-        dataset = VQADataset(
-            root=ds_collections[ds_name]['root'],
-            data=ds_collections[ds_name]['question'],
-            prompt=prompt,
-        )
-
-        outputs = {}
-        for _, (question_id, question, images, conversation, annotations) in tqdm(enumerate(dataset)):
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images,
-                prompt=conversation,
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-
-            outputs[f'v1_{question_id}'] = pred
-
-        print(f'Evaluating {ds_name} ...')
-        results_file = os.path.join(args.out_dir, 'results.json')
-        json.dump(outputs, open(results_file, 'w'))
-        print('Results saved to {}'.format(results_file))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='mmvet')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()

eval/vlm/eval/mmvp/evaluate_mmvp.py

@@ -1,258 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import csv
-import itertools
-import json
-import os
-import random
-
-import torch
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'MMVP': {
-        'root': 'eval/vlm/data/MMVP',
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-    },
-}
-
-
-def collate_fn(batches):
-    questions = [_['question'] for _ in batches]
-    images = [_['images'] for _ in batches]
-    conversations = [_['conversations'] for _ in batches]
-    answers = [_['answer'] for _ in batches]
-    data_ids = [_['data_id'] for _ in batches]
-    options = [_['option'] for _ in batches]
-    return questions, images, conversations, answers, data_ids, options
-
-
-class MMVPDataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, prompt):
-        # run for each subject
-        meta_path = os.path.join(root, 'Questions.csv')
-        with open(meta_path, 'r') as file:
-            csv_reader = csv.DictReader(file)
-            data = [row for row in csv_reader]
-
-        self.data = data
-        self.root = root
-        self.prompt = prompt
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        data = self.data[idx]
-        data_id = data['lndex'] if 'lndex' in data else data['Index']
-        question = data['Question']
-        image = os.path.join(self.root + '/MMVP Images', data_id + '.jpg')
-        image = Image.open(image).convert('RGB')
-
-        options = data['Options'].split('(b)')
-        options[0] = options[0].replace('(a)', '').strip()
-        options[1] = options[1].replace('(b)', '').strip()
-
-        answer = data['Correct Answer'] if 'Correct Answer' in data else None
-        answer = answer.replace('(a)', 'A').replace('(b)', 'B').replace('(c)', 'C').replace('(d)', 'D')
-        choice_list = []
-        new_options = {}
-        multiple_choices = ['A', 'B', 'C', 'D']
-        for i, c in enumerate(options):
-            choice_list.append('{}. {}'.format(multiple_choices[i], c.strip()))
-            new_options[multiple_choices[i]] = c.strip()
-        choice_txt = '\n'.join(choice_list)
-
-        images = [image]
-
-        if len(choice_txt) > 0:
-            question += '\n' + choice_txt
-        question += '\n' + self.prompt
-
-        images, conversation = process_conversation(images, question)
-
-        return {
-            'question': question,
-            'images': images,
-            'conversations': conversation,
-            'answer': answer,
-            'option': new_options,
-            'data_id': data_id
-        }
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def post_process(pred, option):
-    pred = pred.strip()
-    option_candidate = list(option.keys())
-    if len(pred) == 1:
-        return pred
-    elif len(pred) != 1 and pred[0] in option_candidate:
-        return pred[0]
-    elif len(pred) != 1 and pred[0] not in option_candidate:
-        for k, v in option.items():
-            if v in pred:
-                return k
-
-    return pred
-
-
-def evaluate_chat_model():
-    prompt = "Answer with the option's letter from the given choices directly."
-    random.seed(args.seed)
-
-    for ds_name in args.datasets:
-        dataset = MMVPDataset(
-            root=ds_collections[ds_name]['root'],  # hf dataset path.
-            prompt=prompt,
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        outputs = []
-        for _, (questions, images, conversations, answers, data_ids, options) in tqdm(enumerate(dataloader)):
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images[0], # batch=1
-                prompt=conversations[0], # batch=1
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-
-            if len(options[0]) == 0:
-                preds = [pred]
-            else:
-                preds = [post_process(pred, options[0])]
-
-            for question, pred, answer, data_id in zip(questions, preds, answers, data_ids):
-                outputs.append({
-                    'question': question,
-                    'answer': pred,
-                    'gt_answers': answer,
-                    'data_id': data_id
-                })
-
-        torch.distributed.barrier()
-
-        world_size = torch.distributed.get_world_size()
-        merged_outputs = [None for _ in range(world_size)]
-        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
-
-        merged_outputs = [json.loads(_) for _ in merged_outputs]
-        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
-
-        if torch.distributed.get_rank() == 0:
-            print(f'Evaluating {ds_name} ...')
-            results_file = 'results.jsonl'
-            output_path = os.path.join(args.out_dir, results_file)
-            writer = open(output_path, 'w')
-
-            num_correct, num_total = 0, 0
-            index, round_correct = 0, 0
-            for item in merged_outputs:
-                writer.write(json.dumps(item) + '\n')
-                answer = item['answer']
-                gt_answer = item['gt_answers']
-
-                index += 1
-
-                if answer == gt_answer:
-                    round_correct += 1
-                if index == 2:
-                    index = 0
-                    if round_correct == 2:
-                        num_correct += 1
-                    round_correct = 0
-
-                    num_total += 1
-
-            writer.close()
-            print('Results saved to {}'.format(output_path))
-            print(f'The accuracy is {num_correct/num_total}')
-
-            results_file = 'results.txt'
-            output_path = os.path.join(args.out_dir, results_file)
-            writer = open(output_path, 'w')
-            writer.write(f'The accuracy is {num_correct/num_total}')
-            writer.close()
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='MMVP')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()

eval/vlm/eval/pope/eval_pope.py

@@ -1,120 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import json
-import os
-import numpy as np
-
-
-def eval_pope(answers, label_file):
-    label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
-
-    for answer in answers:
-        text = answer['text']
-
-        # Only keep the first sentence
-        if text.find('.') != -1:
-            text = text.split('.')[0]
-
-        text = text.replace(',', '')
-        words = text.split(' ')
-        if 'No' in words or 'not' in words or 'no' in words:
-            answer['text'] = 'no'
-        else:
-            answer['text'] = 'yes'
-
-    for i in range(len(label_list)):
-        if label_list[i] == 'no':
-            label_list[i] = 0
-        else:
-            label_list[i] = 1
-
-    pred_list = []
-    for answer in answers:
-        if answer['text'] == 'no':
-            pred_list.append(0)
-        else:
-            pred_list.append(1)
-
-    pos = 1
-    neg = 0
-    yes_ratio = pred_list.count(1) / len(pred_list)
-
-    TP, TN, FP, FN = 0, 0, 0, 0
-    for pred, label in zip(pred_list, label_list):
-        if pred == pos and label == pos:
-            TP += 1
-        elif pred == pos and label == neg:
-            FP += 1
-        elif pred == neg and label == neg:
-            TN += 1
-        elif pred == neg and label == pos:
-            FN += 1
-
-    ret_message = ""
-    print('TP\tFP\tTN\tFN\t')
-    print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN))
-    ret_message += 'TP\tFP\tTN\tFN\t\n'
-    ret_message += '{}\t{}\t{}\t{}\n'.format(TP, FP, TN, FN)
-
-    precision = float(TP) / float(TP + FP)
-    recall = float(TP) / float(TP + FN)
-    f1 = 2 * precision * recall / (precision + recall)
-    acc = (TP + TN) / (TP + TN + FP + FN)
-    print('Accuracy: {}'.format(acc))
-    print('Precision: {}'.format(precision))
-    print('Recall: {}'.format(recall))
-    print('F1 score: {}'.format(f1))
-    print('Yes ratio: {}'.format(yes_ratio))
-    print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio))
-
-    ret_message += 'Accuracy: {}\n'.format(acc)
-    ret_message += 'Precision: {}\n'.format(precision)
-    ret_message += 'Recall: {}\n'.format(recall)
-    ret_message += 'F1 score: {}\n'.format(f1)
-    ret_message += 'Yes ratio: {}\n'.format(yes_ratio)
-    ret_message += '%.3f, %.3f, %.3f, %.3f, %.3f\n' % (f1, acc, precision, recall, yes_ratio)
-    return f1, ret_message
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--annotation-dir', type=str)
-    parser.add_argument('--question-file', type=str)
-    parser.add_argument('--result-file', type=str)
-    parser.add_argument('--out-dir', type=str)
-    args = parser.parse_args()
-
-    questions = [json.loads(line) for line in open(args.question_file)]
-    questions = {question['question_id']: question for question in questions}
-    answers = json.loads(open(args.result_file).read())
-    avg_f1 = []
-    ret_message = ""
-    for file in os.listdir(args.annotation_dir):
-        assert file.startswith('coco_pope_')
-        assert file.endswith('.json')
-        category = file[10:-5]
-        cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category]
-        print('Category: {}, # samples: {}'.format(category, len(cur_answers)))
-        ret_message += 'Category: {}, # samples: {}\n'.format(category, len(cur_answers))
-        f1, ret = eval_pope(cur_answers, os.path.join(args.annotation_dir, file))
-        ret_message += ret
-        print('====================================')
-        ret_message += '====================================\n'
-        avg_f1.append(f1)
-    print(f"Avg F1 score: {np.array(avg_f1).mean()}")
-    ret_message += f"Avg F1 score: {np.array(avg_f1).mean()}\n"
-
-    writer = open(os.path.join(args.out_dir, "results.txt"), 'w')
-    print(f"write results to file {os.path.join(args.out_dir, 'results.txt')}")
-    writer.write(ret_message)
-    writer.close()

eval/vlm/eval/pope/evaluate_pope.py

@@ -1,231 +0,0 @@
-# Copyright (c) 2023 OpenGVLab
-# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates.
-# SPDX-License-Identifier: MIT
-#
-# This file has been modified by ByteDance Ltd. and/or its affiliates. on 2025-05-20.
-#
-# Original file was released under MIT, with the full license text
-# available at https://github.com/OpenGVLab/InternVL/blob/main/LICENSE.
-#
-# This modified file is released under the same license.
-
-import argparse
-import itertools
-import json
-import os
-import random
-import re
-
-import torch
-from eval.vlm.utils import load_model_and_tokenizer, build_transform, process_conversation
-from PIL import Image
-from tqdm import tqdm
-
-ds_collections = {
-    'pope': {
-        'root': 'eval/vlm/data/pope/val2014',
-        'question': 'eval/vlm/data/pope/llava_pope_test.jsonl',
-        'metric': None,
-        'max_new_tokens': 100,
-        'min_new_tokens': 1,
-    },
-    'pope_cot': {
-        'root': 'eval/vlm/data/pope/val2014',
-        'question': 'eval/vlm/data/pope/llava_pope_test.jsonl',
-        'metric': None,
-        'max_new_tokens': 1000,
-        'min_new_tokens': 1,
-    }
-}
-
-
-COT_INSTRUCTION = (
-    'Your task is to answer the question below. '
-    "Give step by step reasoning before you answer, and when you're ready to answer, "
-    "please use the format \"Final answer: ..\""
-    '\n\n'
-    'Question:'
-    '\n\n'
-    '{question}'
-)
-
-
-def extract_answer(text):
-    match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE)
-    if match:
-        return match.group(2).strip()
-    return text
-
-
-def collate_fn(batches):
-    questions = [_['question'] for _ in batches]
-    images = [_['images'] for _ in batches]
-    conversations = [_['conversations'] for _ in batches]
-    question_ids = [_['question_id'] for _ in batches]
-    annotations = [_['annotation'] for _ in batches]
-
-    return questions, images, conversations, question_ids, annotations
-
-
-class VQADataset(torch.utils.data.Dataset):
-
-    def __init__(self, root, data, prompt):
-        self.root = root
-        self.data = open(data).readlines()
-        self.prompt = prompt
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        data = json.loads(self.data[idx].strip())
-        image, question, question_id, annotation = data['image'], data[
-            'text'], data['question_id'], data.get('answer', None)
-
-        image = os.path.join(self.root, image)
-        image = Image.open(image).convert('RGB')
-        images = [image]
-
-        llava_prompt = 'Answer the question using a single word or phrase.'
-        assert llava_prompt in question
-        question = question.replace(llava_prompt, self.prompt).strip()
-
-        if args.cot:
-            question = COT_INSTRUCTION.format(question=question)
-        
-        images, conversation = process_conversation(images, question)
-
-        return {
-            'question_id': question_id,
-            'question': question,
-            'images': images,
-            'conversations': conversation,
-            'annotation': annotation
-        }
-
-
-class InferenceSampler(torch.utils.data.sampler.Sampler):
-
-    def __init__(self, size):
-        self._size = int(size)
-        assert size > 0
-        self._rank = torch.distributed.get_rank()
-        self._world_size = torch.distributed.get_world_size()
-        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
-
-    @staticmethod
-    def _get_local_indices(total_size, world_size, rank):
-        shard_size = total_size // world_size
-        left = total_size % world_size
-        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
-
-        begin = sum(shard_sizes[:rank])
-        end = min(sum(shard_sizes[:rank + 1]), total_size)
-        return range(begin, end)
-
-    def __iter__(self):
-        yield from self._local_indices
-
-    def __len__(self):
-        return len(self._local_indices)
-
-
-def evaluate_chat_model():
-    prompt = '' if args.cot else 'Answer the question using a single word or phrase.'
-    random.seed(args.seed)
-
-    for ds_name in args.datasets:
-        dataset = VQADataset(
-            root=ds_collections[ds_name]['root'],
-            data=ds_collections[ds_name]['question'],
-            prompt=prompt,
-        )
-        dataloader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            sampler=InferenceSampler(len(dataset)),
-            batch_size=args.batch_size,
-            num_workers=args.num_workers,
-            pin_memory=True,
-            drop_last=False,
-            collate_fn=collate_fn,
-        )
-
-        outputs = []
-        for _, (questions, images, conversations, question_ids, annotations) in tqdm(enumerate(dataloader)):
-            pred = model.chat(
-                tokenizer, 
-                new_token_ids,
-                image_transform,
-                images=images[0], # batch=1
-                prompt=conversations[0], # batch=1
-                max_length=ds_collections[ds_name]['max_new_tokens'], # TODO: how to use ds_collections[ds_name]['min_new_tokens']
-            )
-            pred_orig = pred
-            if args.cot:
-                pred = extract_answer(pred).strip()
-            answers = [pred]
-
-            for question_id, answer, annotation in zip(question_ids, answers, annotations):
-                outputs.append({
-                    'question_id': question_id,
-                    'text': pred,
-                    'text_orig': pred_orig,
-                    'metadata': {},
-                })
-
-        torch.distributed.barrier()
-
-        world_size = torch.distributed.get_world_size()
-        merged_outputs = [None for _ in range(world_size)]
-        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
-
-        merged_outputs = [json.loads(_) for _ in merged_outputs]
-        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
-
-        if torch.distributed.get_rank() == 0:
-            print(f'Evaluating {ds_name} ...')
-            results_file = os.path.join(args.out_dir, 'results.json')
-            json.dump(merged_outputs, open(results_file, 'w'))
-            print('Results saved to {}'.format(results_file))
-            cmd = 'python eval/vlm/eval/pope/eval_pope.py ' \
-                  '--annotation-dir eval/vlm/data/pope/coco ' \
-                  '--question-file eval/vlm/data/pope/llava_pope_test.jsonl ' \
-                  f'--result-file {results_file} ' \
-                  f'--out-dir {args.out_dir}'
-            print(cmd)
-            os.system(cmd)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--datasets', type=str, default='pope')
-    parser.add_argument('--batch-size', type=int, default=1)
-    parser.add_argument('--num-workers', type=int, default=1)
-    parser.add_argument('--out-dir', type=str, default='results')
-    parser.add_argument('--seed', type=int, default=0)
-    parser.add_argument('--model-path', type=str, default='hf/BAGEL-7B-MoT/')
-    parser.add_argument('--cot', action='store_true')
-    args = parser.parse_args()
-
-    if not os.path.exists(args.out_dir):
-        os.makedirs(args.out_dir, exist_ok=True)
-
-    args.datasets = args.datasets.split(',')
-    print('datasets:', args.datasets)
-    assert args.batch_size == 1, 'Only batch size 1 is supported'
-
-    torch.distributed.init_process_group(
-        backend='nccl',
-        world_size=int(os.getenv('WORLD_SIZE', '1')),
-        rank=int(os.getenv('RANK', '0')),
-    )
-
-    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
-
-    model, tokenizer, new_token_ids = load_model_and_tokenizer(args)
-    image_transform = build_transform()
-
-    total_params = sum(p.numel() for p in model.parameters()) / 1e9
-    print(f'[test] total_params: {total_params}B')
-
-    evaluate_chat_model()