Upload folder using huggingface_hub

README.md

@@ -0,0 +1,59 @@
+---
+library_name: peft
+license: other
+base_model: llava-hf/llava-v1.6-mistral-7b-hf
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: train_2025-04-21-17-35-28
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2025-04-21-17-35-28
+
+This model is a fine-tuned version of [llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) on the docker_command_dataset dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 3.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.49.0
+- Pytorch 2.6.0+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

adapter_config.json

@@ -0,0 +1,251 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "llava-hf/llava-v1.6-mistral-7b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "language_model.model.layers.2.self_attn.o_proj",
+    "language_model.model.layers.30.self_attn.v_proj",
+    "language_model.model.layers.18.self_attn.q_proj",
+    "language_model.model.layers.9.self_attn.v_proj",
+    "language_model.model.layers.30.self_attn.k_proj",
+    "language_model.model.layers.0.self_attn.o_proj",
+    "language_model.model.layers.19.mlp.up_proj",
+    "language_model.model.layers.20.self_attn.v_proj",
+    "language_model.model.layers.12.mlp.up_proj",
+    "language_model.model.layers.11.mlp.up_proj",
+    "language_model.model.layers.5.self_attn.q_proj",
+    "language_model.model.layers.18.mlp.gate_proj",
+    "language_model.model.layers.4.self_attn.v_proj",
+    "language_model.model.layers.12.mlp.down_proj",
+    "language_model.model.layers.17.mlp.down_proj",
+    "language_model.model.layers.13.self_attn.k_proj",
+    "language_model.model.layers.11.self_attn.k_proj",
+    "language_model.model.layers.15.self_attn.k_proj",
+    "language_model.model.layers.2.self_attn.k_proj",
+    "language_model.model.layers.23.self_attn.v_proj",
+    "language_model.model.layers.31.mlp.down_proj",
+    "language_model.model.layers.0.mlp.up_proj",
+    "language_model.model.layers.5.mlp.gate_proj",
+    "language_model.model.layers.8.self_attn.q_proj",
+    "language_model.model.layers.12.self_attn.o_proj",
+    "language_model.model.layers.10.self_attn.q_proj",
+    "language_model.model.layers.28.self_attn.q_proj",
+    "language_model.model.layers.5.self_attn.v_proj",
+    "language_model.model.layers.16.self_attn.v_proj",
+    "language_model.model.layers.1.mlp.up_proj",
+    "language_model.model.layers.7.mlp.down_proj",
+    "language_model.model.layers.18.self_attn.v_proj",
+    "language_model.model.layers.23.self_attn.k_proj",
+    "language_model.model.layers.11.self_attn.v_proj",
+    "language_model.model.layers.8.mlp.down_proj",
+    "language_model.model.layers.1.self_attn.q_proj",
+    "language_model.model.layers.18.mlp.up_proj",
+    "language_model.model.layers.2.mlp.up_proj",
+    "language_model.model.layers.20.self_attn.q_proj",
+    "language_model.model.layers.11.self_attn.q_proj",
+    "language_model.model.layers.12.self_attn.q_proj",
+    "language_model.model.layers.3.mlp.up_proj",
+    "language_model.model.layers.25.self_attn.v_proj",
+    "language_model.model.layers.8.self_attn.o_proj",
+    "language_model.model.layers.25.mlp.down_proj",
+    "language_model.model.layers.28.self_attn.k_proj",
+    "language_model.model.layers.29.mlp.gate_proj",
+    "language_model.model.layers.16.mlp.gate_proj",
+    "language_model.model.layers.7.mlp.up_proj",
+    "language_model.model.layers.4.self_attn.k_proj",
+    "language_model.model.layers.31.self_attn.v_proj",
+    "language_model.model.layers.25.self_attn.o_proj",
+    "language_model.model.layers.28.mlp.up_proj",
+    "language_model.model.layers.26.mlp.down_proj",
+    "language_model.model.layers.17.self_attn.v_proj",
+    "language_model.model.layers.5.mlp.down_proj",
+    "language_model.model.layers.3.self_attn.o_proj",
+    "language_model.model.layers.28.self_attn.o_proj",
+    "language_model.model.layers.28.self_attn.v_proj",
+    "language_model.model.layers.0.self_attn.k_proj",
+    "language_model.model.layers.16.mlp.up_proj",
+    "language_model.model.layers.13.self_attn.q_proj",
+    "language_model.model.layers.7.self_attn.k_proj",
+    "language_model.model.layers.31.self_attn.o_proj",
+    "language_model.model.layers.10.self_attn.o_proj",
+    "language_model.model.layers.28.mlp.down_proj",
+    "language_model.model.layers.20.mlp.gate_proj",
+    "language_model.model.layers.7.self_attn.o_proj",
+    "language_model.model.layers.26.mlp.up_proj",
+    "language_model.model.layers.13.self_attn.v_proj",
+    "language_model.model.layers.13.mlp.down_proj",
+    "language_model.model.layers.29.mlp.down_proj",
+    "language_model.model.layers.17.self_attn.q_proj",
+    "language_model.model.layers.19.mlp.gate_proj",
+    "language_model.model.layers.9.mlp.down_proj",
+    "language_model.model.layers.7.mlp.gate_proj",
+    "language_model.model.layers.26.self_attn.q_proj",
+    "language_model.model.layers.14.mlp.gate_proj",
+    "language_model.model.layers.24.mlp.gate_proj",
+    "language_model.model.layers.9.self_attn.q_proj",
+    "language_model.model.layers.18.self_attn.k_proj",
+    "language_model.model.layers.4.self_attn.o_proj",
+    "language_model.model.layers.10.self_attn.k_proj",
+    "language_model.model.layers.29.self_attn.k_proj",
+    "language_model.model.layers.1.mlp.gate_proj",
+    "language_model.model.layers.2.self_attn.q_proj",
+    "language_model.model.layers.16.self_attn.q_proj",
+    "language_model.model.layers.13.mlp.up_proj",
+    "language_model.model.layers.30.mlp.down_proj",
+    "language_model.model.layers.21.mlp.gate_proj",
+    "language_model.model.layers.12.mlp.gate_proj",
+    "language_model.model.layers.13.self_attn.o_proj",
+    "language_model.model.layers.14.self_attn.q_proj",
+    "language_model.model.layers.15.self_attn.v_proj",
+    "language_model.model.layers.25.self_attn.k_proj",
+    "language_model.model.layers.24.mlp.up_proj",
+    "language_model.model.layers.26.self_attn.v_proj",
+    "language_model.model.layers.27.mlp.gate_proj",
+    "language_model.model.layers.15.mlp.down_proj",
+    "language_model.model.layers.29.mlp.up_proj",
+    "language_model.model.layers.15.self_attn.o_proj",
+    "language_model.model.layers.17.mlp.up_proj",
+    "language_model.model.layers.18.mlp.down_proj",
+    "language_model.model.layers.14.self_attn.o_proj",
+    "language_model.model.layers.18.self_attn.o_proj",
+    "language_model.model.layers.24.mlp.down_proj",
+    "language_model.model.layers.15.mlp.gate_proj",
+    "language_model.model.layers.10.mlp.gate_proj",
+    "language_model.model.layers.30.mlp.gate_proj",
+    "language_model.model.layers.17.mlp.gate_proj",
+    "language_model.model.layers.21.self_attn.o_proj",
+    "language_model.model.layers.27.mlp.down_proj",
+    "language_model.model.layers.10.mlp.down_proj",
+    "language_model.model.layers.11.self_attn.o_proj",
+    "language_model.model.layers.14.mlp.down_proj",
+    "language_model.model.layers.19.self_attn.v_proj",
+    "language_model.model.layers.24.self_attn.q_proj",
+    "language_model.model.layers.31.self_attn.q_proj",
+    "language_model.model.layers.9.self_attn.k_proj",
+    "language_model.model.layers.9.mlp.up_proj",
+    "language_model.model.layers.0.self_attn.q_proj",
+    "language_model.model.layers.21.self_attn.q_proj",
+    "language_model.model.layers.22.mlp.down_proj",
+    "language_model.model.layers.25.self_attn.q_proj",
+    "language_model.model.layers.7.self_attn.q_proj",
+    "language_model.model.layers.27.self_attn.o_proj",
+    "language_model.model.layers.28.mlp.gate_proj",
+    "language_model.model.layers.8.mlp.gate_proj",
+    "language_model.model.layers.19.self_attn.k_proj",
+    "language_model.model.layers.26.self_attn.o_proj",
+    "language_model.model.layers.23.mlp.down_proj",
+    "language_model.model.layers.0.mlp.down_proj",
+    "language_model.model.layers.2.mlp.down_proj",
+    "language_model.model.layers.19.self_attn.o_proj",
+    "language_model.model.layers.21.mlp.up_proj",
+    "language_model.model.layers.27.self_attn.k_proj",
+    "language_model.model.layers.2.mlp.gate_proj",
+    "language_model.model.layers.15.mlp.up_proj",
+    "language_model.model.layers.1.mlp.down_proj",
+    "language_model.model.layers.6.self_attn.v_proj",
+    "language_model.model.layers.26.self_attn.k_proj",
+    "language_model.model.layers.23.mlp.up_proj",
+    "language_model.model.layers.7.self_attn.v_proj",
+    "language_model.model.layers.3.self_attn.k_proj",
+    "language_model.model.layers.5.mlp.up_proj",
+    "language_model.model.layers.27.self_attn.q_proj",
+    "language_model.model.layers.11.mlp.down_proj",
+    "language_model.model.layers.19.self_attn.q_proj",
+    "language_model.model.layers.16.self_attn.k_proj",
+    "language_model.model.layers.17.self_attn.o_proj",
+    "language_model.model.layers.27.mlp.up_proj",
+    "language_model.model.layers.4.mlp.down_proj",
+    "language_model.model.layers.22.mlp.up_proj",
+    "language_model.model.layers.19.mlp.down_proj",
+    "language_model.model.layers.3.self_attn.v_proj",
+    "language_model.model.layers.17.self_attn.k_proj",
+    "language_model.model.layers.8.self_attn.k_proj",
+    "language_model.model.layers.20.mlp.down_proj",
+    "language_model.model.layers.29.self_attn.q_proj",
+    "language_model.model.layers.14.self_attn.v_proj",
+    "language_model.model.layers.25.mlp.gate_proj",
+    "language_model.model.layers.29.self_attn.o_proj",
+    "language_model.model.layers.16.self_attn.o_proj",
+    "language_model.model.layers.16.mlp.down_proj",
+    "language_model.model.layers.23.self_attn.q_proj",
+    "language_model.model.layers.30.self_attn.q_proj",
+    "language_model.model.layers.6.mlp.gate_proj",
+    "language_model.model.layers.10.self_attn.v_proj",
+    "language_model.model.layers.12.self_attn.k_proj",
+    "language_model.model.layers.21.mlp.down_proj",
+    "language_model.model.layers.31.mlp.gate_proj",
+    "language_model.model.layers.25.mlp.up_proj",
+    "language_model.model.layers.5.self_attn.o_proj",
+    "language_model.model.layers.15.self_attn.q_proj",
+    "language_model.model.layers.4.mlp.gate_proj",
+    "language_model.model.layers.29.self_attn.v_proj",
+    "language_model.model.layers.30.mlp.up_proj",
+    "language_model.model.layers.12.self_attn.v_proj",
+    "language_model.model.layers.30.self_attn.o_proj",
+    "language_model.model.layers.4.self_attn.q_proj",
+    "language_model.model.layers.8.mlp.up_proj",
+    "language_model.model.layers.31.mlp.up_proj",
+    "language_model.model.layers.6.self_attn.q_proj",
+    "language_model.model.layers.6.mlp.down_proj",
+    "language_model.model.layers.4.mlp.up_proj",
+    "language_model.model.layers.31.self_attn.k_proj",
+    "language_model.model.layers.24.self_attn.o_proj",
+    "language_model.model.layers.22.self_attn.v_proj",
+    "language_model.model.layers.22.self_attn.q_proj",
+    "language_model.model.layers.21.self_attn.k_proj",
+    "language_model.model.layers.2.self_attn.v_proj",
+    "language_model.model.layers.22.mlp.gate_proj",
+    "language_model.model.layers.6.mlp.up_proj",
+    "language_model.model.layers.6.self_attn.k_proj",
+    "language_model.model.layers.26.mlp.gate_proj",
+    "language_model.model.layers.24.self_attn.k_proj",
+    "language_model.model.layers.3.self_attn.q_proj",
+    "language_model.model.layers.14.mlp.up_proj",
+    "language_model.model.layers.1.self_attn.k_proj",
+    "language_model.model.layers.9.mlp.gate_proj",
+    "language_model.model.layers.24.self_attn.v_proj",
+    "language_model.model.layers.1.self_attn.v_proj",
+    "language_model.model.layers.11.mlp.gate_proj",
+    "language_model.model.layers.0.mlp.gate_proj",
+    "language_model.model.layers.9.self_attn.o_proj",
+    "language_model.model.layers.8.self_attn.v_proj",
+    "language_model.model.layers.20.mlp.up_proj",
+    "language_model.model.layers.21.self_attn.v_proj",
+    "language_model.model.layers.22.self_attn.o_proj",
+    "language_model.model.layers.20.self_attn.k_proj",
+    "language_model.model.layers.0.self_attn.v_proj",
+    "language_model.model.layers.1.self_attn.o_proj",
+    "language_model.model.layers.3.mlp.gate_proj",
+    "language_model.model.layers.3.mlp.down_proj",
+    "language_model.model.layers.6.self_attn.o_proj",
+    "language_model.model.layers.13.mlp.gate_proj",
+    "language_model.model.layers.20.self_attn.o_proj",
+    "language_model.model.layers.22.self_attn.k_proj",
+    "language_model.model.layers.23.self_attn.o_proj",
+    "language_model.model.layers.5.self_attn.k_proj",
+    "language_model.model.layers.10.mlp.up_proj",
+    "language_model.model.layers.14.self_attn.k_proj",
+    "language_model.model.layers.27.self_attn.v_proj",
+    "language_model.model.layers.23.mlp.gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd4a7bb5d5c126c0322e7b8e437c498f751c2b0d4984f8ea1ceb63058f802aa5
+size 83952016

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<image>": 32000,
+  "<pad>": 32001
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.6423841059602649,
+    "num_input_tokens_seen": 1916832,
+    "total_flos": 8.57490291718226e+16,
+    "train_loss": 1.3051114364997627,
+    "train_runtime": 5464.1136,
+    "train_samples_per_second": 1.326,
+    "train_steps_per_second": 0.083
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<<SYS>>\n' + message['content'][0]['text'] + '\n<</SYS>>\n\n' }}{% elif message['role'] == 'user' %}{{ '[INST] ' }}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] }}{% endfor %}{{' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'][0]['text'] + '</s> '}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"
+}

llamaboard_config.yaml

@@ -0,0 +1,78 @@
+top.booster: auto
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: LLaVA-NeXT-Mistral-7B-Chat
+top.quantization_bit: '4'
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: llava_next_mistral
+train.additional_target: ''
+train.apollo_rank: 16
+train.apollo_scale: 32
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 2
+train.compute_type: fp16
+train.create_new_adapter: false
+train.cutoff_len: 2048
+train.dataset:
+- docker_command_dataset
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{"optim": "adamw_torch"}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 2e-4
+train.logging_steps: 5
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '0.3'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '3.0'
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: []
+train.save_steps: 100
+train.swanlab_api_key: ''
+train.swanlab_link: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: false
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

preprocessor_config.json

@@ -0,0 +1,52 @@
+{
+  "aspect_ratio_setting": "anyres",
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "LlavaNextImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "processor_class": "LlavaNextProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}

processor_config.json

@@ -0,0 +1,7 @@
+{
+  "image_token": "<image>",
+  "num_additional_image_tokens": 1,
+  "patch_size": 14,
+  "processor_class": "LlavaNextProcessor",
+  "vision_feature_select_strategy": "default"
+}

running_log.txt

@@ -0,0 +1,516 @@
+[INFO|2025-04-21 17:36:49] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/config.json
+
+[INFO|2025-04-21 17:36:49] configuration_utils.py:771 >> Model config LlavaNextConfig {
+  "_name_or_path": "llava-hf/llava-v1.6-mistral-7b-hf",
+  "architectures": [
+    "LlavaNextForConditionalGeneration"
+  ],
+  "ignore_index": -100,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_seq_length": 576,
+  "image_token_index": 32000,
+  "model_type": "llava_next",
+  "multimodal_projector_bias": true,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+    "architectures": [
+      "MistralForCausalLM"
+    ],
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "model_type": "mistral",
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 32064
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.49.0",
+  "use_image_newline_parameter": true,
+  "vision_config": {
+    "hidden_size": 1024,
+    "image_size": 336,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vocab_size": 32064
+}
+
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer.model
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer.json
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file added_tokens.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/added_tokens.json
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/special_tokens_map.json
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer_config.json
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2050 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-21 17:36:49] tokenization_utils_base.py:2313 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-21 17:36:50] processing_utils.py:816 >> loading configuration file processor_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/processor_config.json
+
+[INFO|2025-04-21 17:36:50] image_processing_base.py:381 >> loading configuration file preprocessor_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/preprocessor_config.json
+
+[WARNING|2025-04-21 17:36:50] logging.py:329 >> Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.48, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+
+[INFO|2025-04-21 17:36:50] image_processing_base.py:434 >> Image processor LlavaNextImageProcessor {
+  "aspect_ratio_setting": "anyres",
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "LlavaNextImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "processor_class": "LlavaNextProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}
+
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer.model
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer.json
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file added_tokens.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/added_tokens.json
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/special_tokens_map.json
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/tokenizer_config.json
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2050 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-21 17:36:50] tokenization_utils_base.py:2313 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-21 17:36:50] processing_utils.py:816 >> loading configuration file processor_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/processor_config.json
+
+[INFO|2025-04-21 17:36:51] processing_utils.py:876 >> Processor LlavaNextProcessor:
+- image_processor: LlavaNextImageProcessor {
+  "aspect_ratio_setting": "anyres",
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "LlavaNextImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "processor_class": "LlavaNextProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}
+
+- tokenizer: LlamaTokenizerFast(name_or_path='llava-hf/llava-v1.6-mistral-7b-hf', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>', 'image_token': '<image>'}, clean_up_tokenization_spaces=False, added_tokens_decoder={
+	0: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	1: AddedToken("<s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	2: AddedToken("</s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	32000: AddedToken("<image>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+	32001: AddedToken("<pad>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
+}
+)
+
+{
+  "image_token": "<image>",
+  "num_additional_image_tokens": 1,
+  "patch_size": 14,
+  "processor_class": "LlavaNextProcessor",
+  "vision_feature_select_strategy": "default"
+}
+
+
+[INFO|2025-04-21 17:36:51] logging.py:157 >> Loading dataset MattCoddity/dockerNLcommands...
+
+[INFO|2025-04-21 17:36:53] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/config.json
+
+[INFO|2025-04-21 17:36:53] configuration_utils.py:771 >> Model config LlavaNextConfig {
+  "_name_or_path": "llava-hf/llava-v1.6-mistral-7b-hf",
+  "architectures": [
+    "LlavaNextForConditionalGeneration"
+  ],
+  "ignore_index": -100,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_seq_length": 576,
+  "image_token_index": 32000,
+  "model_type": "llava_next",
+  "multimodal_projector_bias": true,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+    "architectures": [
+      "MistralForCausalLM"
+    ],
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "model_type": "mistral",
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 32064
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.49.0",
+  "use_image_newline_parameter": true,
+  "vision_config": {
+    "hidden_size": 1024,
+    "image_size": 336,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vocab_size": 32064
+}
+
+
+[INFO|2025-04-21 17:36:53] logging.py:157 >> Quantizing model to 4 bit with bitsandbytes.
+
+[INFO|2025-04-21 17:36:53] modeling_utils.py:3982 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/model.safetensors.index.json
+
+[INFO|2025-04-21 17:36:53] modeling_utils.py:1633 >> Instantiating LlavaNextForConditionalGeneration model under default dtype torch.float16.
+
+[INFO|2025-04-21 17:36:53] configuration_utils.py:1140 >> Generate config GenerationConfig {}
+
+
+[INFO|2025-04-21 17:36:54] modeling_utils.py:1633 >> Instantiating CLIPVisionModel model under default dtype torch.float16.
+
+[INFO|2025-04-21 17:36:54] modeling_utils.py:1633 >> Instantiating MistralForCausalLM model under default dtype torch.float16.
+
+[INFO|2025-04-21 17:36:54] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+[INFO|2025-04-21 17:37:57] modeling_utils.py:4970 >> All model checkpoint weights were used when initializing LlavaNextForConditionalGeneration.
+
+
+[INFO|2025-04-21 17:37:57] modeling_utils.py:4978 >> All the weights of LlavaNextForConditionalGeneration were initialized from the model checkpoint at llava-hf/llava-v1.6-mistral-7b-hf.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlavaNextForConditionalGeneration for predictions without further training.
+
+[INFO|2025-04-21 17:37:58] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/generation_config.json
+
+[INFO|2025-04-21 17:37:58] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Casting multimodal projector outputs in torch.float16.
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Fine-tuning method: LoRA
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Found linear modules: q_proj,v_proj,k_proj,gate_proj,up_proj,o_proj,down_proj
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Set vision model not trainable: ['vision_tower'].
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> Set multi model projector not trainable: multi_modal_projector.
+
+[INFO|2025-04-21 17:37:58] logging.py:157 >> trainable params: 20,971,520 || all params: 7,587,719,168 || trainable%: 0.2764
+
+[INFO|2025-04-21 17:37:58] trainer.py:746 >> Using auto half precision backend
+
+[WARNING|2025-04-21 17:37:58] trainer.py:781 >> No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.
+
+[INFO|2025-04-21 17:37:59] trainer.py:2405 >> ***** Running training *****
+
+[INFO|2025-04-21 17:37:59] trainer.py:2406 >>   Num examples = 2,415
+
+[INFO|2025-04-21 17:37:59] trainer.py:2407 >>   Num Epochs = 3
+
+[INFO|2025-04-21 17:37:59] trainer.py:2408 >>   Instantaneous batch size per device = 2
+
+[INFO|2025-04-21 17:37:59] trainer.py:2411 >>   Total train batch size (w. parallel, distributed & accumulation) = 16
+
+[INFO|2025-04-21 17:37:59] trainer.py:2412 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-04-21 17:37:59] trainer.py:2413 >>   Total optimization steps = 453
+
+[INFO|2025-04-21 17:37:59] trainer.py:2414 >>   Number of trainable parameters = 20,971,520
+
+[WARNING|2025-04-21 17:38:00] logging.py:329 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
+
+[INFO|2025-04-21 17:42:39] logging.py:157 >> {'loss': 10.8058, 'learning_rate': 1.9998e-04, 'epoch': 0.03, 'throughput': 350.83}
+
+[INFO|2025-04-21 17:47:20] logging.py:157 >> {'loss': 2.5613, 'learning_rate': 1.9985e-04, 'epoch': 0.07, 'throughput': 350.51}
+
+[INFO|2025-04-21 17:52:02] logging.py:157 >> {'loss': 1.4579, 'learning_rate': 1.9959e-04, 'epoch': 0.10, 'throughput': 350.51}
+
+[INFO|2025-04-21 17:56:43] logging.py:157 >> {'loss': 1.1407, 'learning_rate': 1.9922e-04, 'epoch': 0.13, 'throughput': 350.55}
+
+[INFO|2025-04-21 18:01:24] logging.py:157 >> {'loss': 0.7902, 'learning_rate': 1.9873e-04, 'epoch': 0.17, 'throughput': 350.53}
+
+[INFO|2025-04-21 18:06:04] logging.py:157 >> {'loss': 0.9830, 'learning_rate': 1.9812e-04, 'epoch': 0.20, 'throughput': 350.53}
+
+[INFO|2025-04-21 18:10:45] logging.py:157 >> {'loss': 0.9299, 'learning_rate': 1.9739e-04, 'epoch': 0.23, 'throughput': 350.57}
+
+[INFO|2025-04-21 18:15:26] logging.py:157 >> {'loss': 0.6314, 'learning_rate': 1.9655e-04, 'epoch': 0.26, 'throughput': 350.66}
+
+[INFO|2025-04-21 18:20:06] logging.py:157 >> {'loss': 0.8311, 'learning_rate': 1.9559e-04, 'epoch': 0.30, 'throughput': 350.69}
+
+[INFO|2025-04-21 18:24:46] logging.py:157 >> {'loss': 0.5493, 'learning_rate': 1.9451e-04, 'epoch': 0.33, 'throughput': 350.73}
+
+[INFO|2025-04-21 18:29:27] logging.py:157 >> {'loss': 0.4694, 'learning_rate': 1.9332e-04, 'epoch': 0.36, 'throughput': 350.78}
+
+[INFO|2025-04-21 18:34:07] logging.py:157 >> {'loss': 0.5595, 'learning_rate': 1.9202e-04, 'epoch': 0.40, 'throughput': 350.83}
+
+[INFO|2025-04-21 18:38:47] logging.py:157 >> {'loss': 0.2787, 'learning_rate': 1.9061e-04, 'epoch': 0.43, 'throughput': 350.84}
+
+[INFO|2025-04-21 18:43:28] logging.py:157 >> {'loss': 0.5269, 'learning_rate': 1.8971e-04, 'epoch': 0.46, 'throughput': 350.83}
+
+[INFO|2025-04-21 18:48:09] logging.py:157 >> {'loss': 0.7782, 'learning_rate': 1.8812e-04, 'epoch': 0.50, 'throughput': 350.82}
+
+[INFO|2025-04-21 18:52:48] logging.py:157 >> {'loss': 0.5458, 'learning_rate': 1.8643e-04, 'epoch': 0.53, 'throughput': 350.83}
+
+[INFO|2025-04-21 18:57:27] logging.py:157 >> {'loss': 0.3148, 'learning_rate': 1.8463e-04, 'epoch': 0.56, 'throughput': 350.84}
+
+[INFO|2025-04-21 19:02:08] logging.py:157 >> {'loss': 0.3010, 'learning_rate': 1.8274e-04, 'epoch': 0.60, 'throughput': 350.83}
+
+[INFO|2025-04-21 19:06:49] logging.py:157 >> {'loss': 0.6369, 'learning_rate': 1.8074e-04, 'epoch': 0.63, 'throughput': 350.82}
+
+[INFO|2025-04-21 19:09:03] trainer.py:2657 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-04-21 19:09:03] image_processing_base.py:261 >> Image processor saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/preprocessor_config.json
+
+[INFO|2025-04-21 19:09:03] tokenization_utils_base.py:2500 >> tokenizer config file saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/tokenizer_config.json
+
+[INFO|2025-04-21 19:09:03] tokenization_utils_base.py:2509 >> Special tokens file saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/special_tokens_map.json
+
+[INFO|2025-04-21 19:09:03] processing_utils.py:638 >> chat template saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/chat_template.json
+
+[INFO|2025-04-21 19:09:03] processing_utils.py:644 >> processor saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/processor_config.json
+
+[INFO|2025-04-21 19:09:03] trainer.py:3942 >> Saving model checkpoint to saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28
+
+[INFO|2025-04-21 19:09:03] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--llava-hf--llava-v1.6-mistral-7b-hf/snapshots/144bfb964d4eef1502a22af4c5ff20d0d4a94cc1/config.json
+
+[INFO|2025-04-21 19:09:03] configuration_utils.py:771 >> Model config LlavaNextConfig {
+  "architectures": [
+    "LlavaNextForConditionalGeneration"
+  ],
+  "ignore_index": -100,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_seq_length": 576,
+  "image_token_index": 32000,
+  "model_type": "llava_next",
+  "multimodal_projector_bias": true,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+    "architectures": [
+      "MistralForCausalLM"
+    ],
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "model_type": "mistral",
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 32064
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.49.0",
+  "use_image_newline_parameter": true,
+  "vision_config": {
+    "hidden_size": 1024,
+    "image_size": 336,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vocab_size": 32064
+}
+
+
+[INFO|2025-04-21 19:09:04] tokenization_utils_base.py:2500 >> tokenizer config file saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/tokenizer_config.json
+
+[INFO|2025-04-21 19:09:04] tokenization_utils_base.py:2509 >> Special tokens file saved in saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28/special_tokens_map.json
+
+[WARNING|2025-04-21 19:09:04] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-04-21 19:09:04] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-04-21 19:09:04] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {
+    "image_token": "<image>"
+  },
+  "image_token": "<image>",
+  "legacy": true,
+  "max_length": null,
+  "model_max_length": 2048,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "LlavaNextProcessor",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.6423841059602649,
+    "num_input_tokens_seen": 1916832,
+    "total_flos": 8.57490291718226e+16,
+    "train_loss": 1.3051114364997627,
+    "train_runtime": 5464.1136,
+    "train_samples_per_second": 1.326,
+    "train_steps_per_second": 0.083
+}

trainer_log.jsonl

@@ -0,0 +1,20 @@
+{"current_steps": 5, "total_steps": 453, "loss": 10.8058, "lr": 0.0001999783578606323, "epoch": 0.033112582781456956, "percentage": 1.1, "elapsed_time": "0:04:40", "remaining_time": "6:59:06", "throughput": 350.83, "total_tokens": 98464}
+{"current_steps": 10, "total_steps": 453, "loss": 2.5613, "lr": 0.00019984613426472932, "epoch": 0.06622516556291391, "percentage": 2.21, "elapsed_time": "0:09:21", "remaining_time": "6:54:42", "throughput": 350.51, "total_tokens": 196880}
+{"current_steps": 15, "total_steps": 453, "loss": 1.4579, "lr": 0.00019959386925858942, "epoch": 0.09933774834437085, "percentage": 3.31, "elapsed_time": "0:14:02", "remaining_time": "6:50:15", "throughput": 350.51, "total_tokens": 295472}
+{"current_steps": 20, "total_steps": 453, "loss": 1.1407, "lr": 0.0001992218661313415, "epoch": 0.13245033112582782, "percentage": 4.42, "elapsed_time": "0:18:44", "remaining_time": "6:45:39", "throughput": 350.55, "total_tokens": 394096}
+{"current_steps": 25, "total_steps": 453, "loss": 0.7902, "lr": 0.00019873057212894398, "epoch": 0.16556291390728478, "percentage": 5.52, "elapsed_time": "0:23:24", "remaining_time": "6:40:52", "throughput": 350.53, "total_tokens": 492464}
+{"current_steps": 30, "total_steps": 453, "loss": 0.983, "lr": 0.00019812057791647686, "epoch": 0.1986754966887417, "percentage": 6.62, "elapsed_time": "0:28:05", "remaining_time": "6:36:08", "throughput": 350.53, "total_tokens": 590896}
+{"current_steps": 35, "total_steps": 453, "loss": 0.9299, "lr": 0.0001973926168680066, "epoch": 0.23178807947019867, "percentage": 7.73, "elapsed_time": "0:32:46", "remaining_time": "6:31:23", "throughput": 350.57, "total_tokens": 689328}
+{"current_steps": 40, "total_steps": 453, "loss": 0.6314, "lr": 0.00019654756418487667, "epoch": 0.26490066225165565, "percentage": 8.83, "elapsed_time": "0:37:27", "remaining_time": "6:26:43", "throughput": 350.66, "total_tokens": 788032}
+{"current_steps": 45, "total_steps": 453, "loss": 0.8311, "lr": 0.00019558643584348476, "epoch": 0.2980132450331126, "percentage": 9.93, "elapsed_time": "0:42:06", "remaining_time": "6:21:50", "throughput": 350.69, "total_tokens": 886144}
+{"current_steps": 50, "total_steps": 453, "loss": 0.5493, "lr": 0.00019451038737381077, "epoch": 0.33112582781456956, "percentage": 11.04, "elapsed_time": "0:46:47", "remaining_time": "6:17:08", "throughput": 350.73, "total_tokens": 984672}
+{"current_steps": 55, "total_steps": 453, "loss": 0.4694, "lr": 0.00019332071247016476, "epoch": 0.36423841059602646, "percentage": 12.14, "elapsed_time": "0:51:28", "remaining_time": "6:12:26", "throughput": 350.78, "total_tokens": 1083232}
+{"current_steps": 60, "total_steps": 453, "loss": 0.5595, "lr": 0.00019201884143582495, "epoch": 0.3973509933774834, "percentage": 13.25, "elapsed_time": "0:56:07", "remaining_time": "6:07:39", "throughput": 350.83, "total_tokens": 1181568}
+{"current_steps": 65, "total_steps": 453, "loss": 0.2787, "lr": 0.0001906063394634356, "epoch": 0.4304635761589404, "percentage": 14.35, "elapsed_time": "1:00:48", "remaining_time": "6:02:56", "throughput": 350.84, "total_tokens": 1279936}
+{"current_steps": 70, "total_steps": 453, "loss": 0.5269, "lr": 0.00018970643640796642, "epoch": 0.46357615894039733, "percentage": 15.45, "elapsed_time": "1:05:29", "remaining_time": "5:58:19", "throughput": 350.83, "total_tokens": 1378544}
+{"current_steps": 75, "total_steps": 453, "loss": 0.7782, "lr": 0.00018812051176267307, "epoch": 0.4966887417218543, "percentage": 16.56, "elapsed_time": "1:10:09", "remaining_time": "5:53:37", "throughput": 350.82, "total_tokens": 1476896}
+{"current_steps": 80, "total_steps": 453, "loss": 0.5458, "lr": 0.00018642864300065767, "epoch": 0.5298013245033113, "percentage": 17.66, "elapsed_time": "1:14:49", "remaining_time": "5:48:50", "throughput": 350.83, "total_tokens": 1574912}
+{"current_steps": 85, "total_steps": 453, "loss": 0.3148, "lr": 0.00018463286419478255, "epoch": 0.5629139072847682, "percentage": 18.76, "elapsed_time": "1:19:28", "remaining_time": "5:44:05", "throughput": 350.84, "total_tokens": 1673056}
+{"current_steps": 90, "total_steps": 453, "loss": 0.301, "lr": 0.00018273533434521263, "epoch": 0.5960264900662252, "percentage": 19.87, "elapsed_time": "1:24:09", "remaining_time": "5:39:26", "throughput": 350.83, "total_tokens": 1771536}
+{"current_steps": 95, "total_steps": 453, "loss": 0.6369, "lr": 0.0001807383347837268, "epoch": 0.6291390728476821, "percentage": 20.97, "elapsed_time": "1:28:50", "remaining_time": "5:34:46", "throughput": 350.82, "total_tokens": 1869952}
+{"current_steps": 97, "total_steps": 453, "epoch": 0.6423841059602649, "percentage": 21.41, "elapsed_time": "1:31:04", "remaining_time": "5:34:13", "throughput": 350.8, "total_tokens": 1916832}

trainer_state.json

@@ -0,0 +1,195 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6423841059602649,
+  "eval_steps": 500,
+  "global_step": 97,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.033112582781456956,
+      "grad_norm": 26.791088104248047,
+      "learning_rate": 0.0001999783578606323,
+      "loss": 10.8058,
+      "num_input_tokens_seen": 98464,
+      "step": 5
+    },
+    {
+      "epoch": 0.06622516556291391,
+      "grad_norm": 22.559255599975586,
+      "learning_rate": 0.00019984613426472932,
+      "loss": 2.5613,
+      "num_input_tokens_seen": 196880,
+      "step": 10
+    },
+    {
+      "epoch": 0.09933774834437085,
+      "grad_norm": 8.285969734191895,
+      "learning_rate": 0.00019959386925858942,
+      "loss": 1.4579,
+      "num_input_tokens_seen": 295472,
+      "step": 15
+    },
+    {
+      "epoch": 0.13245033112582782,
+      "grad_norm": 7.880527496337891,
+      "learning_rate": 0.0001992218661313415,
+      "loss": 1.1407,
+      "num_input_tokens_seen": 394096,
+      "step": 20
+    },
+    {
+      "epoch": 0.16556291390728478,
+      "grad_norm": 14.62392807006836,
+      "learning_rate": 0.00019873057212894398,
+      "loss": 0.7902,
+      "num_input_tokens_seen": 492464,
+      "step": 25
+    },
+    {
+      "epoch": 0.1986754966887417,
+      "grad_norm": 12.557646751403809,
+      "learning_rate": 0.00019812057791647686,
+      "loss": 0.983,
+      "num_input_tokens_seen": 590896,
+      "step": 30
+    },
+    {
+      "epoch": 0.23178807947019867,
+      "grad_norm": 12.462843894958496,
+      "learning_rate": 0.0001973926168680066,
+      "loss": 0.9299,
+      "num_input_tokens_seen": 689328,
+      "step": 35
+    },
+    {
+      "epoch": 0.26490066225165565,
+      "grad_norm": 3.7140164375305176,
+      "learning_rate": 0.00019654756418487667,
+      "loss": 0.6314,
+      "num_input_tokens_seen": 788032,
+      "step": 40
+    },
+    {
+      "epoch": 0.2980132450331126,
+      "grad_norm": 16.123748779296875,
+      "learning_rate": 0.00019558643584348476,
+      "loss": 0.8311,
+      "num_input_tokens_seen": 886144,
+      "step": 45
+    },
+    {
+      "epoch": 0.33112582781456956,
+      "grad_norm": 7.482938289642334,
+      "learning_rate": 0.00019451038737381077,
+      "loss": 0.5493,
+      "num_input_tokens_seen": 984672,
+      "step": 50
+    },
+    {
+      "epoch": 0.36423841059602646,
+      "grad_norm": 4.410764694213867,
+      "learning_rate": 0.00019332071247016476,
+      "loss": 0.4694,
+      "num_input_tokens_seen": 1083232,
+      "step": 55
+    },
+    {
+      "epoch": 0.3973509933774834,
+      "grad_norm": 6.899860858917236,
+      "learning_rate": 0.00019201884143582495,
+      "loss": 0.5595,
+      "num_input_tokens_seen": 1181568,
+      "step": 60
+    },
+    {
+      "epoch": 0.4304635761589404,
+      "grad_norm": 3.364258050918579,
+      "learning_rate": 0.0001906063394634356,
+      "loss": 0.2787,
+      "num_input_tokens_seen": 1279936,
+      "step": 65
+    },
+    {
+      "epoch": 0.46357615894039733,
+      "grad_norm": 20.896175384521484,
+      "learning_rate": 0.00018970643640796642,
+      "loss": 0.5269,
+      "num_input_tokens_seen": 1378544,
+      "step": 70
+    },
+    {
+      "epoch": 0.4966887417218543,
+      "grad_norm": 3.4167935848236084,
+      "learning_rate": 0.00018812051176267307,
+      "loss": 0.7782,
+      "num_input_tokens_seen": 1476896,
+      "step": 75
+    },
+    {
+      "epoch": 0.5298013245033113,
+      "grad_norm": 10.354905128479004,
+      "learning_rate": 0.00018642864300065767,
+      "loss": 0.5458,
+      "num_input_tokens_seen": 1574912,
+      "step": 80
+    },
+    {
+      "epoch": 0.5629139072847682,
+      "grad_norm": 3.3909523487091064,
+      "learning_rate": 0.00018463286419478255,
+      "loss": 0.3148,
+      "num_input_tokens_seen": 1673056,
+      "step": 85
+    },
+    {
+      "epoch": 0.5960264900662252,
+      "grad_norm": 13.916143417358398,
+      "learning_rate": 0.00018273533434521263,
+      "loss": 0.301,
+      "num_input_tokens_seen": 1771536,
+      "step": 90
+    },
+    {
+      "epoch": 0.6291390728476821,
+      "grad_norm": 4.097564697265625,
+      "learning_rate": 0.0001807383347837268,
+      "loss": 0.6369,
+      "num_input_tokens_seen": 1869952,
+      "step": 95
+    },
+    {
+      "epoch": 0.6423841059602649,
+      "num_input_tokens_seen": 1916832,
+      "step": 97,
+      "total_flos": 8.57490291718226e+16,
+      "train_loss": 1.3051114364997627,
+      "train_runtime": 5464.1136,
+      "train_samples_per_second": 1.326,
+      "train_steps_per_second": 0.083
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 453,
+  "num_input_tokens_seen": 1916832,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.57490291718226e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fa7685ef62ad79b8a271f32dc871c3861c9fd6b862e0819d6ca0e143f6d38cc
+size 5688

training_args.yaml

@@ -0,0 +1,36 @@
+cutoff_len: 2048
+dataset: docker_command_dataset
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+double_quantization: true
+finetuning_type: lora
+flash_attn: auto
+fp16: true
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 0.0002
+logging_steps: 5
+lora_alpha: 16
+lora_dropout: 0
+lora_rank: 8
+lora_target: all
+lr_scheduler_type: cosine
+max_grad_norm: 0.3
+max_samples: 100000
+model_name_or_path: llava-hf/llava-v1.6-mistral-7b-hf
+num_train_epochs: 3.0
+optim: adamw_torch
+output_dir: saves/LLaVA-NeXT-Mistral-7B-Chat/lora/train_2025-04-21-17-35-28
+packing: false
+per_device_train_batch_size: 2
+plot_loss: true
+preprocessing_num_workers: 16
+quantization_bit: 4
+quantization_method: bitsandbytes
+report_to: none
+save_steps: 100
+stage: sft
+template: llava_next_mistral
+trust_remote_code: true
+warmup_steps: 0