Hugging Face's logo

danielhanchen
/
llama-vision-new

Text Generation
Transformers
Safetensors
llama
text-generation-inference
4-bit precision
bitsandbytes
Model card Files Community
danielhanchen commited on
Commit
d595a4b
·
verified ·
1 Parent(s): 54e11c2

Upload LlamaForCausalLM

Browse files
Files changed (3) hide show
  1. config.json +35 -100
  2. generation_config.json +1 -0
  3. model.safetensors +3 -0
config.json CHANGED
@@ -1,9 +1,24 @@
1
  {
2
  "architectures": [
3
- "MllamaForConditionalGeneration"
4
  ],
5
- "image_token_index": 128256,
6
- "model_type": "mllama",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  "quantization_config": {
8
  "_load_in_4bit": true,
9
  "_load_in_8bit": false,
@@ -13,110 +28,30 @@
13
  "bnb_4bit_use_double_quant": true,
14
  "llm_int8_enable_fp32_cpu_offload": false,
15
  "llm_int8_has_fp16_weight": false,
16
- "llm_int8_skip_modules": null,
 
 
 
 
 
17
  "llm_int8_threshold": 6.0,
18
  "load_in_4bit": true,
19
  "load_in_8bit": false,
20
  "quant_method": "bitsandbytes"
21
  },
22
- "text_config": {
23
- "cross_attention_layers": [
24
- 3,
25
- 8,
26
- 13,
27
- 18,
28
- 23,
29
- 28,
30
- 33,
31
- 38
32
- ],
33
- "dropout": 0,
34
- "eos_token_id": [
35
- 128001,
36
- 128008,
37
- 128009
38
- ],
39
- "hidden_act": "silu",
40
- "hidden_size": 4096,
41
- "initializer_range": 0.02,
42
- "intermediate_size": 14336,
43
- "max_position_embeddings": 131072,
44
- "model_type": "mllama_text_model",
45
- "num_attention_heads": 32,
46
- "num_hidden_layers": 40,
47
- "num_key_value_heads": 8,
48
- "rms_norm_eps": 1e-05,
49
- "rope_scaling": {
50
- "factor": 8.0,
51
- "high_freq_factor": 4.0,
52
- "low_freq_factor": 1.0,
53
- "original_max_position_embeddings": 8192,
54
- "rope_type": "llama3"
55
- },
56
- "rope_theta": 500000.0,
57
- "torch_dtype": "float16",
58
- "use_cache": true,
59
- "vocab_size": 128256
60
  },
 
 
61
  "torch_dtype": "float16",
62
  "transformers_version": "4.50.0.dev0",
63
  "unsloth_fixed": true,
64
- "vision_config": {
65
- "attention_heads": 16,
66
- "hidden_act": "gelu",
67
- "hidden_size": 1280,
68
- "image_size": 560,
69
- "initializer_range": 0.02,
70
- "intermediate_layers_indices": [
71
- 3,
72
- 7,
73
- 15,
74
- 23,
75
- 30
76
- ],
77
- "intermediate_size": 5120,
78
- "max_num_tiles": 4,
79
- "model_type": "mllama_vision_model",
80
- "norm_eps": 1e-05,
81
- "num_channels": 3,
82
- "num_global_layers": 8,
83
- "num_hidden_layers": 32,
84
- "patch_size": 14,
85
- "supported_aspect_ratios": [
86
- [
87
- 1,
88
- 1
89
- ],
90
- [
91
- 1,
92
- 2
93
- ],
94
- [
95
- 1,
96
- 3
97
- ],
98
- [
99
- 1,
100
- 4
101
- ],
102
- [
103
- 2,
104
- 1
105
- ],
106
- [
107
- 2,
108
- 2
109
- ],
110
- [
111
- 3,
112
- 1
113
- ],
114
- [
115
- 4,
116
- 1
117
- ]
118
- ],
119
- "torch_dtype": "float16",
120
- "vision_output_dim": 7680
121
- }
122
  }
 
1
  {
2
  "architectures": [
3
+ "LlamaForCausalLM"
4
  ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 128000,
8
+ "eos_token_id": 128009,
9
+ "head_dim": 64,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 8192,
14
+ "max_position_embeddings": 131072,
15
+ "mlp_bias": false,
16
+ "model_type": "llama",
17
+ "num_attention_heads": 32,
18
+ "num_hidden_layers": 16,
19
+ "num_key_value_heads": 8,
20
+ "pad_token_id": 128004,
21
+ "pretraining_tp": 1,
22
  "quantization_config": {
23
  "_load_in_4bit": true,
24
  "_load_in_8bit": false,
 
28
  "bnb_4bit_use_double_quant": true,
29
  "llm_int8_enable_fp32_cpu_offload": false,
30
  "llm_int8_has_fp16_weight": false,
31
+ "llm_int8_skip_modules": [
32
+ "lm_head",
33
+ "multi_modal_projector",
34
+ "merger",
35
+ "modality_projection"
36
+ ],
37
  "llm_int8_threshold": 6.0,
38
  "load_in_4bit": true,
39
  "load_in_8bit": false,
40
  "quant_method": "bitsandbytes"
41
  },
42
+ "rms_norm_eps": 1e-05,
43
+ "rope_scaling": {
44
+ "factor": 32.0,
45
+ "high_freq_factor": 4.0,
46
+ "low_freq_factor": 1.0,
47
+ "original_max_position_embeddings": 8192,
48
+ "rope_type": "llama3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  },
50
+ "rope_theta": 500000.0,
51
+ "tie_word_embeddings": true,
52
  "torch_dtype": "float16",
53
  "transformers_version": "4.50.0.dev0",
54
  "unsloth_fixed": true,
55
+ "use_cache": true,
56
+ "vocab_size": 128256
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  }
generation_config.json CHANGED
@@ -6,6 +6,7 @@
6
  128008,
7
  128009
8
  ],
 
9
  "pad_token_id": 128004,
10
  "temperature": 0.6,
11
  "top_p": 0.9,
 
6
  128008,
7
  128009
8
  ],
9
+ "max_length": 131072,
10
  "pad_token_id": 128004,
11
  "temperature": 0.6,
12
  "top_p": 0.9,
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd9396be0bcca6a235d24591ca1d35892da09b7134513adf6791a97683670cca
3
+ size 1027676697