Hugging Face's logo

from-our-page
/
BigLlama-2-120B

Text Generation
Transformers
Safetensors
llama
mergekit
Merge
text-generation-inference
Model card Files Community
ampdot commited on
Commit
bfc39fc
·
verified ·
1 Parent(s): f1e27dc

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +55 -0
  2. config.json +29 -0
  3. mergekit_config.yml +25 -0
  4. model-00001-of-00050.safetensors +3 -0
  5. model-00002-of-00050.safetensors +3 -0
  6. model-00003-of-00050.safetensors +3 -0
  7. model-00004-of-00050.safetensors +3 -0
  8. model-00005-of-00050.safetensors +3 -0
  9. model-00006-of-00050.safetensors +3 -0
  10. model-00007-of-00050.safetensors +3 -0
  11. model-00008-of-00050.safetensors +3 -0
  12. model-00009-of-00050.safetensors +3 -0
  13. model-00010-of-00050.safetensors +3 -0
  14. model-00011-of-00050.safetensors +3 -0
  15. model-00012-of-00050.safetensors +3 -0
  16. model-00013-of-00050.safetensors +3 -0
  17. model-00014-of-00050.safetensors +3 -0
  18. model-00015-of-00050.safetensors +3 -0
  19. model-00016-of-00050.safetensors +3 -0
  20. model-00017-of-00050.safetensors +3 -0
  21. model-00018-of-00050.safetensors +3 -0
  22. model-00019-of-00050.safetensors +3 -0
  23. model-00020-of-00050.safetensors +3 -0
  24. model-00021-of-00050.safetensors +3 -0
  25. model-00022-of-00050.safetensors +3 -0
  26. model-00023-of-00050.safetensors +3 -0
  27. model-00024-of-00050.safetensors +3 -0
  28. model-00025-of-00050.safetensors +3 -0
  29. model-00026-of-00050.safetensors +3 -0
  30. model-00027-of-00050.safetensors +3 -0
  31. model-00028-of-00050.safetensors +3 -0
  32. model-00029-of-00050.safetensors +3 -0
  33. model-00030-of-00050.safetensors +3 -0
  34. model-00031-of-00050.safetensors +3 -0
  35. model-00032-of-00050.safetensors +3 -0
  36. model-00033-of-00050.safetensors +3 -0
  37. model-00034-of-00050.safetensors +3 -0
  38. model-00035-of-00050.safetensors +3 -0
  39. model-00036-of-00050.safetensors +3 -0
  40. model-00037-of-00050.safetensors +3 -0
  41. model-00038-of-00050.safetensors +3 -0
  42. model-00039-of-00050.safetensors +3 -0
  43. model-00040-of-00050.safetensors +3 -0
  44. model-00041-of-00050.safetensors +3 -0
  45. model-00042-of-00050.safetensors +3 -0
  46. model-00043-of-00050.safetensors +3 -0
  47. model-00044-of-00050.safetensors +3 -0
  48. model-00045-of-00050.safetensors +3 -0
  49. model-00046-of-00050.safetensors +3 -0
  50. model-00047-of-00050.safetensors +3 -0
README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model:
3
+ - meta-llama/Llama-2-70b-hf
4
+ library_name: transformers
5
+ tags:
6
+ - mergekit
7
+ - merge
8
+
9
+ ---
10
+ # BigLlama-2-120B
11
+
12
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
13
+
14
+ ## Merge Details
15
+ ### Merge Method
16
+
17
+ This model was merged using the Passthrough merge method.
18
+
19
+ ### Models Merged
20
+
21
+ The following models were included in the merge:
22
+ * [meta-llama/Llama-2-70b-hf](https://huggingface.co/meta-llama/Llama-2-70b-hf)
23
+
24
+ ### Configuration
25
+
26
+ The following YAML configuration was used to produce this model:
27
+
28
+ ```yaml
29
+ slices:
30
+ - sources:
31
+ - layer_range: [0, 20]
32
+ model: meta-llama/Llama-2-70b-hf
33
+ - sources:
34
+ - layer_range: [10, 30]
35
+ model: meta-llama/Llama-2-70b-hf
36
+ - sources:
37
+ - layer_range: [20, 40]
38
+ model: meta-llama/Llama-2-70b-hf
39
+ - sources:
40
+ - layer_range: [30, 50]
41
+ model: meta-llama/Llama-2-70b-hf
42
+ - sources:
43
+ - layer_range: [40, 60]
44
+ model: meta-llama/Llama-2-70b-hf
45
+ - sources:
46
+ - layer_range: [50, 70]
47
+ model: meta-llama/Llama-2-70b-hf
48
+ - sources:
49
+ - layer_range: [60, 80]
50
+ model: meta-llama/Llama-2-70b-hf
51
+ merge_method: passthrough
52
+ dtype: float16
53
+
54
+
55
+ ```
config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "head_dim": 128,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 8192,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 28672,
14
+ "max_position_embeddings": 4096,
15
+ "mlp_bias": false,
16
+ "model_type": "llama",
17
+ "num_attention_heads": 64,
18
+ "num_hidden_layers": 140,
19
+ "num_key_value_heads": 8,
20
+ "pretraining_tp": 1,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_scaling": null,
23
+ "rope_theta": 10000.0,
24
+ "tie_word_embeddings": false,
25
+ "torch_dtype": "float16",
26
+ "transformers_version": "4.51.1",
27
+ "use_cache": true,
28
+ "vocab_size": 32000
29
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ slices:
2
+ - sources:
3
+ - layer_range: [0, 20]
4
+ model: meta-llama/Llama-2-70b-hf
5
+ - sources:
6
+ - layer_range: [10, 30]
7
+ model: meta-llama/Llama-2-70b-hf
8
+ - sources:
9
+ - layer_range: [20, 40]
10
+ model: meta-llama/Llama-2-70b-hf
11
+ - sources:
12
+ - layer_range: [30, 50]
13
+ model: meta-llama/Llama-2-70b-hf
14
+ - sources:
15
+ - layer_range: [40, 60]
16
+ model: meta-llama/Llama-2-70b-hf
17
+ - sources:
18
+ - layer_range: [50, 70]
19
+ model: meta-llama/Llama-2-70b-hf
20
+ - sources:
21
+ - layer_range: [60, 80]
22
+ model: meta-llama/Llama-2-70b-hf
23
+ merge_method: passthrough
24
+ dtype: float16
25
+
model-00001-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf7323537f811f1070f6b9a77a7b6f80bf15161c91e0286bc97436b44d065c8f
3
+ size 4940991048
model-00002-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7332fcf67f0d3415f825d22e679a0a4d56dee928251bc96f5a69fb41f72a29ac
3
+ size 4831906200
model-00003-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c2600bfe81b2a36757f2e6435f47fe36cbef536f4f954603522abf5cbae6f6d
3
+ size 4966190736
model-00004-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2353caac7b690e32acba259b69b1912e518eb2543bf2a60f7cf0632b457153e
3
+ size 4831906200
model-00005-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a8c4cab4da1fc22ab57982048244aef850f037b8fdea598ab5e67047d980d4c
3
+ size 4999711712
model-00006-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce01fac975348b691fc24ea2a97b6af9036aab66d3663da7b55be4395d4969ee
3
+ size 4798385224
model-00007-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee228028b98b52fb9678e542b960dc6c027b8ed3f54fbf68e39ec2d571e0dce6
3
+ size 4831906200
model-00008-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01586e3f55b6c82db1494adc236ae741631fac9f2240b019337e99a04f0f9787
3
+ size 4966174240
model-00009-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f68a567edac1697d08f7b552a74760179e473a8d0a854a0cc83f668f702467a
3
+ size 4664183880
model-00010-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e40e478636d46cabc9419086b29c8d2d2fdb5afec875494f2d564b332b42e5b
3
+ size 4831906200
model-00011-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a7db1ab50d4dc500b582cb5d0eff1b12b4d79c292541bfdb749345ab80dbcf3
3
+ size 4999711712
model-00012-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:298a3759d9bacc5ea8ec2f8a0dcae21ab14a0d71988d6147a128fbc38e1e2010
3
+ size 4798385224
model-00013-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2c923b914776675e31c3c39e093d369de5b8fddfae4b004569fc447b6215fbb
3
+ size 4831906200
model-00014-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5175d93d506fc5cb9d27e5618a3105476217569020b6b3b98b5c137624c90620
3
+ size 4966190736
model-00015-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:641fbe1dfae1c43ade39042a8e63224886bafedcb0d97d7ba78dd8bbe3aa408b
3
+ size 4831906200
model-00016-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6be82bb122e359a7132ff5b2488393771c3848638452c991572de1b8c27da1d
3
+ size 4664167376
model-00017-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a786ea047c85e898ad6d23a8a3d62c749ef7187b8766728178b14d40f35f9c36
3
+ size 4999711712
model-00018-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22198e24f9b8712ed9e96f6e64ec3d8fc47de9b682c8fe12a8ffb49ac2e1def3
3
+ size 4798385224
model-00019-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e1377091cc754b3fdf11b703e1f35b2ef51fe52d2c4a9cf602e72fbe0104e7b
3
+ size 4831906200
model-00020-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:743fbbf1e3c3a08777dfba8b680b83b20d318adf7ee5ad49097ed80981f57537
3
+ size 4966190736
model-00021-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36c129026ce2db67059bc99d2d87388b3673e818aebe0347bdbdccac4ec47b95
3
+ size 4831906200
model-00022-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65ac80857a9bd77bd0a6493d63e9d960398f358c07122b145989de251d59a9fb
3
+ size 4966190736
model-00023-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c576afb440944c71dcc079f054491e54fe32da412e7e62307ca9a9661eab0d2d
3
+ size 4982901368
model-00024-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d4f788f6a86d32471aef7b847bdf9be322aa650a94f5371fbf0c2735b7a54f9
3
+ size 4982934368
model-00025-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7c2a2c080a2208096914da4c7bbb0af6910e518b9db7652fbd23adfeb007435
3
+ size 4999711712
model-00026-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce4ecb8e05923258493d0a6a177461e2b73cda4f9dc913b69546c05d5a9a7e31
3
+ size 4798385224
model-00027-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b6deaedce6062dcb001f033c6d215cb308f9188b79f1f4bcc0838aa0effb511
3
+ size 4831906200
model-00028-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b81d3c732e512ca532ffe3fff392abda233db4fa233f7f16efc2699c505edcd
3
+ size 4966190736
model-00029-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70010e8420b87ac4d8d046fe67fb56c8e3687f97c900e636cfe89d9be2051530
3
+ size 4831906200
model-00030-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bd9959c7e73edd4df2bef5e67ad6fe11e68292a86066c7e4be13ff9761224cc
3
+ size 4966174232
model-00031-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d04cb861916a484e8301c29288346b6660504fff5926837762ab001b989c21e4
3
+ size 4966140544
model-00032-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbaf33cdd2b9b5e982cb43b465c85aabc8d7850d8cbb12ee3921403290685fb4
3
+ size 4999711712
model-00033-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb715d0f1b79242bfb88cbc64b42141969abfb526532fd80942904ef3069ae93
3
+ size 4999711720
model-00034-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8bfb87747ea5963626976809df22341e63593a0796ca13de48688ec1f0fcc43
3
+ size 4798385240
model-00035-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a2f3ba50cf42ceb8e4e26599b84267fae3671e2bd9389d14881956c0d7ad25c
3
+ size 4831906208
model-00036-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:046ae704bf9c3d52819c5fecbb02475912986d81c1f9f30f7b898071ff53fb82
3
+ size 4966190744
model-00037-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e90f27cddf8b5a150051f8685e24b74cca2967f26140a8adcc82ef02dc0e2e94
3
+ size 4831906208
model-00038-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e483fe7057140b763efc75e1696b2ef5b4e32ae0b17bd320d40061440eca37d
3
+ size 4664167384
model-00039-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9d2b20696f172b38363548842ace79de0a4c8108cdc890c916a329bb354b63e
3
+ size 4999711744
model-00040-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3adf34b927428c31397c7fcc5498b0bdfd384de5d6dbf87cc99cf16b2d42ddf0
3
+ size 4798385256
model-00041-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d10a4000e3a97c65ab0dbffc8f60a66c158dbb5fcd1cb34870a2f5e9b22f895
3
+ size 4831906216
model-00042-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f466dd7357fb60e99f5230b8474866230daa099621cfcb4e15b02955951a9475
3
+ size 4966190768
model-00043-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76057d09796d4725ef3a28656509b9183ef4c0d20486e9efc94ae2820b6d107f
3
+ size 4831906224
model-00044-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0ee2fb67ddea9fbb0e7afca94c4a8898860aa32fd43f42d5f12d0f436071878
3
+ size 4966190768
model-00045-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2edee6adffccce9f26cd549960931bd6bf2959dc5a209441c9a7700d23b0b65
3
+ size 4982901400
model-00046-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a838b39f282fff8071bd36fd71ad723053bff05dd71bd0959bb93fefecb288d2
3
+ size 4999711720
model-00047-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6adac4067021d09a2af47cad40aa168d36068a776db8bff910c0f559966eb9f8
3
+ size 4647373576