Hugging Face's logo

AndreaUnibo
/
NQ_JetMoE_rank_lstm_train_depth3_n2

Safetensors
Model card Files Community
AndreaUnibo commited on
Commit
1005ce2
·
verified ·
1 Parent(s): 6878033

Training in progress, step 1000

Browse files
Files changed (4) hide show
  1. adapter_config.json +30 -30
  2. adapter_model.safetensors +1 -1
  3. tokenizer.json +6 -1
  4. training_args.bin +1 -1
adapter_config.json CHANGED
@@ -20,45 +20,45 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "model.layers.2.self_attention.kv_proj",
24
- "model.lstm_gates.0.input_gate",
25
- "model.lstm_gates.0.forget_gate",
26
- "model.layers.13.self_attention.kv_proj",
27
- "model.layers.11.self_attention.kv_proj",
28
- "model.layers.0.self_attention.kv_proj",
29
- "model.lstm_gates.1.forget_gate",
30
- "model.layers.19.mlp.router.layer",
31
- "model.layers.20.self_attention.kv_proj",
32
  "model.layers.14.self_attention.kv_proj",
33
- "model.layers.18.mlp.router.layer",
34
- "model.layers.19.self_attention.experts.router.layer",
 
 
35
  "model.layers.1.mlp.router.perturbation",
36
- "model.layers.12.self_attention.kv_proj",
 
37
  "model.layers.15.self_attention.kv_proj",
 
 
 
38
  "model.layers.10.self_attention.kv_proj",
39
- "model.layers.1.self_attention.kv_proj",
40
- "model.layers.19.self_attention.kv_proj",
41
- "model.lstm_gates.1.input_gate",
42
- "model.layers.23.self_attention.kv_proj",
43
- "model.layers.4.self_attention.kv_proj",
44
- "model.layers.16.self_attention.kv_proj",
45
- "model.layers.1.mlp.router.layer",
46
- "model.layers.1.self_attention.experts.router.layer",
47
  "model.layers.18.self_attention.kv_proj",
 
 
 
 
48
  "model.layers.18.self_attention.experts.router.layer",
49
- "model.layers.5.self_attention.kv_proj",
50
- "model.layers.9.self_attention.kv_proj",
51
- "model.lstm_gates.0.output_gate",
52
- "model.lstm_gates.1.transform",
53
  "model.lstm_gates.1.output_gate",
54
- "model.layers.3.self_attention.kv_proj",
55
- "model.layers.21.self_attention.kv_proj",
56
  "model.lstm_gates.0.transform",
57
- "model.layers.7.self_attention.kv_proj",
 
 
 
58
  "model.layers.8.self_attention.kv_proj",
59
- "model.layers.17.self_attention.kv_proj",
60
- "model.layers.6.self_attention.kv_proj",
61
- "model.layers.22.self_attention.kv_proj"
 
 
 
 
 
 
 
 
 
62
  ],
63
  "task_type": "CAUSAL_LM",
64
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
 
 
 
 
 
 
 
23
  "model.layers.14.self_attention.kv_proj",
24
+ "model.layers.9.self_attention.kv_proj",
25
+ "model.lstm_gates.1.forget_gate",
26
+ "model.lstm_gates.0.input_gate",
27
+ "model.layers.6.self_attention.kv_proj",
28
  "model.layers.1.mlp.router.perturbation",
29
+ "model.layers.5.self_attention.kv_proj",
30
+ "model.lstm_gates.1.transform",
31
  "model.layers.15.self_attention.kv_proj",
32
+ "model.layers.17.self_attention.kv_proj",
33
+ "model.layers.20.self_attention.kv_proj",
34
+ "model.layers.18.mlp.router.layer",
35
  "model.layers.10.self_attention.kv_proj",
 
 
 
 
 
 
 
 
36
  "model.layers.18.self_attention.kv_proj",
37
+ "model.layers.19.mlp.router.layer",
38
+ "model.layers.22.self_attention.kv_proj",
39
+ "model.layers.13.self_attention.kv_proj",
40
+ "model.layers.7.self_attention.kv_proj",
41
  "model.layers.18.self_attention.experts.router.layer",
42
+ "model.layers.11.self_attention.kv_proj",
 
 
 
43
  "model.lstm_gates.1.output_gate",
 
 
44
  "model.lstm_gates.0.transform",
45
+ "model.lstm_gates.1.input_gate",
46
+ "model.layers.12.self_attention.kv_proj",
47
+ "model.lstm_gates.0.output_gate",
48
+ "model.layers.0.self_attention.kv_proj",
49
  "model.layers.8.self_attention.kv_proj",
50
+ "model.layers.1.mlp.router.layer",
51
+ "model.layers.19.self_attention.kv_proj",
52
+ "model.layers.1.self_attention.kv_proj",
53
+ "model.layers.1.self_attention.experts.router.layer",
54
+ "model.layers.23.self_attention.kv_proj",
55
+ "model.layers.16.self_attention.kv_proj",
56
+ "model.layers.19.self_attention.experts.router.layer",
57
+ "model.layers.21.self_attention.kv_proj",
58
+ "model.layers.4.self_attention.kv_proj",
59
+ "model.layers.3.self_attention.kv_proj",
60
+ "model.lstm_gates.0.forget_gate",
61
+ "model.layers.2.self_attention.kv_proj"
62
  ],
63
  "task_type": "CAUSAL_LM",
64
  "use_dora": false,
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:58a4ddddb426ade54feb5e08c12032d6cf63981b7603a3ca8b216d49fa72a0d7
3
  size 6303624
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e18f8d0f04672204db022133c5c3c8f57a26502da01a2ed8c4d182bef16f7279
3
  size 6303624
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 600,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": null,
10
  "added_tokens": [
11
  {
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:be7b21be1fa18703adb0dc65b197e01ad9b705ef6f3ef7718f6ad9b2f05df9fe
3
  size 5176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:460d887ec2d1c325b8ea84a7abc2ead2cce6cc5954c0522d4600a6814e67cf27
3
  size 5176