dev-v2-20240827022505

README.md

@@ -1,7 +1,7 @@
 ---
 library_name: transformers
-license: mit
-base_model: microsoft/deberta-v2-xlarge-mnli
+license: apache-2.0
+base_model: tasksource/deberta-small-long-nli
 tags:
 - generated_from_trainer
 metrics:
@@ -19,14 +19,14 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ms-deberta-v2-xlarge-mnli-finetuned-pt
 
-This model is a fine-tuned version of [microsoft/deberta-v2-xlarge-mnli](https://huggingface.co/microsoft/deberta-v2-xlarge-mnli) on the None dataset.
+This model is a fine-tuned version of [tasksource/deberta-small-long-nli](https://huggingface.co/tasksource/deberta-small-long-nli) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7274
-- Accuracy: 0.8571
-- Precision: 0.4286
-- Recall: 0.5
-- F1: 0.4615
-- Ratio: 0.0
+- Loss: 0.2954
+- Accuracy: 1.0
+- Precision: 1.0
+- Recall: 1.0
+- F1: 1.0
+- Ratio: 0.11
 
 ## Model description
 
@@ -46,23 +46,64 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 4
-- eval_batch_size: 4
+- train_batch_size: 16
+- eval_batch_size: 16
 - seed: 42
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 8
+- total_train_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.06
 - lr_scheduler_warmup_steps: 4
-- num_epochs: 3
+- num_epochs: 1
 - label_smoothing_factor: 0.1
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Accuracy | Precision | Recall | F1     | Ratio |
-|:-------------:|:------:|:----:|:---------------:|:--------:|:---------:|:------:|:------:|:-----:|
-| 0.8996        | 1.5385 | 10   | 0.6120          | 0.8571   | 0.4286    | 0.5    | 0.4615 | 0.0   |
+| Training Loss | Epoch  | Step | Validation Loss | Accuracy | Precision | Recall | F1     | Ratio  |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:---------:|:------:|:------:|:------:|
+| 1.4129        | 0.0237 | 10   | 0.5425          | 0.89     | 0.445     | 0.5    | 0.4709 | 0.0    |
+| 0.5102        | 0.0474 | 20   | 0.4968          | 0.89     | 0.445     | 0.5    | 0.4709 | 0.0    |
+| 0.4597        | 0.0711 | 30   | 0.4763          | 0.88     | 0.6225    | 0.5395 | 0.5471 | 0.0327 |
+| 0.4975        | 0.0948 | 40   | 0.4605          | 0.87     | 0.6658    | 0.6614 | 0.6636 | 0.1067 |
+| 0.4639        | 0.1185 | 50   | 0.4434          | 0.8947   | 0.7355    | 0.5850 | 0.6125 | 0.0367 |
+| 0.4687        | 0.1422 | 60   | 0.4557          | 0.892    | 0.7177    | 0.6498 | 0.6747 | 0.0727 |
+| 0.4489        | 0.1659 | 70   | 0.4353          | 0.9293   | 0.8174    | 0.8275 | 0.8224 | 0.114  |
+| 0.4318        | 0.1896 | 80   | 0.4269          | 0.924    | 0.8010    | 0.8325 | 0.8156 | 0.1233 |
+| 0.4723        | 0.2133 | 90   | 0.4202          | 0.9173   | 0.7832    | 0.8580 | 0.8140 | 0.1447 |
+| 0.4052        | 0.2370 | 100  | 0.4016          | 0.9307   | 0.8207    | 0.8309 | 0.8257 | 0.114  |
+| 0.4284        | 0.2607 | 110  | 0.4115          | 0.9187   | 0.7855    | 0.8906 | 0.8255 | 0.1593 |
+| 0.3635        | 0.2844 | 120  | 0.3963          | 0.94     | 0.8308    | 0.9052 | 0.8625 | 0.1393 |
+| 0.3894        | 0.3081 | 130  | 0.3910          | 0.944    | 0.8409    | 0.9075 | 0.8699 | 0.1353 |
+| 0.3537        | 0.3318 | 140  | 0.3598          | 0.9693   | 0.8983    | 0.9642 | 0.9277 | 0.1313 |
+| 0.3776        | 0.3555 | 150  | 0.3868          | 0.944    | 0.8313    | 0.9685 | 0.8823 | 0.166  |
+| 0.3626        | 0.3791 | 160  | 0.3235          | 0.9887   | 0.9699    | 0.9724 | 0.9711 | 0.1107 |
+| 0.3683        | 0.4028 | 170  | 0.3272          | 0.99     | 0.9583    | 0.9944 | 0.9754 | 0.12   |
+| 0.3358        | 0.4265 | 180  | 0.3321          | 0.9873   | 0.9484    | 0.9929 | 0.9692 | 0.1227 |
+| 0.3435        | 0.4502 | 190  | 0.3370          | 0.982    | 0.9297    | 0.9899 | 0.9571 | 0.128  |
+| 0.3613        | 0.4739 | 200  | 0.3136          | 0.9893   | 0.9728    | 0.9728 | 0.9728 | 0.11   |
+| 0.3323        | 0.4976 | 210  | 0.3193          | 0.9887   | 0.9533    | 0.9936 | 0.9723 | 0.1213 |
+| 0.3181        | 0.5213 | 220  | 0.3078          | 0.9947   | 0.9970    | 0.9758 | 0.9861 | 0.1047 |
+| 0.3043        | 0.5450 | 230  | 0.3047          | 0.9947   | 0.9970    | 0.9758 | 0.9861 | 0.1047 |
+| 0.3139        | 0.5687 | 240  | 0.3101          | 0.996    | 0.9825    | 0.9978 | 0.9899 | 0.114  |
+| 0.3247        | 0.5924 | 250  | 0.3048          | 0.9947   | 0.9970    | 0.9758 | 0.9861 | 0.1047 |
+| 0.3217        | 0.6161 | 260  | 0.3126          | 0.9913   | 0.9635    | 0.9951 | 0.9786 | 0.1187 |
+| 0.3071        | 0.6398 | 270  | 0.3021          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3048        | 0.6635 | 280  | 0.3048          | 0.9973   | 0.9882    | 0.9985 | 0.9933 | 0.1127 |
+| 0.3054        | 0.6872 | 290  | 0.2996          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3182        | 0.7109 | 300  | 0.2979          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3059        | 0.7346 | 310  | 0.3103          | 0.9927   | 0.9688    | 0.9959 | 0.9818 | 0.1173 |
+| 0.3044        | 0.7583 | 320  | 0.2991          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3002        | 0.7820 | 330  | 0.2967          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2957        | 0.8057 | 340  | 0.2967          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2971        | 0.8294 | 350  | 0.2968          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2964        | 0.8531 | 360  | 0.2970          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.297         | 0.8768 | 370  | 0.2969          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3039        | 0.9005 | 380  | 0.2968          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.3002        | 0.9242 | 390  | 0.2960          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2968        | 0.9479 | 400  | 0.2956          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2956        | 0.9716 | 410  | 0.2955          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
+| 0.2959        | 0.9953 | 420  | 0.2954          | 1.0      | 1.0       | 1.0    | 1.0    | 0.11   |
 
 
 ### Framework versions

config.json

@@ -1,42 +1,567 @@
 {
-  "_name_or_path": "microsoft/deberta-v2-xlarge-mnli",
+  "_name_or_path": "tasksource/deberta-small-long-nli",
   "architectures": [
     "DebertaV2ForSequenceClassification"
   ],
-  "attention_head_size": 64,
   "attention_probs_dropout_prob": 0.1,
-  "conv_act": "gelu",
-  "conv_kernel_size": 3,
+  "classifiers_size": [
+    3,
+    2,
+    2,
+    2,
+    2,
+    2,
+    1,
+    2,
+    3,
+    2,
+    2,
+    2,
+    3,
+    3,
+    3,
+    3,
+    1,
+    3,
+    3,
+    2,
+    2,
+    3,
+    2,
+    6,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2,
+    2,
+    2,
+    5,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2,
+    2,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2,
+    2,
+    2,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    20,
+    50,
+    3,
+    3,
+    4,
+    2,
+    8,
+    2,
+    3,
+    2,
+    2,
+    4,
+    20,
+    3,
+    3,
+    3,
+    3,
+    3,
+    174,
+    2,
+    2,
+    2,
+    2,
+    41,
+    3,
+    51,
+    2,
+    2,
+    2,
+    2,
+    16,
+    17,
+    18,
+    2,
+    2,
+    2,
+    3,
+    8,
+    3,
+    7,
+    7,
+    11,
+    3,
+    3,
+    42,
+    12,
+    4,
+    100,
+    13,
+    100,
+    8,
+    1,
+    20,
+    2,
+    2,
+    4,
+    5,
+    3,
+    4,
+    14,
+    2,
+    6,
+    4,
+    2,
+    1,
+    3,
+    10,
+    3,
+    10,
+    4,
+    2,
+    7,
+    6,
+    28,
+    3,
+    6,
+    5,
+    7,
+    4,
+    6,
+    3,
+    2,
+    2,
+    2,
+    6,
+    2,
+    7,
+    20,
+    2,
+    2,
+    4,
+    2,
+    9,
+    13,
+    4,
+    2,
+    3,
+    2,
+    3,
+    2,
+    2,
+    2,
+    2,
+    4,
+    1,
+    2,
+    1,
+    2,
+    49,
+    40,
+    10,
+    4,
+    1,
+    2,
+    2,
+    1,
+    5,
+    2,
+    3,
+    2,
+    2,
+    12,
+    3,
+    3,
+    2,
+    19,
+    3,
+    1,
+    2,
+    2,
+    2,
+    2,
+    2,
+    1,
+    2,
+    2,
+    1,
+    1,
+    2,
+    3,
+    2,
+    1,
+    4,
+    3,
+    1,
+    1,
+    1,
+    2,
+    3,
+    2,
+    3,
+    1,
+    1,
+    2,
+    1,
+    2,
+    2,
+    2,
+    2,
+    3,
+    2,
+    2,
+    2,
+    1,
+    3,
+    2,
+    2,
+    1,
+    1,
+    1,
+    1,
+    2,
+    1,
+    1,
+    1,
+    1,
+    4,
+    1,
+    1,
+    1,
+    1,
+    3,
+    1,
+    3,
+    1,
+    2,
+    2,
+    1,
+    2,
+    3,
+    3,
+    2,
+    1,
+    3,
+    1,
+    1,
+    3,
+    1,
+    3,
+    2,
+    1,
+    1,
+    1,
+    2,
+    2,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    2,
+    2,
+    2,
+    2,
+    77,
+    3,
+    2,
+    2,
+    1,
+    1,
+    2,
+    3,
+    18,
+    13,
+    2,
+    2,
+    2,
+    2,
+    2,
+    4,
+    2,
+    24,
+    3,
+    2,
+    2,
+    1,
+    2,
+    2,
+    3,
+    1,
+    2,
+    3,
+    2,
+    3,
+    3,
+    2,
+    2,
+    4,
+    1,
+    17,
+    3,
+    3,
+    2,
+    3,
+    2,
+    3,
+    3,
+    2,
+    1,
+    1,
+    3,
+    2,
+    9,
+    2,
+    1,
+    1,
+    1,
+    1,
+    1,
+    2,
+    2,
+    3,
+    2,
+    3,
+    4,
+    3,
+    3,
+    2,
+    1,
+    1,
+    2,
+    1,
+    1,
+    6,
+    2,
+    3,
+    3,
+    1,
+    1,
+    2,
+    2,
+    10,
+    2,
+    2,
+    2,
+    5,
+    5,
+    5,
+    5,
+    5,
+    5,
+    5,
+    5,
+    5,
+    5,
+    4,
+    1,
+    7,
+    6,
+    6,
+    6,
+    6,
+    6,
+    6,
+    3,
+    2,
+    1,
+    2,
+    2,
+    2,
+    3,
+    3,
+    3,
+    1
+  ],
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 1536,
+  "hidden_size": 768,
   "id2label": {
-    "0": "CONTRADICTION",
-    "1": "NEUTRAL",
-    "2": "ENTAILMENT"
+    "0": "entailment",
+    "1": "neutral",
+    "2": "contradiction"
   },
   "initializer_range": 0.02,
-  "intermediate_size": 6144,
+  "intermediate_size": 3072,
   "label2id": {
-    "CONTRADICTION": 0,
-    "ENTAILMENT": 2,
-    "NEUTRAL": 1
+    "contradiction": 2,
+    "entailment": 0,
+    "neutral": 1
   },
   "layer_norm_eps": 1e-07,
-  "max_position_embeddings": 512,
+  "max_position_embeddings": 1680,
   "max_relative_positions": -1,
   "model_type": "deberta-v2",
   "norm_rel_ebd": "layer_norm",
-  "num_attention_heads": 24,
-  "num_hidden_layers": 24,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
   "pad_token_id": 0,
   "pooler_dropout": 0,
   "pooler_hidden_act": "gelu",
-  "pooler_hidden_size": 1536,
-  "pooling": {
-    "dropout": 0,
-    "hidden_act": "gelu"
-  },
+  "pooler_hidden_size": 768,
   "pos_att_type": [
     "p2c",
     "c2p"
@@ -45,6 +570,538 @@
   "position_buckets": 256,
   "relative_attention": true,
   "share_att_key": true,
+  "tasks": [
+    "glue/mnli",
+    "glue/qnli",
+    "glue/rte",
+    "glue/wnli",
+    "glue/mrpc",
+    "glue/qqp",
+    "glue/stsb",
+    "super_glue/boolq",
+    "super_glue/cb",
+    "super_glue/multirc",
+    "super_glue/wic",
+    "super_glue/axg",
+    "anli/a1",
+    "anli/a2",
+    "anli/a3",
+    "sick/label",
+    "sick/relatedness",
+    "sick/entailment_AB",
+    "snli",
+    "scitail/snli_format",
+    "hans",
+    "WANLI",
+    "recast/recast_ner",
+    "recast/recast_kg_relations",
+    "recast/recast_puns",
+    "recast/recast_verbcorner",
+    "recast/recast_sentiment",
+    "recast/recast_verbnet",
+    "recast/recast_factuality",
+    "recast/recast_megaveridicality",
+    "probability_words_nli/reasoning_2hop",
+    "probability_words_nli/reasoning_1hop",
+    "probability_words_nli/usnli",
+    "nan-nli",
+    "nli_fever",
+    "breaking_nli",
+    "conj_nli",
+    "fracas",
+    "dialogue_nli",
+    "mpe",
+    "dnc",
+    "recast_white/fnplus",
+    "recast_white/sprl",
+    "recast_white/dpr",
+    "joci",
+    "robust_nli/IS_CS",
+    "robust_nli/LI_LI",
+    "robust_nli/ST_WO",
+    "robust_nli/PI_SP",
+    "robust_nli/PI_CD",
+    "robust_nli/ST_SE",
+    "robust_nli/ST_NE",
+    "robust_nli/ST_LM",
+    "robust_nli_is_sd",
+    "robust_nli_li_ts",
+    "add_one_rte",
+    "imppres/implicature_quantifiers/log",
+    "imppres/implicature_numerals_2_3/log",
+    "imppres/implicature_gradable_adjective/log",
+    "imppres/implicature_connectives/log",
+    "imppres/implicature_modals/log",
+    "imppres/implicature_numerals_10_100/log",
+    "imppres/implicature_gradable_verb/log",
+    "hlgd",
+    "paws/labeled_final",
+    "paws/labeled_swap",
+    "medical_questions_pairs",
+    "model-written-evals",
+    "truthful_qa/multiple_choice",
+    "fig-qa",
+    "bigbench/bbq_lite_json",
+    "bigbench/english_proverbs",
+    "bigbench/figure_of_speech_detection",
+    "bigbench/emoji_movie",
+    "bigbench/date_understanding",
+    "bigbench/metaphor_understanding",
+    "bigbench/logical_fallacy_detection",
+    "bigbench/analogical_similarity",
+    "bigbench/mnist_ascii",
+    "bigbench/elementary_math_qa",
+    "bigbench/snarks",
+    "bigbench/social_support",
+    "bigbench/logical_deduction",
+    "bigbench/emojis_emotion_prediction",
+    "bigbench/sports_understanding",
+    "bigbench/cifar10_classification",
+    "bigbench/tracking_shuffled_objects",
+    "bigbench/physics",
+    "bigbench/mathematical_induction",
+    "bigbench/movie_dialog_same_or_different",
+    "bigbench/goal_step_wikihow",
+    "bigbench/strategyqa",
+    "bigbench/identify_math_theorems",
+    "bigbench/gre_reading_comprehension",
+    "bigbench/novel_concepts",
+    "bigbench/conceptual_combinations",
+    "bigbench/hyperbaton",
+    "bigbench/strange_stories",
+    "bigbench/contextual_parametric_knowledge_conflicts",
+    "bigbench/implicatures",
+    "bigbench/penguins_in_a_table",
+    "bigbench/logical_sequence",
+    "bigbench/simple_ethical_questions",
+    "bigbench/dyck_languages",
+    "bigbench/geometric_shapes",
+    "bigbench/irony_identification",
+    "bigbench/intent_recognition",
+    "bigbench/logical_args",
+    "bigbench/known_unknowns",
+    "bigbench/formal_fallacies_syllogisms_negation",
+    "bigbench/suicide_risk",
+    "bigbench/crash_blossom",
+    "bigbench/logic_grid_puzzle",
+    "bigbench/analytic_entailment",
+    "bigbench/dark_humor_detection",
+    "bigbench/timedial",
+    "bigbench/presuppositions_as_nli",
+    "bigbench/arithmetic",
+    "bigbench/implicit_relations",
+    "bigbench/understanding_fables",
+    "bigbench/salient_translation_error_detection",
+    "bigbench/anachronisms",
+    "bigbench/moral_permissibility",
+    "bigbench/abstract_narrative_understanding",
+    "bigbench/misconceptions",
+    "bigbench/nonsense_words_grammar",
+    "bigbench/code_line_description",
+    "bigbench/sentence_ambiguity",
+    "bigbench/disambiguation_qa",
+    "bigbench/crass_ai",
+    "bigbench/similarities_abstraction",
+    "bigbench/authorship_verification",
+    "bigbench/phrase_relatedness",
+    "bigbench/color",
+    "bigbench/hhh_alignment",
+    "bigbench/metaphor_boolean",
+    "bigbench/fantasy_reasoning",
+    "bigbench/ruin_names",
+    "bigbench/cause_and_effect",
+    "bigbench/temporal_sequences",
+    "bigbench/navigate",
+    "bigbench/symbol_interpretation",
+    "bigbench/key_value_maps",
+    "bigbench/entailed_polarity",
+    "bigbench/riddle_sense",
+    "bigbench/discourse_marker_prediction",
+    "bigbench/reasoning_about_colored_objects",
+    "bigbench/empirical_judgments",
+    "bigbench/fact_checker",
+    "bigbench/movie_recommendation",
+    "bigbench/checkmate_in_one",
+    "bigbench/epistemic_reasoning",
+    "bigbench/vitaminc_fact_verification",
+    "bigbench/general_knowledge",
+    "bigbench/identify_odd_metaphor",
+    "bigbench/physical_intuition",
+    "bigbench/winowhy",
+    "bigbench/cs_algorithms",
+    "bigbench/undo_permutation",
+    "bigbench/evaluating_information_essentiality",
+    "bigbench/unit_interpretation",
+    "bigbench/question_selection",
+    "bigbench/international_phonetic_alphabet_nli",
+    "bigbench/play_dialog_same_or_different",
+    "bigbench/real_or_fake_text",
+    "bigbench/human_organs_senses",
+    "bigbench/hindu_knowledge",
+    "bigbench/social_iqa",
+    "bigbench/odd_one_out",
+    "bigbench/causal_judgment",
+    "cos_e/v1.0",
+    "cosmos_qa",
+    "dream",
+    "openbookqa",
+    "qasc",
+    "quartz",
+    "quail",
+    "head_qa/en",
+    "sciq",
+    "social_i_qa",
+    "wiki_hop/original",
+    "wiqa",
+    "piqa",
+    "hellaswag",
+    "super_glue/copa",
+    "balanced-copa",
+    "e-CARE",
+    "art",
+    "winogrande/winogrande_xl",
+    "codah/codah",
+    "ai2_arc/ARC-Easy/challenge",
+    "ai2_arc/ARC-Challenge/challenge",
+    "definite_pronoun_resolution",
+    "swag/regular",
+    "math_qa",
+    "glue/cola",
+    "glue/sst2",
+    "utilitarianism",
+    "amazon_counterfactual/en",
+    "insincere-questions",
+    "toxic_conversations",
+    "TuringBench",
+    "trec",
+    "vitaminc",
+    "hope_edi/english",
+    "rumoureval_2019/RumourEval2019",
+    "ethos/binary",
+    "ethos/multilabel",
+    "tweet_eval/offensive",
+    "tweet_eval/sentiment",
+    "tweet_eval/irony",
+    "tweet_eval/hate",
+    "tweet_eval/emotion",
+    "tweet_eval/emoji",
+    "tweet_eval/stance_abortion",
+    "tweet_eval/stance_atheism",
+    "tweet_eval/stance_climate",
+    "tweet_eval/stance_feminist",
+    "tweet_eval/stance_hillary",
+    "discovery/discovery",
+    "pragmeval/squinky-formality",
+    "pragmeval/emobank-arousal",
+    "pragmeval/squinky-implicature",
+    "pragmeval/squinky-informativeness",
+    "pragmeval/switchboard",
+    "pragmeval/verifiability",
+    "pragmeval/mrda",
+    "pragmeval/emobank-valence",
+    "pragmeval/emobank-dominance",
+    "pragmeval/persuasiveness-strength",
+    "pragmeval/persuasiveness-relevance",
+    "pragmeval/pdtb",
+    "pragmeval/gum",
+    "pragmeval/stac",
+    "pragmeval/persuasiveness-specificity",
+    "pragmeval/sarcasm",
+    "pragmeval/persuasiveness-eloquence",
+    "pragmeval/emergent",
+    "pragmeval/persuasiveness-premisetype",
+    "pragmeval/persuasiveness-claimtype",
+    "silicone/meld_e",
+    "silicone/dyda_e",
+    "silicone/iemocap",
+    "silicone/sem",
+    "silicone/meld_s",
+    "silicone/oasis",
+    "silicone/maptask",
+    "silicone/dyda_da",
+    "lex_glue/eurlex",
+    "lex_glue/scotus",
+    "lex_glue/ledgar",
+    "lex_glue/unfair_tos",
+    "lex_glue/case_hold",
+    "language-identification",
+    "imdb",
+    "rotten_tomatoes",
+    "ag_news",
+    "yelp_review_full/yelp_review_full",
+    "financial_phrasebank/sentences_allagree",
+    "poem_sentiment",
+    "dbpedia_14/dbpedia_14",
+    "amazon_polarity/amazon_polarity",
+    "app_reviews",
+    "hate_speech18",
+    "sms_spam",
+    "humicroedit/subtask-1",
+    "humicroedit/subtask-2",
+    "snips_built_in_intents",
+    "hate_speech_offensive",
+    "yahoo_answers_topics",
+    "stackoverflow-questions",
+    "hyperpartisan_news",
+    "sciie",
+    "citation_intent",
+    "go_emotions/simplified",
+    "scicite",
+    "liar",
+    "lexical_relation_classification/CogALexV",
+    "lexical_relation_classification/EVALution",
+    "lexical_relation_classification/K&H+N",
+    "lexical_relation_classification/BLESS",
+    "lexical_relation_classification/ROOT09",
+    "linguisticprobing/obj_number",
+    "linguisticprobing/bigram_shift",
+    "linguisticprobing/subj_number",
+    "linguisticprobing/sentence_length",
+    "linguisticprobing/odd_man_out",
+    "linguisticprobing/tree_depth",
+    "linguisticprobing/top_constituents",
+    "linguisticprobing/coordination_inversion",
+    "linguisticprobing/past_present",
+    "crowdflower/sentiment_nuclear_power",
+    "crowdflower/tweet_global_warming",
+    "crowdflower/political-media-message",
+    "crowdflower/text_emotion",
+    "crowdflower/corporate-messaging",
+    "crowdflower/political-media-audience",
+    "crowdflower/airline-sentiment",
+    "crowdflower/political-media-bias",
+    "crowdflower/economic-news",
+    "ethics/commonsense",
+    "ethics/deontology",
+    "ethics/justice",
+    "ethics/virtue",
+    "emo/emo2019",
+    "google_wellformed_query",
+    "tweets_hate_speech_detection",
+    "has_part",
+    "blog_authorship_corpus/gender",
+    "blog_authorship_corpus/age",
+    "blog_authorship_corpus/job",
+    "open_question_type",
+    "health_fact",
+    "commonsense_qa",
+    "mc_taco",
+    "ade_corpus_v2/Ade_corpus_v2_classification",
+    "discosense",
+    "circa",
+    "phrase_similarity",
+    "scientific-exaggeration-detection",
+    "quarel",
+    "fever-evidence-related",
+    "numer_sense",
+    "dynasent/dynabench.dynasent.r1.all/r1",
+    "dynasent/dynabench.dynasent.r2.all/r2",
+    "Sarcasm_News_Headline",
+    "sem_eval_2010_task_8",
+    "auditor_review",
+    "medmcqa",
+    "Dynasent_Disagreement",
+    "Politeness_Disagreement",
+    "SBIC_Disagreement",
+    "SChem_Disagreement",
+    "Dilemmas_Disagreement",
+    "logiqa",
+    "wiki_qa",
+    "cycic_classification",
+    "cycic_multiplechoice",
+    "sts-companion",
+    "commonsense_qa_2.0",
+    "lingnli",
+    "monotonicity-entailment",
+    "arct",
+    "scinli",
+    "naturallogic",
+    "onestop_qa",
+    "moral_stories/full",
+    "prost",
+    "dynahate",
+    "syntactic-augmentation-nli",
+    "autotnli",
+    "CONDAQA",
+    "webgpt_comparisons",
+    "synthetic-instruct-gptj-pairwise",
+    "scruples",
+    "wouldyourather",
+    "defeasible-nli/atomic",
+    "defeasible-nli/snli",
+    "help-nli",
+    "nli-veridicality-transitivity",
+    "lonli",
+    "dadc-limit-nli",
+    "FLUTE",
+    "strategy-qa",
+    "summarize_from_feedback/comparisons",
+    "folio",
+    "tomi-nli",
+    "avicenna",
+    "SHP",
+    "MedQA-USMLE-4-options-hf",
+    "wikimedqa/medwiki",
+    "cicero",
+    "CREAK",
+    "mutual",
+    "NeQA",
+    "quote-repetition",
+    "redefine-math",
+    "puzzte",
+    "implicatures",
+    "race/middle",
+    "race/high",
+    "race-c",
+    "spartqa-yn",
+    "spartqa-mchoice",
+    "temporal-nli",
+    "riddle_sense",
+    "clcd-english",
+    "twentyquestions",
+    "reclor",
+    "counterfactually-augmented-imdb",
+    "counterfactually-augmented-snli",
+    "cnli",
+    "boolq-natural-perturbations",
+    "acceptability-prediction",
+    "equate",
+    "ScienceQA_text_only",
+    "ekar_english",
+    "implicit-hate-stg1",
+    "chaos-mnli-ambiguity",
+    "headline_cause/en_simple",
+    "logiqa-2.0-nli",
+    "oasst2_dense_flat/quality",
+    "oasst2_dense_flat/toxicity",
+    "oasst2_dense_flat/helpfulness",
+    "mindgames",
+    "ambient",
+    "path-naturalness-prediction",
+    "civil_comments/toxicity",
+    "civil_comments/severe_toxicity",
+    "civil_comments/obscene",
+    "civil_comments/threat",
+    "civil_comments/insult",
+    "civil_comments/identity_attack",
+    "civil_comments/sexual_explicit",
+    "cloth",
+    "dgen",
+    "I2D2",
+    "args_me",
+    "Touche23-ValueEval",
+    "starcon",
+    "banking77",
+    "ConTRoL-nli",
+    "tracie",
+    "sherliic",
+    "sen-making/1",
+    "sen-making/2",
+    "winowhy",
+    "robustLR",
+    "v1/gen_train234_test2to10",
+    "logical-fallacy",
+    "parade",
+    "cladder",
+    "subjectivity",
+    "MOH",
+    "VUAC",
+    "sharc_modified/mod",
+    "conceptrules_v2",
+    "disrpt/eng.dep.scidtb.rels",
+    "zero-shot-label-nli",
+    "com2sense",
+    "scone",
+    "winodict",
+    "fool-me-twice",
+    "monli",
+    "corr2cause",
+    "lsat_qa/all",
+    "apt",
+    "twitter-financial-news-sentiment",
+    "icl-symbol-tuning-instruct",
+    "SpaceNLI",
+    "propsegment/nli",
+    "HatemojiBuild",
+    "regset",
+    "esci",
+    "chatbot_arena_conversations",
+    "dnd_style_intents",
+    "FLD.v2/default",
+    "FLD.v2/star",
+    "SDOH-NLI",
+    "scifact_entailment",
+    "feasibilityQA",
+    "simple_pair",
+    "AdjectiveScaleProbe-nli",
+    "resnli",
+    "SpaRTUN",
+    "ReSQ",
+    "semantic_fragments_nli",
+    "dataset_train_nli",
+    "stepgame",
+    "nlgraph",
+    "oasst2_pairwise_rlhf_reward",
+    "hh-rlhf/helpful-base",
+    "hh-rlhf/helpful-online",
+    "hh-rlhf/helpful-rejection-sampled",
+    "hh-rlhf/harmless-base",
+    "ruletaker",
+    "PARARULE-Plus",
+    "proofwriter",
+    "logical-entailment",
+    "nope",
+    "LogicNLI",
+    "contract-nli/contractnli_a/seg",
+    "contract-nli/contractnli_b/full",
+    "nli4ct_semeval2024",
+    "lsat-ar",
+    "lsat-rc",
+    "biosift-nli",
+    "brainteasers/WP",
+    "brainteasers/SP",
+    "persuasion",
+    "AmbigNQ-clarifying-question",
+    "SIGA-nli",
+    "FOL-nli",
+    "goal-step-wikihow/order",
+    "PARADISE",
+    "doc-nli",
+    "mctest-nli",
+    "patent-phrase-similarity",
+    "natural-language-satisfiability",
+    "idioms-nli",
+    "lifecycle-entailment",
+    "HelpSteer/helpfulness",
+    "HelpSteer/correctness",
+    "HelpSteer/coherence",
+    "HelpSteer/complexity",
+    "HelpSteer/verbosity",
+    "HelpSteer2/helpfulness",
+    "HelpSteer2/correctness",
+    "HelpSteer2/coherence",
+    "HelpSteer2/complexity",
+    "HelpSteer2/verbosity",
+    "MSciNLI",
+    "UltraFeedback-paired",
+    "AES2-essay-scoring",
+    "english-grading/cohesion",
+    "english-grading/syntax",
+    "english-grading/vocabulary",
+    "english-grading/phraseology",
+    "english-grading/grammar",
+    "english-grading/conventions",
+    "wice",
+    "hover",
+    "tasksource_dpo_pairs",
+    "seahorse_summarization_evaluation",
+    "missing-item-prediction/contrastive",
+    "babi_nli",
+    "gen_debiased_nli",
+    "imppres/presupposition",
+    "/prag",
+    "blimp-2"
+  ],
   "torch_dtype": "float32",
   "transformers_version": "4.44.2",
   "type_vocab_size": 0,

finetune.py

@@ -58,7 +58,7 @@ def gpu_function():
 
 
 
-GPU_CONFIG = modal.gpu.A100(count=2, size="80GB")
+GPU_CONFIG = modal.gpu.A100(count=1, size="80GB")
 
 @app.function(
     gpu=GPU_CONFIG, # self.params.gpu,
@@ -77,8 +77,12 @@ def run_finetune(data):
     print("CUDA device count:", torch.cuda.device_count())
 
     import pandas as pd
-    open('./features_ms_deberta_v2.json', 'w').write(data)
-    df = pd.read_json('./features_ms_deberta_v2.json', lines=False)
+    open('./features_ms_deberta_v3.json', 'w').write(data)
+    df = pd.read_json('./features_ms_deberta_v3.json', lines=False)
+    dfs = []
+    for _ in range(50):
+        dfs.append(df)
+    df = pd.concat(dfs, ignore_index=True)
 
     from datasets import Dataset
     from transformers import (
@@ -87,6 +91,7 @@ def run_finetune(data):
         
         Trainer, TrainingArguments ,EvalPrediction,DataCollatorWithPadding
     )
+    from transformers import EarlyStoppingCallback
     import numpy as np
     from itertools import chain
     import re
@@ -114,7 +119,7 @@ def run_finetune(data):
     model_name = "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7"
     # model_name = "MoritzLaurer/bge-m3-zeroshot-v2.0"
     # model_name = "cross-encoder/nli-deberta-v3-base"
-    model_name = "microsoft/deberta-v2-xlarge-mnli"
+    model_name = "tasksource/deberta-small-long-nli"
 
     # Define the directory where the output/results will be saved
     output_dir = "./"
@@ -167,7 +172,7 @@ def run_finetune(data):
     # In[7]:
 
 
-    train_data, test_data = train_test_split(df, test_size=0.3, random_state=42)
+    train_data, test_data = train_test_split(df, test_size=0.1, random_state=42)
 
     # Shuffle the train_data DataFrame and create a new DataFrame with shuffled rows
     train_shuffle_df = train_data
@@ -225,9 +230,9 @@ def run_finetune(data):
     training_args = TrainingArguments(
     output_dir=output_dir,           # Output directory
     logging_dir=output_dir + "/logs",# Output directory for logging
-    num_train_epochs=3,              # Total number of training epochs
-    per_device_train_batch_size=2,  # Batch size per device during training
-    per_device_eval_batch_size=2,    # Batch size for evaluation
+    num_train_epochs=1,              # Total number of training epochs
+    per_device_train_batch_size=16,  # Batch size per device during training
+    per_device_eval_batch_size=16,    # Batch size for evaluation
     warmup_steps=4,                  # Number of warmup steps for learning rate scheduler
     weight_decay=0.01,               # Strength of weight decay
     gradient_accumulation_steps=2,   # The number of steps whose gradients are accumulated
@@ -241,10 +246,13 @@ def run_finetune(data):
     logging_first_step=True,
     do_eval=True,
     hub_model_id="rafaelsandroni/ms-deberta-v2-xlarge-mnli-finetuned-pt",
+    load_best_model_at_end=True,
     )
 
 
     # In[15]:
+    
+    callbacks = [EarlyStoppingCallback(early_stopping_patience=3)]
 
 
     trainer = Trainer(
@@ -253,7 +261,8 @@ def run_finetune(data):
         train_dataset=train_dataset,     # Training dataset
         eval_dataset=test_dataset,      # Evaluation dataset
         tokenizer=tokenizer,
-        compute_metrics=compute_metrics
+        compute_metrics=compute_metrics,
+        callbacks=callbacks
     )
 
 
@@ -275,15 +284,6 @@ def run_finetune(data):
     v = 2
     commit = f"dev-v{v}-{t}"
     trainer.push_to_hub(commit, token=token)
-    # predict
-    for i, row in test_shuffle_df.iterrows():
-        pred_model = trainer.model([row["premise"], row["hypothesis"]])
-        print(pred_model, row["class"])
-    # In[ ]:
-
-
-    model.eval()
-
 
 
 
@@ -294,7 +294,7 @@ def run():
     import pandas as pd
     t0 = time.time()
     #df = pd.read_json('./features_ms_deberta_v2.json', lines=False)
-    with open('./features_ms_deberta_v2.json') as f:
+    with open('./features_ms_deberta_v3.json') as f:
         data = f.read()
     run_finetune.remote(data)
     

logs/events.out.tfevents.1724725103.modal.2.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c64adf7268903d8e1a57092586040cdc767d5cbba6e9ec9fa5e474de10deed1
+size 54531

logs/events.out.tfevents.1724725505.modal.2.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d6bc56408583b74aeb274c40e1aa47606b5336d596ef2f968c5e4f9f3998d4a
+size 609

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e19f1cdad97fb8d9e7d868cf21b586c53b3b1ac414d4ff676b34fcc245fafee
-size 3547884268
+oid sha256:6334b199dda92d20aa4e2d1408025de9b46734324097a006e55556eff6ef2943
+size 567601628

special_tokens_map.json

@@ -1,10 +1,46 @@
 {
-  "bos_token": "[CLS]",
-  "cls_token": "[CLS]",
-  "eos_token": "[SEP]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "[UNK]",
     "lstrip": false,

spm.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5598d5e96f339a8d980c15f9afd405a2e5e1be7db41de3ed13b0f03fac1e8c17
-size 2447305
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -47,12 +47,19 @@
   "do_lower_case": false,
   "eos_token": "[SEP]",
   "mask_token": "[MASK]",
-  "model_max_length": 512,
+  "max_length": 1680,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "[SEP]",
   "sp_model_kwargs": {},
   "split_by_punct": false,
+  "stride": 0,
   "tokenizer_class": "DebertaV2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]",
   "vocab_type": "spm"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:919b327360e865115024a79bfb9a7c9c2a4976a465b7ea7dbab24e13a14eb474
+oid sha256:a8f36e6350dbd0271210951e69cf655fcf20e165b523f134f106da097aeaf403
 size 5176