Upload folder using huggingface_hub

README.md

@@ -35,11 +35,6 @@ extra_gated_description: >-
 pipeline_tag: image-text-to-text
 ---
 
-Checkpoint of Mistral-Small-3.1-24B-Instruct-2503 with FP8 per-tensor quantization in the Mistral-format. Please run with vLLM like so:
-```
-vllm serve nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8 --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice --limit_mm_per_prompt 'image=10'
-```
-
 # Model Card for Mistral-Small-3.1-24B-Instruct-2503
 
 Building upon Mistral Small 3 (2501), Mistral Small 3.1 (2503) **adds state-of-the-art vision understanding** and enhances **long context capabilities up to 128k tokens** without compromising text performance. 

params.json

@@ -27,7 +27,43 @@
     "image_size": 1540
   },
   "quantization": {
-    "qformat_weight": "fp8_e4m3",
-    "activation_scheme": "dynamic"
+    "config_groups": {
+      "group_0": {
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "strategy": "token",
+          "symmetric": true,
+          "type": "float"
+        },
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "minmax",
+          "observer_kwargs": {},
+          "strategy": "tensor",
+          "symmetric": true,
+          "type": "float"
+        }
+      }
+    },
+    "format": "float-quantized",
+    "ignore": [
+      "lm_head",
+      "output"
+    ],
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed"
   }
-}
+}