|
|
|
|
|
|
|
|
|
@@ -230,7 +230,7 @@
|
|
#define GGML_MAX_CONTEXTS 64
|
|
#define GGML_MAX_SRC 10
|
|
#ifndef GGML_MAX_NAME
|
|
-#define GGML_MAX_NAME 64
|
|
+#define GGML_MAX_NAME 128
|
|
#endif
|
|
#define GGML_MAX_OP_PARAMS 64
|
|
#define GGML_DEFAULT_N_THREADS 4
|
|
|
|
|
|
|
|
|
|
@@ -212,6 +212,10 @@ enum llm_arch {
|
|
LLM_ARCH_JAIS,
|
|
LLM_ARCH_NEMOTRON,
|
|
LLM_ARCH_EXAONE,
|
|
+ LLM_ARCH_FLUX,
|
|
+ LLM_ARCH_SD1,
|
|
+ LLM_ARCH_SDXL,
|
|
+ LLM_ARCH_CLIP_G,
|
|
LLM_ARCH_UNKNOWN,
|
|
};
|
|
|
|
@@ -259,6 +263,10 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
|
{ LLM_ARCH_JAIS, "jais" },
|
|
{ LLM_ARCH_NEMOTRON, "nemotron" },
|
|
{ LLM_ARCH_EXAONE, "exaone" },
|
|
+ { LLM_ARCH_FLUX, "flux" },
|
|
+ { LLM_ARCH_SD1, "sd1" },
|
|
+ { LLM_ARCH_SDXL, "sdxl" },
|
|
+ { LLM_ARCH_CLIP_G, "clip_g" },
|
|
{ LLM_ARCH_UNKNOWN, "(unknown)" },
|
|
};
|
|
|
|
@@ -1337,6 +1345,9 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
|
|
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
|
|
},
|
|
},
|
|
+ { LLM_ARCH_CLIP_G, {
|
|
+ // No explicit mappings (handled by tensor name patterns)
|
|
+ }},
|
|
{ LLM_ARCH_FLUX, {}},
|
|
{ LLM_ARCH_SD1, {}},
|
|
{ LLM_ARCH_SDXL, {}},
|
|
@@ -4629,6 +4640,12 @@ static void llm_load_hparams(
|
|
// get general kv
|
|
ml.get_key(LLM_KV_GENERAL_NAME, model.name, false);
|
|
|
|
+ // Disable LLM metadata for image models (exclude CLIP_G)
|
|
+ if (model.arch == LLM_ARCH_FLUX || model.arch == LLM_ARCH_SD1 || model.arch == LLM_ARCH_SDXL) {
|
|
+ model.ftype = ml.ftype;
|
|
+ return;
|
|
+ }
|
|
+
|
|
// get hparams kv
|
|
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);
|
|
|
|
@@ -15827,6 +15844,160 @@ static void llama_tensor_dequantize_internal(
|
|
workers.clear();
|
|
}
|
|
|
|
+static ggml_type img_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
|
|
+ const std::string name = ggml_get_name(tensor);
|
|
+ const llm_arch arch = qs.model.arch;
|
|
+
|
|
+ // Sanity check
|
|
+ if (
|
|
+ (name.find("model.diffusion_model.") != std::string::npos) ||
|
|
+ (name.find("first_stage_model.") != std::string::npos) ||
|
|
+ (name.find("single_transformer_blocks.") != std::string::npos)
|
|
+ ) {
|
|
+ throw std::runtime_error("Invalid input GGUF file. This is not a supported UNET model");
|
|
+ }
|
|
+
|
|
+ // Block unsupported quant types
|
|
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_Q4_0_4_4 ||
|
|
+ ftype == LLAMA_FTYPE_MOSTLY_Q4_0_4_8 || ftype == LLAMA_FTYPE_MOSTLY_Q4_0_8_8) {
|
|
+ throw std::runtime_error("Invalid quantization type for image model (Not supported)");
|
|
+ }
|
|
+
|
|
+ // CLIP_G-specific rules
|
|
+ if (arch == LLM_ARCH_CLIP_G) {
|
|
+ // Keep layer norms and embeddings in F32
|
|
+ if (
|
|
+ name.find("ln_final") != std::string::npos ||
|
|
+ name.find("positional_embedding") != std::string::npos ||
|
|
+ name.find("logit_scale") != std::string::npos
|
|
+ ) {
|
|
+ new_type = GGML_TYPE_F32;
|
|
+ }
|
|
+ // Keep token embeddings in F16
|
|
+ else if (name.find("token_embedding") != std::string::npos) {
|
|
+ new_type = GGML_TYPE_F16;
|
|
+ }
|
|
+ // Quantize attention layers (combined QKV)
|
|
+ else if (name.find("attn.in_proj_weight") != std::string::npos) {
|
|
+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {
|
|
+ new_type = GGML_TYPE_Q4_K;
|
|
+ } else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {
|
|
+ new_type = GGML_TYPE_Q5_K;
|
|
+ } else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) {
|
|
+ new_type = GGML_TYPE_Q6_K;
|
|
+ }
|
|
+ }
|
|
+ // Quantize MLP layers
|
|
+ else if (
|
|
+ name.find("mlp.c_fc.weight") != std::string::npos ||
|
|
+ name.find("mlp.c_proj.weight") != std::string::npos
|
|
+ ) {
|
|
+ new_type = (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;
|
|
+ }
|
|
+ return new_type;
|
|
+ }
|
|
+
|
|
+ // Existing rules for image models (FLUX/SD1/SDXL)
|
|
+ if ( // Tensors to keep in FP32
|
|
+ (arch == LLM_ARCH_FLUX) && (
|
|
+ (name.find("img_in.") != std::string::npos) ||
|
|
+ (name.find("time_in.in_layer.") != std::string::npos) ||
|
|
+ (name.find("vector_in.in_layer.") != std::string::npos) ||
|
|
+ (name.find("guidance_in.in_layer.") != std::string::npos) ||
|
|
+ (name.find("final_layer.linear.") != std::string::npos)
|
|
+ ) || (arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL) && (
|
|
+ (name.find("conv_in.") != std::string::npos) ||
|
|
+ (name.find("conv_out.") != std::string::npos) ||
|
|
+ (name == "input_blocks.0.0.weight") ||
|
|
+ (name == "out.2.weight")
|
|
+ )) {
|
|
+ new_type = GGML_TYPE_F32;
|
|
+ } else if ( // Tensors to keep in FP16
|
|
+ (arch == LLM_ARCH_FLUX) && (
|
|
+ (name.find("txt_in.") != std::string::npos) ||
|
|
+ (name.find("time_in.") != std::string::npos) ||
|
|
+ (name.find("vector_in.") != std::string::npos) ||
|
|
+ (name.find("guidance_in.") != std::string::npos) ||
|
|
+ (name.find("final_layer.") != std::string::npos)
|
|
+ ) || (arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL) && (
|
|
+ (name.find("class_embedding.") != std::string::npos) ||
|
|
+ (name.find("time_embedding.") != std::string::npos) ||
|
|
+ (name.find("add_embedding.") != std::string::npos) ||
|
|
+ (name.find("time_embed.") != std::string::npos) ||
|
|
+ (name.find("label_emb.") != std::string::npos) ||
|
|
+ (name.find("proj_in.") != std::string::npos) ||
|
|
+ (name.find("proj_out.") != std::string::npos)
|
|
+ )) {
|
|
+ new_type = GGML_TYPE_F16;
|
|
+ }
|
|
+
|
|
+ // Existing quantization rules for image models...
|
|
+ return new_type;
|
|
+}
|
|
+
|
|
static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
|
|
const std::string name = ggml_get_name(tensor);
|
|
|
|
const llm_arch arch = qs.model.arch;
|
|
+ if (arch == LLM_ARCH_FLUX || arch == LLM_ARCH_SD1 || arch == LLM_ARCH_SDXL || arch == LLM_ARCH_CLIP_G) {
|
|
+ return img_tensor_get_type(qs, new_type, tensor, ftype);
|
|
+ }
|
|
const auto tn = LLM_TN(arch);
|
|
|
|
auto use_more_bits = [](int i_layer, int n_layers) -> bool { |