Create run.py

run.py

@@ -0,0 +1,79 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+# 检查是否有可用的 GPU
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# 模型名称
+model_name = "Qwen/Qwen2.5-3B-Instruct"
+
+# 加载分词器
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+# 再次加载 LoRA 模型
+base_model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True).to(device)
+lora_model = PeftModel.from_pretrained(base_model, "wangyiqun/qwen25_3b_instruct_lora_vulgarity_finetuned")
+
+# 推理示例
+input_text = "Hello"
+input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to(lora_model.device)
+output = lora_model.generate(input_ids, max_new_tokens=50, do_sample=True, top_p=0.95, temperature=0.35)
+output_text = tokenizer.decode(output[0], skip_special_tokens=True)
+print(output_text)
+    
+
+
+
+#!pip install gradio
+import gradio as gr
+
+# 假设这些变量已经定义
+# tokenizer, model
+
+# 对话函数
+def chatbot(input_text, history):
+    if history is None:
+        history = []
+    # 将当前输入添加到历史记录中
+    history.append({"role": "user", "content": input_text})
+    messages = [
+        {"role": "system", "content": "根据用户友好的打招呼,和用户对骂，注意上下文语义连贯"}
+    ] + history
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    global lora_model
+    model = lora_model
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=512
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    # 将模型回复添加到历史记录中
+    history.append({"role": "assistant", "content": response})
+    # 转换为 Gradio Chatbot 所需的格式
+    chat_history = [(msg["content"] if msg["role"] == "user" else None,
+                     msg["content"] if msg["role"] == "assistant" else None) for msg in history if
+                    msg["role"] in ["user", "assistant"]]
+    return chat_history, history
+
+
+# Gradio 界面
+iface = gr.Interface(
+    fn=chatbot,
+    inputs=[gr.Textbox(label="输入你的问题"), gr.State()],
+    outputs=[gr.Chatbot(label="聊天历史"), gr.State()],
+    title="Qwen2.5-finetune-骂人专家",
+    description="Qwen2.5-finetune-骂人专家"
+)
+
+iface.launch(share=True, inbrowser=False, debug=True)