Add Gradio app and requirements

.ipynb_checkpoints/app-checkpoint.py

@@ -0,0 +1,20 @@
+from flask import Flask, request, jsonify
+from inference import load_model, classify_text
+
+app = Flask(__name__)
+
+# 加载模型
+model, tokenizer = load_model()
+
+@app.route('/predict', methods=['POST'])
+def predict():
+    data = request.json
+    text = data.get("text", "")
+    if not text:
+        return jsonify({"error": "No text provided"}), 400
+    # 进行推理
+    prediction = classify_text(text, model, tokenizer)
+    return jsonify({"result": prediction})
+
+if __name__ == '__main__':
+    app.run(debug=True)

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -0,0 +1,4 @@
+flask
+transformers
+torch
+gunicorn

app.py

@@ -1,45 +1,78 @@
 import gradio as gr
 import torch
-from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModel
+from transformers import GPT2Model, ViTModel, GPT2Tokenizer, ViTFeatureExtractor
 from PIL import Image
+import requests
+import os
 
-# 加载自定义的多模态模型
-model_name = "Muhusjf/ViT-GPT2-multimodal-model"
+# 定义多模态模型
+class MultiModalModel(torch.nn.Module):
+    def __init__(self, gpt2_model_name="gpt2", vit_model_name="google/vit-base-patch16-224-in21k"):
+        super(MultiModalModel, self).__init__()
+        self.gpt2 = GPT2Model.from_pretrained(gpt2_model_name)
+        self.vit = ViTModel.from_pretrained(vit_model_name)
+        self.classifier = torch.nn.Linear(self.gpt2.config.hidden_size + self.vit.config.hidden_size, 2)
 
-# 加载分词器和特征提取器
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")
+    def forward(self, input_ids, attention_mask, pixel_values):
+        gpt2_outputs = self.gpt2(input_ids=input_ids, attention_mask=attention_mask)
+        text_features = gpt2_outputs.last_hidden_state[:, -1, :]
+
+        vit_outputs = self.vit(pixel_values=pixel_values)
+        image_features = vit_outputs.last_hidden_state[:, 0, :]
+
+        fused_features = torch.cat((text_features, image_features), dim=1)
+        logits = self.classifier(fused_features)
+        return logits
 
 # 加载模型
-model = AutoModel.from_pretrained(model_name)
+def load_model():
+    model_name = "Muhusjf/ViT-GPT2-multimodal-model"
+    model = MultiModalModel()
+    # 下载模型权重
+    model_url = f"https://huggingface.co/{model_name}/resolve/main/pytorch_model.bin"
+    model_path = "./pytorch_model.bin"
+    if not os.path.exists(model_path):
+        response = requests.get(model_url)
+        with open(model_path, "wb") as f:
+            f.write(response.content)
+    # 加载权重
+    model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
+    model.eval()
+    return model
+
+# 初始化模型和加载器
+model = load_model()
+tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+tokenizer.pad_token = tokenizer.eos_token
+feature_extractor = ViTFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")
 
-# 自定义多模态推理函数
-def multimodal_pipeline(image, text):
-    # 图像特征提取
+# 定义推理函数
+def predict(image, text):
+    # 处理图像
+    image = Image.fromarray(image)
     image_features = feature_extractor(images=image, return_tensors="pt")
-    
-    # 文本编码
-    text_inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=128)
 
-    # 获取文本和图像特征
-    with torch.no_grad():
-        text_outputs = model(**text_inputs)
-        image_outputs = model(**image_features)
+    # 处理文本
+    inputs = tokenizer.encode_plus(
+        f"Question: {text} Answer:",
+        return_tensors="pt",
+        max_length=128,
+        truncation=True,
+        padding="max_length"
+    )
+
+    input_ids = inputs["input_ids"]
+    attention_mask = inputs["attention_mask"]
+    pixel_values = image_features["pixel_values"]
 
-    # 简单融合特征（可以根据需要实现更复杂的融合策略）
-    fused_features = torch.cat((text_outputs.last_hidden_state[:, -1, :], image_outputs.last_hidden_state[:, 0, :]), dim=1)
+    # 推理
+    with torch.no_grad():
+        logits = model(input_ids, attention_mask, pixel_values)
+        prediction = torch.argmax(logits, dim=1).item()
+        label = "yes" if prediction == 1 else "no"
 
-    # 示例分类结果
-    result = "Positive" if torch.mean(fused_features) > 0 else "Negative"
-    return result
+    return label
 
 # 创建 Gradio 界面
-iface = gr.Interface(
-    fn=multimodal_pipeline,
-    inputs=["image", "text"],
-    outputs="text",
-    title="Multi-modal Sentiment Analysis"
-)
-
-# 启动 Gradio 应用
+iface = gr.Interface(fn=predict, inputs=["image", "text"], outputs="text", title="Multi-modal Inference")
 iface.launch()

requirements.txt

@@ -2,3 +2,4 @@ torch
 transformers
 gradio
 pillow
+requests