Update app.py

app.py

@@ -1,28 +1,17 @@
-import os
-import gc
-import psutil
 import cachetools
 from pydantic import BaseModel
 from llama_cpp import Llama
 from concurrent.futures import ThreadPoolExecutor, as_completed
-import re
-import httpx
 import asyncio
 import gradio as gr
-import torch
+import os
 from dotenv import load_dotenv
-from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse
+from fastapi import FastAPI, JSONResponse
 import uvicorn
 from threading import Thread
-import gptcache
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.feature_extraction.text import TfidfVectorizer
-import nltk
-from nltk.corpus import stopwords
-import wget
-
-nltk.download('stopwords')
+import psutil
+import gc
+import torch
 
 load_dotenv()
 HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
@@ -34,7 +23,6 @@ global_data = {
     'tokensxx': {
         'eos': '<|end_of-text|>',
         'pad': '<pad>',
-        'padding': '<pad>',
         'unk': '<unk>',
         'bos': '<|begin_of_text|>',
         'sep': '<|sep|>',
@@ -47,7 +35,6 @@ global_data = {
     'tokens': {
         'eos': 'eos_token',
         'pad': 'pad_token',
-        'padding': 'padding_token',
         'unk': 'unk_token',
         'bos': 'bos_token',
         'sep': 'sep_token',
@@ -200,7 +187,6 @@ global_data = {
     "n_layer_dense_lead": {},
     "expert_weights_scale": {},
     "rope_yarn_log_mul": {},
-    'model_type': {},
     'eval': {},
     'time': {},
     'token': {},
@@ -229,111 +215,135 @@ global_data = {
     'load_model': {},
     'end': {},
     'llama_perf_context_print': {},
-    'llm_load_print_meta': {}
+    'llm_load_print_meta': {},
+    'model_type': {}
 }
 
 model_configs = [
-    {"repo_id": "Hjgugugjhuhjggg/testing_semifinal-Q2_K-GGUF", "filename": "testing_semifinal-q2_k.gguf", "name": "testing"},
-    {"repo_id": "bartowski/Llama-3.2-3B-Instruct-uncensored-GGUF", "filename": "Llama-3.2-3B-Instruct-uncensored-Q2_K.gguf", "name": "Llama-3.2-3B-Instruct"},
-    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B-Q2_K-GGUF", "filename": "Meta-Llama-3.1-70B-Q2_K.gguf", "name": "Meta-Llama-3.1-13B"}
+    {
+        "repo_id": "Hjgugugjhuhjggg/testing_semifinal-Q2_K-GGUF",
+        "filename": "testing_semifinal-q2_k.gguf",
+        "name": "testing"
+    },
+    {
+        "repo_id": "bartowski/Llama-3.2-3B-Instruct-uncensored-GGUF",
+        "filename": "Llama-3.2-3B-Instruct-uncensored-Q2_K.gguf",
+        "name": "Llama-3.2-3B-Instruct"
+    }
 ]
 
+class ModelManager:
+    def __init__(self):
+        self.models = {}
+
+    def load_model(self, model_config):
+        if model_config['name'] not in self.models:
+            try:
+                self.models[model_config['name']] = Llama.from_pretrained(
+                    repo_id=model_config['repo_id'],
+                    filename=model_config['filename'],
+                    use_auth_token=HUGGINGFACE_TOKEN,
+                    n_threads=8,
+                    use_gpu=False
+                )
+            except Exception as e:
+                pass
+
+    def load_all_models(self):
+        with ThreadPoolExecutor() as executor:
+            for config in model_configs:
+                executor.submit(self.load_model, config)
+        return self.models
+
+model_manager = ModelManager()
+global_data['models'] = model_manager.load_all_models()
+
+class ChatRequest(BaseModel):
+    message: str
+
 def normalize_input(input_text):
-    stop_words = set(stopwords.words('english'))
-    words = input_text.split()
-    filtered_words = [word for word in words if word.lower() not in stop_words]
-    return " ".join(filtered_words)
+    return input_text.strip()
 
-async def load_models():
-    tasks = []
-    for model in model_configs:
-        model_path = os.path.join("models", model["filename"])
-        if not os.path.exists(model_path):
-            url = f"https://huggingface.co/{model['repo_id']}/resolve/main/{model['filename']}"
-            tasks.append(download_model(url, model_path))
-    await asyncio.gather(*tasks)
-    for model in model_configs:
-        model_path = os.path.join("models", model["filename"])
-        global_data['models'][model["name"]] = Llama(model_path)
+def remove_duplicates(text):
+    lines = text.split('\n')
+    unique_lines = []
+    seen_lines = set()
+    for line in lines:
+        if line not in seen_lines:
+            unique_lines.append(line)
+            seen_lines.add(line)
+    return '\n'.join(unique_lines)
 
-async def download_model(url, model_path):
-    wget.download(url, model_path)
+def cache_response(func):
+    def wrapper(*args, **kwargs):
+        cache_key = f"{args}-{kwargs}"
+        if cache_key in cache:
+            return cache[cache_key]
+        response = func(*args, **kwargs)
+        cache[cache_key] = response
+        return response
+    return wrapper
 
-async def generate_model_response(model, inputs):
+@cache_response
+def generate_model_response(model, inputs):
     try:
-        response = await model.generate(inputs)
-        return response
+        response = model(inputs)
+        return remove_duplicates(response['choices'][0]['text'])
     except Exception as e:
-        return {"error": str(e)}
+        return ""
 
-def get_best_response(responses):
-    if not responses:
-        return {"error": "No valid responses from models."}
-    scores = [response['score'] for response in responses]
-    best_score_index = scores.index(max(scores))
-    return responses[best_score_index]
+def remove_repetitive_responses(responses):
+    unique_responses = {}
+    for response in responses:
+        if response['model'] not in unique_responses:
+            unique_responses[response['model']] = response['response']
+    return unique_responses
 
 async def process_message(message):
     inputs = normalize_input(message)
-    tasks = [generate_model_response(model, inputs) for model in global_data['models'].values()]
-    responses = await asyncio.gather(*tasks)
-    best_response = get_best_response(responses)
-    return best_response
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(generate_model_response, model, inputs)
+            for model in global_data['models'].values()
+        ]
+        responses = [
+            {'model': model_name, 'response': future.result()}
+            for model_name, future in zip(global_data['models'].keys(), as_completed(futures))
+        ]
+    unique_responses = remove_repetitive_responses(responses)
+    formatted_response = ""
+    for model, response in unique_responses.items():
+        formatted_response += f"**{model}:**\n{response}\n\n"
+    return formatted_response
+
+app = FastAPI()
+
+@app.post("/generate")
+async def generate(request: ChatRequest):
+    try:
+        response = await process_message(request.message)
+        return JSONResponse(content={"response": response})
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)})
 
 def run_uvicorn():
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    try:
+        uvicorn.run(app, host="0.0.0.0", port=7860)
+    except Exception as e:
+        print(f"Error al ejecutar uvicorn: {e}")
 
 iface = gr.Interface(
     fn=process_message,
     inputs=gr.Textbox(lines=2, placeholder="Enter your message here..."),
     outputs=gr.Markdown(),
     title="Multi-Model LLM API (CPU Optimized)",
-    description=""
+    description="Optimized version using GPU and memory management techniques."
 )
 
 def run_gradio():
     iface.launch(server_port=7862, prevent_thread_lock=True)
 
-def release_resources():
-    try:
-        torch.cuda.empty_cache()
-        gc.collect()
-    except Exception as e:
-        print(f"Failed to release resources: {e}")
-
-def resource_manager():
-    MAX_RAM_PERCENT = 1
-    MAX_CPU_PERCENT = 1
-    MAX_GPU_PERCENT = 1
-    MAX_RAM_MB = 1
-
-    while True:
-        try:
-            virtual_mem = psutil.virtual_memory()
-            current_ram_percent = virtual_mem.percent
-            current_ram_mb = virtual_mem.used / (1024 * 1024)
-
-            if current_ram_percent > MAX_RAM_PERCENT or current_ram_mb > MAX_RAM_MB:
-                release_resources()
-
-            current_cpu_percent = psutil.cpu_percent()
-            if current_cpu_percent > MAX_CPU_PERCENT:
-                psutil.Process(os.getpid()).nice()
-
-            if torch.cuda.is_available():
-                gpu = torch.cuda.current_device()
-                gpu_mem = torch.cuda.memory_percent(gpu)
-
-                if gpu_mem > MAX_GPU_PERCENT:
-                    release_resources()
-
-        except Exception as e:
-            print(f"Error in resource manager: {e}")
-
-resource_manager()
-
 if __name__ == "__main__":
-    asyncio.run(load_models())
     Thread(target=run_uvicorn).start()
     Thread(target=run_gradio).start()
     asyncio.get_event_loop().run_forever()