use simplified code instead

app.py

@@ -1,20 +1,14 @@
 import gradio as gr
 import spaces
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import pipeline
 
-# Load the model and tokenizer
-tokenizer = AutoTokenizer.from_pretrained("TheBloke/Chronoboros-33B-GPTQ")
-model = AutoModelForCausalLM.from_pretrained("TheBloke/Chronoboros-33B-GPTQ", device_map="auto")
-
-# Set a valid pad_token_id to avoid generation errors
-model.generation_config.pad_token_id = tokenizer.eos_token_id
-
-model.eval()  # Ensure the model is in evaluation mode
+# Create the text generation pipeline.
+# If you're running on GPU, you can specify device=0 (or use device_map="auto" if supported).
+pipe = pipeline("text-generation", model="TheBloke/Chronoboros-33B-GPTQ", device=0)
 
 @spaces.GPU
 def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
-    # Build the prompt using conversation history
+    # Build the prompt from system message and conversation history.
     prompt = f"{system_message}\n"
     for user_text, assistant_text in history:
         if user_text:
@@ -22,44 +16,28 @@ def respond(message, history: list[tuple[str, str]], system_message, max_tokens,
         if assistant_text:
             prompt += f"Assistant: {assistant_text}\n"
     prompt += f"User: {message}\nAssistant: "
-
-    # Tokenize the prompt
-    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device)
     
-    # Generate the response with no gradients
-    with torch.no_grad():
-        output_ids = model.generate(
-            input_ids,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            top_p=top_p,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id,  # also pass it here to be safe
-        )
+    # Generate a response using the pipeline.
+    # The pipeline returns a list of dictionaries; we take the generated text from the first output.
+    output = pipe(prompt, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p)
+    full_text = output[0]["generated_text"]
     
-    # Extract the new tokens (tokens generated after the prompt)
-    new_tokens = output_ids[0][input_ids.shape[1]:]
+    # Remove the prompt from the generated text to isolate the response.
+    response_text = full_text[len(prompt):]
     
-    # Stream output in chunks (here yielding every 5 tokens)
+    # Simulate streaming output in chunks (e.g., 5 characters at a time).
     chunk_size = 5
-    for i in range(0, new_tokens.shape[0], chunk_size):
-        current_response = tokenizer.decode(new_tokens[: i + chunk_size], skip_special_tokens=True)
-        yield current_response
+    for i in range(0, len(response_text), chunk_size):
+        yield response_text[: i + chunk_size]
 
-# Configure the ChatInterface with additional inputs
+# Configure the ChatInterface with additional inputs.
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
     ],
 )