|
|
import gradio as gr |
|
|
import torch |
|
|
from transformers import ( |
|
|
AutoModelForCausalLM, |
|
|
AutoTokenizer, |
|
|
TextIteratorStreamer, |
|
|
) |
|
|
import os |
|
|
from threading import Thread |
|
|
import spaces |
|
|
import time |
|
|
import subprocess |
|
|
|
|
|
subprocess.run( |
|
|
"pip install flash-attn --no-build-isolation", |
|
|
env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"}, |
|
|
shell=True, |
|
|
) |
|
|
hf_token = os.getenv("HF_TOKEN") |
|
|
|
|
|
token = hf_token |
|
|
|
|
|
|
|
|
model = AutoModelForCausalLM.from_pretrained( |
|
|
"microsoft/Phi-3-mini-128k-instruct", |
|
|
token=token, |
|
|
trust_remote_code=True, |
|
|
) |
|
|
tok = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", token=token) |
|
|
terminators = [ |
|
|
tok.eos_token_id, |
|
|
] |
|
|
|
|
|
if torch.cuda.is_available(): |
|
|
device = torch.device("cuda") |
|
|
print(f"Using GPU: {torch.cuda.get_device_name(device)}") |
|
|
else: |
|
|
device = torch.device("cpu") |
|
|
print("Using CPU") |
|
|
|
|
|
model = model.to(device) |
|
|
|
|
|
|
|
|
|
|
|
@spaces.GPU(duration=60) |
|
|
def chat(message, history, temperature, do_sample, max_tokens): |
|
|
chat = [] |
|
|
for item in history: |
|
|
chat.append({"role": "user", "content": item[0]}) |
|
|
if item[1] is not None: |
|
|
chat.append({"role": "assistant", "content": item[1]}) |
|
|
chat.append({"role": "user", "content": message}) |
|
|
messages = tok.apply_chat_template(chat, tokenize=False, add_generation_prompt=True) |
|
|
model_inputs = tok([messages], return_tensors="pt").to(device) |
|
|
streamer = TextIteratorStreamer( |
|
|
tok, timeout=20.0, skip_prompt=True, skip_special_tokens=True |
|
|
) |
|
|
generate_kwargs = dict( |
|
|
model_inputs, |
|
|
streamer=streamer, |
|
|
max_new_tokens=max_tokens, |
|
|
do_sample=True, |
|
|
temperature=temperature, |
|
|
eos_token_id=terminators, |
|
|
) |
|
|
|
|
|
if temperature == 0: |
|
|
generate_kwargs["do_sample"] = False |
|
|
|
|
|
t = Thread(target=model.generate, kwargs=generate_kwargs) |
|
|
t.start() |
|
|
|
|
|
partial_text = "" |
|
|
for new_text in streamer: |
|
|
partial_text += new_text |
|
|
yield partial_text |
|
|
|
|
|
yield partial_text |
|
|
|
|
|
|
|
|
demo = gr.ChatInterface( |
|
|
respond, |
|
|
additional_inputs=[ |
|
|
gr.Textbox( |
|
|
value=( |
|
|
"You are an assistant for controlling PTZ cameras.\n" |
|
|
"When the user gives you a clear command, please JUST respond in the following format:\n" |
|
|
"Camera:<camera_id>. Tracking_Target:<target_name> placement:<position> speed:<speed> only_ptz_action:<only_ptz_action> tracking_action:<tracking_action>.\n" |
|
|
"If multiple cameras are specified, provide separate lines for each camera.\n" |
|
|
"Only provide commands for the cameras specified by the user.\n" |
|
|
"Do not include additional cameras that the user did not mention.\n" |
|
|
"Ensure all field names are spelled correctly.\n\n" |
|
|
"The available placements are ONLY: top_left, top_middle, top_right, center_left, center_middle, center_right, bottom_left, bottom_middle, bottom_right.\n" |
|
|
"The available speed options are ONLY: slow, medium, fast.\n" |
|
|
"The available only_ptz_actions are ONLY: turn_right, turn_left, tilt_up, tilt_down, zoom_in, zoom_out, stop.\n" |
|
|
"The available tracking_actions are ONLY: tracking.\n\n" |
|
|
"Default Values:\n" |
|
|
"- camera_id: default\n" |
|
|
"- tracking_target: default\n" |
|
|
"- placement: center_middle\n" |
|
|
"- speed: medium\n" |
|
|
"- only_ptz_action: default\n" |
|
|
"- tracking_action: default\n\n" |
|
|
"Rules for Defaults:\n" |
|
|
"1. If the camera_id is not specified, use the default value `default`.\n" |
|
|
"2. If the tracking_target is not specified, use the default value `default`.\n" |
|
|
"3. If the position information is incomplete or not specified, default the placement to `center_middle`.\n" |
|
|
"4. If only a general direction is specified, interpret it as the middle of that direction.\n" |
|
|
" For example, 'top' is interpreted as 'top_middle' and 'left' as 'center_left'.\n" |
|
|
"5. If the speed is not specified, default to `medium`.\n" |
|
|
"6. If the only_ptz_action is not specified, default to `default`.\n" |
|
|
"7. If the tracking_action is not specified, default to `default`.\n" |
|
|
"8. Camera IDs are restricted to 1, 2, 3, and 4. If an invalid camera_id is provided, use `default`.\n" |
|
|
"9. If the user specifies 'all camera' or 'all cameras', apply the command to all cameras (1-4).\n\n" |
|
|
"**Special Action Handling**:\n" |
|
|
"- If only `camera_id` and `only_ptz_action` are specified (all other fields are `default`), execute only the specified `only_ptz_action`.\n" |
|
|
"- If only `camera_id` and `tracking_action` are specified (all other fields are `default`), execute only the specified `tracking_action`.\n" |
|
|
"- When tracking is involved, set `tracking_action` to `tracking`.\n\n" |
|
|
"Examples:\n" |
|
|
"User: Please set camera 1 to track target Alice at bottom_right with speed fast and action turn_right.\n" |
|
|
"Assistant: Camera:1. Tracking_Target:Alice placement:bottom_right speed:fast only_ptz_action:turn_right tracking_action:default.\n\n" |
|
|
"User: Please set camera 3 to track target Bob at top with speed slow.\n" |
|
|
"Assistant: Camera:3. Tracking_Target:Bob placement:top_middle speed:slow only_ptz_action:default tracking_action:tracking.\n\n" |
|
|
"User: Please set camera 2 to track target Carol.\n" |
|
|
"Assistant: Camera:2. Tracking_Target:Carol placement:center_middle speed:medium only_ptz_action:default tracking_action:tracking.\n\n" |
|
|
"User: Please track target Dave at left.\n" |
|
|
"Assistant: Camera:default. Tracking_Target:Dave placement:center_left speed:medium only_ptz_action:default tracking_action:tracking.\n\n" |
|
|
"User: Please control camera 4.\n" |
|
|
"Assistant: Camera:4. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:default tracking_action:default.\n\n" |
|
|
"User: Please start recording.\n" |
|
|
"Assistant: Camera:default. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:default tracking_action:default.\n\n" |
|
|
"User: Camera 5 action turn_left.\n" |
|
|
"Assistant: Camera:default. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_left tracking_action:default.\n\n" |
|
|
"User: camera 1 and 2 turn right.\n" |
|
|
"Assistant:\n" |
|
|
"Camera:1. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_right tracking_action:default.\n" |
|
|
"Camera:2. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_right tracking_action:default.\n\n" |
|
|
"Respond with ONLY the Assistant's output. Do NOT add any extra text." |
|
|
|
|
|
), |
|
|
label="System message" |
|
|
) |
|
|
, |
|
|
gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), |
|
|
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), |
|
|
gr.Slider( |
|
|
minimum=0.1, |
|
|
maximum=1.0, |
|
|
value=0.95, |
|
|
step=0.05, |
|
|
label="Top-p (nucleus sampling)", |
|
|
), |
|
|
], |
|
|
) |
|
|
demo.launch() |
|
|
|
