Mr-Vicky-01/AI-scanner

INFERENCE

from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
import torch
import json

# Load model and tokenizer
tokenizer = AutoTokenizer.from_pretrained("Mr-Vicky-01/AI-scanner")
model = AutoModelForCausalLM.from_pretrained("Mr-Vicky-01/AI-scanner")

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)

sys_prompt = """<|im_start|>system\nYou are Securitron, an AI assistant specialized in detecting vulnerabilities in source code. Analyze the provided code and provide a structured report on any security issues found.<|im_end|>"""

user_prompt = """
CODE FOR SCANNING
"""

prompt = f"""{sys_prompt}
<|im_start|>user
{user_prompt}<|im_end|>
<|im_start|>assistant
"""

encodeds = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.to(device)

text_streamer = TextStreamer(tokenizer, skip_prompt=True)

response = model.generate(
    input_ids=encodeds,
    streamer=text_streamer,
    max_new_tokens=512,
    use_cache=True,
    pad_token_id=151645,
    eos_token_id=151645,
    num_return_sequences=1
)
    
output = json.loads(tokenizer.decode(response[0]).split('<|im_start|>assistant')[-1].split('<|im_end|>')[0].strip())