Files changed (1) hide show
  1. README.md +67 -3
README.md CHANGED
@@ -1,3 +1,67 @@
1
- ---
2
- license: apache-2.0
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ ---
4
+
5
+ ### Generation
6
+
7
+ The following is the sample code for inference.
8
+
9
+ ```python
10
+
11
+ from llava.model.builder import load_pretrained_model
12
+ from llava.mm_utils import process_images, tokenizer_image_token
13
+ from llava.constants import DEFAULT_IMAGE_TOKEN
14
+
15
+ from PIL import Image
16
+ import torch
17
+ import time
18
+ import warnings
19
+ import json
20
+
21
+ # export PYTHONPATH="/thestack/LLM4CodeBeta/LLaVA-NeXT-FLAME:$PYTHONPATH"
22
+
23
+ warnings.filterwarnings("ignore")
24
+
25
+ pretrained = "/root/nfs3/flame_ft/res/checkpoints/flame-google_siglip-so400m-patch14-384-deepseek-ai_deepseek-coder-6.7b-instruct-mlp2x_gelu-selectlayer-2-onevision-1-pretrain_mmcoder-3NODE-Date1212-STAGE2v9-2-data_1220_no_code_v1-inst_data-STAGE2v9-eos-16k-1220-FINETUNE-2-data_1220/no_code_v1-inst_data-v5_v6-eos-16k-1223"
26
+
27
+ model_name = "flame"
28
+ device = "cuda"
29
+ device_map = "auto"
30
+ llava_model_args = {
31
+ "multimodal": True,
32
+ "attn_implementation": None,
33
+ }
34
+ tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map,**llava_model_args)
35
+ model.config.tokenizer_padding_side = 'left' # Use left padding for batch processing
36
+ # model.config.image_aspect_ratio = "resize"
37
+ model.eval()
38
+
39
+ url = "/root/nfs2/flame_ft/datasets/data_1220/TESTING_DATA/TEST80/imgs/000000034/000000034.png"
40
+ image = Image.open(url)
41
+ image_tensor = process_images([image], image_processor, model.config)
42
+ image_tensor = [_image.to(dtype=torch.float16, device=device) for _image in image_tensor]
43
+
44
+ prompt = "Below is an image of the page to create. Generate React code and styles to replicate the design, including layout, typography, and styling. Format your response as follows:'// CSS\n[CSS/SCSS code]\n\n// [React Implementation (JS/TS/JSX/TSX)]\n[Component code]'.\n\n ### Input Image:\n{image}\n\n### Response:\n"
45
+
46
+ input_ids = tokenizer_image_token(prompt, tokenizer, return_tensors='pt')
47
+ input_ids = input_ids.unsqueeze(0)
48
+ input_ids=input_ids.to(device)
49
+ image_sizes = [image.size]
50
+ modalities = ["image"]
51
+
52
+ cont = model.generate(
53
+ input_ids,
54
+ images=image_tensor,
55
+ image_sizes=image_sizes,
56
+ modalities=modalities, # Added this line with the modalities
57
+ do_sample=True,
58
+ num_beams=5,
59
+ temperature=0.1,
60
+ max_new_tokens=4096,
61
+ top_p=0.95,
62
+ repetition_penalty=1.05
63
+ )
64
+
65
+ text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
66
+
67
+ ```