Initial commit: LLaMA-based itinerary generator

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,111 @@
+# LLaMA Itinerary Generator
+
+A custom fine-tuned version of LLaMA-2 for generating detailed travel itineraries.
+
+## Prerequisites
+
+1. Access to Hugging Face's LLaMA-2 model (requires approval from Meta)
+2. Python 3.8 or higher
+3. CUDA-capable GPU with at least 16GB VRAM
+4. Hugging Face account and token
+
+## Setup
+
+1. Create a virtual environment:
+```bash
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+2. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+3. Configure your Hugging Face token:
+```bash
+huggingface-cli login
+```
+
+## Project Structure
+
+```
+.
+├── config/
+│   └── config.json         # Training configuration
+├── data/
+│   └── itineraries.json    # Training data
+└── src/
+    └── train.py           # Training script
+```
+
+## Training Data
+
+The training data in `data/itineraries.json` contains examples of travel itineraries with the following structure:
+- Destination
+- Duration
+- Preferences
+- Budget
+- Detailed day-by-day itinerary
+
+## Training the Model
+
+1. Review and adjust the configuration in `config/config.json` if needed.
+
+2. Start training:
+```bash
+python src/train.py
+```
+
+The script will:
+- Load the LLaMA-2 base model
+- Fine-tune it on the itinerary dataset
+- Save checkpoints during training
+- Export the final model
+
+## Model Details
+
+This model is fine-tuned to generate travel itineraries based on:
+- Destination
+- Duration of stay
+- Travel preferences
+- Budget constraints
+
+The model learns to:
+- Structure day-by-day itineraries
+- Balance activities based on preferences
+- Consider budget constraints
+- Include practical details like transportation and check-in/out
+
+## Output Format
+
+The model generates itineraries in a structured format:
+```
+Day 1:
+- Activity 1
+- Activity 2
+...
+
+Day 2:
+- Activity 1
+- Activity 2
+...
+```
+
+## Monitoring Training
+
+Training progress can be monitored using TensorBoard:
+```bash
+tensorboard --logdir output/runs
+```
+
+## Model Deployment
+
+After training, the model will be saved in the `output` directory. You can upload it to Hugging Face Hub using:
+```bash
+huggingface-cli upload rahmanazhar/Travereel-Model-V1 output/
+```
+
+## License
+
+This project uses LLaMA 2 which is licensed under the LLAMA 2 Community License Agreement.

config/config.json

@@ -0,0 +1,25 @@
+{
+    "model_config": {
+        "base_model": "meta-llama/Llama-2-7b-hf",
+        "max_length": 512,
+        "learning_rate": 2e-5,
+        "num_epochs": 3,
+        "batch_size": 4,
+        "gradient_accumulation_steps": 4,
+        "warmup_steps": 100
+    },
+    "training_config": {
+        "output_dir": "output",
+        "logging_steps": 10,
+        "save_steps": 100,
+        "evaluation_strategy": "steps",
+        "eval_steps": 100,
+        "save_total_limit": 3,
+        "fp16": true
+    },
+    "data_config": {
+        "train_file": "data/itineraries.json",
+        "validation_split": 0.1,
+        "max_train_samples": null
+    }
+}

data/itineraries.json

@@ -0,0 +1,23 @@
+[
+    {
+        "destination": "Bali, Indonesia",
+        "duration": 5,
+        "preferences": "Beach activities, cultural experiences, local cuisine",
+        "budget": "$1000",
+        "itinerary": "Day 1:\n- Arrive in Denpasar International Airport\n- Check-in at hotel in Seminyak\n- Evening walk on Seminyak Beach\n- Dinner at local warung\n\nDay 2:\n- Morning yoga session\n- Visit Tanah Lot Temple\n- Sunset dinner at Jimbaran Beach\n\nDay 3:\n- Day trip to Ubud\n- Visit Monkey Forest\n- Traditional dance performance\n- Art market shopping\n\nDay 4:\n- Nusa Penida island tour\n- Snorkeling at Crystal Bay\n- Visit Kelingking Beach\n- Local seafood dinner\n\nDay 5:\n- Morning spa treatment\n- Visit Uluwatu Temple\n- Final beach sunset\n- Departure"
+    },
+    {
+        "destination": "Tokyo, Japan",
+        "duration": 4,
+        "preferences": "Modern attractions, traditional culture, food exploration",
+        "budget": "$1500",
+        "itinerary": "Day 1:\n- Arrive at Narita Airport\n- Check-in at hotel in Shinjuku\n- Evening exploration of Shinjuku\n- Dinner at Robot Restaurant\n\nDay 2:\n- Tsukiji Outer Market tour\n- Visit Senso-ji Temple\n- Explore Akihabara\n- Sushi-making class\n\nDay 3:\n- TeamLab Borderless\n- Harajuku shopping\n- Meiji Shrine visit\n- Shibuya Crossing & dinner\n\nDay 4:\n- Morning at Ueno Park\n- Tokyo Sky Tree\n- Final shopping\n- Departure"
+    },
+    {
+        "destination": "Paris, France",
+        "duration": 3,
+        "preferences": "Art, history, romantic spots",
+        "budget": "$1200",
+        "itinerary": "Day 1:\n- Arrive at Charles de Gaulle Airport\n- Check-in at hotel near Le Marais\n- Visit Eiffel Tower\n- Seine River dinner cruise\n\nDay 2:\n- Louvre Museum morning\n- Notre-Dame Cathedral view\n- Montmartre walk\n- Evening at Moulin Rouge\n\nDay 3:\n- Palace of Versailles\n- Shopping at Champs-Élysées\n- Arc de Triomphe sunset\n- Departure"
+    }
+]

requirements.txt

@@ -0,0 +1,11 @@
+torch>=2.0.0
+transformers>=4.30.0
+datasets>=2.12.0
+accelerate>=0.20.0
+bitsandbytes>=0.39.0
+trl>=0.4.7
+peft>=0.4.0
+evaluate>=0.4.0
+tensorboard>=2.13.0
+scipy>=1.10.0
+scikit-learn>=1.2.2

sample_model.txt

@@ -0,0 +1,4 @@
+This is a sample model file for testing Hugging Face upload functionality.
+Model Name: Travereel-Model-V1
+Version: 1.0
+Type: Test Model

src/generate.py

@@ -0,0 +1,87 @@
+import torch
+from transformers import LlamaForCausalLM, LlamaTokenizer
+import argparse
+import json
+
+class ItineraryGenerator:
+    def __init__(self, model_path: str):
+        self.tokenizer = LlamaTokenizer.from_pretrained(model_path)
+        self.model = LlamaForCausalLM.from_pretrained(
+            model_path,
+            torch_dtype=torch.float16,
+            device_map="auto"
+        )
+        self.model.eval()
+
+    def generate_itinerary(
+        self,
+        destination: str,
+        duration: int,
+        preferences: str,
+        budget: str,
+        max_length: int = 1024,
+        temperature: float = 0.7,
+        top_p: float = 0.9,
+    ) -> str:
+        prompt = f"""Generate a detailed travel itinerary for {destination} for {duration} days.
+Preferences: {preferences}
+Budget: {budget}
+
+Detailed Itinerary:"""
+
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **inputs,
+                max_length=max_length,
+                temperature=temperature,
+                top_p=top_p,
+                num_return_sequences=1,
+                pad_token_id=self.tokenizer.eos_token_id
+            )
+
+        generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        # Extract only the generated itinerary part
+        itinerary = generated_text[len(prompt):]
+        return itinerary.strip()
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate travel itineraries using fine-tuned LLaMA model")
+    parser.add_argument("--model_path", type=str, required=True, help="Path to the fine-tuned model")
+    parser.add_argument("--destination", type=str, required=True, help="Travel destination")
+    parser.add_argument("--duration", type=int, required=True, help="Number of days")
+    parser.add_argument("--preferences", type=str, required=True, help="Travel preferences")
+    parser.add_argument("--budget", type=str, required=True, help="Travel budget")
+    parser.add_argument("--output", type=str, help="Output file path (optional)")
+    
+    args = parser.parse_args()
+    
+    generator = ItineraryGenerator(args.model_path)
+    
+    itinerary = generator.generate_itinerary(
+        destination=args.destination,
+        duration=args.duration,
+        preferences=args.preferences,
+        budget=args.budget
+    )
+    
+    output = {
+        "destination": args.destination,
+        "duration": args.duration,
+        "preferences": args.preferences,
+        "budget": args.budget,
+        "generated_itinerary": itinerary
+    }
+    
+    if args.output:
+        with open(args.output, 'w') as f:
+            json.dump(output, f, indent=2)
+        print(f"Itinerary saved to {args.output}")
+    else:
+        print("\nGenerated Itinerary:")
+        print("=" * 50)
+        print(itinerary)
+
+if __name__ == "__main__":
+    main()

src/train.py

@@ -0,0 +1,102 @@
+import torch
+from transformers import (
+    LlamaForCausalLM,
+    LlamaTokenizer,
+    TrainingArguments,
+    Trainer,
+    DataCollatorForLanguageModeling
+)
+from datasets import load_dataset
+import os
+import json
+from typing import Dict, List
+
+class ItineraryDataset(torch.utils.data.Dataset):
+    def __init__(self, data_path: str, tokenizer, max_length: int = 512):
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.examples = self._load_data(data_path)
+        
+    def _load_data(self, data_path: str) -> List[Dict]:
+        with open(data_path, 'r') as f:
+            return json.load(f)
+    
+    def __len__(self):
+        return len(self.examples)
+    
+    def __getitem__(self, idx):
+        example = self.examples[idx]
+        prompt = f"""Generate a detailed travel itinerary for {example['destination']} for {example['duration']} days.
+Preferences: {example['preferences']}
+Budget: {example['budget']}"""
+        
+        target = example['itinerary']
+        
+        # Combine prompt and target with special tokens
+        combined = f"{prompt}\n{target}</s>"
+        
+        # Tokenize
+        encodings = self.tokenizer(
+            combined,
+            truncation=True,
+            max_length=self.max_length,
+            padding="max_length",
+            return_tensors="pt"
+        )
+        
+        return {
+            "input_ids": encodings["input_ids"][0],
+            "attention_mask": encodings["attention_mask"][0],
+            "labels": encodings["input_ids"][0].clone()
+        }
+
+def train_itinerary_model(
+    model_name: str = "meta-llama/Llama-2-7b-hf",
+    data_path: str = "data/itineraries.json",
+    output_dir: str = "output",
+    num_epochs: int = 3,
+    batch_size: int = 4,
+    learning_rate: float = 2e-5,
+):
+    # Initialize tokenizer and model
+    tokenizer = LlamaTokenizer.from_pretrained(model_name)
+    model = LlamaForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=torch.float16,
+        device_map="auto"
+    )
+    
+    # Load dataset
+    dataset = ItineraryDataset(data_path, tokenizer)
+    
+    # Training arguments
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        num_train_epochs=num_epochs,
+        per_device_train_batch_size=batch_size,
+        gradient_accumulation_steps=4,
+        learning_rate=learning_rate,
+        warmup_steps=100,
+        logging_steps=10,
+        save_steps=100,
+        fp16=True,
+        report_to="tensorboard"
+    )
+    
+    # Initialize trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=dataset,
+        data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)
+    )
+    
+    # Train the model
+    trainer.train()
+    
+    # Save the model
+    trainer.save_model()
+    tokenizer.save_pretrained(output_dir)
+
+if __name__ == "__main__":
+    train_itinerary_model()