Upload 2 files

app.py

@@ -0,0 +1,103 @@
+import streamlit as st
+import torch
+from transformers import AutoTokenizer
+import whisper
+import subprocess
+import os
+import pandas as pd
+from dl import PitchEvaluationModel  # Import model
+
+def download_youtube_video(url, output_file="pitch_video.mp4"):
+    """Download YouTube video using yt-dlp."""
+    if "youtube.com" not in url and "youtu.be" not in url:
+        st.error("❌ Invalid URL! Please enter a valid YouTube link.")
+        return None
+    try:
+        command = ["yt-dlp", "-f", "mp4", "-o", output_file, url]
+        subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        return output_file
+    except subprocess.CalledProcessError:
+        st.error("❌ Failed to download the video. Please check the URL and try again.")
+        return None
+
+def transcribe_video(video_file):
+    """Transcribe video using Whisper."""
+    try:
+        model = whisper.load_model("base")
+        result = model.transcribe(video_file)
+        return result["text"]
+    except Exception as e:
+        st.error("❌ An error occurred during transcription.")
+        return ""
+
+def load_model():
+    """Load the trained model."""
+    try:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model = PitchEvaluationModel("bert-base-uncased").to(device)
+        model.load_state_dict(torch.load("best_pitch_model.pt", map_location=device))
+        model.eval()
+        tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+        return model, tokenizer, device
+    except Exception as e:
+        st.error("❌ Failed to load the model.")
+        return None, None, None
+
+def evaluate_pitch(transcript, model, tokenizer, device):
+    """Evaluate transcript using the trained model."""
+    try:
+        inputs = tokenizer(transcript, return_tensors="pt", truncation=True, padding="max_length", max_length=512)
+        input_ids, attention_mask = inputs["input_ids"].to(device), inputs["attention_mask"].to(device)
+        with torch.no_grad():
+            clarity, team, traction = model(input_ids, attention_mask)
+        return torch.argmax(clarity).item() + 1, torch.argmax(team).item() + 1, torch.argmax(traction).item() + 1
+    except Exception as e:
+        st.error("❌ Error in evaluation process.")
+        return None, None, None
+
+# Streamlit App UI
+st.set_page_config(page_title="Pitch Evaluation App", layout="wide")
+st.title("🚀 Pitch Evaluation")
+
+option = st.radio("Choose Input Method", ("YouTube URL", "Upload File"), horizontal=True)
+
+if option == "YouTube URL":
+    url = st.text_input("🎥 Enter YouTube URL")
+    if st.button("Download and Transcribe", use_container_width=True):
+        video_file = download_youtube_video(url)
+        if video_file:
+            transcript = transcribe_video(video_file)
+            st.text_area("📜 Transcript", transcript, height=200)
+elif option == "Upload File":
+    uploaded_file = st.file_uploader("📂 Upload Video", type=["mp4"], help="Upload a video file for transcription and evaluation.")
+    if uploaded_file is not None:
+        if uploaded_file.type != "video/mp4":
+            st.error("❌ Invalid file format! Please upload an MP4 file.")
+        else:
+            with open("uploaded_video.mp4", "wb") as f:
+                f.write(uploaded_file.getbuffer())
+            transcript = transcribe_video("uploaded_video.mp4")
+            st.text_area("📜 Transcript", transcript, height=200)
+
+if 'transcript' in locals() and transcript:
+    model, tokenizer, device = load_model()
+    if model is not None:
+        clarity, team, traction = evaluate_pitch(transcript, model, tokenizer, device)
+        if None not in (clarity, team, traction):
+            # Create a DataFrame for the scoring table
+            categories = ["Clarity & Conciseness", "Team-Market Fit", "Traction / Validation"]
+            scores = [clarity, team, traction]
+            descriptions = [
+                "Extremely clear, direct, and easy to follow;no fluff, just essential details." if clarity == 5 else "Mostly clear, with only minor unnecessary details." if clarity == 4 else "Somewhat clear but includes extra details or minor distractions." if clarity == 3 else "Lacks clarity; hard to follow; too much fluff or filler." if clarity == 2 else "Unclear, rambling, and difficult to understand.",
+                "Founders have highly relevant skills & experience to execute this successfully." if team == 5 else "Founders have good experience but may lack some key skills." if team == 4 else "Some relevant experience but gaps in expertise." if team == 3 else "Limited relevant experience; execution ability is questionable." if team == 2 else "No clear expertise in this space; team seems unqualified.",
+                "Strong proof of demand (users, revenue, engagement, partnerships, etc.)." if traction == 5 else "Good early validation with promising signs of demand." if traction == 4 else "Some traction but not yet convincing." if traction == 3 else "Weak or vague traction, with little evidence of demand." if traction == 2 else "No validation or proof that people want this."
+            ]
+            df = pd.DataFrame({"Category": categories, "Score (1-5)": scores, "Evaluation": descriptions})
+
+            st.write("## 📊 Evaluation Results")
+            st.table(df)
+
+        if ((clarity + team + traction)/3) >=3.5:
+            st.write("## 🎉 Congrats! You have a high possibility to be accepted")
+        else:
+            st.write("## 🙌 Need More Practice, but don't give up!")

dl.py

@@ -0,0 +1,210 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import cohen_kappa_score
+import numpy as np
+import torch.optim.lr_scheduler as lr_scheduler
+from datasets import load_dataset
+
+# 1. Dataset Preparation
+class PitchDataset(Dataset):
+    def __init__(self, texts, clarity_scores, team_scores, traction_scores, tokenizer, max_length=512):
+        self.texts = texts
+        self.clarity_scores = clarity_scores
+        self.team_scores = team_scores
+        self.traction_scores = traction_scores
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        
+    def __len__(self):
+        return len(self.texts)
+        
+    def __getitem__(self, idx):
+        text = str(self.texts[idx])
+        encoding = self.tokenizer(
+            text,
+            max_length=self.max_length,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt"
+        )
+        # Convert scores to zero-based index (1-5 → 0-4)
+        clarity_score = self.clarity_scores[idx] - 1
+        team_score = self.team_scores[idx] - 1
+        traction_score = self.traction_scores[idx] - 1
+        
+        return {
+            'input_ids': encoding['input_ids'].flatten(),
+            'attention_mask': encoding['attention_mask'].flatten(),
+            'clarity_score': torch.tensor(clarity_score, dtype=torch.long),
+            'team_score': torch.tensor(team_score, dtype=torch.long),
+            'traction_score': torch.tensor(traction_score, dtype=torch.long)
+        }
+
+# 2. Model Definition (Using Bert model with gradient checkpointing)
+class PitchEvaluationModel(nn.Module):
+    def __init__(self, pretrained_model="bert-base-uncased", use_gradient_checkpointing=True):
+        super().__init__()
+        self.encoder = AutoModel.from_pretrained(pretrained_model)
+        if use_gradient_checkpointing:
+            self.encoder.gradient_checkpointing_enable()
+        self.dropout = nn.Dropout(0.3)
+        self.clarity_classifier = nn.Linear(self.encoder.config.hidden_size, 5)
+        self.team_classifier = nn.Linear(self.encoder.config.hidden_size, 5)
+        self.traction_classifier = nn.Linear(self.encoder.config.hidden_size, 5)
+        
+    def forward(self, input_ids, attention_mask):
+        outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
+        pooled_output = outputs.last_hidden_state[:, 0]  # Using CLS token
+        pooled_output = self.dropout(pooled_output)
+        
+        clarity_logits = self.clarity_classifier(pooled_output)
+        team_logits = self.team_classifier(pooled_output)
+        traction_logits = self.traction_classifier(pooled_output)
+        
+        return clarity_logits, team_logits, traction_logits
+
+# 3. Training Function (Includes learning rate scheduling and multi-task weighted loss)
+def train_model(model, train_loader, val_loader, device, epochs=10,
+                clarity_weight=1.0, team_weight=1.0, traction_weight=1.0):
+    optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5)
+    # Use StepLR, decrease learning rate every 2 epochs
+    scheduler = lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.8)
+    criterion = nn.CrossEntropyLoss()
+    best_qwk = 0
+
+    for epoch in range(epochs):
+        model.train()
+        total_loss = 0
+        
+        for batch in train_loader:
+            input_ids = batch['input_ids'].to(device)
+            attention_mask = batch['attention_mask'].to(device)
+            clarity_scores = batch['clarity_score'].to(device)
+            team_scores = batch['team_score'].to(device)
+            traction_scores = batch['traction_score'].to(device)
+            
+            optimizer.zero_grad()
+            
+            clarity_logits, team_logits, traction_logits = model(input_ids, attention_mask)
+            
+            clarity_loss = criterion(clarity_logits, clarity_scores)
+            team_loss = criterion(team_logits, team_scores)
+            traction_loss = criterion(traction_logits, traction_scores)
+            
+            # Multi-task weighted loss
+            loss = clarity_weight * clarity_loss + team_weight * team_loss + traction_weight * traction_loss
+            loss.backward()
+            optimizer.step()
+            
+            total_loss += loss.item()
+        
+        scheduler.step()  # Adjust learning rate
+        
+        avg_loss = total_loss / len(train_loader)
+        print(f"Epoch {epoch+1}/{epochs}, Loss: {avg_loss:.4f}")
+        
+        # Evaluate on validation set and save the best model
+        qwk = evaluate_model(model, val_loader, device)
+        if qwk > best_qwk:
+            best_qwk = qwk
+            torch.save(model.state_dict(), "best_pitch_model.pt")
+            print(f"Model saved with QWK: {best_qwk:.4f}")
+    
+    return model
+
+# 4. Evaluation Function (Calculates QWK)
+def evaluate_model(model, data_loader, device):
+    model.eval()
+    
+    all_clarity_preds = []
+    all_team_preds = []
+    all_traction_preds = []
+    
+    all_clarity_true = []
+    all_team_true = []
+    all_traction_true = []
+    
+    with torch.no_grad():
+        for batch in data_loader:
+            input_ids = batch['input_ids'].to(device)
+            attention_mask = batch['attention_mask'].to(device)
+            
+            clarity_logits, team_logits, traction_logits = model(input_ids, attention_mask)
+            
+            # Convert predictions back to 1-5 scale
+            clarity_preds = torch.argmax(clarity_logits, dim=1).cpu().numpy() + 1
+            team_preds = torch.argmax(team_logits, dim=1).cpu().numpy() + 1
+            traction_preds = torch.argmax(traction_logits, dim=1).cpu().numpy() + 1
+            
+            all_clarity_preds.extend(clarity_preds)
+            all_team_preds.extend(team_preds)
+            all_traction_preds.extend(traction_preds)
+            
+            all_clarity_true.extend((batch['clarity_score'].cpu().numpy() + 1))
+            all_team_true.extend((batch['team_score'].cpu().numpy() + 1))
+            all_traction_true.extend((batch['traction_score'].cpu().numpy() + 1))
+    
+    clarity_qwk = cohen_kappa_score(all_clarity_true, all_clarity_preds, weights='quadratic')
+    team_qwk = cohen_kappa_score(all_team_true, all_team_preds, weights='quadratic')
+    traction_qwk = cohen_kappa_score(all_traction_true, all_traction_preds, weights='quadratic')
+    
+    overall_qwk = (clarity_qwk + team_qwk + traction_qwk) / 3
+    
+    print("Evaluation Results:")
+    print(f"Clarity QWK: {clarity_qwk:.4f}")
+    print(f"Team Market Fit QWK: {team_qwk:.4f}")
+    print(f"Traction QWK: {traction_qwk:.4f}")
+    print(f"Overall QWK: {overall_qwk:.4f}")
+    
+    return overall_qwk
+
+# 5. Main Function (Includes training, validation, and test set evaluation)
+def main():
+    # Load data
+    dataset = load_dataset("jasonhwan/yc-startup-pitches-with-scores", split="train")
+    df = dataset.to_pandas()
+
+    df.to_csv("yc_startup_pitches.csv", index=False)
+
+    df = pd.read_csv("yc_startup_pitches.csv")
+    
+    # Extract text and scores
+    texts = df['transcript'].values
+    clarity_scores = df['clarity_score'].values
+    team_scores = df['team_market_fit_score'].values
+    traction_scores = df['traction_validation_score'].values
+    
+    # Split dataset: 70% training, 15% validation, 15% testing
+    train_texts, temp_texts, train_clarity, temp_clarity, train_team, temp_team, train_traction, temp_traction = train_test_split(
+        texts, clarity_scores, team_scores, traction_scores, test_size=0.3, random_state=42
+    )
+    val_texts, test_texts, val_clarity, test_clarity, val_team, test_team, val_traction, test_traction = train_test_split(
+        temp_texts, temp_clarity, temp_team, temp_traction, test_size=0.5, random_state=42
+    )
+    
+    # Initialize tokenizer
+    tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased")
+    
+    train_dataset = PitchDataset(train_texts, train_clarity, train_team, train_traction, tokenizer)
+    val_dataset = PitchDataset(val_texts, val_clarity, val_team, val_traction, tokenizer)
+    test_dataset = PitchDataset(test_texts, test_clarity, test_team, test_traction, tokenizer)
+    
+    # Create DataLoaders
+    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=8)
+    test_loader = DataLoader(test_dataset, batch_size=8)
+    
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model = PitchEvaluationModel(pretrained_model="bert-base-uncased", use_gradient_checkpointing=True)
+    model.to(device)
+    
+    model = train_model(model, train_loader, val_loader, device, epochs=10)
+    
+    model.load_state_dict(torch.load("best_pitch_model.pt"))
+
+if __name__ == "__main__":
+    main()