submit pull for merge (#1)

- submit pull for merge (85bdb4e2f53788772ec789554db9a859d4a957e8)

.gitattributes

@@ -37,3 +37,20 @@ input/baldwin-letter-1.jpg filter=lfs diff=lfs merge=lfs -text
 input/baldwin-letter-2.jpg filter=lfs diff=lfs merge=lfs -text
 input/magellan-travels.jpg filter=lfs diff=lfs merge=lfs -text
 input/okeefe-menu.pdf filter=lfs diff=lfs merge=lfs -text
+backup/input/baldwin-letter-1.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/baldwin-letter-2.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/letter-1.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/letter-2.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/letter-3.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/magellan-travels.jpg filter=lfs diff=lfs merge=lfs -text
+backup/input/menu.pdf filter=lfs diff=lfs merge=lfs -text
+backup/input/The[[:space:]]Magician,[[:space:]]or[[:space:]]Bottle[[:space:]]Cungerer.jpeg filter=lfs diff=lfs merge=lfs -text
+input/a-la-carte.pdf filter=lfs diff=lfs merge=lfs -text
+input/handwritten-letter.jpg filter=lfs diff=lfs merge=lfs -text
+input/letter-1.jpg filter=lfs diff=lfs merge=lfs -text
+input/letter-2.jpg filter=lfs diff=lfs merge=lfs -text
+input/letter-3.jpg filter=lfs diff=lfs merge=lfs -text
+input/magician-satire.jpg filter=lfs diff=lfs merge=lfs -text
+input/menu.pdf filter=lfs diff=lfs merge=lfs -text
+input/The[[:space:]]Magician,[[:space:]]or[[:space:]]Bottle[[:space:]]Cungerer.jpeg filter=lfs diff=lfs merge=lfs -text
+output/ymca-letter.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 title: Historical OCR
-emoji: 🚀
+emoji: 📜
 colorFrom: red
 colorTo: green
 sdk: streamlit
@@ -22,15 +22,78 @@ This application uses Mistral AI's OCR capabilities to transcribe and extract in
 - Structured output generation using Mistral models
 - Interactive web interface with Streamlit
 - Supports historical documents and manuscripts
+- PDF preview functionality for better user experience
+- Smart handling of large PDFs with automatic page limiting
+- Robust error handling with helpful messages
+- Image preprocessing options for enhanced OCR accuracy
+
+## Project Structure
+
+The project is organized as follows:
+
+```
+Historical OCR - Project Structure
+
+┌─ Main Applications
+│  ├─ app.py                        # Standard Streamlit interface for OCR processing
+│  └─ streamlit_app.py              # Educational modular version with learning components
+│
+├─ Core Functionality
+│  ├─ structured_ocr.py             # Main OCR processing engine with Mistral AI integration
+│  ├─ ocr_utils.py                  # Utility functions for OCR text and image processing
+│  ├─ pdf_ocr.py                    # PDF-specific document processing functionality
+│  └─ config.py                     # Configuration settings and API keys
+│
+├─ Testing & Development
+│  ├─ simple_test.py                # Basic OCR functionality test
+│  ├─ test_pdf.py                   # PDF processing test
+│  ├─ test_pdf_preview.py           # PDF preview generation test
+│  └─ prepare_for_hf.py             # Prepare project for Hugging Face deployment
+│
+├─ Scripts
+│  ├─ run_local.sh                  # Launch standard or educational app locally
+│  ├─ run_large_files.sh            # Process large documents with optimized settings
+│  └─ setup_git.sh                  # Configure Git repositories
+│
+├─ Educational Modules (streamlit/)
+│  ├─ modules/
+│  │  ├─ module1.py                 # Introduction and Problematization
+│  │  ├─ module2.py                 # Historical Typography & OCR Challenges
+│  │  ├─ module3.py                 # Document Analysis Techniques
+│  │  ├─ module4.py                 # Processing Methods
+│  │  ├─ module5.py                 # Research Applications
+│  │  └─ module6.py                 # Future Directions
+│  │
+│  ├─ modular_app.py                # Learning module framework
+│  ├─ layout.py                     # UI components for educational interface
+│  └─ process_file.py               # File processing for educational app
+│
+├─ UI Components (ui/)
+│  └─ layout.py                     # Shared UI components and styling
+│
+├─ Data Directories
+│  ├─ input/                        # Sample documents for testing/demo
+│  └─ output/                       # Output directory for processed files
+│
+└─ Dependencies
+   ├─ requirements.txt              # Python package dependencies
+   └─ packages.txt                  # System-level dependencies
+```
 
 ## Setup for Local Development
 
-1. This directory is standalone and can be moved anywhere
-2. Install dependencies:
+1. Clone this repository
+2. Install system dependencies:
+   - For PDF processing, you need poppler:
+     - On macOS: `brew install poppler`
+     - On Ubuntu/Debian: `apt-get install poppler-utils`
+     - On Windows: Download from [poppler releases](https://github.com/oschwartz10612/poppler-windows/releases/) and add to PATH
+   - For text recognition: `tesseract-ocr`
+3. Install Python dependencies:
 ```
 pip install -r requirements.txt
 ```
-3. Set up your Mistral API key:
+4. Set up your Mistral API key:
    - Option 1: Create a `.env` file in this directory and add your Mistral API key:
      ```
      MISTRAL_API_KEY=your_api_key_here
@@ -40,7 +103,7 @@ pip install -r requirements.txt
      export MISTRAL_API_KEY=your_api_key_here
      ```
    - Get your API key from [Mistral AI Console](https://console.mistral.ai/api-keys/)
-4. Run the Streamlit app using the script:
+5. Run the Streamlit app using the script:
 ```
 ./run_local.sh
 ```
@@ -52,12 +115,32 @@ streamlit run app.py
 ## Usage
 
 1. Upload an image or PDF file using the file uploader
-2. Select processing options in the sidebar (e.g., use vision model)
+2. Select processing options in the sidebar (e.g., use vision model, image preprocessing)
 3. Click "Process Document" to analyze the file
 4. View the structured results and extract information
 
+## Application Versions
+
+Two versions of the application are available:
+
+1. **Standard Version** (`app.py`): Focused on document processing with a clean interface
+2. **Educational Version** (`streamlit_app.py`): Enhanced with educational modules and interactive components
+
+To run the educational version:
+```
+streamlit run streamlit_app.py
+```
+
 ## Deployment on Hugging Face Spaces
 
-This app is designed to be deployed on Hugging Face Spaces. The `README.md` contains the necessary configuration metadata.
+This app is designed to be deployed on Hugging Face Spaces. To deploy:
+
+1. Fork this repository to your GitHub account or directly create a new Space on [Hugging Face](https://huggingface.co/spaces)
+2. Connect your GitHub repository to your Hugging Face Space for automatic deployment
+3. Add your Mistral API key as a secret in your Hugging Face Space settings:
+   - Secret name: `HF_MISTRAL_API_KEY`
+   - Secret value: Your Mistral API key
+
+The `README.md` contains the necessary configuration metadata for Hugging Face Spaces.
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at [Hugging Face Spaces documentation](https://huggingface.co/docs/hub/spaces-config-reference)

app.py

@@ -2,24 +2,105 @@ import os
 import streamlit as st
 import json
 import sys
+import time
 from pathlib import Path
 import tempfile
+import io
+from pdf2image import convert_from_bytes
+from PIL import Image, ImageEnhance, ImageFilter
+import cv2
+import numpy as np
 
 # Import the StructuredOCR class and config from the local files
 from structured_ocr import StructuredOCR
 from config import MISTRAL_API_KEY
 
+# Check for modular UI components
+try:
+    from ui.layout import tool_container, key_concept, research_question
+    MODULAR_UI = True
+except ImportError:
+    MODULAR_UI = False
+
 # Set page configuration
 st.set_page_config(
     page_title="Historical OCR",
-    page_icon="🚀",
+    page_icon="📜",
     layout="wide",
     initial_sidebar_state="expanded"
 )
 
+# Enable caching for expensive operations
+@st.cache_data(ttl=3600, show_spinner=False)
+def convert_pdf_to_images(pdf_bytes, dpi=150):
+    """Convert PDF bytes to a list of images with caching"""
+    try:
+        return convert_from_bytes(pdf_bytes, dpi=dpi)
+    except Exception as e:
+        st.error(f"Error converting PDF: {str(e)}")
+        return []
+
+@st.cache_data(ttl=3600, show_spinner=False)
+def preprocess_image(image_bytes, preprocessing_options):
+    """Preprocess image with selected options"""
+    # Convert bytes to OpenCV format
+    image = Image.open(io.BytesIO(image_bytes))
+    img_array = np.array(image)
+    
+    # Apply preprocessing based on selected options
+    if preprocessing_options.get("grayscale", False):
+        img_array = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
+        img_array = cv2.cvtColor(img_array, cv2.COLOR_GRAY2RGB)
+    
+    if preprocessing_options.get("contrast", 0) != 0:
+        contrast_factor = 1 + (preprocessing_options.get("contrast", 0) / 10)
+        image = Image.fromarray(img_array)
+        enhancer = ImageEnhance.Contrast(image)
+        image = enhancer.enhance(contrast_factor)
+        img_array = np.array(image)
+    
+    if preprocessing_options.get("denoise", False):
+        img_array = cv2.fastNlMeansDenoisingColored(img_array, None, 10, 10, 7, 21)
+        
+    if preprocessing_options.get("threshold", False):
+        # Convert to grayscale if not already
+        if len(img_array.shape) == 3:
+            gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
+        else:
+            gray = img_array
+        # Apply adaptive threshold
+        binary = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, 
+                                      cv2.THRESH_BINARY, 11, 2)
+        # Convert back to RGB
+        img_array = cv2.cvtColor(binary, cv2.COLOR_GRAY2RGB)
+        
+    # Convert back to PIL Image
+    processed_image = Image.fromarray(img_array)
+    
+    # Convert to bytes
+    byte_io = io.BytesIO()
+    processed_image.save(byte_io, format='PNG')
+    byte_io.seek(0)
+    
+    return byte_io.getvalue()
+
 # Define functions
-def process_file(uploaded_file, use_vision=True):
-    """Process the uploaded file and return the OCR results"""
+def process_file(uploaded_file, use_vision=True, preprocessing_options=None):
+    """Process the uploaded file and return the OCR results
+    
+    Args:
+        uploaded_file: The uploaded file to process
+        use_vision: Whether to use vision model
+        preprocessing_options: Dictionary of preprocessing options
+    """
+    if preprocessing_options is None:
+        preprocessing_options = {}
+        
+    # Show progress indicator
+    progress_bar = st.progress(0)
+    status_text = st.empty()
+    status_text.text("Preparing file for processing...")
+    
     # Save the uploaded file to a temporary file
     with tempfile.NamedTemporaryFile(delete=False, suffix=Path(uploaded_file.name).suffix) as tmp:
         tmp.write(uploaded_file.getvalue())
@@ -29,6 +110,8 @@ def process_file(uploaded_file, use_vision=True):
         # Check if API key is available
         if not MISTRAL_API_KEY:
             # Return dummy data if no API key
+            progress_bar.progress(100)
+            status_text.empty()
             return {
                 "file_name": uploaded_file.name,
                 "topics": ["Sample Document"],
@@ -38,7 +121,11 @@ def process_file(uploaded_file, use_vision=True):
                     "content": "This is sample content. To process real documents, please set the MISTRAL_API_KEY environment variable."
                 }
             }
-            
+        
+        # Update progress
+        progress_bar.progress(20)
+        status_text.text("Initializing OCR processor...")
+        
         # Initialize OCR processor
         processor = StructuredOCR()
         
@@ -46,9 +133,53 @@ def process_file(uploaded_file, use_vision=True):
         file_ext = Path(uploaded_file.name).suffix.lower()
         file_type = "pdf" if file_ext == ".pdf" else "image"
         
-        # Process the file
-        result = processor.process_file(temp_path, file_type=file_type, use_vision=use_vision)
+        # Apply preprocessing if needed
+        if any(preprocessing_options.values()) and file_type == "image":
+            status_text.text("Applying image preprocessing...")
+            processed_bytes = preprocess_image(uploaded_file.getvalue(), preprocessing_options)
+            
+            # Save processed image to temp file
+            with tempfile.NamedTemporaryFile(delete=False, suffix=Path(uploaded_file.name).suffix) as proc_tmp:
+                proc_tmp.write(processed_bytes)
+                temp_path = proc_tmp.name
+        
+        # Get file size in MB
+        file_size_mb = os.path.getsize(temp_path) / (1024 * 1024)
+        
+        # Check if file exceeds API limits (50 MB)
+        if file_size_mb > 50:
+            st.error(f"File too large ({file_size_mb:.1f} MB). Maximum file size allowed by Mistral API is 50MB.")
+            return {
+                "file_name": uploaded_file.name,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "confidence_score": 0.0,
+                "error": f"File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB",
+                "ocr_contents": {
+                    "error": f"Failed to process file: File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB",
+                    "partial_text": "Document could not be processed due to size limitations."
+                }
+            }
+        
+        # Update progress
+        progress_bar.progress(40)
+        status_text.text("Processing document with OCR...")
+        
+        # Process the file with file size information for automatic page limiting
+        # Make sure we're using the latest mistral-ocr model 
+        # See https://docs.mistral.ai/capabilities/document/ for more info
+        result = processor.process_file(temp_path, file_type=file_type, use_vision=use_vision, file_size_mb=file_size_mb)
+        
+        # Complete progress
+        progress_bar.progress(100)
+        status_text.empty()
+        
         return result
+    except Exception as e:
+        progress_bar.progress(100)
+        status_text.empty()
+        st.error(f"Error during processing: {str(e)}")
+        raise
     finally:
         # Clean up the temporary file
         if os.path.exists(temp_path):
@@ -57,122 +188,414 @@ def process_file(uploaded_file, use_vision=True):
 # App title and description
 st.title("Historical Document OCR")
 st.subheader("Powered by Mistral AI")
-st.markdown("""
-This application uses Mistral AI's OCR capabilities to transcribe and extract information from historical documents.
-Upload an image or PDF file to get started.
-""")
+
+# Create main layout with tabs and columns
+main_tab1, main_tab2 = st.tabs(["Document Processing", "About"])
+
+with main_tab1:
+    # Create a two-column layout for file upload and preview
+    upload_col, preview_col = st.columns([1, 1])
+    
+    # File uploader in the left column
+    with upload_col:
+        st.markdown("""
+        Upload an image or PDF file to get started. 
+        
+        Using the latest `mistral-ocr-latest` model for advanced document understanding.
+        """)
+        uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], help="Limit 50MB per file")
 
 # Sidebar with options
 with st.sidebar:
     st.header("Options")
+    
+    # Model options
+    st.subheader("Model Settings")
     use_vision = st.checkbox("Use Vision Model", value=True, 
                             help="For image files, use the vision model for improved analysis (may be slower)")
     
-    st.markdown("---")
-    st.subheader("About")
+    # Image preprocessing options (collapsible)
+    st.subheader("Image Preprocessing")
+    with st.expander("Preprocessing Options"):
+        preprocessing_options = {}
+        preprocessing_options["grayscale"] = st.checkbox("Convert to Grayscale", 
+                                                        help="Convert image to grayscale before OCR")
+        preprocessing_options["threshold"] = st.checkbox("Apply Thresholding", 
+                                                      help="Apply adaptive thresholding to enhance text")
+        preprocessing_options["denoise"] = st.checkbox("Denoise Image", 
+                                                     help="Remove noise from the image")
+        preprocessing_options["contrast"] = st.slider("Adjust Contrast", -5, 5, 0, 
+                                                    help="Adjust image contrast (-5 to +5)")
+    
+    # PDF options (collapsible)
+    st.subheader("PDF Options")
+    with st.expander("PDF Settings"):
+        pdf_dpi = st.slider("PDF Resolution (DPI)", 72, 300, 150, 
+                          help="Higher DPI gives better quality but slower processing")
+        max_pages = st.number_input("Maximum Pages to Process", 1, 20, 5, 
+                                  help="Limit number of pages to process")
+
+# About tab content
+with main_tab2:
     st.markdown("""
-    This app uses Mistral AI's OCR API to extract text from historical documents.
+    ### About This Application
+    
+    This app uses [Mistral AI's Document OCR](https://docs.mistral.ai/capabilities/document/) to extract text and images from historical documents.
     
     It can process:
     - Image files (jpg, png, etc.)
-    - PDF documents
+    - PDF documents (multi-page support)
     
-    The extracted content is processed into structured data based on the document type.
+    The extracted content is processed into structured data based on the document type, combining:
+    - Text extraction with `mistral-ocr-latest`
+    - Analysis with language models
+    - Layout preservation with images
+    
+    View results in three formats:
+    - Structured HTML view
+    - Raw JSON (for developers)
+    - Markdown with images (preserves document layout)
+    
+    **New Features:**
+    - Image preprocessing for better OCR quality
+    - PDF resolution and page controls
+    - Progress tracking during processing
     """)
 
-# File uploader
-uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"])
-
-if uploaded_file is not None:
-    # Display the uploaded file
-    st.subheader("Uploaded Document")
-    file_ext = Path(uploaded_file.name).suffix.lower()
-    
-    col1, col2 = st.columns([1, 1])
-    
-    with col1:
-        if file_ext == ".pdf":
-            st.info("Processing PDF document...")
-            # For PDFs, you might show preview of first page
-            st.write(f"File: {uploaded_file.name}")
-        else:
-            st.image(uploaded_file, use_container_width=True)
+with main_tab1:
+    if uploaded_file is not None:
+        # Check file size (cap at 50MB)
+        file_size_mb = len(uploaded_file.getvalue()) / (1024 * 1024)
+        
+        if file_size_mb > 50:
+            with upload_col:
+                st.error(f"File too large ({file_size_mb:.1f} MB). Maximum file size is 50MB.")
+            st.stop()
+        
+        file_ext = Path(uploaded_file.name).suffix.lower()
+        
+        # Display document preview in preview column
+        with preview_col:
+            st.subheader("Document Preview")
+            if file_ext == ".pdf":
+                try:
+                    # Convert first page of PDF to image for preview
+                    pdf_bytes = uploaded_file.getvalue()
+                    images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                    
+                    if images:
+                        # Convert PIL image to bytes for Streamlit
+                        first_page = images[0]
+                        img_bytes = io.BytesIO()
+                        first_page.save(img_bytes, format='JPEG')
+                        img_bytes.seek(0)
+                        
+                        # Display the PDF preview
+                        st.image(img_bytes, caption=f"PDF Preview: {uploaded_file.name}", use_container_width=True)
+                    else:
+                        st.info(f"PDF uploaded: {uploaded_file.name}")
+                except Exception:
+                    # Simply show the file name without an error message
+                    st.info(f"PDF uploaded: {uploaded_file.name}")
+                    st.info("Click 'Process Document' to analyze the content.")
+            else:
+                st.image(uploaded_file, use_container_width=True)
 
-    # Process button
-    process_button = st.button("Process Document")
-    
-    if process_button:
-        with st.spinner("Processing document..."):
+        # Add image preprocessing preview in a collapsible section if needed
+        if any(preprocessing_options.values()) and uploaded_file.type.startswith('image/'):
+            with st.expander("Image Preprocessing Preview"):
+                preview_cols = st.columns(2)
+                
+                with preview_cols[0]:
+                    st.markdown("**Original Image**")
+                    st.image(uploaded_file, use_container_width=True)
+                
+                with preview_cols[1]:
+                    st.markdown("**Preprocessed Image**")
+                    try:
+                        processed_bytes = preprocess_image(uploaded_file.getvalue(), preprocessing_options)
+                        st.image(io.BytesIO(processed_bytes), use_container_width=True)
+                    except Exception as e:
+                        st.error(f"Error in preprocessing: {str(e)}")
+        
+        # Process button - flush left with similar padding as file browser
+        with upload_col:
+            process_button = st.button("Process Document", use_container_width=True)
+        
+        # Results section
+        if process_button:
             try:
-                # Process the file
-                result = process_file(uploaded_file, use_vision)
+                # Get max_pages or default if not available
+                max_pages_value = max_pages if 'max_pages' in locals() else None
                 
-                # Display the results
-                with col2:
-                    st.subheader("Extracted Information")
-                    
-                    # Display file info
-                    st.write(f"**File Name:** {result.get('file_name', uploaded_file.name)}")
-                    
-                    # Display languages if available
-                    if 'languages' in result:
-                        languages = [lang for lang in result['languages'] if lang is not None]
-                        if languages:
-                            st.write(f"**Languages Detected:** {', '.join(languages)}")
-                    
-                    # Display topics if available
-                    if 'topics' in result and result['topics']:
-                        st.write(f"**Topics:** {', '.join(result['topics'])}")
+                # Call process_file with all options
+                result = process_file(uploaded_file, use_vision, preprocessing_options)
                 
-                # Display the OCR contents
-                st.subheader("Document Contents")
-                if 'ocr_contents' in result:
-                    # Create tabs for different views
-                    tab1, tab2 = st.tabs(["Structured View", "Raw JSON"])
+                # Create results tabs for better organization
+                results_tab1, results_tab2 = st.tabs(["Document Analysis", "Technical Details"])
+                
+                with results_tab1:
+                    # Create two columns for metadata and content
+                    meta_col, content_col = st.columns([1, 2])
                     
-                    with tab1:
-                        # Display in a more user-friendly format based on the content structure
-                        if isinstance(result['ocr_contents'], dict):
-                            for section, content in result['ocr_contents'].items():
-                                if content:  # Only display non-empty sections
-                                    st.markdown(f"#### {section.replace('_', ' ').title()}")
-                                    
-                                    if isinstance(content, str):
-                                        st.markdown(content)
-                                    elif isinstance(content, list):
-                                        for item in content:
-                                            if isinstance(item, str):
-                                                st.markdown(f"- {item}")
-                                            elif isinstance(item, dict):
-                                                st.json(item)
-                                    elif isinstance(content, dict):
-                                        for k, v in content.items():
-                                            st.markdown(f"**{k}:** {v}")
+                    with meta_col:
+                        st.subheader("Document Metadata")
+                        st.success("**Document processed successfully**")
+                        
+                        # Display file info
+                        st.write(f"**File Name:** {result.get('file_name', uploaded_file.name)}")
+                        
+                        # Display info if only limited pages were processed
+                        if 'limited_pages' in result:
+                            st.info(f"Processed {result['limited_pages']['processed']} of {result['limited_pages']['total']} pages")
+                        
+                        # Display languages if available
+                        if 'languages' in result:
+                            languages = [lang for lang in result['languages'] if lang is not None]
+                            if languages:
+                                st.write(f"**Languages:** {', '.join(languages)}")
+                                
+                        # Confidence score if available
+                        if 'confidence_score' in result:
+                            confidence = result['confidence_score']
+                            st.write(f"**OCR Confidence:** {confidence:.1%}")
+                        
+                        # Display topics if available
+                        if 'topics' in result and result['topics']:
+                            st.write(f"**Topics:** {', '.join(result['topics'])}")
                     
-                    with tab2:
-                        # Show the raw JSON for developers
-                        st.json(result)
-                else:
-                    st.error("No OCR content was extracted from the document.")
+                    with content_col:
+                        st.subheader("Document Contents")
+                        if 'ocr_contents' in result:
+                            # Check if there are images in the OCR result
+                            has_images = False
+                            if 'raw_response' in result:
+                                try:
+                                    has_images = any(page.images for page in result['raw_response'].pages)
+                                except Exception:
+                                    has_images = False
+                            
+                            # Create tabs for different views
+                            if has_images:
+                                view_tab1, view_tab2, view_tab3 = st.tabs(["Structured View", "Raw JSON", "With Images"])
+                            else:
+                                view_tab1, view_tab2 = st.tabs(["Structured View", "Raw JSON"])
+                            
+                            with view_tab1:
+                                # Display in a more user-friendly format based on the content structure
+                                html_content = ""
+                                if isinstance(result['ocr_contents'], dict):
+                                    for section, content in result['ocr_contents'].items():
+                                        if content:  # Only display non-empty sections
+                                            section_title = f"<h4>{section.replace('_', ' ').title()}</h4>"
+                                            html_content += section_title
+                                            
+                                            if isinstance(content, str):
+                                                html_content += f"<p>{content}</p>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                st.markdown(content)
+                                            elif isinstance(content, list):
+                                                html_list = "<ul>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                for item in content:
+                                                    if isinstance(item, str):
+                                                        html_list += f"<li>{item}</li>"
+                                                        st.markdown(f"- {item}")
+                                                    elif isinstance(item, dict):
+                                                        html_list += f"<li>{json.dumps(item)}</li>"
+                                                        st.json(item)
+                                                html_list += "</ul>"
+                                                html_content += html_list
+                                            elif isinstance(content, dict):
+                                                html_dict = "<dl>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                for k, v in content.items():
+                                                    html_dict += f"<dt><strong>{k}</strong></dt><dd>{v}</dd>"
+                                                    st.markdown(f"**{k}:** {v}")
+                                                html_dict += "</dl>"
+                                                html_content += html_dict
+                                
+                                # Add download button in a smaller section
+                                with st.expander("Export Content"):
+                                    # Alternative download button
+                                    html_bytes = html_content.encode()
+                                    st.download_button(
+                                        label="Download as HTML",
+                                        data=html_bytes,
+                                        file_name="document_content.html",
+                                        mime="text/html"
+                                    )
+                            
+                            with view_tab2:
+                                # Show the raw JSON for developers
+                                st.json(result)
+                            
+                            if has_images:
+                                with view_tab3:
+                                    # Show loading indicator while preparing images
+                                    with st.spinner("Preparing document with embedded images..."):
+                                        try:
+                                            # Import function
+                                            try:
+                                                from ocr_utils import get_combined_markdown
+                                            except ImportError:
+                                                st.error("Required module ocr_utils not found.")
+                                                st.stop()
+                                            
+                                            # Check if raw_response is available
+                                            if 'raw_response' not in result:
+                                                st.warning("Raw OCR response not available. Cannot display images.")
+                                                st.stop()
+                                            
+                                            # Validate the raw_response structure before processing
+                                            if not hasattr(result['raw_response'], 'pages'):
+                                                st.warning("Invalid OCR response format. Cannot display images.")
+                                                st.stop()
+                                                
+                                            # Get the combined markdown with images
+                                            # Set a flag to compress images if needed
+                                            compress_images = True
+                                            max_image_width = 800  # Maximum width for images
+                                            
+                                            try:
+                                                # First try to get combined markdown with compressed images
+                                                if compress_images and hasattr(result['raw_response'], 'pages'):
+                                                    from ocr_utils import get_combined_markdown_compressed
+                                                    combined_markdown = get_combined_markdown_compressed(
+                                                        result['raw_response'], 
+                                                        max_width=max_image_width,
+                                                        quality=85
+                                                    )
+                                                else:
+                                                    # Fall back to regular method if compression not available
+                                                    combined_markdown = get_combined_markdown(result['raw_response'])
+                                            except (ImportError, AttributeError):
+                                                # Fall back to regular method
+                                                combined_markdown = get_combined_markdown(result['raw_response'])
+                                            
+                                            if not combined_markdown or combined_markdown.strip() == "":
+                                                st.warning("No image content found in the document.")
+                                                st.stop()
+                                            
+                                            # Check if there are many images that might cause loading issues
+                                            image_count = sum(len(page.images) for page in result['raw_response'].pages if hasattr(page, 'images'))
+                                            
+                                            # Add warning for image-heavy documents
+                                            if image_count > 10:
+                                                st.warning(f"This document contains {image_count} images. Rendering may take longer than usual.")
+                                                
+                                            # Add CSS to ensure proper spacing and handling of text and images
+                                            st.markdown("""
+                                            <style>
+                                            .markdown-text-container {
+                                                padding: 10px;
+                                                background-color: #f9f9f9;
+                                                border-radius: 5px;
+                                            }
+                                            .markdown-text-container img {
+                                                margin: 15px 0;
+                                                max-width: 100%;
+                                                border: 1px solid #ddd;
+                                                border-radius: 4px;
+                                                display: block;
+                                            }
+                                            .markdown-text-container p {
+                                                margin-bottom: 16px;
+                                                line-height: 1.6;
+                                            }
+                                            /* Add lazy loading for images to improve performance */
+                                            .markdown-text-container img {
+                                                loading: lazy;
+                                            }
+                                            </style>
+                                            """, unsafe_allow_html=True)
+                                            
+                                            # For very image-heavy documents, show images in a paginated way
+                                            if image_count > 20:
+                                                # Show image content in a paginated way
+                                                st.write("Document contains many images. Showing in a paginated format:")
+                                                
+                                                # Split the combined markdown by page separators
+                                                pages = combined_markdown.split("---")
+                                                
+                                                # Create a page selector
+                                                page_num = st.selectbox("Select page to view:", 
+                                                                     options=list(range(1, len(pages)+1)),
+                                                                     index=0)
+                                                
+                                                # Display only the selected page
+                                                st.markdown(f"""
+                                                <div class="markdown-text-container">
+                                                {pages[page_num-1]}
+                                                </div>
+                                                """, unsafe_allow_html=True)
+                                                
+                                                # Add note about pagination
+                                                st.info(f"Showing page {page_num} of {len(pages)}. Select a different page from the dropdown above.")
+                                            else:
+                                                # Wrap the markdown in a div with the class for styling
+                                                st.markdown(f"""
+                                                <div class="markdown-text-container">
+                                                {combined_markdown}
+                                                </div>
+                                                """, unsafe_allow_html=True)
+                                            
+                                            # Add a download button for the combined content
+                                            st.download_button(
+                                                label="Download with Images (HTML)",
+                                                data=f"""
+                                                <html>
+                                                <head>
+                                                    <style>
+                                                    body {{ font-family: Arial, sans-serif; line-height: 1.6; }}
+                                                    img {{ max-width: 100%; margin: 15px 0; }}
+                                                    </style>
+                                                </head>
+                                                <body>
+                                                {combined_markdown}
+                                                </body>
+                                                </html>
+                                                """,
+                                                file_name="document_with_images.html",
+                                                mime="text/html"
+                                            )
+                                            
+                                        except Exception as e:
+                                            st.error(f"Could not display document with images: {str(e)}")
+                                            st.info("Try refreshing or processing the document again.")
+                        else:
+                            st.error("No OCR content was extracted from the document.")
+                
+                with results_tab2:
+                    st.subheader("Raw Processing Results")
+                    st.json(result)
                     
             except Exception as e:
                 st.error(f"Error processing document: {str(e)}")
-else:
-    # Display sample images when no file is uploaded
-    st.info("Upload a document to get started.")
-    
-    # Show example images
-    st.subheader("Example Documents")
-    col1, col2, col3 = st.columns([1, 1, 1])
-    
-    # Find sample images from the input directory to display
-    input_dir = Path(__file__).parent / "input"
-    sample_images = []
-    if input_dir.exists():
-        sample_images = list(input_dir.glob("*.jpg"))[:3]  # Limit to 3 samples
-    
-    if sample_images:
-        for i, img_path in enumerate(sample_images):
-            col = [col1, col2, col3][i % 3]
-            with col:
-                st.image(str(img_path), caption=img_path.name, use_container_width=True)
+    else:
+        # Display sample images in the main area when no file is uploaded
+        st.info("Upload a document to get started using the file uploader above.")
+        
+        # Show example images in a grid
+        st.subheader("Example Documents")
+        
+        # Add a sample images container
+        with st.container():
+            # Find sample images from the input directory to display
+            input_dir = Path(__file__).parent / "input"
+            sample_images = []
+            if input_dir.exists():
+                # Find valid jpg files (with size > 50KB to avoid placeholders)
+                sample_images = [
+                    path for path in input_dir.glob("*.jpg") 
+                    if path.stat().st_size > 50000
+                ][:3]  # Limit to 3 samples
+            
+            if sample_images:
+                columns = st.columns(3)
+                for i, img_path in enumerate(sample_images):
+                    with columns[i % 3]:
+                        try:
+                            st.image(str(img_path), caption=img_path.name, use_container_width=True)
+                        except Exception as e:
+                            st.error(f"Error loading image {img_path.name}: {str(e)}")

backup/app.py

@@ -0,0 +1,535 @@
+import os
+import streamlit as st
+import json
+import sys
+import time
+from pathlib import Path
+import tempfile
+import io
+from pdf2image import convert_from_bytes
+from PIL import Image, ImageEnhance, ImageFilter
+import cv2
+import numpy as np
+
+# Import the StructuredOCR class and config from the local files
+from structured_ocr import StructuredOCR
+from config import MISTRAL_API_KEY
+
+# Set page configuration
+st.set_page_config(
+    page_title="Historical OCR",
+    page_icon="🚀",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Enable caching for expensive operations
+@st.cache_data(ttl=3600, show_spinner=False)
+def convert_pdf_to_images(pdf_bytes, dpi=150):
+    """Convert PDF bytes to a list of images with caching"""
+    try:
+        return convert_from_bytes(pdf_bytes, dpi=dpi)
+    except Exception as e:
+        st.error(f"Error converting PDF: {str(e)}")
+        return []
+
+@st.cache_data(ttl=3600, show_spinner=False)
+def preprocess_image(image_bytes, preprocessing_options):
+    """Preprocess image with selected options"""
+    # Convert bytes to OpenCV format
+    image = Image.open(io.BytesIO(image_bytes))
+    img_array = np.array(image)
+    
+    # Apply preprocessing based on selected options
+    if preprocessing_options.get("grayscale", False):
+        img_array = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
+        img_array = cv2.cvtColor(img_array, cv2.COLOR_GRAY2RGB)
+    
+    if preprocessing_options.get("contrast", 0) != 0:
+        contrast_factor = 1 + (preprocessing_options.get("contrast", 0) / 10)
+        image = Image.fromarray(img_array)
+        enhancer = ImageEnhance.Contrast(image)
+        image = enhancer.enhance(contrast_factor)
+        img_array = np.array(image)
+    
+    if preprocessing_options.get("denoise", False):
+        img_array = cv2.fastNlMeansDenoisingColored(img_array, None, 10, 10, 7, 21)
+        
+    if preprocessing_options.get("threshold", False):
+        # Convert to grayscale if not already
+        if len(img_array.shape) == 3:
+            gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
+        else:
+            gray = img_array
+        # Apply adaptive threshold
+        binary = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, 
+                                      cv2.THRESH_BINARY, 11, 2)
+        # Convert back to RGB
+        img_array = cv2.cvtColor(binary, cv2.COLOR_GRAY2RGB)
+        
+    # Convert back to PIL Image
+    processed_image = Image.fromarray(img_array)
+    
+    # Convert to bytes
+    byte_io = io.BytesIO()
+    processed_image.save(byte_io, format='PNG')
+    byte_io.seek(0)
+    
+    return byte_io.getvalue()
+
+# Define functions
+def process_file(uploaded_file, use_vision=True, preprocessing_options=None):
+    """Process the uploaded file and return the OCR results
+    
+    Args:
+        uploaded_file: The uploaded file to process
+        use_vision: Whether to use vision model
+        preprocessing_options: Dictionary of preprocessing options
+    """
+    if preprocessing_options is None:
+        preprocessing_options = {}
+        
+    # Show progress indicator
+    progress_bar = st.progress(0)
+    status_text = st.empty()
+    status_text.text("Preparing file for processing...")
+    
+    # Save the uploaded file to a temporary file
+    with tempfile.NamedTemporaryFile(delete=False, suffix=Path(uploaded_file.name).suffix) as tmp:
+        tmp.write(uploaded_file.getvalue())
+        temp_path = tmp.name
+    
+    try:
+        # Check if API key is available
+        if not MISTRAL_API_KEY:
+            # Return dummy data if no API key
+            progress_bar.progress(100)
+            status_text.empty()
+            return {
+                "file_name": uploaded_file.name,
+                "topics": ["Sample Document"],
+                "languages": ["English"],
+                "ocr_contents": {
+                    "title": "Sample Document",
+                    "content": "This is sample content. To process real documents, please set the MISTRAL_API_KEY environment variable."
+                }
+            }
+        
+        # Update progress
+        progress_bar.progress(20)
+        status_text.text("Initializing OCR processor...")
+        
+        # Initialize OCR processor
+        processor = StructuredOCR()
+        
+        # Determine file type from extension
+        file_ext = Path(uploaded_file.name).suffix.lower()
+        file_type = "pdf" if file_ext == ".pdf" else "image"
+        
+        # Apply preprocessing if needed
+        if any(preprocessing_options.values()) and file_type == "image":
+            status_text.text("Applying image preprocessing...")
+            processed_bytes = preprocess_image(uploaded_file.getvalue(), preprocessing_options)
+            
+            # Save processed image to temp file
+            with tempfile.NamedTemporaryFile(delete=False, suffix=Path(uploaded_file.name).suffix) as proc_tmp:
+                proc_tmp.write(processed_bytes)
+                temp_path = proc_tmp.name
+        
+        # Get file size in MB
+        file_size_mb = os.path.getsize(temp_path) / (1024 * 1024)
+        
+        # Check if file exceeds API limits (50 MB)
+        if file_size_mb > 50:
+            st.error(f"File too large ({file_size_mb:.1f} MB). Maximum file size allowed by Mistral API is 50MB.")
+            return {
+                "file_name": uploaded_file.name,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "confidence_score": 0.0,
+                "error": f"File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB",
+                "ocr_contents": {
+                    "error": f"Failed to process file: File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB",
+                    "partial_text": "Document could not be processed due to size limitations."
+                }
+            }
+        
+        # Update progress
+        progress_bar.progress(40)
+        status_text.text("Processing document with OCR...")
+        
+        # Process the file with file size information for automatic page limiting
+        # Make sure we're using the latest mistral-ocr model 
+        # See https://docs.mistral.ai/capabilities/document/ for more info
+        result = processor.process_file(temp_path, file_type=file_type, use_vision=use_vision, file_size_mb=file_size_mb)
+        
+        # Complete progress
+        progress_bar.progress(100)
+        status_text.empty()
+        
+        return result
+    except Exception as e:
+        progress_bar.progress(100)
+        status_text.empty()
+        st.error(f"Error during processing: {str(e)}")
+        raise
+    finally:
+        # Clean up the temporary file
+        if os.path.exists(temp_path):
+            os.unlink(temp_path)
+
+# App title and description
+st.title("Historical Document OCR")
+st.subheader("Powered by Mistral AI")
+
+# Create main layout with tabs and columns
+main_tab1, main_tab2 = st.tabs(["Document Processing", "About"])
+
+with main_tab1:
+    # Create a two-column layout for file upload and preview
+    upload_col, preview_col = st.columns([1, 1])
+    
+    # File uploader in the left column
+    with upload_col:
+        st.markdown("""
+        Upload an image or PDF file to get started. 
+        
+        Using the latest `mistral-ocr-latest` model for advanced document understanding.
+        """)
+        uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"])
+
+# Sidebar with options
+with st.sidebar:
+    st.header("Options")
+    
+    # Model options
+    st.subheader("Model Settings")
+    use_vision = st.checkbox("Use Vision Model", value=True, 
+                            help="For image files, use the vision model for improved analysis (may be slower)")
+    
+    # Image preprocessing options (collapsible)
+    st.subheader("Image Preprocessing")
+    with st.expander("Preprocessing Options"):
+        preprocessing_options = {}
+        preprocessing_options["grayscale"] = st.checkbox("Convert to Grayscale", 
+                                                        help="Convert image to grayscale before OCR")
+        preprocessing_options["threshold"] = st.checkbox("Apply Thresholding", 
+                                                      help="Apply adaptive thresholding to enhance text")
+        preprocessing_options["denoise"] = st.checkbox("Denoise Image", 
+                                                     help="Remove noise from the image")
+        preprocessing_options["contrast"] = st.slider("Adjust Contrast", -5, 5, 0, 
+                                                    help="Adjust image contrast (-5 to +5)")
+    
+    # PDF options (collapsible)
+    st.subheader("PDF Options")
+    with st.expander("PDF Settings"):
+        pdf_dpi = st.slider("PDF Resolution (DPI)", 72, 300, 150, 
+                          help="Higher DPI gives better quality but slower processing")
+        max_pages = st.number_input("Maximum Pages to Process", 1, 20, 5, 
+                                  help="Limit number of pages to process")
+
+# About tab content
+with main_tab2:
+    st.markdown("""
+    ### About This Application
+    
+    This app uses [Mistral AI's Document OCR](https://docs.mistral.ai/capabilities/document/) to extract text and images from historical documents.
+    
+    It can process:
+    - Image files (jpg, png, etc.)
+    - PDF documents (multi-page support)
+    
+    The extracted content is processed into structured data based on the document type, combining:
+    - Text extraction with `mistral-ocr-latest`
+    - Analysis with language models
+    - Layout preservation with images
+    
+    View results in three formats:
+    - Structured HTML view
+    - Raw JSON (for developers)
+    - Markdown with images (preserves document layout)
+    
+    **New Features:**
+    - Image preprocessing for better OCR quality
+    - PDF resolution and page controls
+    - Progress tracking during processing
+    """)
+
+with main_tab1:
+    if uploaded_file is not None:
+        # Check file size (cap at 50MB)
+        file_size_mb = len(uploaded_file.getvalue()) / (1024 * 1024)
+        
+        if file_size_mb > 50:
+            with upload_col:
+                st.error(f"File too large ({file_size_mb:.1f} MB). Maximum file size is 50MB.")
+            st.stop()
+        
+        file_ext = Path(uploaded_file.name).suffix.lower()
+        
+        # Display document preview in preview column
+        with preview_col:
+            st.subheader("Document Preview")
+            if file_ext == ".pdf":
+                try:
+                    # Convert first page of PDF to image for preview
+                    pdf_bytes = uploaded_file.getvalue()
+                    images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                    
+                    if images:
+                        # Convert PIL image to bytes for Streamlit
+                        first_page = images[0]
+                        img_bytes = io.BytesIO()
+                        first_page.save(img_bytes, format='JPEG')
+                        img_bytes.seek(0)
+                        
+                        # Display the PDF preview
+                        st.image(img_bytes, caption=f"PDF Preview: {uploaded_file.name}", use_container_width=True)
+                    else:
+                        st.info(f"PDF uploaded: {uploaded_file.name}")
+                except Exception:
+                    # Simply show the file name without an error message
+                    st.info(f"PDF uploaded: {uploaded_file.name}")
+                    st.info("Click 'Process Document' to analyze the content.")
+            else:
+                st.image(uploaded_file, use_container_width=True)
+
+        # Add image preprocessing preview in a collapsible section if needed
+        if any(preprocessing_options.values()) and uploaded_file.type.startswith('image/'):
+            with st.expander("Image Preprocessing Preview"):
+                preview_cols = st.columns(2)
+                
+                with preview_cols[0]:
+                    st.markdown("**Original Image**")
+                    st.image(uploaded_file, use_container_width=True)
+                
+                with preview_cols[1]:
+                    st.markdown("**Preprocessed Image**")
+                    try:
+                        processed_bytes = preprocess_image(uploaded_file.getvalue(), preprocessing_options)
+                        st.image(io.BytesIO(processed_bytes), use_container_width=True)
+                    except Exception as e:
+                        st.error(f"Error in preprocessing: {str(e)}")
+        
+        # Process button - flush left with similar padding as file browser
+        with upload_col:
+            process_button = st.button("Process Document", use_container_width=True)
+        
+        # Results section
+        if process_button:
+            try:
+                # Get max_pages or default if not available
+                max_pages_value = max_pages if 'max_pages' in locals() else None
+                
+                # Call process_file with all options
+                result = process_file(uploaded_file, use_vision, preprocessing_options)
+                
+                # Create results tabs for better organization
+                results_tab1, results_tab2 = st.tabs(["Document Analysis", "Technical Details"])
+                
+                with results_tab1:
+                    # Create two columns for metadata and content
+                    meta_col, content_col = st.columns([1, 2])
+                    
+                    with meta_col:
+                        st.subheader("Document Metadata")
+                        st.success("**Document processed successfully**")
+                        
+                        # Display file info
+                        st.write(f"**File Name:** {result.get('file_name', uploaded_file.name)}")
+                        
+                        # Display info if only limited pages were processed
+                        if 'limited_pages' in result:
+                            st.info(f"Processed {result['limited_pages']['processed']} of {result['limited_pages']['total']} pages")
+                        
+                        # Display languages if available
+                        if 'languages' in result:
+                            languages = [lang for lang in result['languages'] if lang is not None]
+                            if languages:
+                                st.write(f"**Languages:** {', '.join(languages)}")
+                                
+                        # Confidence score if available
+                        if 'confidence_score' in result:
+                            confidence = result['confidence_score']
+                            st.write(f"**OCR Confidence:** {confidence:.1%}")
+                        
+                        # Display topics if available
+                        if 'topics' in result and result['topics']:
+                            st.write(f"**Topics:** {', '.join(result['topics'])}")
+                    
+                    with content_col:
+                        st.subheader("Document Contents")
+                        if 'ocr_contents' in result:
+                            # Check if there are images in the OCR result
+                            has_images = False
+                            if 'raw_response' in result:
+                                try:
+                                    has_images = any(page.images for page in result['raw_response'].pages)
+                                except Exception:
+                                    has_images = False
+                            
+                            # Create tabs for different views
+                            if has_images:
+                                view_tab1, view_tab2, view_tab3 = st.tabs(["Structured View", "Raw JSON", "With Images"])
+                            else:
+                                view_tab1, view_tab2 = st.tabs(["Structured View", "Raw JSON"])
+                            
+                            with view_tab1:
+                                # Display in a more user-friendly format based on the content structure
+                                html_content = ""
+                                if isinstance(result['ocr_contents'], dict):
+                                    for section, content in result['ocr_contents'].items():
+                                        if content:  # Only display non-empty sections
+                                            section_title = f"<h4>{section.replace('_', ' ').title()}</h4>"
+                                            html_content += section_title
+                                            
+                                            if isinstance(content, str):
+                                                html_content += f"<p>{content}</p>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                st.markdown(content)
+                                            elif isinstance(content, list):
+                                                html_list = "<ul>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                for item in content:
+                                                    if isinstance(item, str):
+                                                        html_list += f"<li>{item}</li>"
+                                                        st.markdown(f"- {item}")
+                                                    elif isinstance(item, dict):
+                                                        html_list += f"<li>{json.dumps(item)}</li>"
+                                                        st.json(item)
+                                                html_list += "</ul>"
+                                                html_content += html_list
+                                            elif isinstance(content, dict):
+                                                html_dict = "<dl>"
+                                                st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                                for k, v in content.items():
+                                                    html_dict += f"<dt><strong>{k}</strong></dt><dd>{v}</dd>"
+                                                    st.markdown(f"**{k}:** {v}")
+                                                html_dict += "</dl>"
+                                                html_content += html_dict
+                                
+                                # Add download button in a smaller section
+                                with st.expander("Export Content"):
+                                    # Alternative download button
+                                    html_bytes = html_content.encode()
+                                    st.download_button(
+                                        label="Download as HTML",
+                                        data=html_bytes,
+                                        file_name="document_content.html",
+                                        mime="text/html"
+                                    )
+                            
+                            with view_tab2:
+                                # Show the raw JSON for developers
+                                st.json(result)
+                            
+                            if has_images:
+                                with view_tab3:
+                                    # Show loading indicator while preparing images
+                                    with st.spinner("Preparing document with embedded images..."):
+                                        try:
+                                            # Import function
+                                            try:
+                                                from ocr_utils import get_combined_markdown
+                                            except ImportError:
+                                                st.error("Required module ocr_utils not found.")
+                                                st.stop()
+                                            
+                                            # Check if raw_response is available
+                                            if 'raw_response' not in result:
+                                                st.warning("Raw OCR response not available. Cannot display images.")
+                                                st.stop()
+                                            
+                                            # Validate the raw_response structure before processing
+                                            if not hasattr(result['raw_response'], 'pages'):
+                                                st.warning("Invalid OCR response format. Cannot display images.")
+                                                st.stop()
+                                                
+                                            # Get the combined markdown with images
+                                            combined_markdown = get_combined_markdown(result['raw_response'])
+                                            
+                                            if not combined_markdown or combined_markdown.strip() == "":
+                                                st.warning("No image content found in the document.")
+                                                st.stop()
+                                            
+                                            # Add CSS to ensure proper spacing and handling of text and images
+                                            st.markdown("""
+                                            <style>
+                                            .markdown-text-container {
+                                                padding: 10px;
+                                                background-color: #f9f9f9;
+                                                border-radius: 5px;
+                                            }
+                                            .markdown-text-container img {
+                                                margin: 15px 0;
+                                                max-width: 100%;
+                                                border: 1px solid #ddd;
+                                                border-radius: 4px;
+                                                display: block;
+                                            }
+                                            .markdown-text-container p {
+                                                margin-bottom: 16px;
+                                                line-height: 1.6;
+                                            }
+                                            </style>
+                                            """, unsafe_allow_html=True)
+                                            
+                                            # Wrap the markdown in a div with the class for styling
+                                            st.markdown(f"""
+                                            <div class="markdown-text-container">
+                                            {combined_markdown}
+                                            </div>
+                                            """, unsafe_allow_html=True)
+                                            
+                                            # Add a download button for the combined content
+                                            st.download_button(
+                                                label="Download with Images (HTML)",
+                                                data=f"""
+                                                <html>
+                                                <head>
+                                                    <style>
+                                                    body {{ font-family: Arial, sans-serif; line-height: 1.6; }}
+                                                    img {{ max-width: 100%; margin: 15px 0; }}
+                                                    </style>
+                                                </head>
+                                                <body>
+                                                {combined_markdown}
+                                                </body>
+                                                </html>
+                                                """,
+                                                file_name="document_with_images.html",
+                                                mime="text/html"
+                                            )
+                                            
+                                        except Exception as e:
+                                            st.error(f"Could not display document with images: {str(e)}")
+                                            st.info("Try refreshing or processing the document again.")
+                        else:
+                            st.error("No OCR content was extracted from the document.")
+                
+                with results_tab2:
+                    st.subheader("Raw Processing Results")
+                    st.json(result)
+                    
+            except Exception as e:
+                st.error(f"Error processing document: {str(e)}")
+    else:
+        # Display sample images in the main area when no file is uploaded
+        st.info("Upload a document to get started using the file uploader above.")
+        
+        # Show example images in a grid
+        st.subheader("Example Documents")
+        
+        # Add a sample images container
+        with st.container():
+            # Find sample images from the input directory to display
+            input_dir = Path(__file__).parent / "input"
+            sample_images = []
+            if input_dir.exists():
+                sample_images = list(input_dir.glob("*.jpg"))[:3]  # Limit to 3 samples
+            
+            if sample_images:
+                columns = st.columns(3)
+                for i, img_path in enumerate(sample_images):
+                    with columns[i % 3]:
+                        st.image(str(img_path), caption=img_path.name, use_container_width=True)

backup/config.py

@@ -0,0 +1,17 @@
+# config.py
+"""
+Configuration file for Mistral OCR processing.
+Contains API key and other settings.
+"""
+import os
+
+# Your Mistral API key - get from Hugging Face secrets or environment variable
+# The priority order is: HF_SPACES environment var > regular environment var > empty string
+# Note: No default API key is provided for security reasons
+MISTRAL_API_KEY = os.environ.get("HF_MISTRAL_API_KEY",  # First check HF-specific env var
+                  os.environ.get("MISTRAL_API_KEY", ""))  # Then check regular env var
+
+# Model settings 
+OCR_MODEL = "mistral-ocr-latest"
+TEXT_MODEL = "ministral-8b-latest"
+VISION_MODEL = "pixtral-12b-latest"

backup/input/menu.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d96008f374f5be8046b569c868e33f4e5a0e5e166c245d324b44140c7e6c2e
+size 2554815

backup/ocr_utils.py

@@ -0,0 +1,136 @@
+"""
+Utility functions for OCR processing with Mistral AI.
+Contains helper functions for working with OCR responses and image handling.
+"""
+
+import json
+import base64
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from mistralai import DocumentURLChunk, ImageURLChunk, TextChunk
+
+def replace_images_in_markdown(markdown_str: str, images_dict: dict) -> str:
+    """
+    Replace image placeholders in markdown with base64-encoded images.
+
+    Args:
+        markdown_str: Markdown text containing image placeholders
+        images_dict: Dictionary mapping image IDs to base64 strings
+
+    Returns:
+        Markdown text with images replaced by base64 data
+    """
+    for img_name, base64_str in images_dict.items():
+        markdown_str = markdown_str.replace(
+            f"![{img_name}]({img_name})", f"![{img_name}]({base64_str})"
+        )
+    return markdown_str
+
+def get_combined_markdown(ocr_response) -> str:
+    """
+    Combine OCR text and images into a single markdown document.
+    Ensures proper spacing between text and images.
+
+    Args:
+        ocr_response: Response from OCR processing containing text and images
+            See https://docs.mistral.ai/capabilities/document/ for API reference
+
+    Returns:
+        Combined markdown string with embedded images
+    """
+    markdowns: list[str] = []
+    # Extract images from page
+    for page in ocr_response.pages:
+        image_data = {}
+        for img in page.images:
+            image_data[img.id] = img.image_base64
+        
+        # Replace image placeholders with actual images
+        page_markdown = replace_images_in_markdown(page.markdown, image_data)
+        
+        # Ensure proper spacing between paragraphs and images
+        # Add extra newlines between paragraphs to improve rendering
+        page_markdown = page_markdown.replace("\n", "\n\n")
+        
+        # Add page separator for multi-page documents
+        markdowns.append(page_markdown)
+        
+    # Join pages with clear separators for multi-page documents
+    return "\n\n---\n\n".join(markdowns)
+
+def encode_image_for_api(image_path: Union[str, Path]) -> str:
+    """
+    Encode an image as base64 for API use.
+    
+    Args:
+        image_path: Path to the image file
+        
+    Returns:
+        Base64 data URL for the image
+    """
+    # Convert to Path object if string
+    image_file = Path(image_path) if isinstance(image_path, str) else image_path
+    
+    # Verify image exists
+    if not image_file.is_file():
+        raise FileNotFoundError(f"Image file not found: {image_file}")
+    
+    # Encode image as base64
+    encoded = base64.b64encode(image_file.read_bytes()).decode()
+    return f"data:image/jpeg;base64,{encoded}"
+
+def process_image_with_ocr(client, image_path: Union[str, Path], model: str = "mistral-ocr-latest"):
+    """
+    Process an image with OCR and return the response.
+    
+    Args:
+        client: Mistral AI client
+        image_path: Path to the image file
+        model: OCR model to use
+        
+    Returns:
+        OCR response object
+    """
+    # Encode image as base64
+    base64_data_url = encode_image_for_api(image_path)
+    
+    # Process image with OCR
+    image_response = client.ocr.process(
+        document=ImageURLChunk(image_url=base64_data_url),
+        model=model
+    )
+    
+    return image_response
+
+def ocr_response_to_json(ocr_response, indent: int = 4) -> str:
+    """
+    Convert OCR response to a formatted JSON string.
+    
+    Args:
+        ocr_response: OCR response object
+        indent: Indentation level for JSON formatting
+        
+    Returns:
+        Formatted JSON string
+    """
+    # Convert response to JSON
+    response_dict = json.loads(ocr_response.model_dump_json())
+    return json.dumps(response_dict, indent=indent)
+
+# For display in notebooks
+try:
+    from IPython.display import Markdown, display
+    
+    def display_ocr_with_images(ocr_response):
+        """
+        Display OCR response with embedded images in IPython environments.
+        
+        Args:
+            ocr_response: OCR response object
+        """
+        combined_markdown = get_combined_markdown(ocr_response)
+        display(Markdown(combined_markdown))
+except ImportError:
+    # IPython not available
+    pass

backup/pdf_ocr.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""
+PDFOCR - Module for processing PDF files with OCR and extracting structured data.
+"""
+
+import json
+from pathlib import Path
+from structured_ocr import StructuredOCR
+
+class PDFOCR:
+    """Class for processing PDF files with OCR and extracting structured data."""
+    
+    def __init__(self, api_key=None):
+        """Initialize the PDF OCR processor."""
+        self.processor = StructuredOCR(api_key=api_key)
+    
+    def process_pdf(self, pdf_path, use_vision=True):
+        """
+        Process a PDF file with OCR and extract structured data.
+        
+        Args:
+            pdf_path: Path to the PDF file
+            use_vision: Whether to use vision model for improved analysis
+            
+        Returns:
+            Dictionary with structured OCR results
+        """
+        pdf_path = Path(pdf_path)
+        if not pdf_path.exists():
+            raise FileNotFoundError(f"PDF file not found: {pdf_path}")
+        
+        return self.processor.process_file(pdf_path, file_type="pdf", use_vision=use_vision)
+    
+    def save_json_output(self, pdf_path, output_path, use_vision=True):
+        """
+        Process a PDF file and save the structured output as JSON.
+        
+        Args:
+            pdf_path: Path to the PDF file
+            output_path: Path where to save the JSON output
+            use_vision: Whether to use vision model for improved analysis
+            
+        Returns:
+            Path to the saved JSON file
+        """
+        # Process the PDF
+        result = self.process_pdf(pdf_path, use_vision=use_vision)
+        
+        # Save the result to JSON
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        with open(output_path, 'w') as f:
+            json.dump(result, f, indent=2)
+        
+        return output_path
+
+# For testing directly
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 2:
+        print("Usage: python pdf_ocr.py <pdf_path> [output_path]")
+        sys.exit(1)
+        
+    pdf_path = sys.argv[1]
+    output_path = sys.argv[2] if len(sys.argv) > 2 else None
+    
+    processor = PDFOCR()
+    
+    if output_path:
+        result_path = processor.save_json_output(pdf_path, output_path)
+        print(f"Results saved to: {result_path}")
+    else:
+        result = processor.process_pdf(pdf_path)
+        print(json.dumps(result, indent=2))

backup/requirements.txt

@@ -0,0 +1,10 @@
+streamlit>=1.43.2
+mistralai>=0.0.7
+pydantic>=2.0.0
+pycountry>=23.12.11
+pillow>=10.0.0
+python-multipart>=0.0.6
+pdf2image>=1.17.0
+pytesseract>=0.3.10
+opencv-python-headless>=4.6.0
+numpy>=1.23.5

backup/structured_ocr.py

@@ -0,0 +1,414 @@
+import os
+import sys
+import time
+from enum import Enum
+from pathlib import Path
+import json
+import base64
+import pycountry
+import logging
+from pydantic import BaseModel
+from mistralai import Mistral
+from mistralai import DocumentURLChunk, ImageURLChunk, TextChunk
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, 
+                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Import utilities for OCR processing
+try:
+    from ocr_utils import replace_images_in_markdown, get_combined_markdown
+except ImportError:
+    # Define fallback functions if module not found
+    def replace_images_in_markdown(markdown_str, images_dict):
+        for img_name, base64_str in images_dict.items():
+            markdown_str = markdown_str.replace(
+                f"![{img_name}]({img_name})", f"![{img_name}]({base64_str})"
+            )
+        return markdown_str
+        
+    def get_combined_markdown(ocr_response):
+        markdowns = []
+        for page in ocr_response.pages:
+            image_data = {}
+            for img in page.images:
+                image_data[img.id] = img.image_base64
+            markdowns.append(replace_images_in_markdown(page.markdown, image_data))
+        return "\n\n".join(markdowns)
+
+# Import config directly (now local to historical-ocr)
+from config import MISTRAL_API_KEY, OCR_MODEL, TEXT_MODEL, VISION_MODEL
+
+# Create language enum for structured output
+languages = {lang.alpha_2: lang.name for lang in pycountry.languages if hasattr(lang, 'alpha_2')}
+
+class LanguageMeta(Enum.__class__):
+    def __new__(metacls, cls, bases, classdict):
+        for code, name in languages.items():
+            classdict[name.upper().replace(' ', '_')] = name
+        return super().__new__(metacls, cls, bases, classdict)
+
+class Language(Enum, metaclass=LanguageMeta):
+    pass
+
+class StructuredOCRModel(BaseModel):
+    file_name: str
+    topics: list[str]
+    languages: list[Language]
+    ocr_contents: dict
+
+class StructuredOCR:
+    def __init__(self, api_key=None):
+        """Initialize the OCR processor with API key"""
+        self.api_key = api_key or MISTRAL_API_KEY
+        self.client = Mistral(api_key=self.api_key)
+    
+    def process_file(self, file_path, file_type=None, use_vision=True, max_pages=None, file_size_mb=None, custom_pages=None):
+        """Process a file and return structured OCR results
+        
+        Args:
+            file_path: Path to the file to process
+            file_type: 'pdf' or 'image' (will be auto-detected if None)
+            use_vision: Whether to use vision model for improved analysis
+            max_pages: Optional limit on number of pages to process
+            file_size_mb: Optional file size in MB (used for automatic page limiting)
+            custom_pages: Optional list of specific page numbers to process
+            
+        Returns:
+            Dictionary with structured OCR results
+        """
+        # Convert file_path to Path object if it's a string
+        file_path = Path(file_path)
+        
+        # Auto-detect file type if not provided
+        if file_type is None:
+            suffix = file_path.suffix.lower()
+            file_type = "pdf" if suffix == ".pdf" else "image"
+        
+        # Get file size if not provided
+        if file_size_mb is None and file_path.exists():
+            file_size_mb = file_path.stat().st_size / (1024 * 1024)  # Convert bytes to MB
+            
+        # Check if file exceeds API limits (50 MB)
+        if file_size_mb and file_size_mb > 50:
+            logging.warning(f"File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB")
+            return {
+                "file_name": file_path.name,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "confidence_score": 0.0,
+                "error": f"File size {file_size_mb:.2f} MB exceeds API limit of 50 MB",
+                "ocr_contents": {
+                    "error": f"Failed to process file: File size {file_size_mb:.2f} MB exceeds Mistral API limit of 50 MB",
+                    "partial_text": "Document could not be processed due to size limitations."
+                }
+            }
+            
+        # For PDF files, limit pages based on file size if no explicit limit is given
+        if file_type == "pdf" and file_size_mb and max_pages is None and custom_pages is None:
+            if file_size_mb > 100:  # Very large files
+                max_pages = 3
+            elif file_size_mb > 50:  # Large files
+                max_pages = 5
+            elif file_size_mb > 20:  # Medium files
+                max_pages = 10
+            else:  # Small files
+                max_pages = None  # Process all pages
+        
+        # Start processing timer
+        start_time = time.time()
+        
+        # Read and process the file
+        if file_type == "pdf":
+            result = self._process_pdf(file_path, use_vision, max_pages, custom_pages)
+        else:
+            result = self._process_image(file_path, use_vision)
+            
+        # Add processing time information
+        processing_time = time.time() - start_time
+        result['processing_time'] = processing_time
+        
+        # Add a default confidence score if not present
+        if 'confidence_score' not in result:
+            result['confidence_score'] = 0.85  # Default confidence
+            
+        return result
+    
+    def _process_pdf(self, file_path, use_vision=True, max_pages=None, custom_pages=None):
+        """Process a PDF file with OCR
+        
+        Args:
+            file_path: Path to the PDF file
+            use_vision: Whether to use vision model
+            max_pages: Optional limit on the number of pages to process
+            custom_pages: Optional list of specific page numbers to process
+        """
+        logger = logging.getLogger("pdf_processor")
+        logger.info(f"Processing PDF: {file_path}")
+        
+        try:
+            # Upload the PDF file
+            logger.info("Uploading PDF file to Mistral API")
+            uploaded_file = self.client.files.upload(
+                file={
+                    "file_name": file_path.stem,
+                    "content": file_path.read_bytes(),
+                },
+                purpose="ocr",
+            )
+            
+            # Get a signed URL for the uploaded file
+            signed_url = self.client.files.get_signed_url(file_id=uploaded_file.id, expiry=1)
+            
+            # Process the PDF with OCR
+            logger.info(f"Processing PDF with OCR using {OCR_MODEL}")
+            pdf_response = self.client.ocr.process(
+                document=DocumentURLChunk(document_url=signed_url.url), 
+                model=OCR_MODEL, 
+                include_image_base64=True
+            )
+            
+            # Limit pages if requested
+            pages_to_process = pdf_response.pages
+            total_pages = len(pdf_response.pages)
+            limited_pages = False
+            
+            logger.info(f"PDF has {total_pages} total pages")
+            
+            # Handle custom page selection if provided
+            if custom_pages:
+                # Convert to 0-based indexing and filter valid page numbers
+                valid_indices = [i-1 for i in custom_pages if 0 < i <= total_pages]
+                if valid_indices:
+                    pages_to_process = [pdf_response.pages[i] for i in valid_indices]
+                    limited_pages = True
+                    logger.info(f"Processing {len(valid_indices)} custom-selected pages")
+            # Otherwise handle max_pages limit
+            elif max_pages and total_pages > max_pages:
+                pages_to_process = pages_to_process[:max_pages]
+                limited_pages = True
+                logger.info(f"Processing only first {max_pages} pages out of {total_pages} total pages")
+            
+            # Calculate average confidence score based on OCR response if available
+            confidence_score = 0.0
+            try:
+                # Some OCR APIs provide confidence scores
+                confidence_values = []
+                for page in pages_to_process:
+                    if hasattr(page, 'confidence'):
+                        confidence_values.append(page.confidence)
+                
+                if confidence_values:
+                    confidence_score = sum(confidence_values) / len(confidence_values)
+                else:
+                    confidence_score = 0.85  # Default if no confidence scores available
+            except:
+                confidence_score = 0.85  # Default fallback
+                
+            # Combine pages' markdown into a single string
+            all_markdown = "\n\n".join([page.markdown for page in pages_to_process])
+            
+            # Extract structured data using the appropriate model
+            if use_vision:
+                # Get base64 of first page for vision model
+                first_page_image = None
+                if pages_to_process and pages_to_process[0].images:
+                    first_page_image = pages_to_process[0].images[0].image_base64
+                
+                if first_page_image:
+                    # Use vision model
+                    logger.info(f"Using vision model: {VISION_MODEL}")
+                    result = self._extract_structured_data_with_vision(first_page_image, all_markdown, file_path.name)
+                else:
+                    # Fall back to text-only model if no image available
+                    logger.info(f"No images in PDF, falling back to text model: {TEXT_MODEL}")
+                    result = self._extract_structured_data_text_only(all_markdown, file_path.name)
+            else:
+                # Use text-only model
+                logger.info(f"Using text-only model: {TEXT_MODEL}")
+                result = self._extract_structured_data_text_only(all_markdown, file_path.name)
+            
+            # Add page limit info to result if needed
+            if limited_pages:
+                result['limited_pages'] = {
+                    'processed': len(pages_to_process),
+                    'total': total_pages
+                }
+                
+            # Add confidence score
+            result['confidence_score'] = confidence_score
+                
+            # Store the raw OCR response for image rendering
+            result['raw_response'] = pdf_response
+            
+            logger.info(f"PDF processing completed successfully")
+            return result
+            
+        except Exception as e:
+            logger.error(f"Error processing PDF: {str(e)}")
+            # Return basic result on error
+            return {
+                "file_name": file_path.name,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "confidence_score": 0.0,
+                "error": str(e),
+                "ocr_contents": {
+                    "error": f"Failed to process PDF: {str(e)}",
+                    "partial_text": "Document could not be fully processed."
+                }
+            }
+    
+    def _process_image(self, file_path, use_vision=True):
+        """Process an image file with OCR"""
+        logger = logging.getLogger("image_processor")
+        logger.info(f"Processing image: {file_path}")
+        
+        try:
+            # Read and encode the image file
+            logger.info("Encoding image for API")
+            encoded_image = base64.b64encode(file_path.read_bytes()).decode()
+            base64_data_url = f"data:image/jpeg;base64,{encoded_image}"
+            
+            # Process the image with OCR
+            logger.info(f"Processing image with OCR using {OCR_MODEL}")
+            image_response = self.client.ocr.process(
+                document=ImageURLChunk(image_url=base64_data_url), 
+                model=OCR_MODEL,
+                include_image_base64=True
+            )
+            
+            # Get the OCR markdown from the first page
+            image_ocr_markdown = image_response.pages[0].markdown if image_response.pages else ""
+            
+            # Calculate confidence score if available
+            confidence_score = 0.85  # Default value
+            try:
+                if hasattr(image_response.pages[0], 'confidence'):
+                    confidence_score = image_response.pages[0].confidence
+            except:
+                pass
+                
+            # Extract structured data using the appropriate model
+            if use_vision:
+                logger.info(f"Using vision model: {VISION_MODEL}")
+                result = self._extract_structured_data_with_vision(base64_data_url, image_ocr_markdown, file_path.name)
+            else:
+                logger.info(f"Using text-only model: {TEXT_MODEL}")
+                result = self._extract_structured_data_text_only(image_ocr_markdown, file_path.name)
+                
+            # Add confidence score
+            result['confidence_score'] = confidence_score
+                
+            # Store the raw OCR response for image rendering
+            result['raw_response'] = image_response
+            
+            logger.info("Image processing completed successfully")
+            return result
+            
+        except Exception as e:
+            logger.error(f"Error processing image: {str(e)}")
+            # Return basic result on error
+            return {
+                "file_name": file_path.name,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "confidence_score": 0.0,
+                "error": str(e),
+                "ocr_contents": {
+                    "error": f"Failed to process image: {str(e)}",
+                    "partial_text": "Image could not be processed."
+                }
+            }
+    
+    def _extract_structured_data_with_vision(self, image_base64, ocr_markdown, filename):
+        """Extract structured data using vision model"""
+        try:
+            # Parse with vision model with a timeout
+            chat_response = self.client.chat.parse(
+                model=VISION_MODEL,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            ImageURLChunk(image_url=image_base64),
+                            TextChunk(text=(
+                                f"This is a historical document's OCR in markdown:\n"
+                                f"<BEGIN_IMAGE_OCR>\n{ocr_markdown}\n<END_IMAGE_OCR>.\n"
+                                f"Convert this into a structured JSON response with the OCR contents in a sensible dictionary. "
+                                f"Extract topics, languages, and organize the content logically."
+                            ))
+                        ],
+                    },
+                ],
+                response_format=StructuredOCRModel,
+                temperature=0
+            )
+            
+            # Convert the response to a dictionary
+            result = json.loads(chat_response.choices[0].message.parsed.json())
+            
+            # Ensure languages is a list of strings, not Language enum objects
+            if 'languages' in result:
+                result['languages'] = [str(lang) for lang in result.get('languages', [])]
+            
+        except Exception as e:
+            # Fall back to text-only model if vision model fails
+            print(f"Vision model failed: {str(e)}. Falling back to text-only model.")
+            result = self._extract_structured_data_text_only(ocr_markdown, filename)
+            
+        return result
+    
+    def _extract_structured_data_text_only(self, ocr_markdown, filename):
+        """Extract structured data using text-only model"""
+        try:
+            # Parse with text-only model with a timeout
+            chat_response = self.client.chat.parse(
+                model=TEXT_MODEL,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": f"This is a historical document's OCR in markdown:\n"
+                                  f"<BEGIN_IMAGE_OCR>\n{ocr_markdown}\n<END_IMAGE_OCR>.\n"
+                                  f"Convert this into a structured JSON response with the OCR contents. "
+                                  f"Extract topics, languages, and organize the content logically."
+                    },
+                ],
+                response_format=StructuredOCRModel,
+                temperature=0
+            )
+            
+            # Convert the response to a dictionary
+            result = json.loads(chat_response.choices[0].message.parsed.json())
+            
+            # Ensure languages is a list of strings, not Language enum objects
+            if 'languages' in result:
+                result['languages'] = [str(lang) for lang in result.get('languages', [])]
+            
+        except Exception as e:
+            # Create a basic result if parsing fails
+            print(f"Text model failed: {str(e)}. Creating basic result.")
+            result = {
+                "file_name": filename,
+                "topics": ["Document"],
+                "languages": ["English"],
+                "ocr_contents": {
+                    "raw_text": ocr_markdown
+                }
+            }
+            
+        return result
+
+# For testing directly
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 2:
+        print("Usage: python structured_ocr.py <file_path>")
+        sys.exit(1)
+        
+    file_path = sys.argv[1]
+    processor = StructuredOCR()
+    result = processor.process_file(file_path)
+    
+    print(json.dumps(result, indent=2))

config.py

@@ -5,10 +5,13 @@ Contains API key and other settings.
 """
 import os
 
-# Your Mistral API key - get from environment variable
-MISTRAL_API_KEY = os.environ.get("MISTRAL_API_KEY", "")
+# Your Mistral API key - get from Hugging Face secrets or environment variable
+# The priority order is: HF_SPACES environment var > regular environment var > empty string
+# Note: No default API key is provided for security reasons
+MISTRAL_API_KEY = os.environ.get("HF_MISTRAL_API_KEY",  # First check HF-specific env var
+                  os.environ.get("MISTRAL_API_KEY", ""))  # Then check regular env var
 
-# Model settings
+# Model settings 
 OCR_MODEL = "mistral-ocr-latest"
 TEXT_MODEL = "ministral-8b-latest"
 VISION_MODEL = "pixtral-12b-latest"

input/a-la-carte.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d96008f374f5be8046b569c868e33f4e5a0e5e166c245d324b44140c7e6c2e
+size 2554815

input/menu.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d96008f374f5be8046b569c868e33f4e5a0e5e166c245d324b44140c7e6c2e
+size 2554815

modules/content/__init__.py

@@ -0,0 +1,36 @@
+"""
+Module initialization file for the workshop modules.
+"""
+from . import module1, module2, module3, module4, module5, module6
+
+# Module registry for easy access by module number
+modules = {
+    1: module1,
+    2: module2,
+    3: module3,
+    4: module4,
+    5: module5,
+    6: module6
+}
+
+# Module names for navigation and display
+module_names = [
+    "Introduction",
+    "Text-Image Relations",
+    "OCR Technology",
+    "Methodological Approaches", 
+    "Interactive OCR",
+    "Conclusion"
+]
+
+def get_module(module_number):
+    """Get a module by its number (1-6)"""
+    if module_number in modules:
+        return modules[module_number]
+    raise ValueError(f"Unknown module number: {module_number}")
+
+def get_module_name(module_number):
+    """Get a module name by its number (1-6)"""
+    if 1 <= module_number <= len(module_names):
+        return module_names[module_number - 1]
+    return f"Module {module_number}"

modules/content/module1.py

@@ -0,0 +1,85 @@
+import streamlit as st
+from layout import gray_container, blue_container, yellow_container, card_grid, key_concept
+
+def render():
+    """Module 1: Introduction and Problematization"""
+    
+    st.title("Module 1: Introduction and Problematization")
+    
+    # Workshop overview in gray container
+    overview_content = """
+    <h3>Workshop Overview</h3>
+    <p>
+    This interactive workshop explores the application of OCR technology to historical documents,
+    combining theoretical understanding with practical experiences. Designed for historians, 
+    archivists, and digital humanities scholars, it offers both conceptual frameworks and hands-on skills.
+    </p>
+    """
+    gray_container(overview_content)
+    
+    # For historians section with blue background
+    historians_content = """
+    <h3>For Historians:</h3>
+    <p>
+    How might OCR technology transform our access to and interpretation of historical 
+    documents? What new research questions become possible when large archives 
+    become machine-readable?
+    </p>
+    """
+    blue_container(historians_content)
+    
+    # What is OCR section with yellow background
+    ocr_content = """
+    <h3>What is OCR?</h3>
+    <p>
+    Optical Character Recognition (OCR) technology enables computers to extract text from images and documents. 
+    Modern OCR uses AI vision models to understand both the text and its visual context.
+    </p>
+    """
+    yellow_container(ocr_content)
+    
+    # What you'll learn section
+    st.subheader("What You'll Learn")
+    
+    # Create cards for the learning outcomes
+    cards = [
+        """
+        <h4>Conceptual Understanding</h4>
+        <ul>
+            <li>Text-image relationships in historical documents</li>
+            <li>Evolution of OCR technology</li>
+            <li>AI vision models for document analysis</li>
+            <li>Historical typography challenges</li>
+        </ul>
+        """,
+        
+        """
+        <h4>Methodological Approaches</h4>
+        <ul>
+            <li>Critical frameworks for OCR research</li>
+            <li>Hybrid computational methods</li>
+            <li>Error analysis and interpretation</li>
+            <li>Contextual reading strategies</li>
+        </ul>
+        """,
+        
+        """
+        <h4>Practical Skills</h4>
+        <ul>
+            <li>Processing historical documents</li>
+            <li>Analyzing extracted information</li>
+            <li>Integrating OCR into workflows</li>
+            <li>Building searchable archives</li>
+        </ul>
+        """
+    ]
+    
+    card_grid(cards)
+    
+    # Add a key concept
+    concept_content = """
+    <h4>Workshop Structure</h4>
+    <p>This workshop combines theory and practice through six modules, each building on the previous ones.</p>
+    <p>Navigate between modules using the buttons at the bottom of the page.</p>
+    """
+    key_concept(concept_content)

modules/content/module2.py

@@ -0,0 +1,88 @@
+import streamlit as st
+from layout import gray_container, card_grid, key_concept, research_question
+
+def render():
+    """Module 2: Text-Image Relations in Historical Archives"""
+    
+    st.title("Module 2: Text-Image Relations in Historical Archives")
+    
+    col1, col2 = st.columns([1, 1])
+    
+    with col1:
+        textual_content = """
+        <h3>Textual Elements</h3>
+        <ul>
+            <li><strong>Typography</strong>: Varying fonts, sizes, and styles</li>
+            <li><strong>Layout</strong>: Columns, margins, and spacing</li>
+            <li><strong>Marginalia</strong>: Notes, comments, and additions</li>
+            <li><strong>Decorative Text</strong>: Illuminated letters and calligraphy</li>
+        </ul>
+        """
+        gray_container(textual_content)
+        
+        visual_content = """
+        <h3>Visual Elements</h3>
+        <ul>
+            <li><strong>Illustrations</strong>: Diagrams, maps, and artistic representations</li>
+            <li><strong>Watermarks</strong>: Hidden identifiers that locate documents</li>
+            <li><strong>Damage</strong>: Tears, stains, and fading affecting legibility</li>
+            <li><strong>Material Features</strong>: Paper quality and physical dimensions</li>
+        </ul>
+        """
+        gray_container(visual_content)
+    
+    with col2:
+        interdependence_content = """
+        <h3>Interdependence</h3>
+        <p>The relationship between text and image in historical documents exists on a complex spectrum:</p>
+        <ul>
+            <li>Text functions as image (decorative headings)</li>
+            <li>Images function as text (symbolic representations)</li>
+            <li>Layout creates meaning through visual organization</li>
+            <li>Material conditions affect both textual and visual elements</li>
+        </ul>
+        """
+        gray_container(interdependence_content)
+        
+        # Display an example image
+        st.image("https://upload.wikimedia.org/wikipedia/commons/thumb/0/0c/Book_of_Kells_folio_292r.jpg/800px-Book_of_Kells_folio_292r.jpg", 
+                caption="Book of Kells - Example of text-image integration")
+    
+    # OCR Challenges section
+    challenge_content = """
+    <h3>OCR Challenges</h3>
+    <p>These complex text-image relationships create particular challenges for OCR:</p>
+    """
+    gray_container(challenge_content)
+    
+    # Cards for OCR challenges
+    cards = [
+        """
+        <h4>Distinguishing Text from Decoration</h4>
+        <p>Where does ornamental text end and functional text begin?</p>
+        """,
+        
+        """
+        <h4>Handling Illustrations</h4>
+        <p>Should they be processed as images or described as text?</p>
+        """,
+        
+        """
+        <h4>Interpreting Layout</h4>
+        <p>How to capture the significance of spacing and organization?</p>
+        """,
+        
+        """
+        <h4>Preserving Context</h4>
+        <p>Maintaining the relationship between textual and visual elements</p>
+        """
+    ]
+    
+    card_grid(cards)
+    
+    # Research question box
+    research_content = """
+    <h4>Research Question</h4>
+    <p>How do we approach documents where the visual presentation is as meaningful as the textual content itself?</p>
+    """
+    research_question(research_content)

modules/content/module3.py

@@ -0,0 +1,106 @@
+import streamlit as st
+from pathlib import Path
+from layout import gray_container, tool_container, key_concept, research_question
+
+def render():
+    """Module 3: OCR Technology and Historical Documents"""
+    
+    st.title("Module 3: OCR Technology and Historical Documents")
+    
+    col1, col2 = st.columns([1, 1])
+    
+    with col1:
+        traditional_content = """
+        <h3>Traditional OCR Approaches</h3>
+        <ol>
+            <li><strong>Pattern Matching</strong>: Early OCR compared characters to templates</li>
+            <li><strong>Feature Extraction</strong>: Identifying key features of characters</li>
+            <li><strong>Statistical Models</strong>: Using probabilities to improve recognition</li>
+        </ol>
+        """
+        gray_container(traditional_content)
+        
+        modern_content = """
+        <h3>Modern AI-Enhanced OCR</h3>
+        <ol>
+            <li><strong>Neural Networks</strong>: Deep learning models trained on vast datasets</li>
+            <li><strong>Computer Vision</strong>: Advanced image processing techniques</li>
+            <li><strong>Language Models</strong>: Contextual understanding to resolve ambiguities</li>
+            <li><strong>Multimodal Models</strong>: Integration of text, layout, and visual understanding</li>
+        </ol>
+        """
+        gray_container(modern_content)
+    
+    with col2:
+        challenges_content = """
+        <h3>Challenges with Historical Documents</h3>
+        <p>Historical materials present unique difficulties:</p>
+        <ul>
+            <li><strong>Typography Variation</strong>: Non-standardized fonts and styles</li>
+            <li><strong>Historical Language</strong>: Archaic vocabulary and grammar</li>
+            <li><strong>Layout Complexity</strong>: Non-linear arrangements</li>
+            <li><strong>Document Degradation</strong>: Fading, tears, stains, and damage</li>
+            <li><strong>Material Artifacts</strong>: Paper texture, binding shadows, etc.</li>
+        </ul>
+        """
+        gray_container(challenges_content)
+        
+        # Display OCR processing diagram
+        st.image("https://cdn.dribbble.com/users/412119/screenshots/16353886/media/82e593c60a5e4d460db917236eab6ece.jpg", 
+                caption="OCR processing layers")
+    
+    # Key concept section
+    concept_content = """
+    <h3>Vision-Enhanced OCR</h3>
+    <p>Modern OCR systems like those based on Mistral-7B-Vision combine:</p>
+    <ol>
+        <li>Image understanding capabilities to process the visual aspects</li>
+        <li>Text recognition to extract characters accurately</li>
+        <li>Layout analysis to understand structure</li>
+        <li>Contextual language processing for improved accuracy</li>
+    </ol>
+    <p>This multimodal approach dramatically improves OCR results on historical documents compared to traditional OCR.</p>
+    """
+    key_concept(concept_content)
+    
+    # Technical details in a tool container
+    tech_content = """
+    <h3>Technical Evolution of OCR</h3>
+    <p><strong>Traditional OCR Pipeline:</strong></p>
+    <ol>
+        <li>Preprocessing (binarization, noise removal)</li>
+        <li>Layout analysis (segmentation)</li>
+        <li>Character recognition (pattern matching)</li>
+        <li>Post-processing (spell checking)</li>
+    </ol>
+    
+    <p><strong>Modern LLM-Vision Pipeline:</strong></p>
+    <ol>
+        <li>Image normalization</li>
+        <li>Image embedding via vision encoder</li>
+        <li>Integration with language model</li>
+        <li>Joint inference across modalities</li>
+        <li>Structured extraction of information</li>
+    </ol>
+    """
+    tool_container(tech_content)
+    
+    # Research question
+    research_content = """
+    <h4>Consider This:</h4>
+    <p>How might the capabilities of vision-language models change our approach to digitizing historical archives?</p>
+    """
+    research_question(research_content)
+    
+    # Display history if available
+    if 'processing_history' in st.session_state and st.session_state.processing_history:
+        with st.expander("Your OCR Processing History"):
+            st.markdown("You've already processed the following documents:")
+            
+            for item in st.session_state.processing_history:
+                st.markdown(f"**{item['fileName']}**")
+                col1, col2 = st.columns(2)
+                with col1:
+                    st.write(f"**Topics:** {', '.join(item['result'].get('topics', ['Unknown']))}")
+                with col2:
+                    st.write(f"**Vision model used:** {'Yes' if item['useVision'] else 'No'}")

modules/content/module4.py

@@ -0,0 +1,124 @@
+import streamlit as st
+from pathlib import Path
+from layout import gray_container, tool_container, key_concept, quote
+
+def render():
+    """Module 4: Methodological Approaches"""
+    
+    st.title("Module 4: Methodological Approaches")
+    
+    col1, col2 = st.columns([1, 1])
+    
+    with col1:
+        hybrid_content = """
+        <h3>Hybrid Methodologies</h3>
+        
+        <h4>1. Computational + Human Reading</h4>
+        <ul>
+            <li>OCR for initial processing and discovery</li>
+            <li>Human review for context and interpretation</li>
+            <li>Iterative refinement of computational outputs</li>
+        </ul>
+        
+        <h4>2. Close + Distant Reading</h4>
+        <ul>
+            <li>Distant reading through large-scale OCR processing</li>
+            <li>Close reading of selected passages</li>
+            <li>Zooming between scales of analysis</li>
+        </ul>
+        """
+        gray_container(hybrid_content)
+        
+        # Check if the diagram image is available and display it
+        input_dir = Path(__file__).parent.parent / "input"
+        diagram_path = input_dir / "diagram.jpg"
+        
+        if diagram_path.exists():
+            try:
+                from PIL import Image
+                with Image.open(diagram_path) as img:
+                    st.image(img, caption="Historical VLM architecture", use_column_width=True)
+            except Exception:
+                # If there's an error, just show a placeholder
+                st.image("https://placekitten.com/800/400", caption="Historical VLM architecture placeholder")
+        else:
+            # If the file doesn't exist, show a placeholder
+            st.image("https://placekitten.com/800/400", caption="Historical VLM architecture placeholder")
+    
+    with col2:
+        mistral_content = """
+        <h3>Mistral-OCR-Latest: State-of-the-Art</h3>
+        
+        <p>The Mistral-OCR model represents a significant advancement:</p>
+        <ul>
+            <li><strong>Multimodal Understanding</strong>: Processes both visual and textual information</li>
+            <li><strong>Contextual Awareness</strong>: Considers historical context</li>
+            <li><strong>Layout Recognition</strong>: Preserves complex document structures</li>
+            <li><strong>Historical Font Adaptation</strong>: Trained on diverse historical typography</li>
+        </ul>
+        """
+        gray_container(mistral_content)
+        
+        # Check if the workflow image is available and display it
+        workflow_path = input_dir / "workflow.jpg"
+        
+        if workflow_path.exists():
+            try:
+                from PIL import Image
+                with Image.open(workflow_path) as img:
+                    st.image(img, caption="Mistral OCR workflow", use_column_width=True)
+            except Exception:
+                # If there's an error, just show a placeholder
+                st.image("https://placekitten.com/800/400", caption="Mistral OCR workflow placeholder")
+        else:
+            # If the file doesn't exist, show a placeholder
+            st.image("https://placekitten.com/800/400", caption="Mistral OCR workflow placeholder")
+    
+    # Practical workflow section
+    workflow_content = """
+    <h3>Practical Workflow</h3>
+    
+    <p>A typical historical OCR workflow with Mistral-OCR includes:</p>
+    <ol>
+        <li><strong>Selection</strong>: Choosing appropriate documents</li>
+        <li><strong>Preprocessing</strong>: Enhancing images before OCR</li>
+        <li><strong>OCR Processing</strong>: Running documents through vision-enhanced OCR</li>
+        <li><strong>Post-processing</strong>: Cleaning up outputs and structured extraction</li>
+        <li><strong>Verification</strong>: Cross-checking results against originals</li>
+        <li><strong>Integration</strong>: Incorporating OCR outputs into research materials</li>
+    </ol>
+    """
+    tool_container(workflow_content)
+    
+    # Methodological considerations
+    st.subheader("Methodological Considerations")
+    
+    col1, col2 = st.columns([1, 1])
+    
+    with col1:
+        advantages_content = """
+        <h4>Advantages of Hybrid Approaches</h4>
+        <ul>
+            <li>Balance between automation and expert judgment</li>
+            <li>Ability to process large volumes while preserving detail</li>
+            <li>Context-sensitive analysis of complex documents</li>
+            <li>Iterative improvement of results</li>
+        </ul>
+        """
+        gray_container(advantages_content)
+    
+    with col2:
+        limitations_content = """
+        <h4>Limitations and Challenges</h4>
+        <ul>
+            <li>OCR errors requiring expert correction</li>
+            <li>Bias in training data affecting recognition</li>
+            <li>Complexity in evaluating OCR quality</li>
+            <li>Technical infrastructure requirements</li>
+        </ul>
+        """
+        gray_container(limitations_content)
+    
+    # Quote
+    quote_content = "The most powerful digital humanities work occurs at the intersection of computational methods and traditional humanistic inquiry."
+    quote(quote_content, "Dr. Sarah E. Bond, Digital Humanities Scholar")

modules/content/module5.py

@@ -0,0 +1,547 @@
+import streamlit as st
+import io
+import tempfile
+from pathlib import Path
+from datetime import datetime
+from layout import tool_container, key_concept, research_question, upload_container
+import sys
+
+# Import the necessary modules for OCR processing
+sys.path.append(str(Path(__file__).parent.parent))
+try:
+    from process_file import process_file as process_file_util
+    process_file = process_file_util
+except ImportError:
+    # Fallback if process_file is not available
+    def process_file(uploaded_file, use_vision=True, custom_prompt=None):
+        """Fallback function for processing files"""
+        st.warning("Using mock processing function. Real OCR functionality is not available.")
+        return {
+            "file_name": uploaded_file.name,
+            "languages": ["English"],
+            "topics": ["History", "Document"],
+            "ocr_contents": {
+                "content": f"This is mock OCR content for {uploaded_file.name}. Vision model: {use_vision}"
+            }
+        }
+
+def render():
+    """Module 5: Interactive OCR Experiment"""
+    
+    st.title("Module 5: Interactive OCR Experiment")
+    
+    # Introduction to the interactive experiment
+    intro_content = """
+    <h3>Interactive OCR Experiment</h3>
+    <p>
+    This interactive experiment allows you to process historical documents with OCR and analyze the results.
+    Try different settings and compare the outcomes to understand the strengths and limitations of OCR technology.
+    </p>
+    """
+    st.markdown(intro_content, unsafe_allow_html=True)
+    
+    # Create tabs for different activities
+    experiment_tab, compare_tab, analyze_tab = st.tabs(["Process Documents", "Compare Results", "Analysis Guide"])
+    
+    # Try to import PDF tools if available
+    try:
+        from pdf2image import convert_from_bytes
+        pdf_support = True
+    except ImportError:
+        pdf_support = False
+        st.warning("PDF preview functionality is limited. The pdf2image module is required for PDF previews.")
+    
+    with experiment_tab:
+        # Create a two-column layout
+        col1, col2 = st.columns([1, 1])
+        
+        with col1:
+            # Tool container for document selection and options
+            st.subheader("Step 1: Select Document & Options")
+            
+            # Processing options
+            use_vision = st.checkbox("Use Vision Model", value=True,
+                                 help="Use the vision model for improved analysis")
+            
+            # Additional prompt
+            st.markdown("### Custom Research Prompt (Optional)")
+            st.markdown("""Provide additional instructions to guide the OCR analysis. 
+            Focus on specific aspects of historical research you're interested in.""")
+            custom_prompt = st.text_area("Research Prompt",
+                                       placeholder="E.g., Focus on identifying dates and historical figures...",
+                                       help="Optional instructions to guide the analysis")
+            
+            # Sample document selection
+            input_dir = Path(__file__).parent.parent / "input"
+            
+            if input_dir.exists():
+                sample_files = list(input_dir.glob("*.jpg")) + list(input_dir.glob("*.png")) + list(input_dir.glob("*.pdf"))
+                
+                if sample_files:
+                    st.markdown("#### Sample Documents")
+                    sample_options = ["Upload my own document"] + [f.name for f in sample_files]
+                    sample_choice = st.selectbox("Choose a document:", sample_options)
+                    
+                    if sample_choice != "Upload my own document":
+                        # Process the selected sample file
+                        selected_file = next((f for f in sample_files if f.name == sample_choice), None)
+                        
+                        if selected_file:
+                            # Store the selected sample file in session state
+                            with open(selected_file, "rb") as f:
+                                file_bytes = f.read()
+                            
+                            st.session_state.sample_file = {
+                                "name": selected_file.name,
+                                "bytes": file_bytes
+                            }
+                            
+                            # Preview the selected sample
+                            if selected_file.suffix.lower() == ".pdf" and pdf_support:
+                                try:
+                                    with st.spinner("Generating PDF preview..."):
+                                        images = convert_from_bytes(file_bytes, first_page=1, last_page=1, dpi=150)
+                                        if images:
+                                            st.image(images[0], caption=f"Preview: {selected_file.name}", use_column_width=True)
+                                except Exception:
+                                    st.info(f"PDF selected: {selected_file.name}")
+                            else:
+                                # For images display directly
+                                try:
+                                    from PIL import Image
+                                    img = Image.open(io.BytesIO(file_bytes))
+                                    st.image(img, caption=f"Preview: {selected_file.name}", use_column_width=True)
+                                except Exception:
+                                    st.info(f"Selected: {selected_file.name}")
+                    else:
+                        # Clear the sample file if "Upload my own" is selected
+                        if 'sample_file' in st.session_state:
+                            del st.session_state.sample_file
+                        
+                        # Display file uploader
+                        upload_html = """
+                        <h4>Upload a document to get started</h4>
+                        <p>Supported formats: PDF, JPG, PNG</p>
+                        """
+                        
+                        upload_container(upload_html)
+                        uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+                        
+                        if uploaded_file is not None:
+                            # Display preview of the uploaded file
+                            file_ext = Path(uploaded_file.name).suffix.lower()
+                            
+                            if file_ext == ".pdf" and pdf_support:
+                                try:
+                                    # Convert first page of PDF to image for preview
+                                    pdf_bytes = uploaded_file.getvalue()
+                                    with st.spinner("Generating PDF preview..."):
+                                        images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                                        if images:
+                                            st.image(images[0], caption=f"PDF Preview: {uploaded_file.name}", use_column_width=True)
+                                        else:
+                                            st.info(f"PDF uploaded: {uploaded_file.name}")
+                                except Exception:
+                                    st.info(f"PDF uploaded: {uploaded_file.name}")
+                            elif file_ext != ".pdf":
+                                st.image(uploaded_file, use_column_width=True)
+                            else:
+                                st.info(f"PDF uploaded: {uploaded_file.name}")
+                else:
+                    # No sample files, just show the uploader
+                    upload_html = """
+                    <h4>Upload a document to get started</h4>
+                    <p>Supported formats: PDF, JPG, PNG</p>
+                    """
+                    
+                    upload_container(upload_html)
+                    uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+                    
+                    if uploaded_file is not None:
+                        # Display the file preview
+                        file_ext = Path(uploaded_file.name).suffix.lower()
+                        if file_ext == ".pdf" and pdf_support:
+                            try:
+                                pdf_bytes = uploaded_file.getvalue()
+                                with st.spinner("Generating PDF preview..."):
+                                    images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                                    if images:
+                                        st.image(images[0], caption=f"PDF Preview: {uploaded_file.name}", use_column_width=True)
+                            except Exception:
+                                st.info(f"PDF uploaded: {uploaded_file.name}")
+                        elif file_ext != ".pdf":
+                            st.image(uploaded_file, use_column_width=True)
+                        else:
+                            st.info(f"PDF uploaded: {uploaded_file.name}")
+            else:
+                # No input directory
+                upload_html = """
+                <h4>Upload a document to get started</h4>
+                <p>Supported formats: PDF, JPG, PNG</p>
+                """
+                
+                upload_container(upload_html)
+                uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+            
+            # Process button
+            st.subheader("Step 2: Process the Document")
+            
+            # Get the file to process (either uploaded or sample)
+            file_to_process = None
+            if 'sample_file' in st.session_state and sample_choice != "Upload my own document":
+                # Create a FileUploader-like object from the sample file
+                class SampleFileObject:
+                    def __init__(self, name, data):
+                        self.name = name
+                        self._data = data
+                    
+                    def getvalue(self):
+                        return self._data
+                
+                file_to_process = SampleFileObject(
+                    st.session_state.sample_file["name"],
+                    st.session_state.sample_file["bytes"]
+                )
+            elif 'uploaded_file' in locals() and uploaded_file is not None:
+                file_to_process = uploaded_file
+            
+            # Process button
+            process_button = st.button(
+                "Process Document",
+                disabled=file_to_process is None,
+                use_container_width=True
+            )
+            
+            if process_button and file_to_process is not None:
+                with st.spinner("Processing document..."):
+                    try:
+                        # Process the file
+                        result = process_file(file_to_process, use_vision, custom_prompt=custom_prompt if custom_prompt else None)
+                        
+                        if result:
+                            st.success("Document processed successfully!")
+                            
+                            # Store result in session state for display in the right column
+                            st.session_state.current_result = result
+                            
+                            # Add to processing history
+                            history_item = {
+                                "id": datetime.now().timestamp(),
+                                "fileName": file_to_process.name,
+                                "timestamp": datetime.now().isoformat(),
+                                "result": result,
+                                "useVision": use_vision
+                            }
+                            
+                            if 'processing_history' not in st.session_state:
+                                st.session_state.processing_history = []
+                            
+                            st.session_state.processing_history.append(history_item)
+                            
+                            st.experimental_rerun()
+                        else:
+                            st.error("Failed to process document.")
+                    except Exception as e:
+                        st.error(f"Error processing document: {str(e)}")
+            
+            # Experiment instructions
+            experiment_content = """
+            <h3>Experiment Instructions</h3>
+            <ol>
+                <li><strong>Step 1:</strong> Select a document and choose your options</li>
+                <li><strong>Step 2:</strong> Process the document with the selected options</li>
+                <li><strong>Step 3:</strong> Analyze the results in the panel on the right</li>
+                <li><strong>Step 4:</strong> Try again with different settings (e.g., toggle vision model)</li>
+                <li><strong>Step 5:</strong> Compare results between different runs</li>
+            </ol>
+            """
+            key_concept(experiment_content)
+        
+        with col2:
+            # Results display
+            st.subheader("Step 3: View Results")
+            
+            if 'current_result' in st.session_state and st.session_state.current_result:
+                result = st.session_state.current_result
+                
+                # Display results in a tool container
+                result_html = f"""
+                <h4>Results for: {result.get('file_name', 'Unknown')}</h4>
+                <p><strong>Languages:</strong> {', '.join(result.get('languages', ['Unknown']))}</p>
+                <p><strong>Topics:</strong> {', '.join(result.get('topics', ['Unknown']))}</p>
+                """
+                tool_container(result_html)
+                
+                # Create tabs for different views
+                tab1, tab2 = st.tabs(["Structured View", "Raw JSON"])
+                
+                with tab1:
+                    # Display in a more user-friendly format
+                    if 'ocr_contents' in result:
+                        if isinstance(result['ocr_contents'], dict):
+                            for section, content in result['ocr_contents'].items():
+                                if content:  # Only display non-empty sections
+                                    st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                    
+                                    if isinstance(content, str):
+                                        st.markdown(content)
+                                    elif isinstance(content, list):
+                                        for item in content:
+                                            if isinstance(item, str):
+                                                st.markdown(f"- {item}")
+                                            elif isinstance(item, dict):
+                                                st.json(item)
+                                    elif isinstance(content, dict):
+                                        for k, v in content.items():
+                                            st.markdown(f"**{k}:** {v}")
+                
+                with tab2:
+                    # Show the raw JSON
+                    st.json(result)
+                
+                # Download options
+                st.markdown("### Export Results")
+                
+                col1, col2 = st.columns(2)
+                
+                with col1:
+                    # Export as JSON
+                    import json
+                    json_bytes = json.dumps(result, indent=2).encode()
+                    st.download_button(
+                        label="Download JSON",
+                        data=json_bytes,
+                        file_name="ocr_results.json",
+                        mime="application/json",
+                        use_container_width=True
+                    )
+                
+                with col2:
+                    # Export as text if content is available
+                    if 'ocr_contents' in result and isinstance(result['ocr_contents'], dict) and 'content' in result['ocr_contents']:
+                        text_content = result['ocr_contents']['content']
+                        st.download_button(
+                            label="Download Text",
+                            data=text_content.encode(),
+                            file_name="ocr_text.txt",
+                            mime="text/plain",
+                            use_container_width=True
+                        )
+            else:
+                # Show placeholder when no results are available
+                placeholder_html = """
+                <h4>Results will appear here</h4>
+                <p>Upload and process a document to see the OCR results in this panel.</p>
+                <p>The OCR tool will:</p>
+                <ol>
+                    <li>Extract text from your document</li>
+                    <li>Identify languages and topics</li>
+                    <li>Provide structured content analysis</li>
+                    <li>Generate downloadable results</li>
+                </ol>
+                """
+                tool_container(placeholder_html)
+            
+            # Display processing history if available
+            if 'processing_history' in st.session_state and st.session_state.processing_history:
+                st.subheader("Step 4: Review Processing History")
+                
+                # Most recent result
+                latest = st.session_state.processing_history[-1]
+                latest_html = f"""
+                <h4>Latest Document: {latest['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(latest['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if latest['useVision'] else 'No'}</p>
+                """
+                tool_container(latest_html)
+                
+                # History in expander
+                with st.expander("View Complete Processing History"):
+                    for i, item in enumerate(reversed(st.session_state.processing_history)):
+                        st.markdown(f"""
+                        <div style="background-color: var(--color-gray-700); padding: 0.75rem; border-radius: 0.5rem; margin-bottom: 0.5rem;">
+                        <strong>{item['fileName']}</strong><br>
+                        {datetime.fromisoformat(item['timestamp']).strftime('%Y-%m-%d %H:%M')} - 
+                        Vision model: {'Yes' if item['useVision'] else 'No'}
+                        </div>
+                        """, unsafe_allow_html=True)
+                        
+                        # Option to view a previous result
+                        if st.button(f"View This Result", key=f"view_history_{i}"):
+                            st.session_state.current_result = item['result']
+                            st.experimental_rerun()
+    
+    # Compare tab for side-by-side comparison
+    with compare_tab:
+        st.subheader("Compare OCR Results")
+        
+        if 'processing_history' in st.session_state and len(st.session_state.processing_history) >= 2:
+            st.markdown("""
+            Select two processing results to compare side by side. This allows you to see
+            how different options (like using the vision model) affect OCR quality.
+            """)
+            
+            # Create selection dropdowns for the documents
+            col1, col2 = st.columns(2)
+            with col1:
+                # First document selector
+                doc_options_1 = [f"{i+1}: {item['fileName']} ({'Vision' if item['useVision'] else 'No Vision'})" 
+                               for i, item in enumerate(st.session_state.processing_history)]
+                doc_choice_1 = st.selectbox("First Document:", doc_options_1, key="compare_doc_1")
+                doc_index_1 = int(doc_choice_1.split(":")[0]) - 1
+                
+            with col2:
+                # Second document selector
+                doc_options_2 = [f"{i+1}: {item['fileName']} ({'Vision' if item['useVision'] else 'No Vision'})" 
+                               for i, item in enumerate(st.session_state.processing_history)]
+                default_index = min(1, len(st.session_state.processing_history) - 1)  # Default to second item
+                doc_choice_2 = st.selectbox("Second Document:", doc_options_2, key="compare_doc_2", index=default_index)
+                doc_index_2 = int(doc_choice_2.split(":")[0]) - 1
+            
+            # Retrieve the selected documents
+            doc1 = st.session_state.processing_history[doc_index_1]
+            doc2 = st.session_state.processing_history[doc_index_2]
+            
+            # Show comparison
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                doc1_html = f"""
+                <h4>Document 1: {doc1['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(doc1['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if doc1['useVision'] else 'No'}</p>
+                """
+                tool_container(doc1_html)
+                
+                # Display content summary
+                if 'ocr_contents' in doc1['result'] and isinstance(doc1['result']['ocr_contents'], dict):
+                    if 'content' in doc1['result']['ocr_contents']:
+                        content = doc1['result']['ocr_contents']['content']
+                        st.markdown(f"""
+                        <div style="max-height: 300px; overflow-y: auto; word-wrap: break-word; 
+                                   border: 1px solid #374151; padding: 1rem; background-color: #1f2937;">
+                            {content[:500]}{'...' if len(content) > 500 else ''}
+                        </div>
+                        """, unsafe_allow_html=True)
+            
+            with col2:
+                doc2_html = f"""
+                <h4>Document 2: {doc2['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(doc2['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if doc2['useVision'] else 'No'}</p>
+                """
+                tool_container(doc2_html)
+                
+                # Display content summary
+                if 'ocr_contents' in doc2['result'] and isinstance(doc2['result']['ocr_contents'], dict):
+                    if 'content' in doc2['result']['ocr_contents']:
+                        content = doc2['result']['ocr_contents']['content']
+                        st.markdown(f"""
+                        <div style="max-height: 300px; overflow-y: auto; word-wrap: break-word; 
+                                   border: 1px solid #374151; padding: 1rem; background-color: #1f2937;">
+                            {content[:500]}{'...' if len(content) > 500 else ''}
+                        </div>
+                        """, unsafe_allow_html=True)
+            
+            # Comparison analysis
+            if doc1['fileName'] == doc2['fileName'] and doc1['useVision'] != doc2['useVision']:
+                comparison_content = """
+                <h3>Vision vs. Non-Vision Model Comparison</h3>
+                <p>You're comparing the same document processed with different models. 
+                This is an excellent way to evaluate the impact of vision capabilities on OCR accuracy.</p>
+                
+                <p>Look for these differences:</p>
+                <ul>
+                  <li>Completeness of extracted text</li>
+                  <li>Accuracy of layout understanding</li>
+                  <li>Recognition of complex elements (tables, figures)</li>
+                  <li>Topic and language detection accuracy</li>
+                </ul>
+                """
+                key_concept(comparison_content)
+        else:
+            need_more_content = """
+            <h3>Need More Documents to Compare</h3>
+            <p>Process at least two documents to enable side-by-side comparison. Try processing 
+            the same document with and without the vision model to see the differences in OCR quality.</p>
+            """
+            research_question(need_more_content)
+    
+    # Analysis guide tab
+    with analyze_tab:
+        st.subheader("Analysis Guide")
+        
+        st.markdown("""
+        ### How to Analyze OCR Results
+        
+        When analyzing OCR results from historical documents, consider these key factors:
+        
+        1. **Text Accuracy**
+           - Check for common OCR errors (e.g., mistaking "e" for "c", "l" for "1")
+           - Assess recognition of period-specific typography and writing styles
+           - Evaluate handling of degraded or damaged text areas
+        
+        2. **Structure Preservation**
+           - Does the OCR maintain paragraph and section breaks?
+           - Are columns and tabular data correctly preserved?
+           - How well are page transitions handled?
+        
+        3. **Special Elements**
+           - Recognition of footnotes, marginalia, and annotations
+           - Handling of illustrations, diagrams, and decorative elements
+           - Treatment of watermarks, signatures, and stamps
+        
+        4. **Metadata Extraction**
+           - Accuracy of detected languages, topics, and document type
+           - Identification of dates, names, and key entities
+           - Recognition of document purpose and context
+        """)
+        
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            challenge_content = """
+            <h3>Common OCR Challenges</h3>
+            <ul>
+                <li><strong>Typography Variations</strong>: Historical fonts that differ from modern text</li>
+                <li><strong>Material Degradation</strong>: Fading, stains, tears affecting legibility</li>
+                <li><strong>Handwritten Elements</strong>: Marginalia, signatures, and annotations</li>
+                <li><strong>Complex Layouts</strong>: Multi-column formats and decorative elements</li>
+                <li><strong>Language and Terminology</strong>: Archaic terms and multilingual content</li>
+            </ul>
+            """
+            gray_container(challenge_content)
+        
+        with col2:
+            tips_content = """
+            <h3>Making the Most of OCR Results</h3>
+            <ul>
+                <li><strong>Contextual Reading</strong>: Use context to interpret unclear passages</li>
+                <li><strong>Error Patterns</strong>: Identify and correct systematic OCR errors</li>
+                <li><strong>Hybrid Analysis</strong>: Combine OCR search with close reading</li>
+                <li><strong>Comparative Processing</strong>: Try different settings on documents</li>
+                <li><strong>Iterative Refinement</strong>: Use insights to improve future processing</li>
+            </ul>
+            """
+            gray_container(tips_content)
+        
+        # Show example analysis if there's processing history
+        if 'processing_history' in st.session_state and st.session_state.processing_history:
+            with st.expander("Example Analysis from Your Documents"):
+                # Pick the latest document
+                latest = st.session_state.processing_history[-1]
+                
+                st.markdown(f"""
+                #### Sample Analysis for: {latest['fileName']}
+                
+                **Document Context:**
+                - Languages: {', '.join(latest['result'].get('languages', ['Unknown']))}
+                - Topics: {', '.join(latest['result'].get('topics', ['Unknown']))}
+                - Vision model used: {'Yes' if latest['useVision'] else 'No'}
+                
+                **What to Look For:**
+                1. Check how well the model identified key topics and languages
+                2. Evaluate the completeness of extracted text
+                3. Note any systematic errors in text recognition
+                4. Assess how well document structure was preserved
+                """)

modules/content/module6.py

@@ -0,0 +1,154 @@
+import streamlit as st
+from layout import gray_container, key_concept, quote, tool_container
+from datetime import datetime
+
+def render():
+    """Module 6: Conclusion and Future Directions"""
+    
+    st.title("Module 6: Conclusion and Future Directions")
+    
+    col1, col2 = st.columns([3, 2])
+    
+    with col1:
+        summary_content = """
+        <h3>Workshop Summary</h3>
+        <p>Throughout this workshop, we've explored:</p>
+        <ol>
+            <li><strong>Text-Image Interdependence</strong>: The complex relationship between textual and visual elements</li>
+            <li><strong>OCR Technology</strong>: The evolution of OCR and its application to historical materials</li>
+            <li><strong>Methodological Approaches</strong>: Hybrid strategies for working with historical texts</li>
+            <li><strong>Practical Application</strong>: Hands-on experience with OCR processing tools</li>
+        </ol>
+        """
+        gray_container(summary_content)
+        
+        takeaways_content = """
+        <h3>Key Takeaways</h3>
+        <ol>
+            <li><strong>OCR is Not Perfect</strong>: Even advanced AI models face challenges with historical documents</li>
+            <li><strong>Context Matters</strong>: Vision-enhanced models provide better results by understanding document context</li>
+            <li><strong>Hybrid Approaches</strong>: Combining computational methods with traditional research yields best results</li>
+            <li><strong>Critical Evaluation</strong>: Always evaluate OCR outputs with awareness of limitations</li>
+            <li><strong>Structured Extraction</strong>: Modern OCR goes beyond text recognition to understand document structure</li>
+        </ol>
+        """
+        gray_container(takeaways_content)
+    
+    with col2:
+        # Display workshop statistics if there's processing history
+        if 'processing_history' in st.session_state and st.session_state.processing_history:
+            st.subheader("Your Workshop Statistics")
+            
+            # Calculate statistics
+            total_docs = len(st.session_state.processing_history)
+            vision_docs = len([item for item in st.session_state.processing_history if item['useVision']])
+            non_vision_docs = total_docs - vision_docs
+            
+            # Create metrics for statistics
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                st.metric("Documents Processed", total_docs)
+                st.metric("With Vision Model", vision_docs)
+            
+            with col2:
+                st.metric("Without Vision Model", non_vision_docs)
+            
+            # Topics word cloud
+            if total_docs > 0:
+                st.subheader("Topics Encountered")
+                all_topics = []
+                for item in st.session_state.processing_history:
+                    if 'topics' in item['result']:
+                        all_topics.extend(item['result']['topics'])
+                
+                if all_topics:
+                    # Count topic frequencies
+                    topic_counts = {}
+                    for topic in all_topics:
+                        if topic in topic_counts:
+                            topic_counts[topic] += 1
+                        else:
+                            topic_counts[topic] = 1
+                    
+                    # Display as a horizontal bar chart
+                    st.bar_chart(topic_counts)
+        else:
+            # Show placeholder stats
+            placeholder_content = """
+            <h3>Workshop Outcomes</h3>
+            <p>Complete the interactive OCR experiment in Module 5 to generate your personal workshop statistics.</p>
+            <p>You'll be able to see:</p>
+            <ul>
+                <li>Number of documents processed</li>
+                <li>Comparison of vision vs. non-vision models</li>
+                <li>Topics identified across your documents</li>
+                <li>Performance metrics for your processing tasks</li>
+            </ul>
+            """
+            tool_container(placeholder_content)
+    
+    # Future directions section
+    st.subheader("Future Directions")
+    
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        tech_content = """
+        <h3>Technological Developments</h3>
+        <ul>
+            <li><strong>Multimodal AI models</strong>: Increasingly sophisticated understanding</li>
+            <li><strong>Historical font training</strong>: Models trained on historical typography</li>
+            <li><strong>Document intelligence</strong>: Enhanced understanding of structures</li>
+            <li><strong>Collaborative correction</strong>: Platforms for collective improvement</li>
+        </ul>
+        """
+        gray_container(tech_content)
+        
+    with col2:
+        research_content = """
+        <h3>Research Applications</h3>
+        <ul>
+            <li><strong>Large-scale corpus analysis</strong>: Processing entire archives</li>
+            <li><strong>Multilingual historical research</strong>: Working across languages</li>
+            <li><strong>Image-text integration</strong>: New methodologies for visual analysis</li>
+            <li><strong>Computational paleography</strong>: AI-assisted handwriting analysis</li>
+        </ul>
+        """
+        gray_container(research_content)
+    
+    # Inspiring quote
+    quote_content = "The digital humanities are not about building, they're about sharing. The digital humanities are not about the digital at all. They're all about innovation and disruption. The digital humanities are really an insurgent humanities."
+    quote(quote_content, "Matthew Kirschenbaum, Professor of Digital Humanities")
+    
+    # Additional resources
+    resources_content = """
+    <h3>Additional Resources</h3>
+    <ul>
+        <li><a href="https://docs.mistral.ai/" target="_blank">Mistral AI Documentation</a>: Learn more about the OCR models used in this workshop</li>
+        <li><a href="https://readcoop.eu/transkribus/" target="_blank">Transkribus</a>: Platform for historical document transcription</li>
+        <li><a href="https://ocr-d.de/en/" target="_blank">OCR-D</a>: Coordinated OCR research project for historical documents</li>
+        <li><a href="https://scholar.google.com/scholar?q=historical+OCR" target="_blank">Historical OCR Research Papers</a>: Academic research on historical OCR</li>
+    </ul>
+    """
+    tool_container(resources_content)
+    
+    # Acknowledgments
+    st.subheader("Acknowledgments")
+    
+    acknowledgment_content = """
+    <p>This workshop was designed as an educational resource for historians, archivists, and digital humanities scholars.</p>
+    <p>It demonstrates the integration of modern AI vision-language models with historical research methodologies.</p>
+    <p>Special thanks to the digital humanities community for continued innovation in computational approaches to historical research.</p>
+    """
+    st.markdown(acknowledgment_content, unsafe_allow_html=True)
+    
+    # Restart the workshop button
+    if st.button("Start Workshop Again", use_container_width=True):
+        # Reset the session state to start the workshop again
+        if 'current_module' in st.session_state:
+            st.session_state.current_module = 1
+        
+        # Do not reset the processing history
+        
+        st.experimental_rerun()

modules/educational_module.py

@@ -0,0 +1,547 @@
+import streamlit as st
+import io
+import tempfile
+from pathlib import Path
+from datetime import datetime
+from layout import tool_container, key_concept, research_question, upload_container
+import sys
+
+# Import the necessary modules for OCR processing
+sys.path.append(str(Path(__file__).parent.parent))
+try:
+    from process_file import process_file as process_file_util
+    process_file = process_file_util
+except ImportError:
+    # Fallback if process_file is not available
+    def process_file(uploaded_file, use_vision=True, custom_prompt=None):
+        """Fallback function for processing files"""
+        st.warning("Using mock processing function. Real OCR functionality is not available.")
+        return {
+            "file_name": uploaded_file.name,
+            "languages": ["English"],
+            "topics": ["History", "Document"],
+            "ocr_contents": {
+                "content": f"This is mock OCR content for {uploaded_file.name}. Vision model: {use_vision}"
+            }
+        }
+
+def render():
+    """Module 5: Interactive OCR Experiment"""
+    
+    st.title("Module 5: Interactive OCR Experiment")
+    
+    # Introduction to the interactive experiment
+    intro_content = """
+    <h3>Interactive OCR Experiment</h3>
+    <p>
+    This interactive experiment allows you to process historical documents with OCR and analyze the results.
+    Try different settings and compare the outcomes to understand the strengths and limitations of OCR technology.
+    </p>
+    """
+    st.markdown(intro_content, unsafe_allow_html=True)
+    
+    # Create tabs for different activities
+    experiment_tab, compare_tab, analyze_tab = st.tabs(["Process Documents", "Compare Results", "Analysis Guide"])
+    
+    # Try to import PDF tools if available
+    try:
+        from pdf2image import convert_from_bytes
+        pdf_support = True
+    except ImportError:
+        pdf_support = False
+        st.warning("PDF preview functionality is limited. The pdf2image module is required for PDF previews.")
+    
+    with experiment_tab:
+        # Create a two-column layout
+        col1, col2 = st.columns([1, 1])
+        
+        with col1:
+            # Tool container for document selection and options
+            st.subheader("Step 1: Select Document & Options")
+            
+            # Processing options
+            use_vision = st.checkbox("Use Vision Model", value=True,
+                                 help="Use the vision model for improved analysis")
+            
+            # Additional prompt
+            st.markdown("### Custom Research Prompt (Optional)")
+            st.markdown("""Provide additional instructions to guide the OCR analysis. 
+            Focus on specific aspects of historical research you're interested in.""")
+            custom_prompt = st.text_area("Research Prompt",
+                                       placeholder="E.g., Focus on identifying dates and historical figures...",
+                                       help="Optional instructions to guide the analysis")
+            
+            # Sample document selection
+            input_dir = Path(__file__).parent.parent / "input"
+            
+            if input_dir.exists():
+                sample_files = list(input_dir.glob("*.jpg")) + list(input_dir.glob("*.png")) + list(input_dir.glob("*.pdf"))
+                
+                if sample_files:
+                    st.markdown("#### Sample Documents")
+                    sample_options = ["Upload my own document"] + [f.name for f in sample_files]
+                    sample_choice = st.selectbox("Choose a document:", sample_options)
+                    
+                    if sample_choice != "Upload my own document":
+                        # Process the selected sample file
+                        selected_file = next((f for f in sample_files if f.name == sample_choice), None)
+                        
+                        if selected_file:
+                            # Store the selected sample file in session state
+                            with open(selected_file, "rb") as f:
+                                file_bytes = f.read()
+                            
+                            st.session_state.sample_file = {
+                                "name": selected_file.name,
+                                "bytes": file_bytes
+                            }
+                            
+                            # Preview the selected sample
+                            if selected_file.suffix.lower() == ".pdf" and pdf_support:
+                                try:
+                                    with st.spinner("Generating PDF preview..."):
+                                        images = convert_from_bytes(file_bytes, first_page=1, last_page=1, dpi=150)
+                                        if images:
+                                            st.image(images[0], caption=f"Preview: {selected_file.name}", use_column_width=True)
+                                except Exception:
+                                    st.info(f"PDF selected: {selected_file.name}")
+                            else:
+                                # For images display directly
+                                try:
+                                    from PIL import Image
+                                    img = Image.open(io.BytesIO(file_bytes))
+                                    st.image(img, caption=f"Preview: {selected_file.name}", use_column_width=True)
+                                except Exception:
+                                    st.info(f"Selected: {selected_file.name}")
+                    else:
+                        # Clear the sample file if "Upload my own" is selected
+                        if 'sample_file' in st.session_state:
+                            del st.session_state.sample_file
+                        
+                        # Display file uploader
+                        upload_html = """
+                        <h4>Upload a document to get started</h4>
+                        <p>Supported formats: PDF, JPG, PNG</p>
+                        """
+                        
+                        upload_container(upload_html)
+                        uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+                        
+                        if uploaded_file is not None:
+                            # Display preview of the uploaded file
+                            file_ext = Path(uploaded_file.name).suffix.lower()
+                            
+                            if file_ext == ".pdf" and pdf_support:
+                                try:
+                                    # Convert first page of PDF to image for preview
+                                    pdf_bytes = uploaded_file.getvalue()
+                                    with st.spinner("Generating PDF preview..."):
+                                        images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                                        if images:
+                                            st.image(images[0], caption=f"PDF Preview: {uploaded_file.name}", use_column_width=True)
+                                        else:
+                                            st.info(f"PDF uploaded: {uploaded_file.name}")
+                                except Exception:
+                                    st.info(f"PDF uploaded: {uploaded_file.name}")
+                            elif file_ext != ".pdf":
+                                st.image(uploaded_file, use_column_width=True)
+                            else:
+                                st.info(f"PDF uploaded: {uploaded_file.name}")
+                else:
+                    # No sample files, just show the uploader
+                    upload_html = """
+                    <h4>Upload a document to get started</h4>
+                    <p>Supported formats: PDF, JPG, PNG</p>
+                    """
+                    
+                    upload_container(upload_html)
+                    uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+                    
+                    if uploaded_file is not None:
+                        # Display the file preview
+                        file_ext = Path(uploaded_file.name).suffix.lower()
+                        if file_ext == ".pdf" and pdf_support:
+                            try:
+                                pdf_bytes = uploaded_file.getvalue()
+                                with st.spinner("Generating PDF preview..."):
+                                    images = convert_from_bytes(pdf_bytes, first_page=1, last_page=1, dpi=150)
+                                    if images:
+                                        st.image(images[0], caption=f"PDF Preview: {uploaded_file.name}", use_column_width=True)
+                            except Exception:
+                                st.info(f"PDF uploaded: {uploaded_file.name}")
+                        elif file_ext != ".pdf":
+                            st.image(uploaded_file, use_column_width=True)
+                        else:
+                            st.info(f"PDF uploaded: {uploaded_file.name}")
+            else:
+                # No input directory
+                upload_html = """
+                <h4>Upload a document to get started</h4>
+                <p>Supported formats: PDF, JPG, PNG</p>
+                """
+                
+                upload_container(upload_html)
+                uploaded_file = st.file_uploader("Choose a file", type=["pdf", "png", "jpg", "jpeg"], label_visibility="collapsed")
+            
+            # Process button
+            st.subheader("Step 2: Process the Document")
+            
+            # Get the file to process (either uploaded or sample)
+            file_to_process = None
+            if 'sample_file' in st.session_state and sample_choice != "Upload my own document":
+                # Create a FileUploader-like object from the sample file
+                class SampleFileObject:
+                    def __init__(self, name, data):
+                        self.name = name
+                        self._data = data
+                    
+                    def getvalue(self):
+                        return self._data
+                
+                file_to_process = SampleFileObject(
+                    st.session_state.sample_file["name"],
+                    st.session_state.sample_file["bytes"]
+                )
+            elif 'uploaded_file' in locals() and uploaded_file is not None:
+                file_to_process = uploaded_file
+            
+            # Process button
+            process_button = st.button(
+                "Process Document",
+                disabled=file_to_process is None,
+                use_container_width=True
+            )
+            
+            if process_button and file_to_process is not None:
+                with st.spinner("Processing document..."):
+                    try:
+                        # Process the file
+                        result = process_file(file_to_process, use_vision, custom_prompt=custom_prompt if custom_prompt else None)
+                        
+                        if result:
+                            st.success("Document processed successfully!")
+                            
+                            # Store result in session state for display in the right column
+                            st.session_state.current_result = result
+                            
+                            # Add to processing history
+                            history_item = {
+                                "id": datetime.now().timestamp(),
+                                "fileName": file_to_process.name,
+                                "timestamp": datetime.now().isoformat(),
+                                "result": result,
+                                "useVision": use_vision
+                            }
+                            
+                            if 'processing_history' not in st.session_state:
+                                st.session_state.processing_history = []
+                            
+                            st.session_state.processing_history.append(history_item)
+                            
+                            st.experimental_rerun()
+                        else:
+                            st.error("Failed to process document.")
+                    except Exception as e:
+                        st.error(f"Error processing document: {str(e)}")
+            
+            # Experiment instructions
+            experiment_content = """
+            <h3>Experiment Instructions</h3>
+            <ol>
+                <li><strong>Step 1:</strong> Select a document and choose your options</li>
+                <li><strong>Step 2:</strong> Process the document with the selected options</li>
+                <li><strong>Step 3:</strong> Analyze the results in the panel on the right</li>
+                <li><strong>Step 4:</strong> Try again with different settings (e.g., toggle vision model)</li>
+                <li><strong>Step 5:</strong> Compare results between different runs</li>
+            </ol>
+            """
+            key_concept(experiment_content)
+        
+        with col2:
+            # Results display
+            st.subheader("Step 3: View Results")
+            
+            if 'current_result' in st.session_state and st.session_state.current_result:
+                result = st.session_state.current_result
+                
+                # Display results in a tool container
+                result_html = f"""
+                <h4>Results for: {result.get('file_name', 'Unknown')}</h4>
+                <p><strong>Languages:</strong> {', '.join(result.get('languages', ['Unknown']))}</p>
+                <p><strong>Topics:</strong> {', '.join(result.get('topics', ['Unknown']))}</p>
+                """
+                tool_container(result_html)
+                
+                # Create tabs for different views
+                tab1, tab2 = st.tabs(["Structured View", "Raw JSON"])
+                
+                with tab1:
+                    # Display in a more user-friendly format
+                    if 'ocr_contents' in result:
+                        if isinstance(result['ocr_contents'], dict):
+                            for section, content in result['ocr_contents'].items():
+                                if content:  # Only display non-empty sections
+                                    st.markdown(f"#### {section.replace('_', ' ').title()}")
+                                    
+                                    if isinstance(content, str):
+                                        st.markdown(content)
+                                    elif isinstance(content, list):
+                                        for item in content:
+                                            if isinstance(item, str):
+                                                st.markdown(f"- {item}")
+                                            elif isinstance(item, dict):
+                                                st.json(item)
+                                    elif isinstance(content, dict):
+                                        for k, v in content.items():
+                                            st.markdown(f"**{k}:** {v}")
+                
+                with tab2:
+                    # Show the raw JSON
+                    st.json(result)
+                
+                # Download options
+                st.markdown("### Export Results")
+                
+                col1, col2 = st.columns(2)
+                
+                with col1:
+                    # Export as JSON
+                    import json
+                    json_bytes = json.dumps(result, indent=2).encode()
+                    st.download_button(
+                        label="Download JSON",
+                        data=json_bytes,
+                        file_name="ocr_results.json",
+                        mime="application/json",
+                        use_container_width=True
+                    )
+                
+                with col2:
+                    # Export as text if content is available
+                    if 'ocr_contents' in result and isinstance(result['ocr_contents'], dict) and 'content' in result['ocr_contents']:
+                        text_content = result['ocr_contents']['content']
+                        st.download_button(
+                            label="Download Text",
+                            data=text_content.encode(),
+                            file_name="ocr_text.txt",
+                            mime="text/plain",
+                            use_container_width=True
+                        )
+            else:
+                # Show placeholder when no results are available
+                placeholder_html = """
+                <h4>Results will appear here</h4>
+                <p>Upload and process a document to see the OCR results in this panel.</p>
+                <p>The OCR tool will:</p>
+                <ol>
+                    <li>Extract text from your document</li>
+                    <li>Identify languages and topics</li>
+                    <li>Provide structured content analysis</li>
+                    <li>Generate downloadable results</li>
+                </ol>
+                """
+                tool_container(placeholder_html)
+            
+            # Display processing history if available
+            if 'processing_history' in st.session_state and st.session_state.processing_history:
+                st.subheader("Step 4: Review Processing History")
+                
+                # Most recent result
+                latest = st.session_state.processing_history[-1]
+                latest_html = f"""
+                <h4>Latest Document: {latest['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(latest['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if latest['useVision'] else 'No'}</p>
+                """
+                tool_container(latest_html)
+                
+                # History in expander
+                with st.expander("View Complete Processing History"):
+                    for i, item in enumerate(reversed(st.session_state.processing_history)):
+                        st.markdown(f"""
+                        <div style="background-color: var(--color-gray-700); padding: 0.75rem; border-radius: 0.5rem; margin-bottom: 0.5rem;">
+                        <strong>{item['fileName']}</strong><br>
+                        {datetime.fromisoformat(item['timestamp']).strftime('%Y-%m-%d %H:%M')} - 
+                        Vision model: {'Yes' if item['useVision'] else 'No'}
+                        </div>
+                        """, unsafe_allow_html=True)
+                        
+                        # Option to view a previous result
+                        if st.button(f"View This Result", key=f"view_history_{i}"):
+                            st.session_state.current_result = item['result']
+                            st.experimental_rerun()
+    
+    # Compare tab for side-by-side comparison
+    with compare_tab:
+        st.subheader("Compare OCR Results")
+        
+        if 'processing_history' in st.session_state and len(st.session_state.processing_history) >= 2:
+            st.markdown("""
+            Select two processing results to compare side by side. This allows you to see
+            how different options (like using the vision model) affect OCR quality.
+            """)
+            
+            # Create selection dropdowns for the documents
+            col1, col2 = st.columns(2)
+            with col1:
+                # First document selector
+                doc_options_1 = [f"{i+1}: {item['fileName']} ({'Vision' if item['useVision'] else 'No Vision'})" 
+                               for i, item in enumerate(st.session_state.processing_history)]
+                doc_choice_1 = st.selectbox("First Document:", doc_options_1, key="compare_doc_1")
+                doc_index_1 = int(doc_choice_1.split(":")[0]) - 1
+                
+            with col2:
+                # Second document selector
+                doc_options_2 = [f"{i+1}: {item['fileName']} ({'Vision' if item['useVision'] else 'No Vision'})" 
+                               for i, item in enumerate(st.session_state.processing_history)]
+                default_index = min(1, len(st.session_state.processing_history) - 1)  # Default to second item
+                doc_choice_2 = st.selectbox("Second Document:", doc_options_2, key="compare_doc_2", index=default_index)
+                doc_index_2 = int(doc_choice_2.split(":")[0]) - 1
+            
+            # Retrieve the selected documents
+            doc1 = st.session_state.processing_history[doc_index_1]
+            doc2 = st.session_state.processing_history[doc_index_2]
+            
+            # Show comparison
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                doc1_html = f"""
+                <h4>Document 1: {doc1['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(doc1['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if doc1['useVision'] else 'No'}</p>
+                """
+                tool_container(doc1_html)
+                
+                # Display content summary
+                if 'ocr_contents' in doc1['result'] and isinstance(doc1['result']['ocr_contents'], dict):
+                    if 'content' in doc1['result']['ocr_contents']:
+                        content = doc1['result']['ocr_contents']['content']
+                        st.markdown(f"""
+                        <div style="max-height: 300px; overflow-y: auto; word-wrap: break-word; 
+                                   border: 1px solid #374151; padding: 1rem; background-color: #1f2937;">
+                            {content[:500]}{'...' if len(content) > 500 else ''}
+                        </div>
+                        """, unsafe_allow_html=True)
+            
+            with col2:
+                doc2_html = f"""
+                <h4>Document 2: {doc2['fileName']}</h4>
+                <p><strong>Processed at:</strong> {datetime.fromisoformat(doc2['timestamp']).strftime('%Y-%m-%d %H:%M')}</p>
+                <p><strong>Vision model used:</strong> {'Yes' if doc2['useVision'] else 'No'}</p>
+                """
+                tool_container(doc2_html)
+                
+                # Display content summary
+                if 'ocr_contents' in doc2['result'] and isinstance(doc2['result']['ocr_contents'], dict):
+                    if 'content' in doc2['result']['ocr_contents']:
+                        content = doc2['result']['ocr_contents']['content']
+                        st.markdown(f"""
+                        <div style="max-height: 300px; overflow-y: auto; word-wrap: break-word; 
+                                   border: 1px solid #374151; padding: 1rem; background-color: #1f2937;">
+                            {content[:500]}{'...' if len(content) > 500 else ''}
+                        </div>
+                        """, unsafe_allow_html=True)
+            
+            # Comparison analysis
+            if doc1['fileName'] == doc2['fileName'] and doc1['useVision'] != doc2['useVision']:
+                comparison_content = """
+                <h3>Vision vs. Non-Vision Model Comparison</h3>
+                <p>You're comparing the same document processed with different models. 
+                This is an excellent way to evaluate the impact of vision capabilities on OCR accuracy.</p>
+                
+                <p>Look for these differences:</p>
+                <ul>
+                  <li>Completeness of extracted text</li>
+                  <li>Accuracy of layout understanding</li>
+                  <li>Recognition of complex elements (tables, figures)</li>
+                  <li>Topic and language detection accuracy</li>
+                </ul>
+                """
+                key_concept(comparison_content)
+        else:
+            need_more_content = """
+            <h3>Need More Documents to Compare</h3>
+            <p>Process at least two documents to enable side-by-side comparison. Try processing 
+            the same document with and without the vision model to see the differences in OCR quality.</p>
+            """
+            research_question(need_more_content)
+    
+    # Analysis guide tab
+    with analyze_tab:
+        st.subheader("Analysis Guide")
+        
+        st.markdown("""
+        ### How to Analyze OCR Results
+        
+        When analyzing OCR results from historical documents, consider these key factors:
+        
+        1. **Text Accuracy**
+           - Check for common OCR errors (e.g., mistaking "e" for "c", "l" for "1")
+           - Assess recognition of period-specific typography and writing styles
+           - Evaluate handling of degraded or damaged text areas
+        
+        2. **Structure Preservation**
+           - Does the OCR maintain paragraph and section breaks?
+           - Are columns and tabular data correctly preserved?
+           - How well are page transitions handled?
+        
+        3. **Special Elements**
+           - Recognition of footnotes, marginalia, and annotations
+           - Handling of illustrations, diagrams, and decorative elements
+           - Treatment of watermarks, signatures, and stamps
+        
+        4. **Metadata Extraction**
+           - Accuracy of detected languages, topics, and document type
+           - Identification of dates, names, and key entities
+           - Recognition of document purpose and context
+        """)
+        
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            challenge_content = """
+            <h3>Common OCR Challenges</h3>
+            <ul>
+                <li><strong>Typography Variations</strong>: Historical fonts that differ from modern text</li>
+                <li><strong>Material Degradation</strong>: Fading, stains, tears affecting legibility</li>
+                <li><strong>Handwritten Elements</strong>: Marginalia, signatures, and annotations</li>
+                <li><strong>Complex Layouts</strong>: Multi-column formats and decorative elements</li>
+                <li><strong>Language and Terminology</strong>: Archaic terms and multilingual content</li>
+            </ul>
+            """
+            gray_container(challenge_content)
+        
+        with col2:
+            tips_content = """
+            <h3>Making the Most of OCR Results</h3>
+            <ul>
+                <li><strong>Contextual Reading</strong>: Use context to interpret unclear passages</li>
+                <li><strong>Error Patterns</strong>: Identify and correct systematic OCR errors</li>
+                <li><strong>Hybrid Analysis</strong>: Combine OCR search with close reading</li>
+                <li><strong>Comparative Processing</strong>: Try different settings on documents</li>
+                <li><strong>Iterative Refinement</strong>: Use insights to improve future processing</li>
+            </ul>
+            """
+            gray_container(tips_content)
+        
+        # Show example analysis if there's processing history
+        if 'processing_history' in st.session_state and st.session_state.processing_history:
+            with st.expander("Example Analysis from Your Documents"):
+                # Pick the latest document
+                latest = st.session_state.processing_history[-1]
+                
+                st.markdown(f"""
+                #### Sample Analysis for: {latest['fileName']}
+                
+                **Document Context:**
+                - Languages: {', '.join(latest['result'].get('languages', ['Unknown']))}
+                - Topics: {', '.join(latest['result'].get('topics', ['Unknown']))}
+                - Vision model used: {'Yes' if latest['useVision'] else 'No'}
+                
+                **What to Look For:**
+                1. Check how well the model identified key topics and languages
+                2. Evaluate the completeness of extracted text
+                3. Note any systematic errors in text recognition
+                4. Assess how well document structure was preserved
+                """)

modules/modular_app.py

@@ -0,0 +1,276 @@
+import streamlit as st
+from pathlib import Path
+import sys
+from layout import page_wrapper
+from modules import get_module, get_module_name, module_names
+
+# Set page configuration with dark theme
+st.set_page_config(
+    page_title="Historical OCR Workshop",
+    page_icon="📜",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+# Initialize session state for workshop navigation
+if 'current_module' not in st.session_state:
+    st.session_state.current_module = 1
+
+if 'workshop_started' not in st.session_state:
+    st.session_state.workshop_started = False
+
+if 'processing_history' not in st.session_state:
+    st.session_state.processing_history = []
+
+def navigate_to_module(module_number):
+    """Navigate to a specific module"""
+    st.session_state.current_module = module_number
+    st.rerun()
+
+# Welcome screen if workshop hasn't been started
+if not st.session_state.workshop_started:
+    def welcome_screen():
+        """Renders the welcome/start screen"""
+        # Hero section with eye-catching design
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E3A8A 0%, #2563EB 100%); 
+                    padding: 2rem; border-radius: 0.75rem; text-align: center; 
+                    margin-bottom: 2rem; box-shadow: 0 4px 6px rgba(0,0,0,0.3);">
+            <h1>Historical OCR Workshop</h1>
+            <p style="font-size: 1.25rem;">Unlock the potential of historical documents with modern OCR technology</p>
+        </div>
+        """, unsafe_allow_html=True)
+        
+        # Introduction with cleaner layout
+        col1, col2 = st.columns([3, 2])
+        
+        with col1:
+            st.markdown("""
+            <div style="background-color: #1f2937; padding: 1.5rem; border-radius: 0.75rem; margin-bottom: 1.5rem;">
+            <h3>Workshop Overview</h3>
+            
+            This interactive workshop explores the application of OCR technology to historical documents,
+            combining theoretical understanding with practical experiences. Designed for historians, 
+            archivists, and digital humanities scholars, it offers both conceptual frameworks and hands-on skills.
+            </div>
+            """, unsafe_allow_html=True)
+            
+            st.markdown("""
+            <div style="background-color: #374151; padding: 0.75rem; border-radius: 0.5rem; 
+                        margin: 1rem 0; border-left: 3px solid #3B82F6;">
+            <h4>What is OCR?</h4>
+            Optical Character Recognition (OCR) technology enables computers to extract text from images and documents.
+            Modern OCR uses AI vision models to understand both the text and its visual context, making it powerful for
+            historical research and digital humanities.
+            </div>
+            """, unsafe_allow_html=True)
+            
+        with col2:
+            # Add an engaging research question
+            st.markdown("""
+            <div style="background-color: #1E3A8A; color: white; padding: 0.75rem; 
+                       border-radius: 0.5rem; margin: 1rem 0; border-left: 3px solid #60A5FA;">
+            <h4>For Historians:</h4>
+            How might OCR technology transform our access to and interpretation of historical documents? 
+            What new research questions become possible when large archives become machine-readable?
+            </div>
+            """, unsafe_allow_html=True)
+            
+            # Display a sample historical document image
+            input_dir = Path(__file__).parent / "input"
+            sample_path = input_dir / "magellan-travels.jpg"
+            if sample_path.exists():
+                try:
+                    from PIL import Image
+                    with Image.open(sample_path) as img:
+                        st.image(img, caption="Sample Historical Document", width=300)
+                except Exception:
+                    pass
+        
+        # What you'll learn section with visual learning outcomes
+        st.markdown('<h3>What You\'ll Learn</h3>', unsafe_allow_html=True)
+        
+        # Create three columns for clean layout
+        col1, col2, col3 = st.columns(3)
+        
+        with col1:
+            st.markdown("""
+            <div style="background-color: #1f2937; padding: 1rem; border-radius: 0.5rem;">
+            <h4>Conceptual Understanding</h4>
+            
+            - Text-image relationships in historical documents
+            - Evolution of OCR technology 
+            - AI vision models for document analysis
+            - Historical typography challenges
+            </div>
+            """, unsafe_allow_html=True)
+            
+        with col2:
+            st.markdown("""
+            <div style="background-color: #1f2937; padding: 1rem; border-radius: 0.5rem;">
+            <h4>Methodological Approaches</h4>
+            
+            - Critical frameworks for OCR in historical research
+            - Hybrid computational-traditional methods
+            - Error analysis and interpretation
+            - Contextual reading strategies
+            </div>
+            """, unsafe_allow_html=True)
+            
+        with col3:
+            st.markdown("""
+            <div style="background-color: #1f2937; padding: 1rem; border-radius: 0.5rem;">
+            <h4>Practical Skills</h4>
+            
+            - Processing historical documents with OCR
+            - Analyzing and structuring extracted information
+            - Integrating OCR into research workflows
+            - Building searchable archives
+            </div>
+            """, unsafe_allow_html=True)
+        
+        # Module overview
+        st.markdown('<h3>Workshop Modules</h3>', unsafe_allow_html=True)
+        
+        # First row of modules
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            for i in [1, 3, 5]:
+                st.markdown(f"""
+                <div style="background-color: #1f2937; border-radius: 8px; padding: 16px; 
+                           margin-bottom: 16px; border-top: 4px solid #3B82F6;">
+                    <div style="background-color: #3B82F6; color: white; font-weight: bold; 
+                               padding: 4px 10px; border-radius: 12px; font-size: 0.9rem; 
+                               display: inline-block; margin-bottom: 8px;">Module {i}</div>
+                    <div style="font-weight: 600; margin-bottom: 8px; font-size: 1.1rem; color: white;">
+                        {module_names[i-1]}
+                    </div>
+                    <p>Module {i} of the historical OCR workshop.</p>
+                </div>
+                """, unsafe_allow_html=True)
+        
+        with col2:
+            for i in [2, 4, 6]:
+                st.markdown(f"""
+                <div style="background-color: #1f2937; border-radius: 8px; padding: 16px; 
+                           margin-bottom: 16px; border-top: 4px solid #3B82F6;">
+                    <div style="background-color: #3B82F6; color: white; font-weight: bold; 
+                               padding: 4px 10px; border-radius: 12px; font-size: 0.9rem; 
+                               display: inline-block; margin-bottom: 8px;">Module {i}</div>
+                    <div style="font-weight: 600; margin-bottom: 8px; font-size: 1.1rem; color: white;">
+                        {module_names[i-1]}
+                    </div>
+                    <p>Module {i} of the historical OCR workshop.</p>
+                </div>
+                """, unsafe_allow_html=True)
+        
+        # Inspirational quote
+        st.markdown("""
+        <div style="font-style: italic; color: #D1D5DB; padding: 0.5rem 1rem; 
+                   border-left: 3px solid #4B5563; margin: 1rem 0;">
+        "The digital turn in historical research is not just about converting analog to digital; 
+        it's about transforming how we access, analyze, and interpret the past."
+        <br/><br/>
+        <span style="font-size:0.9rem; text-align:right; display:block;">— Dr. Jane Winters, Professor of Digital Humanities</span>
+        </div>
+        """, unsafe_allow_html=True)
+        
+        # Start button with enhanced styling
+        st.markdown('<div style="text-align: center; margin-top: 2rem;">', unsafe_allow_html=True)
+        col1, col2, col3 = st.columns([1, 2, 1])
+        with col2:
+            if st.button("Begin Workshop Journey", key="start_workshop", type="primary", use_container_width=True):
+                st.session_state.workshop_started = True
+                st.rerun()
+        st.markdown('<p style="text-align:center; margin-top:8px; font-size:0.9rem; color:#666;">No installation required • Start immediately</p>', unsafe_allow_html=True)
+        st.markdown('</div>', unsafe_allow_html=True)
+    
+    # Display the welcome screen (outside modules)
+    welcome_screen()
+else:
+    # Get the current module to display
+    current_module = st.session_state.current_module
+    module = get_module(current_module)
+    
+    # Create navigation callbacks for the page wrapper
+    def nav_to_prev():
+        if current_module > 1:
+            st.session_state.current_module = current_module - 1
+            st.rerun()
+    
+    def nav_to_next():
+        if current_module < 6:
+            st.session_state.current_module = current_module + 1
+            st.rerun()
+    
+    # Create the sidebar navigation
+    with st.sidebar:
+        st.markdown("<h1>Workshop Navigation</h1>", unsafe_allow_html=True)
+        
+        # Visual header
+        st.markdown("<div style='display:flex; align-items:center; margin-bottom:20px;'>", unsafe_allow_html=True)
+        
+        # Show a progress indicator
+        st.markdown(f"<div><b>Your Progress:</b> Module {current_module} of 6</div>", unsafe_allow_html=True)
+        st.progress(current_module / 6)
+        
+        # Module navigation buttons
+        st.markdown("<h3>Modules</h3>", unsafe_allow_html=True)
+        
+        for i, name in enumerate(module_names, 1):
+            btn_style = "primary" if i == current_module else "secondary"
+            if st.button(f"{i}: {name}", key=f"nav_module_{i}", type=btn_style, use_container_width=True):
+                st.session_state.current_module = i
+                st.rerun()
+        
+        # About the workshop in a collapsible section
+        with st.expander("About the Workshop"):
+            st.markdown("""
+            This interactive workshop explores OCR technology for historical documents. 
+            
+            **How to use this workshop:**
+            1. Navigate through modules sequentially
+            2. Expand content sections to read more
+            3. Try the interactive OCR experiment
+            4. Reflect on research questions
+            
+            For help or more information, use the reference materials in Module 6.
+            """)
+            
+        # Processing history if available
+        if st.session_state.processing_history:
+            with st.expander("Your Activity"):
+                st.markdown(f"<b>Documents processed:</b> {len(st.session_state.processing_history)}", unsafe_allow_html=True)
+                
+                # Show the most recent document processed
+                latest = st.session_state.processing_history[-1]
+                st.markdown(f"""
+                <div style="background:#f9f9f9; padding:8px; border-radius:4px; margin-top:10px; color:#333;">
+                    <b>Latest document:</b> {latest['fileName']}<br>
+                    <span style="font-size:0.9rem;">Processed with {' vision model' if latest['useVision'] else ' basic OCR'}</span>
+                </div>
+                """, unsafe_allow_html=True)
+    
+    # Render the current module content using the page wrapper
+    page_wrapper(module.render, current_module)
+
+# At the bottom of the page, create the hidden navigation buttons for the fixed navigation bar
+if st.session_state.workshop_started:
+    # Previous navigation button (hidden, activated by the fixed nav)
+    if st.session_state.current_module > 1:
+        if st.button("←", key=f"nav_prev_{st.session_state.current_module-1}", label_visibility="collapsed"):
+            st.session_state.current_module -= 1
+            st.rerun()
+    
+    # Next navigation button (hidden, activated by the fixed nav)
+    if st.session_state.current_module < 6:
+        if st.button("→", key=f"nav_next_{st.session_state.current_module+1}", label_visibility="collapsed"):
+            st.session_state.current_module += 1
+            st.rerun()
+    
+    # Module navigation dots (hidden, activated by the fixed nav)
+    for i in range(1, 7):
+        if st.button(f"{i}", key=f"nav_dot_{i}", label_visibility="collapsed"):
+            st.session_state.current_module = i
+            st.rerun()

ocr_utils.py

@@ -0,0 +1,212 @@
+"""
+Utility functions for OCR processing with Mistral AI.
+Contains helper functions for working with OCR responses and image handling.
+"""
+
+import json
+import base64
+import io
+from pathlib import Path
+from typing import Dict, List, Optional, Union, Any
+
+try:
+    from PIL import Image
+    PILLOW_AVAILABLE = True
+except ImportError:
+    PILLOW_AVAILABLE = False
+
+from mistralai import DocumentURLChunk, ImageURLChunk, TextChunk
+
+def replace_images_in_markdown(markdown_str: str, images_dict: dict) -> str:
+    """
+    Replace image placeholders in markdown with base64-encoded images.
+
+    Args:
+        markdown_str: Markdown text containing image placeholders
+        images_dict: Dictionary mapping image IDs to base64 strings
+
+    Returns:
+        Markdown text with images replaced by base64 data
+    """
+    for img_name, base64_str in images_dict.items():
+        markdown_str = markdown_str.replace(
+            f"![{img_name}]({img_name})", f"![{img_name}]({base64_str})"
+        )
+    return markdown_str
+
+def get_combined_markdown(ocr_response) -> str:
+    """
+    Combine OCR text and images into a single markdown document.
+    Ensures proper spacing between text and images.
+
+    Args:
+        ocr_response: Response from OCR processing containing text and images
+            See https://docs.mistral.ai/capabilities/document/ for API reference
+
+    Returns:
+        Combined markdown string with embedded images
+    """
+    markdowns: list[str] = []
+    # Extract images from page
+    for page in ocr_response.pages:
+        image_data = {}
+        for img in page.images:
+            image_data[img.id] = img.image_base64
+        
+        # Replace image placeholders with actual images
+        page_markdown = replace_images_in_markdown(page.markdown, image_data)
+        
+        # Ensure proper spacing between paragraphs and images
+        # Add extra newlines between paragraphs to improve rendering
+        page_markdown = page_markdown.replace("\n", "\n\n")
+        
+        # Add page separator for multi-page documents
+        markdowns.append(page_markdown)
+        
+    # Join pages with clear separators for multi-page documents
+    return "\n\n---\n\n".join(markdowns)
+
+def encode_image_for_api(image_path: Union[str, Path]) -> str:
+    """
+    Encode an image as base64 for API use.
+    
+    Args:
+        image_path: Path to the image file
+        
+    Returns:
+        Base64 data URL for the image
+    """
+    # Convert to Path object if string
+    image_file = Path(image_path) if isinstance(image_path, str) else image_path
+    
+    # Verify image exists
+    if not image_file.is_file():
+        raise FileNotFoundError(f"Image file not found: {image_file}")
+    
+    # Encode image as base64
+    encoded = base64.b64encode(image_file.read_bytes()).decode()
+    return f"data:image/jpeg;base64,{encoded}"
+
+def process_image_with_ocr(client, image_path: Union[str, Path], model: str = "mistral-ocr-latest"):
+    """
+    Process an image with OCR and return the response.
+    
+    Args:
+        client: Mistral AI client
+        image_path: Path to the image file
+        model: OCR model to use
+        
+    Returns:
+        OCR response object
+    """
+    # Encode image as base64
+    base64_data_url = encode_image_for_api(image_path)
+    
+    # Process image with OCR
+    image_response = client.ocr.process(
+        document=ImageURLChunk(image_url=base64_data_url),
+        model=model
+    )
+    
+    return image_response
+
+def ocr_response_to_json(ocr_response, indent: int = 4) -> str:
+    """
+    Convert OCR response to a formatted JSON string.
+    
+    Args:
+        ocr_response: OCR response object
+        indent: Indentation level for JSON formatting
+        
+    Returns:
+        Formatted JSON string
+    """
+    # Convert response to JSON
+    response_dict = json.loads(ocr_response.model_dump_json())
+    return json.dumps(response_dict, indent=indent)
+
+def get_combined_markdown_compressed(ocr_response, max_width: int = 800, quality: int = 85) -> str:
+    """
+    Combine OCR text and images into a single markdown document with compressed images.
+    Reduces image sizes to improve performance.
+    
+    Args:
+        ocr_response: Response from OCR processing containing text and images
+        max_width: Maximum width to resize images to (preserves aspect ratio)
+        quality: JPEG quality (0-100) for compression
+            
+    Returns:
+        Combined markdown string with embedded compressed images
+    """
+    if not PILLOW_AVAILABLE:
+        # Fall back to regular method if PIL is not available
+        return get_combined_markdown(ocr_response)
+        
+    markdowns: list[str] = []
+    
+    # Process each page
+    for page in ocr_response.pages:
+        image_data = {}
+        
+        # Process and compress each image
+        for img in page.images:
+            try:
+                # Decode base64 image
+                img_bytes = base64.b64decode(img.image_base64.split(',')[1] if ',' in img.image_base64 else img.image_base64)
+                
+                # Open with PIL
+                pil_img = Image.open(io.BytesIO(img_bytes))
+                
+                # Resize if needed (maintain aspect ratio)
+                original_width, original_height = pil_img.size
+                if original_width > max_width:
+                    ratio = max_width / original_width
+                    new_height = int(original_height * ratio)
+                    pil_img = pil_img.resize((max_width, new_height), Image.LANCZOS)
+                
+                # Convert to bytes with compression
+                buffer = io.BytesIO()
+                format = pil_img.format if pil_img.format else 'JPEG'
+                if format.upper() == 'JPEG' or format.upper() == 'JPG':
+                    pil_img.save(buffer, format=format, quality=quality, optimize=True)
+                else:
+                    # For non-JPEG formats (PNG, etc.)
+                    pil_img.save(buffer, format=format, optimize=True)
+                
+                # Convert back to base64
+                compressed_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
+                mime_type = f"image/{format.lower()}" if format else "image/jpeg"
+                image_data[img.id] = f"data:{mime_type};base64,{compressed_base64}"
+                
+            except Exception as e:
+                # If compression fails, use original image
+                image_data[img.id] = img.image_base64
+        
+        # Replace image placeholders with compressed images
+        page_markdown = replace_images_in_markdown(page.markdown, image_data)
+        
+        # Ensure proper spacing between paragraphs and images
+        page_markdown = page_markdown.replace("\n", "\n\n")
+        
+        # Add page to list
+        markdowns.append(page_markdown)
+        
+    # Join pages with clear separators
+    return "\n\n---\n\n".join(markdowns)
+
+# For display in notebooks
+try:
+    from IPython.display import Markdown, display
+    
+    def display_ocr_with_images(ocr_response):
+        """
+        Display OCR response with embedded images in IPython environments.
+        
+        Args:
+            ocr_response: OCR response object
+        """
+        combined_markdown = get_combined_markdown(ocr_response)
+        display(Markdown(combined_markdown))
+except ImportError:
+    # IPython not available
+    pass

output/recipe_test.json

@@ -0,0 +1,16 @@
+{
+  "file_name": "img-0.jpeg",
+  "topics": [
+    "Cooking",
+    "Recipes",
+    "Baking"
+  ],
+  "languages": [
+    "English"
+  ],
+  "ocr_contents": {
+    "title": "Pecan Butterballs Cookies",
+    "recipe": "1 cup butter, creamy if possible\n1/4 inch honey\n2 \" ounces flour\n1/2 teaspoon salt\n2 \" ounces pecans\n2 cups finely chopped pecans\nForm into small balls, bake at 300 40-45 min roll in uncoated sugar"
+  },
+  "confidence_score": 0.85,
+  "raw_response":