main.py

import markdown
from flask import Flask, render_template, request, redirect, url_for, jsonify, session, flash
from werkzeug.security import generate_password_hash, check_password_hash
import os
import fitz  # PyMuPDF for text-based PDFs
import pytesseract  # OCR for images
import requests  # To call the AI API
from PIL import Image
import sqlite3
from deep_translator import GoogleTranslator
from translate import Translator
from flask_wtf.csrf import CSRFProtect, generate_csrf
import re
import datetime
from datetime import timedelta
import base64
import numpy as np  # For table extraction
import pandas as pd  # For table handling
from sklearn.feature_extraction.text import TfidfVectorizer  # For topic extraction
import spacy  # For NLP tasks
import io  # For in-memory file handling
import uuid  # For generating unique IDs
from tabulate import tabulate
# Import prompts from the prompts module
from prompts import (
    SUMMARY_PROMPT, 
    QUESTIONS_PROMPT, 
    CHAT_SYSTEM_PROMPT, 
    CONTEXT_PROMPT, 
    FALLBACK_QUESTIONS,
    COMMON_RESPONSE_PREFIXES,
    DOCUMENT_CATEGORIES,
    SENTIMENT_WORDS,
    ENTITY_TYPE_MAP,
    CITATION_PATTERNS
)
import zlib  # For compression

# Load environment variables from .env file if it exists
try:
    from dotenv import load_dotenv
    load_dotenv()
    print("Environment variables loaded from .env file")
except ImportError:
    print("python-dotenv not installed. Environment variables not loaded from .env file.")
except Exception as e:
    print(f"Error loading .env file: {str(e)}")

try:
    nlp = spacy.load("en_core_web_sm")
    print("Successfully loaded spaCy model")
except Exception as e:
    print(f"Could not load spaCy model: {str(e)}")
    print("Named entity recognition and advanced NLP features will be limited")
    print("To install the model, run: python -m spacy download en_core_web_sm")
    nlp = None

# Add a helper function to check if NLP is available
def is_nlp_available():
    """Check if NLP functionality is available (spaCy is loaded)"""
    return nlp is not None

app = Flask(__name__)
# Use environment variable for secret key if available, otherwise use a fixed one
app.secret_key = os.environ.get("FLASK_SECRET_KEY", "JaiWn5ackErSta6UxabrAPhewrumeJUhEsWeGe4pE3uthAspujefr8c")
app.config["UPLOAD_FOLDER"] = "uploads"
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=7)  # Session lasts 7 days
app.config['WTF_CSRF_ENABLED'] = True
# Use environment variable for CSRF secret key if available, otherwise use a fixed one
app.config['WTF_CSRF_SECRET_KEY'] = os.environ.get("FLASK_CSRF_SECRET_KEY", "pudr2swA8rUcRuThASAxUcRap5ath7sTabruThEsWuDr6he7r9phu")
app.config['SESSION_COOKIE_SECURE'] = os.environ.get("SESSION_COOKIE_SECURE", "False").lower() == "true"
app.config['SESSION_COOKIE_HTTPONLY'] = True  # Prevent JavaScript access to session cookie
app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'  # CSRF protection

# Add file-based session configuration to prevent cookie size issues
os.makedirs('flask_session', exist_ok=True)  # Create directory for session files
app.config['SESSION_TYPE'] = 'filesystem'
app.config['SESSION_FILE_DIR'] = 'flask_session'
app.config['SESSION_PERMANENT'] = True
app.config['SESSION_USE_SIGNER'] = True

# Initialize the Flask-Session extension
from flask_session import Session
sess = Session(app)

csrf = CSRFProtect(app)

# Exempt certain routes from CSRF protection
csrf.exempt('/translate_content')
csrf.exempt('/translate_summary')
csrf.exempt('/translate_questions')
csrf.exempt('/debug_translate')
csrf.exempt('/chat')  # Exempt chat route from CSRF protection since it's an API endpoint

# Ensure upload folder exists
os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True)

# Password validation regex
PASSWORD_REGEX = re.compile(r'^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)[a-zA-Z\d]{8,}$')

# --- Database Setup ---
def init_db():
    conn = sqlite3.connect("users.db")
    cursor = conn.cursor()
    
    # Create users table if it doesn't exist
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS users (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            email TEXT UNIQUE NOT NULL,
            password TEXT NOT NULL,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            remember_token TEXT
        )
    """)
    
    # Check if remember_token column exists, add it if it doesn't
    cursor.execute("PRAGMA table_info(users)")
    columns = [column[1] for column in cursor.fetchall()]
    
    if "remember_token" not in columns:
        cursor.execute("ALTER TABLE users ADD COLUMN remember_token TEXT")
        print("Added remember_token column to users table")
    
    conn.commit()
    conn.close()

init_db()

# --- Helper Functions ---
def is_valid_email(email):
    email_regex = r'^[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}$'
    return re.match(email_regex, email) is not None

def is_valid_password(password):
    return PASSWORD_REGEX.match(password) is not None

def generate_remember_token():
    return os.urandom(24).hex()

def compress_data(data):
    """Compress text data to save session space, using only base64 encoding"""
    if not data:
        return None
    try:
        # Just use base64 encoding without compression
        if isinstance(data, str):
            data_bytes = data.encode("utf-8")
            encoded_data = base64.b64encode(data_bytes).decode("utf-8")
            return encoded_data
        else:
            # If not a string, return as is
            print(f"Warning: Non-string data passed to compress_data: {type(data)}")
            return data
    except Exception as e:
        print(f"Error compressing data: {str(e)}")
        # Return original data on error
        return data

def decompress_data(data):
    """Decompress text data from session, using only base64 decoding"""
    if not data:
        return ""
    try:
        # Just use base64 decoding without decompression
        if isinstance(data, str):
            try:
                decoded_data = base64.b64decode(data.encode("utf-8")).decode("utf-8")
                return decoded_data
            except:
                # If decoding fails, it might not be encoded data
                print("Warning: Could not decode data, returning as is")
                return data
        else:
            # If not a string, return as is
            print(f"Warning: Non-string data passed to decompress_data: {type(data)}")
            return str(data)
    except Exception as e:
        print(f"Error decompressing data: {str(e)}")
        # Return string representation on error
        return str(data)

def get_from_session(key, default=""):
    """Get data from session with automatic decompression"""
    try:
        if key not in session:
            return default
            
        value = session.get(key, default)
        if value is None:
            return default
            
        # If the value is a string and looks compressed, try to decompress it
        if isinstance(value, str) and value.startswith("__COMPRESSED__"):
            try:
                compressed_data = base64.b64decode(value[14:])
                return zlib.decompress(compressed_data).decode('utf-8')
            except Exception as e:
                print(f"Error decompressing session data for key {key}: {str(e)}")
                return value
        
        return value
    except Exception as e:
        print(f"Error in get_from_session for key {key}: {str(e)}")
        return default

def store_in_session(key, value):
    """Store data in session with automatic compression for large values"""
    try:
        if value is None:
            session[key] = None
            return
            
        # For strings, compress if they're large
        if isinstance(value, str) and len(value) > 1000:
            try:
                compressed = zlib.compress(value.encode('utf-8'))
                b64_compressed = base64.b64encode(compressed).decode('ascii')
                session[key] = f"__COMPRESSED__{b64_compressed}"
            except Exception as e:
                print(f"Error compressing data for key {key}: {str(e)}")
                session[key] = value
        else:
            session[key] = value
            
        # Verify storage was successful
        if key not in session:
            print(f"Warning: Failed to store {key} in session")
    except Exception as e:
        print(f"Error in store_in_session for key {key}: {str(e)}")
        # Make a best effort to store something
        try:
            session[key] = str(value)[:200] + "... (truncated due to error)"
        except:
            pass

def format_questions_html(questions_content):
    """
    Format questions and answers into proper HTML structure with spoiler buttons.
    This function prevents nesting questions inside each other when refreshing.
    """
    # Make sure re module is properly imported
    import re
    
    # If input is empty, return empty string
    if not questions_content or questions_content.strip() == "":
        print("Empty questions content, returning empty string")
        return ""
    
    # Try to extract question-answer pairs
    question_answer_pairs = []
    
    # First check if the content already has the expected HTML structure
    if "<div class=\"question-container\">" in questions_content:
        print("Questions content already has HTML structure, adding show/hide buttons if needed")
        # Check if 'show-answer-btn' already exists in the content
        if "show-answer-btn" not in questions_content:
            # Replace the existing structure with one that includes buttons
            processed_html = re.sub(
                r'<div class="question-container">\s*<h3[^>]*>(.*?)</h3>\s*<div class="answer[^"]*"[^>]*>(.*?)</div>\s*</div>',
                r'<div class="question-container">\n    <h3>\1</h3>\n    <button class="show-answer-btn">\n        <span>Show Answer</span>\n        <i class="fas fa-chevron-down"></i>\n    </button>\n    <div class="answer hidden">\2</div>\n</div>',
                questions_content,
                flags=re.DOTALL
            )
            return processed_html
        return questions_content
        
    # Try different formats if it's not HTML already
    try:
        # First check if the content already has questions and answers
        q_pattern = r'Question:\s*(.*?)(?=Answer:|$)'
        a_pattern = r'Answer:\s*(.*?)(?=Question:|$)'
        
        q_matches = re.findall(q_pattern, questions_content, re.DOTALL)
        a_matches = re.findall(a_pattern, questions_content, re.DOTALL)
        
        # Pair questions with answers
        for i in range(min(len(q_matches), len(a_matches))):
            q = q_matches[i].strip()
            a = a_matches[i].strip()
            question_answer_pairs.append((q, a))
        
        # If no pairs found yet, try splitting by question
        if not question_answer_pairs:
            parts = re.split(r'^\s*Question:', questions_content, flags=re.MULTILINE)
            if len(parts) > 1:
                for part in parts[1:]:  # Skip first empty part
                    lines = part.strip().split('\n')
                    q = lines[0].strip()
                    a = '\n'.join(lines[1:]).strip()
                    if q and a:
                        question_answer_pairs.append((q, a))
    
    except Exception as e:
        print(f"Error parsing questions: {e}")
        # Continue with whatever pairs we found
    
    # If we couldn't extract any pairs, create a default one
    if not question_answer_pairs:
        question_answer_pairs = [
            ("What does this document discuss?", "The document content could not be processed.")
        ]
    
    # Helper function to process markdown-style formatting
    def process_markdown(text):
        # Convert **text** to <strong>text</strong> - more robust pattern matching
        # This pattern handles both inline and multi-word bold formatting
        processed = re.sub(r'\*\*([^*]+?)\*\*', r'<strong>\1</strong>', text)
        
        # Also handle underscores for emphasis (_text_) -> <em>text</em>
        processed = re.sub(r'\_([^_]+?)\_', r'<em>\1</em>', processed)
        
        # Handle bullet points
        processed = re.sub(r'^\s*\*\s+(.+)$', r'<li>\1</li>', processed, flags=re.MULTILINE)
        
        # Wrap consecutive list items in ul tags
        if '<li>' in processed:
            processed = re.sub(r'(<li>.*?</li>)+', r'<ul>\g<0></ul>', processed, flags=re.DOTALL)
        
        return processed
    
    # Build HTML output with Show/Hide buttons
    html_output = ""
    for q, a in question_answer_pairs:
        # Make sure question has "Question:" prefix if it doesn't already
        if not q.lower().startswith("question:"):
            q = f"Question: {q}"
        
        # Apply markdown processing
        q = process_markdown(q)
        a = process_markdown(a)

        # Create clean answer format
        if a.lower().startswith("answer:"):
            # Remove the "Answer:" prefix since we'll add it as a separate element
            a = re.sub(r'^answer:\s*', '', a, flags=re.IGNORECASE)
        
        html_output += f"""<div class="question-container">
    <h3 class="text-purple-700">{q}</h3>
    <button class="show-answer-btn">
        <span>Show Answer</span>
        <i class="fas fa-chevron-down"></i>
    </button>
    <div class="answer hidden">
        <p><span class="answer-label">Answer:</span> {a}</p>
    </div>
</div>
"""
    
    # Add JavaScript for functionality
    html_output += """
<script>
document.addEventListener('DOMContentLoaded', function() {
    // Initialize show/hide buttons
    const buttons = document.querySelectorAll('.show-answer-btn');
    console.log('Found ' + buttons.length + ' Show Answer buttons');
    
    buttons.forEach(function(button) {
        button.addEventListener('click', function() {
            const answer = this.nextElementSibling;
            const span = this.querySelector('span');
            const icon = this.querySelector('i');
            
            if (answer.classList.contains('hidden')) {
                // Show answer
                answer.classList.remove('hidden');
                span.textContent = 'Hide Answer';
                icon.className = 'fas fa-chevron-up';
            } else {
                // Hide answer
                answer.classList.add('hidden');
                span.textContent = 'Show Answer';
                icon.className = 'fas fa-chevron-down';
            }
        });
    });
});
</script>
"""
    
    return html_output

@app.route("/signin", methods=["GET", "POST"])
def signin():
    # Check if user is already logged in
    if "user" in session:
        return redirect(url_for("upload_file"))

    # Check if form is submitted
    if request.method == "POST":
        # Get form data
        email = request.form.get("email")
        password = request.form.get("password")
        
        # Validate form data
        if not email or not password:
            flash("Please fill out all fields.", "error")
            return render_template("signin.html", csrf_token=generate_csrf())
        
        # Check if email is valid
        if not is_valid_email(email):
            flash("Invalid email format.", "error")
            return render_template("signin.html", csrf_token=generate_csrf())

        # Check if password is valid
        if not is_valid_password(password):
            flash("Invalid password format.", "error")
            return render_template("signin.html", csrf_token=generate_csrf())

        try:
            conn = sqlite3.connect("users.db")
            cursor = conn.cursor()
            
            # Check if user exists
            cursor.execute("SELECT * FROM users WHERE email = ?", (email,))
            user = cursor.fetchone()
            
            if user:
                # Check if password is correct
                if check_password_hash(user[2], password):
                    # Generate remember token
                    remember_token = generate_remember_token()
                    
                    # Update user record with remember token
                    cursor.execute("UPDATE users SET remember_token = ? WHERE id = ?", (remember_token, user[0]))
                    conn.commit()
                        
                        # Set session variables
                    session["user"] = email
                    session["remember_token"] = remember_token
                    
                    flash("Logged in successfully!", "success")
                    return redirect(url_for("upload_file"))
                else:
                    flash("Invalid email or password.", "error")
                    return render_template("signin.html", csrf_token=generate_csrf())
            else:
                flash("User not found. Please check your email and password.", "error")
                return render_template("signin.html", csrf_token=generate_csrf())
        except Exception as e:
            print(f"Error logging in: {str(e)}")
            flash("An error occurred. Please try again.", "error")
        finally:
            conn.close()

    return render_template("signin.html", csrf_token=generate_csrf())

@app.route("/signup", methods=["GET", "POST"])
def signup():
    # Check if user is already logged in
    if "user" in session:
        return redirect(url_for("upload_file"))

    # Check if form is submitted
    if request.method == "POST":
        # Get form data
        email = request.form.get("email")
        password = request.form.get("password")
        confirm_password = request.form.get("confirm_password")
        terms = request.form.get("terms")
        
        # Validate form data
        if not email or not password or not confirm_password:
            flash("Please fill out all fields.", "error")
            return render_template("signup.html", csrf_token=generate_csrf())
        
        # Check if email is valid
        if not is_valid_email(email):
            flash("Invalid email format.", "error")
            return render_template("signup.html", csrf_token=generate_csrf())
        
        # Check if passwords match
        if password != confirm_password:
            flash("Passwords do not match.", "error")
            return render_template("signup.html", csrf_token=generate_csrf())
        
        # Check if password is valid
        if not is_valid_password(password):
            flash("Invalid password format. Password must contain at least 8 characters, including uppercase, lowercase, and numbers.", "error")
            return render_template("signup.html", csrf_token=generate_csrf())
            
        # Check if terms are accepted
        if not terms:
            flash("You must accept the Terms of Service and Privacy Policy.", "error")
            return render_template("signup.html", csrf_token=generate_csrf())
            
        try:
            conn = sqlite3.connect("users.db")
            cursor = conn.cursor()
            
            # Check if user already exists
            cursor.execute("SELECT * FROM users WHERE email = ?", (email,))
            user = cursor.fetchone()
            
            if user:
                flash("User already exists. Please sign in instead.", "error")
                return render_template("signup.html", csrf_token=generate_csrf())
            else:
                # Hash the password
                hashed_password = generate_password_hash(password)
                
                # Generate remember token
                remember_token = generate_remember_token()
                
                # Create new user
                cursor.execute(
                    "INSERT INTO users (email, password, remember_token) VALUES (?, ?, ?)",
                    (email, hashed_password, remember_token)
                )
                conn.commit()
                
                # Set session variables
                session["user"] = email
                session["remember_token"] = remember_token
                
                flash("Account created successfully!", "success")
                return redirect(url_for("upload_file"))
        except Exception as e:
            print(f"Error creating user: {str(e)}")
            flash("An error occurred. Please try again.", "error")
        finally:
            conn.close()

    return render_template("signup.html", csrf_token=generate_csrf())

@app.route("/logout")
def logout():
    # Clear remember token from database
    if "user" in session:
        try:
            conn = sqlite3.connect("users.db")
            cursor = conn.cursor()
            cursor.execute("UPDATE users SET remember_token = NULL WHERE email = ?", 
                          (session["user"],))
            conn.commit()
            conn.close()
        except Exception as e:
            print(f"Error clearing remember token: {str(e)}")
            # Continue with logout even if token clearing fails
    
    # Clear session
    session.clear()
    
    # Create response and clear cookie
    response = redirect(url_for("signin"))
    response.delete_cookie('remember_token')
    
    flash("You have been logged out successfully.", "success")
    return response

@app.route("/settings", methods=["GET", "POST"])
def settings():
    """User settings page to manage account and preferences"""
    # Check if user is logged in
    if "user" not in session:
        flash("Please sign in to access settings.", "error")
        return redirect(url_for("signin"))

    user_email = session["user"]
    
    try:
        # Connect to database
        conn = sqlite3.connect("users.db")
        cursor = conn.cursor()
        
        # Get user information
        cursor.execute("SELECT * FROM users WHERE email = ?", (user_email,))
        user = cursor.fetchone()
        
        if not user:
            flash("User not found.", "error")
            return redirect(url_for("signin"))
        
        # Create user settings table if it doesn't exist
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS user_settings (
                user_id INTEGER PRIMARY KEY,
                openai_api_key TEXT,
                google_translate_key TEXT,
                clear_history INTEGER DEFAULT 1,
                save_extracted_text INTEGER DEFAULT 0,
                default_language TEXT DEFAULT 'en',
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id)
            )
        """)
        conn.commit()
        
        # Check if settings exist for this user
        cursor.execute("SELECT * FROM user_settings WHERE user_id = ?", (user[0],))
        user_settings = cursor.fetchone()
        
        # Initialize settings
        openai_api_key = ""
        google_translate_key = ""
        clear_history = True
        save_extracted_text = False
        default_language = "en"
        
        # Load existing settings if available
        if user_settings:
            openai_api_key = user_settings[1] or ""
            google_translate_key = user_settings[2] or ""
            clear_history = bool(user_settings[3])
            save_extracted_text = bool(user_settings[4])
            default_language = user_settings[5] or "en"
        
        # Handle form submission
        if request.method == "POST":
            # Form validation
            current_password = request.form.get("current_password", "")
            new_password = request.form.get("new_password", "")
            confirm_password = request.form.get("confirm_password", "")
            
            # Get API keys
            new_openai_api_key = request.form.get("openai_api_key", "")
            new_google_translate_key = request.form.get("google_translate_key", "")
            
            # Get preferences
            new_clear_history = "clear_history" in request.form
            new_save_extracted_text = "save_extracted_text" in request.form
            new_default_language = request.form.get("default_language", "en")
            
            # Process password change if requested
            if current_password and new_password:
                if new_password != confirm_password:
                    flash("New passwords do not match.", "error")
                else:
                    # Verify current password
                    if check_password_hash(user[2], current_password):
                        # Password validation
                        if is_valid_password(new_password):
                            # Update password
                            hashed_password = generate_password_hash(new_password)
                            cursor.execute("UPDATE users SET password = ? WHERE id = ?", 
                                         (hashed_password, user[0]))
                            conn.commit()
                            flash("Password updated successfully.", "success")
                        else:
                            flash("Password must contain at least 8 characters, including uppercase, lowercase, and numbers.", "error")
                    else:
                        flash("Current password is incorrect.", "error")
            
            # Update user settings
            if user_settings:
                # Update existing settings
                cursor.execute("""
                    UPDATE user_settings 
                    SET openai_api_key = ?, google_translate_key = ?, clear_history = ?, 
                        save_extracted_text = ?, default_language = ?, updated_at = CURRENT_TIMESTAMP
                    WHERE user_id = ?
                """, (new_openai_api_key, new_google_translate_key, int(new_clear_history), 
                      int(new_save_extracted_text), new_default_language, user[0]))
            else:
                # Insert new settings
                cursor.execute("""
                    INSERT INTO user_settings 
                    (user_id, openai_api_key, google_translate_key, clear_history, 
                     save_extracted_text, default_language)
                    VALUES (?, ?, ?, ?, ?, ?)
                """, (user[0], new_openai_api_key, new_google_translate_key, 
                      int(new_clear_history), int(new_save_extracted_text), new_default_language))
            
            conn.commit()
            
            # Update local variables to reflect new settings
            openai_api_key = new_openai_api_key
            google_translate_key = new_google_translate_key
            clear_history = new_clear_history
            save_extracted_text = new_save_extracted_text
            default_language = new_default_language
            
            # Update session language
            session["selected_language"] = default_language
            
            # Apply OpenAI API key to environment if provided
            if new_openai_api_key:
                os.environ["OPENAI_API_KEY"] = new_openai_api_key
            
            # Apply Google Translate API key to environment if provided
            if new_google_translate_key:
                os.environ["GOOGLE_TRANSLATE_API_KEY"] = new_google_translate_key
            
            flash("Settings saved successfully.", "success")
            
        # Close database connection
        conn.close()
        
        # Render settings template with user info and settings
        return render_template("settings.html", 
                             user_email=user_email,
                             openai_api_key=openai_api_key,
                             google_translate_key=google_translate_key,
                             clear_history=clear_history,
                             save_extracted_text=save_extracted_text,
                             default_language=default_language,
                             csrf_token=generate_csrf())
                             
    except Exception as e:
        print(f"Error in settings page: {str(e)}")
        flash(f"An error occurred: {str(e)}", "error")
        return redirect(url_for("upload_file"))

# --- PDF Text Extraction ---
def extract_text_from_pdf(pdf_path):
    try:
        doc = fitz.open(pdf_path)
        extracted_text = ""

        for page in doc:
            try:
                text = page.get_text("text")
                if text.strip():
                    extracted_text += text + "\n"
                else:
                    # Try OCR if text extraction gives no results
                    try:
                        pix = page.get_pixmap()  # Convert page to an image
                        img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
                        ocr_text = pytesseract.image_to_string(img)
                        if ocr_text.strip():
                            extracted_text += ocr_text + "\n"
                            print(f"Used OCR for page {page.number} of PDF")
                    except Exception as ocr_err:
                        print(f"OCR error on page {page.number}: {str(ocr_err)}")
                        # Continue without OCR if it fails
            except Exception as page_err:
                print(f"Error processing page {page.number}: {str(page_err)}")
                # Continue with next page if one fails

        # If we got some text, return it; otherwise use a fallback message
        if extracted_text.strip():
            return extracted_text.strip()
        else:
            print("No text was extracted from the PDF")
            return "Text extraction failed. This may be a scanned or image-based PDF that requires OCR."
    
    except Exception as e:
        print(f"Error opening or processing PDF: {str(e)}")
        # Return a default text so the process can continue
        return "Sample text for processing. The PDF could not be read correctly."



# --- OCR for Scanned PDFs ---
def extract_text_from_image(image_path):
    text = pytesseract.image_to_string(Image.open(image_path))
    return text if text.strip() else "No text detected."

def extract_text_from_txt(txt_path):
    """Extract text from a .txt file"""
    try:
        with open(txt_path, 'r', encoding='utf-8') as file:
            text = file.read()
        return text if text.strip() else "No text found in the text file."
    except UnicodeDecodeError:
        # Try with a different encoding if UTF-8 fails
        try:
            with open(txt_path, 'r', encoding='latin-1') as file:
                text = file.read()
            print("Used latin-1 encoding for text file")
            return text if text.strip() else "No text found in the text file."
        except Exception as e:
            print(f"Error reading text file with latin-1 encoding: {str(e)}")
            return "Could not read the text file. The file may be corrupted."
    except Exception as e:
        print(f"Error reading text file: {str(e)}")
        return "Error reading the text file: " + str(e)


# --- Text Summarization ---
def summarize_text(text):
    try:
        # Limit text length to prevent API issues
        max_text_length = 8000
        original_length = len(text)
        if len(text) > max_text_length:
            print(f"Truncating text for summarization from {original_length} to {max_text_length} characters")
            # Take beginning and end
            half_length = max_text_length // 2
            text = text[:half_length] + "\n...\n" + text[-half_length:]
        
        api_url = "https://api.qewertyy.dev/models"
        payload = {
            "model_id": 5,  # GPT-3.5 Turbo
            "messages": [
                {
                    "role": "user",
                    "content": SUMMARY_PROMPT.format(text=text)
                }
            ]
        }
        headers = {"Content-Type": "application/json"}

        # Limit timeout to avoid hanging
        print(f"Calling summarization API with {len(text)} characters")
        response = requests.post(api_url, json=payload, headers=headers, timeout=15)
        print(f"Summarization API response status: {response.status_code}")

        if response.status_code == 200:
            data = response.json()
            summary = data.get("content", "")
            if summary:
                print(f"Got summary of {len(summary)} characters")
                return summary
            else:
                print("API returned empty summary content")
        else:
            print(f"API summary generation failed: Status code {response.status_code}, Response: {response.text}")
        
        # If we get here, either the request failed or returned no summary
        print("Using fallback summary method")
        
        # Fallback to a basic summary (first few sentences + last few sentences)
        sentences = text.split('.')
        first_part = '.'.join(sentences[:4]) + '.'
        last_part = '.'.join(sentences[-4:]) + '.'
        summary = f"{first_part}\n...\n{last_part}\n\nNote: This is an auto-generated excerpt as the summary API failed."
        
        return summary
    
    except requests.exceptions.Timeout:
        print("Summary API request timed out")
        # Create a simple summary from the text
        sentences = text.split('.')
        if len(sentences) > 8:
            summary = '.'.join(sentences[:4]) + '.\n...\n' + '.'.join(sentences[-4:]) + '.'
        else:
            summary = '.'.join(sentences[:min(4, len(sentences))]) + '.'
        return f"{summary}\n\nNote: This is an auto-generated excerpt as the summary API timed out."
    
    except Exception as e:
        print(f"Error in summarization: {str(e)}")
        # Return a basic summary to prevent workflow interruption
        return "Summary generation failed. The system is experiencing issues with the summarization API."


# --- Question Generation ---
def generate_questions(text):
    try:
        # Limit text length to prevent API issues
        max_text_length = 8000
        original_length = len(text)
        if len(text) > max_text_length:
            print(f"Truncating text for question generation from {original_length} to {max_text_length} characters")
            # Take beginning and middle sections
            third_length = max_text_length // 3
            text = text[:third_length] + "\n...\n" + text[len(text)//2-third_length//2:len(text)//2+third_length//2] + "\n...\n" + text[-third_length:]
        
        # Modified prompt that includes the Show Answer button and emphasizes questions MUST be based on the text content
        questions_prompt = """Generate 10 questions and answers based STRICTLY on the provided text content only. Do not invent information.
Format each question and answer exactly as follows:

<div class="question-container">
<h3 class="text-purple-700">Question: [specific question about the text content]</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> [answer based on information from the text]</p>
</div>
</div>

Important instructions:
1. ONLY ask questions that can be directly answered from the text content provided
2. Make sure questions are specific and not generic or unrelated to the content
3. Make important terms or keywords in answers bold using HTML <strong> tags
4. Each answer must cite specific information from the text
5. Do NOT create questions about topics not covered in the text

Text to generate questions from:
{text}"""
        
        # Try first with the qewertyy API
        try:
            print(f"Attempting to call qewertyy API for question generation")
            api_url = "https://api.qewertyy.dev/models"
            payload = {
            "model_id": 5,  # GPT-3.5 Turbo
            "messages": [
                {
                    "role": "user",
                        "content": questions_prompt.format(text=text)
                }
            ]
        }
            headers = {"Content-Type": "application/json"}

            print("Sending API request for question generation...")
        # Limit timeout to avoid hanging
            response = requests.post(api_url, json=payload, headers=headers, timeout=20)
            print(f"qewertyy API response status: {response.status_code}")

            if response.status_code == 200:
                data = response.json()
                questions = data.get("content", "")
                    
                # Print detailed debugging information
                print(f"API response length: {len(str(data))} characters")
                print(f"Content field length: {len(questions)} characters")
                
                if not questions:
                    print("API returned empty content field")
                elif len(questions) < 50:
                    print(f"API returned very short content: '{questions}'")
                
                # More specific check for valid content
            if questions and "<div class=\"question-container\">" in questions:
                    print(f"Got valid questions of {len(questions)} characters")
                    
                    # Add script if it's not already there
                    if "</script>" not in questions:
                        questions += add_show_answer_script()
                        return questions
                    else:
                        print("API returned invalid HTML structure for questions")
                        # Try to debug what was returned instead
                        content_preview = questions[:200] + "..." if len(questions) > 200 else questions
                        print(f"Content preview: {content_preview}")
            else:
                    print(f"API question generation failed: Status code {response.status_code}, Response: {response.text[:500]}")
                
            # If still here, try with a simplified prompt
            print("Attempting with simplified API prompt...")
            simplified_prompt = """Generate 10 specific questions and detailed answers based ONLY on this text content. 
Each question should be formatted as 'Question: [question]' and each answer as 'Answer: [answer]'. 
Make answers detailed and informative. Use <strong> tags to highlight important terms.

Text content:
{text}"""
            
            simplified_payload = {
                "model_id": 5,  # GPT-3.5 Turbo
                "messages": [
                    {
                        "role": "user",
                        "content": simplified_prompt.format(text=text)
                    }
                ]
            }
            
            response = requests.post(api_url, json=simplified_payload, headers=headers, timeout=20)
            print(f"Simplified API request response status: {response.status_code}")
            
            if response.status_code == 200:
                data = response.json()
                raw_questions = data.get("content", "")
                
                if raw_questions and ("Question:" in raw_questions or "question:" in raw_questions):
                    print(f"Got raw questions content: {len(raw_questions)} characters")
                    # Format the raw questions into HTML
                    formatted_questions = format_questions_html(raw_questions)
                    return formatted_questions
                else:
                    print("Simplified API request also failed to return valid content")
        
            # If first API fails, try with OpenAI API if available
            openai_api_key = os.environ.get('OPENAI_API_KEY')
            if openai_api_key:
                try:
                    print("Attempting to use OpenAI API for question generation")
                    import openai
                    openai.api_key = openai_api_key
                    
                    completion = openai.chat.completions.create(
                        model="gpt-3.5-turbo",
                        messages=[
                            {"role": "system", "content": "You are a helpful assistant that generates specific questions and answers about text content."},
                            {"role": "user", "content": questions_prompt.format(text=text)}
                        ],
                        temperature=0.7,
                        max_tokens=2000
                    )
                    
                    questions = completion.choices[0].message.content
                    if questions and "<div class=\"question-container\">" in questions:
                        # Add script if it's not already there
                        if "</script>" not in questions:
                            questions += add_show_answer_script()
                        return questions
                except Exception as openai_error:
                    print(f"OpenAI API error: {str(openai_error)}")
        
        except Exception as api_error:
            print(f"API error: {str(api_error)}")
        
        # If all APIs fail, use our improved local model for question generation
        try:
            import nltk
            from nltk.tokenize import sent_tokenize
            import re
            
            # Download necessary NLTK data if not present
            try:
                nltk.data.find('tokenizers/punkt')
            except LookupError:
                nltk.download('punkt')
            
            print("Using improved local question generation with NLTK")
            
            # Extract key sentences for question generation
            sentences = sent_tokenize(text)
            
            # Extract paragraphs for broader context
            paragraphs = text.split('\n\n')
            paragraphs = [p for p in paragraphs if len(p.strip()) > 20]  # Only use substantial paragraphs
            
            # Find key sentences that have meaningful content
            key_sentences = []
            
            # Try to find sentences with important keywords or phrases first
            important_patterns = [
                r'\b(main|primary|key|important|significant)\b.*\b(point|concept|idea|finding|result|conclusion)\b',
                r'\b(according to|research shows|study indicates|findings reveal|data suggests|results show)\b',
                r'\b(defined as|refers to|known as|meaning|definition of|concept of)\b',
                r'\b(consists of|composed of|made up of|contains|includes|comprises)\b',
                r'\b(first|second|third|lastly|finally|ultimately|in conclusion)\b'
            ]
            
            # Find sentences matching important patterns
            for pattern in important_patterns:
                matching_sentences = [s for s in sentences if re.search(pattern, s, re.IGNORECASE) and len(s.split()) > 8]
                key_sentences.extend(matching_sentences[:3])  # Limit to 3 matches per pattern
            
            # If we don't have enough key sentences, add more from different parts of the document
            if len(key_sentences) < 8:
                # Evenly sample from beginning, middle, and end of the document
                if len(sentences) > 20:
                    beginning = sentences[:len(sentences)//3]
                    middle = sentences[len(sentences)//3:2*len(sentences)//3]
                    end = sentences[2*len(sentences)//3:]
                    
                    # Select substantial sentences from each section
                    for section in [beginning, middle, end]:
                        substantial_sentences = [s for s in section if len(s.split()) > 8 and s not in key_sentences]
                        if substantial_sentences:
                            # Take sentences at regular intervals
                            step = max(1, len(substantial_sentences) // 3)
                            key_sentences.extend([substantial_sentences[i] for i in range(0, len(substantial_sentences), step)][:3])
                else:
                    # For shorter documents, take all substantial sentences
                    substantial_sentences = [s for s in sentences if len(s.split()) > 8 and s not in key_sentences]
                    key_sentences.extend(substantial_sentences[:10 - len(key_sentences)])
            
            # Limit to 10 unique key sentences
            unique_key_sentences = []
            for sentence in key_sentences:
                if sentence not in unique_key_sentences:
                    unique_key_sentences.append(sentence)
            
            key_sentences = unique_key_sentences[:10]
            
            # Function to get paragraph context for a sentence
            def get_paragraph_context(sentence):
                for paragraph in paragraphs:
                    if sentence in paragraph:
                        return paragraph
                return sentence
            
            # Generate questions from key sentences
            questions_html = ""
            question_count = 0
            
            for i, sentence in enumerate(key_sentences):
                if question_count >= 10:  # Limit to 10 questions
                    break
                    
                # Skip very short sentences
                if len(sentence.split()) < 5:
                    continue
                
                # Get paragraph context for better answers
                context = get_paragraph_context(sentence)
                
                # Extract key terms from the sentence
                words = sentence.lower().split()
                
                # Remove common words and keep potential key terms
                stopwords = ["the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "with", "by", "about", "like", "through", "over", "before", "after", "since", "until", "while", "of", "from", "as", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "can", "could", "will", "would", "shall", "should", "may", "might", "must"]
                key_terms = [word for word in words if len(word) > 3 and word not in stopwords][:5]
                
                # Create different question types based on sentence content
                question_type = "what"  # Default question type
                
                # Determine question type based on content
                if any(w in sentence.lower() for w in ["reason", "why", "because", "cause"]):
                    question_type = "why"
                elif any(w in sentence.lower() for w in ["how", "process", "method", "procedure", "step"]):
                    question_type = "how"
                elif any(w in sentence.lower() for w in ["when", "date", "time", "year", "month", "day"]):
                    question_type = "when"
                elif any(w in sentence.lower() for w in ["where", "location", "place", "country", "city", "region"]):
                    question_type = "where"
                elif any(w in sentence.lower() for w in ["who", "person", "people", "author", "writer", "researcher"]):
                    question_type = "who"
                
                # Extract subject for the question
                subject_phrase = ""
                
                # Try to extract a meaningful subject from the sentence
                if len(sentence.split(",")) > 1:
                    # If there are commas, use the first part as it might contain the main subject
                    first_part = sentence.split(",")[0]
                    if len(first_part.split()) > 3:
                        subject_phrase = " ".join(first_part.split()[:5])
                elif len(sentence.split()) > 5:
                    # Otherwise use the first 5 words or a key term with surrounding words
                    if key_terms:
                        # Find the position of the first key term
                        first_key_term_pos = words.index(key_terms[0]) if key_terms[0] in words else -1
                        if 0 <= first_key_term_pos < len(words) - 3:
                            subject_words = words[max(0, first_key_term_pos-1):min(first_key_term_pos+4, len(words))]
                            subject_phrase = " ".join(subject_words)
                        else:
                            subject_phrase = " ".join(words[:5])
                    else:
                        subject_phrase = " ".join(words[:5])
                else:
                    # For very short sentences, use the whole sentence
                    subject_phrase = sentence
                
                # Handle different question types
                if question_type == "what":
                    if "is" in words or "are" in words:
                        question = f"What {subject_phrase}?"
                    else:
                        question = f"What does the document say about {subject_phrase}?"
                elif question_type == "why":
                    question = f"Why {subject_phrase}?"
                elif question_type == "how":
                    question = f"How {subject_phrase}?"
                elif question_type == "when":
                    question = f"When {subject_phrase}?"
                elif question_type == "where":
                    question = f"Where {subject_phrase}?"
                elif question_type == "who":
                    question = f"Who {subject_phrase}?"
                
                # Clean up the question
                question = question.strip()
                question = re.sub(r'\s+', ' ', question)  # Remove extra spaces
                question = question.capitalize()  # Capitalize first letter
                
                # Make sure the question ends with a question mark
                if not question.endswith("?"):
                    question = question + "?"
                
                # Create an answer with context
                answer = context
                
                # Emphasize key terms in the answer using <strong> tags
                for term in key_terms:
                    if len(term) > 3:  # Only emphasize meaningful terms
                        pattern = re.compile(r'\b' + re.escape(term) + r'\b', re.IGNORECASE)
                        answer = pattern.sub(f'<strong>\\g<0></strong>', answer)
                
                # Add to HTML
                questions_html += f'''
<div class="question-container">
<h3 class="text-purple-700">Question: {question}</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {answer}</p>
</div>
</div>
'''
                question_count += 1
            
            # If we generated at least some questions
            if questions_html:
                # Add the document title question if we have fewer than 10 questions
                if question_count < 10:
                    # Extract the first paragraph as the document summary
                    doc_summary = paragraphs[0] if paragraphs else text[:200]
                    
                    # Create a question about the document's main topic
                    questions_html = f'''
<div class="question-container">
<h3 class="text-purple-700">Question: What is the main topic of this document?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {doc_summary}</p>
</div>
</div>
''' + questions_html
                
                # Add script for show/hide functionality
                questions_html += add_show_answer_script()
                print(f"Generated {question_count} questions using NLTK")
                return questions_html
        
        except Exception as nltk_error:
            print(f"NLTK error: {str(nltk_error)}")
        
        # If all attempts fail, use the fallback questions but with a note
        print("All question generation methods failed. Using fallback questions with note.")
        return generate_fallback_questions(text)
        
    except Exception as e:
        print(f"Error in question generation: {str(e)}")
        # Return basic questions to prevent workflow interruption
        return generate_fallback_questions(text)


def add_show_answer_script():
    """Add JavaScript to handle show/hide functionality for answers"""
    return """
<script>
document.addEventListener('DOMContentLoaded', function() {
    // Initialize show/hide buttons
    const buttons = document.querySelectorAll('.show-answer-btn');
    console.log('Found ' + buttons.length + ' Show Answer buttons');
    
    buttons.forEach(function(button) {
        button.addEventListener('click', function() {
            const answer = this.nextElementSibling;
            const span = this.querySelector('span');
            const icon = this.querySelector('i');
            
            if (answer.classList.contains('hidden')) {
                // Show answer
                answer.classList.remove('hidden');
                span.textContent = 'Hide Answer';
                icon.className = 'fas fa-chevron-up';
            } else {
                // Hide answer
                answer.classList.add('hidden');
                span.textContent = 'Show Answer';
                icon.className = 'fas fa-chevron-down';
            }
        });
    });
});
</script>
"""


def generate_fallback_questions(text):
    """Generate basic document-specific questions when the API fails"""
    # Create a technical notice as the first question
    note_question = """
<div class="question-container">
<h3 class="text-purple-700">Question: Technical Notice About These Questions</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> <strong>NOTE:</strong> These are automatically generated questions based on your document content. The AI question generation system used a simplified approach. For more specific questions, you can use the Chat feature to ask about particular aspects of your document.</p>
</div>
</div>
"""
    
    try:
        # If text is empty or too short, return generic fallback questions
        if not text or len(text) < 100:
            return note_question + FALLBACK_QUESTIONS
            
        # Extract paragraphs for broader context
        paragraphs = text.split('\n\n')
        paragraphs = [p for p in paragraphs if len(p.strip()) > 20]  # Only use substantial paragraphs
        
        # If no substantial paragraphs found, use the original text divided by sentences
        if not paragraphs:
            import re
            paragraphs = re.split(r'(?<=[.!?])\s+', text)
            paragraphs = [p for p in paragraphs if len(p.strip()) > 20]  # Only use substantial paragraphs
            
        # Still no usable content? Use generic fallback questions
        if not paragraphs:
            return note_question + FALLBACK_QUESTIONS
        
        # Create document-specific questions
        questions_html = note_question
        
        # Question 1: Main topic question using first paragraph
        first_paragraph = paragraphs[0] if paragraphs else text[:200]
        questions_html += f"""
<div class="question-container">
<h3 class="text-purple-700">Question: What is the main topic of this document?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {first_paragraph}</p>
</div>
</div>
"""
        
        # Question 2: Content from the middle of the document
        middle_paragraph = paragraphs[len(paragraphs)//2] if len(paragraphs) > 2 else ""
        if middle_paragraph:
            questions_html += f"""
<div class="question-container">
<h3 class="text-purple-700">Question: What details does the document provide in the middle section?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {middle_paragraph}</p>
</div>
</div>
"""
        
        # Question 3: Content from the end of the document (conclusion)
        end_paragraph = paragraphs[-1] if len(paragraphs) > 1 else ""
        if end_paragraph and end_paragraph != first_paragraph:
            questions_html += f"""
<div class="question-container">
<h3 class="text-purple-700">Question: What conclusions or final points does the document make?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {end_paragraph}</p>
</div>
</div>
"""
            
        # Add a question about document length and structure
        doc_stats = f"The document contains approximately {len(text)} characters or roughly {len(text)//5} words. "
        doc_stats += f"It has {len(paragraphs)} paragraphs. "
        
        # Check for any numerical data in the document
        import re
        numbers = re.findall(r'\b\d+(?:\.\d+)?(?:\s*%|\s*percent)?\b', text)
        if numbers:
            doc_stats += f"The document contains numerical data, including: {', '.join(numbers[:5])}."
            
        questions_html += f"""
<div class="question-container">
<h3 class="text-purple-700">Question: What is the structure and length of this document?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> {doc_stats}</p>
</div>
</div>
"""

        # Question about how to get more specific answers
        questions_html += """
<div class="question-container">
<h3 class="text-purple-700">Question: How can I get more specific information about this document?</h3>
<button class="show-answer-btn">
    <span>Show Answer</span>
    <i class="fas fa-chevron-down"></i>
</button>
<div class="answer hidden">
    <p><span class="answer-label">Answer:</span> You can use the <strong>Chat</strong> tab to ask specific questions about your document content. The chat feature provides direct answers based on your document's text. You can also use the <strong>Text</strong> tab to read through the document directly.</p>
</div>
</div>
"""

        # Add script for show/hide functionality
        questions_html += add_show_answer_script()
        return questions_html
        
    except Exception as e:
        print(f"Error generating document-specific fallback questions: {str(e)}")
        # If anything fails, use the original fallback questions
        return note_question + FALLBACK_QUESTIONS


# --- Translation API ---
@app.route("/translate_summary", methods=["POST"])
def translate_summary():
    data = request.json
    target_lang = data.get("language")
    summary = data.get("summary")

    if not target_lang or not summary:
        return jsonify({"error": "Missing data"}), 400

    try:
        translated_text = GoogleTranslator(source="auto", target=target_lang).translate(summary)
        
        # Store the translated summary in session
        store_in_session("translated_summary", translated_text)
        
        return jsonify({"translated_summary": translated_text})
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route("/translate_questions", methods=["POST"])
def translate_questions():
    data = request.json
    target_lang = data.get("language")
    questions = data.get("questions")

    if not target_lang or not questions:
        return jsonify({"error": "Missing data"}), 400

    try:
        translated_text = GoogleTranslator(source="auto", target=target_lang).translate(questions)
        
        # Store the translated questions in session
        store_in_session("translated_questions", translated_text)
        
        return jsonify({"translated_questions": translated_text})
    except Exception as e:
        return jsonify({"error": str(e)}), 500

# --- Home Route ---
@app.route("/", methods=["GET", "POST"])
def home():
    if "user" in session:
        return redirect(url_for("upload_file"))  # Show upload if logged in
    return redirect(url_for("signin"))  # Show sign-in if not logged in


# --- Upload & Process PDF ---
@app.route("/upload", methods=["GET", "POST"])
def upload_file():
    if "user" not in session:
        return redirect(url_for("signin"))

    if request.method == "POST":
        # Validate CSRF token
        if not request.form.get('csrf_token'):
            flash("Invalid form submission. Please try again.", "error")
            return render_template("upload.html", csrf_token=generate_csrf())

        if "file" not in request.files:
            flash("No file selected.", "error")
            return render_template("upload.html", csrf_token=generate_csrf())

        file = request.files["file"]
        if not file or not file.filename:
            flash("No file selected.", "error")
            return render_template("upload.html", csrf_token=generate_csrf())

        # Check file extension
        allowed_extensions = {'.pdf', '.png', '.jpg', '.jpeg', '.txt'}
        file_ext = os.path.splitext(file.filename)[1].lower()
        if file_ext not in allowed_extensions:
            flash("Invalid file type. Please upload a PDF, image, or text file.", "error")
            return render_template("upload.html", csrf_token=generate_csrf())

        try:
            # Get user settings
            user_email = session["user"]
            user_settings = {}
            
            try:
                conn = sqlite3.connect("users.db")
                cursor = conn.cursor()
                
                # Get user ID
                cursor.execute("SELECT id FROM users WHERE email = ?", (user_email,))
                user_id = cursor.fetchone()
                
                if user_id:
                    # Get user settings
                    cursor.execute("SELECT * FROM user_settings WHERE user_id = ?", (user_id[0],))
                    settings = cursor.fetchone()
                    
                    if settings:
                        user_settings = {
                            "clear_history": bool(settings[3]),
                            "save_extracted_text": bool(settings[4]),
                            "default_language": settings[5]
                        }
                conn.close()
            except Exception as db_error:
                print(f"Error getting user settings: {str(db_error)}")
                # Continue with default settings if there's an error
                user_settings = {
                    "clear_history": True,
                    "save_extracted_text": False,
                    "default_language": "en"
                }
                
            # Clear previous file data from session
            session_keys_to_clear = [
                "extracted_text", "summary", "questions", "file_size",
                "translated_summary", "translated_questions", "translated_extracted_text",
                "current_file", "topics", "tables", "highlighted_text",
                "entities", "sentiment", "classification", "citations",
                "file_path", "original_filename", "full_text_path"  # Added full_text_path to ensure complete reset
            ]
            
            # Add chat_history to the keys to clear if user setting says so
            if user_settings.get("clear_history", True):
                session_keys_to_clear.append("chat_history")
            
            # Track which keys were actually cleared for logging
            cleared_keys = []
            for key in session_keys_to_clear:
                if key in session:
                    # Special handling for chat_history to log when it's cleared
                    if key == "chat_history" and session[key]:
                        print(f"Clearing chat history with {len(session[key])} previous conversations")
                    # Special handling for full_text_path to delete the actual file
                    if key == "full_text_path" and os.path.exists(session[key]):
                        try:
                            os.remove(session[key])
                            print(f"Deleted previous full text file: {session[key]}")
                        except Exception as file_error:
                            print(f"Could not delete full text file: {str(file_error)}")
                    # Delete the session key
                    del session[key]
                    cleared_keys.append(key)
            
            print(f"Cleared previous file data from session: {', '.join(cleared_keys)}")
            print(f"Cleared previous file data from session for new upload: {file.filename}")
            
            # Save the file
            unique_filename = f"{uuid.uuid4()}_{file.filename}"
            filepath = os.path.join(app.config["UPLOAD_FOLDER"], unique_filename)
            file.save(filepath)
            print(f"File saved to {filepath}")

            # Extract text based on file type - add error handling
            extracted_text = ""
            print(f"Extracting text from {file_ext} file")
            try:
                if file_ext == '.pdf':
                    extracted_text = extract_text_from_pdf(filepath)
                elif file_ext == '.txt':
                    extracted_text = extract_text_from_txt(filepath)
                else:
                    extracted_text = extract_text_from_image(filepath)
                print(f"Text extraction completed. Got {len(extracted_text)} characters")
            except Exception as extraction_error:
                print(f"Error during text extraction: {str(extraction_error)}")
                # Use fallback text so process can continue
                extracted_text = "Sample text for processing. The document could not be read correctly."

            if not extracted_text or extracted_text == "No text found.":
                print("No text was extracted, using fallback text")
                extracted_text = "Sample text for processing. No text could be extracted from the document."

            # Save extracted text to file if user setting is enabled
            if user_settings.get("save_extracted_text", False):
                text_file_path = f"{filepath}.txt"
                try:
                    with open(text_file_path, 'w', encoding='utf-8') as f:
                        f.write(extracted_text)
                    print(f"Saved extracted text to file: {text_file_path}")
                except Exception as save_error:
                    print(f"Error saving extracted text to file: {str(save_error)}")

            # Generate summary and questions - add robust error handling
            summary = ""
            questions = ""
            questions_html = ""
            
            # Try to generate summary with robust error handling
            print("Generating summary...")
            try:
                summary = summarize_text(extracted_text)
                print(f"Summary generated: {len(summary)} characters")
            except Exception as summary_error:
                print(f"Error during summary generation: {str(summary_error)}")
                # Use fallback summary
                summary = "Summary generation failed. Please try again later."
            
            # Try to generate questions with robust error handling
            print("Generating questions...")
            try:
                questions = generate_questions(extracted_text)
                questions_html = questions  # The questions are already in HTML format
                print(f"Questions generated: {len(questions)} characters")
            except Exception as question_error:
                print(f"Error during question generation: {str(question_error)}")
                # Use fallback questions
                questions = generate_fallback_questions(extracted_text)
                questions_html = questions

            # Extract topics from content - with error handling
            topics = []
            try:
                print("Extracting topics...")
                topics = extract_topics(extracted_text)
                print(f"Topics extracted: {topics}")
            except Exception as topic_error:
                print(f"Error during topic extraction: {str(topic_error)}")
                topics = ["Topic extraction failed"]
            
            # Extract named entities - with error handling
            entities = {}
            try:
                print("Extracting entities...")
                entities = extract_named_entities(extracted_text)
            except Exception as entity_error:
                print(f"Error during entity extraction: {str(entity_error)}")
                entities = {"Error": ["Entity extraction failed"]}
            
            # Classify document - with error handling
            classification = {}
            try:
                print("Classifying document...")
                classification = classify_document(extracted_text, summary)
            except Exception as classify_error:
                print(f"Error during classification: {str(classify_error)}")
                classification = {"category": "Unknown", "confidence": 0}
            
            # Extract citations - with error handling
            citations = []
            try:
                print("Extracting citations...")
                citations = extract_citations(extracted_text)
            except Exception as citation_error:
                print(f"Error during citation extraction: {str(citation_error)}")
                citations = []

            # Store in session for access in other routes with compression
            print("Storing data in session...")
            try:
                # Store the original file path to retrieve later if needed
                session["file_path"] = filepath
                session["original_filename"] = file.filename
                session["current_file"] = file.filename
                
                # Store crucial data first
                try:
                    # Truncate the extracted text to a reasonable size to prevent session bloat
                    max_text_length = 15000
                    if len(extracted_text) > max_text_length:
                        print(f"Truncating extracted_text from {len(extracted_text)} to {max_text_length} characters for session storage")
                        # Keep beginning and end
                        half_length = max_text_length // 2
                        truncated_text = extracted_text[:half_length] + "\n...[text truncated for storage]...\n" + extracted_text[-half_length:]
                        store_in_session("extracted_text", truncated_text)
                        # Store full text on disk for reference
                        text_file_path = f"{filepath}.txt"
                        with open(text_file_path, 'w', encoding='utf-8') as f:
                            f.write(extracted_text)
                        print(f"Saved full extracted text to: {text_file_path}")
                        session["full_text_path"] = text_file_path
                    else:
                        store_in_session("extracted_text", extracted_text)
                        # Since text is short enough, no need for a separate file
                        if "full_text_path" in session:
                            del session["full_text_path"]
                    print(f"Stored extracted_text in session (possibly truncated)")
                except Exception as e:
                    print(f"Failed to store extracted_text: {str(e)}")
                    session["extracted_text"] = "Text storage failed"
                
                try:
                    store_in_session("summary", summary)
                    print("Stored summary in session")
                except Exception as e:
                    print(f"Failed to store summary: {str(e)}")
                    session["summary"] = "Summary storage failed"
                
                try:
                    # Truncate questions if too large
                    max_questions_length = 5000
                    if len(questions_html) > max_questions_length:
                        print(f"Truncating questions from {len(questions_html)} to approximately {max_questions_length} characters")
                        # Find a clean cut-off point by looking for a complete question container
                        cutoff_point = questions_html.rfind('</div>\n</div>', 0, max_questions_length)
                        if cutoff_point > 0:
                            truncated_questions = questions_html[:cutoff_point+12] + "\n<!-- Some questions truncated -->"
                            store_in_session("questions", truncated_questions)
                        else:
                            store_in_session("questions", questions_html[:max_questions_length] + "...")
                    else:
                        store_in_session("questions", questions_html)
                    print("Stored questions in session")
                except Exception as e:
                    print(f"Failed to store questions: {str(e)}")
                    session["questions"] = "Questions storage failed"
                
                # Store the rest of the data (simpler items)
                session["topics"] = topics[:10]  # Limit to top 10 topics
                
                # Simplify entities to reduce size
                simplified_entities = {}
                for category, items in entities.items():
                    simplified_entities[category] = items[:10]  # Limit to top 10 entities per category
                session["entities"] = simplified_entities
                
                session["classification"] = classification
                
                # Limit citations
                session["citations"] = citations[:20]  # Limit to top 20 citations
                
                # Get file size for display
                file_size = os.path.getsize(filepath)
                if file_size < 1024:
                    size_str = f"{file_size} bytes"
                elif file_size < 1024 * 1024:
                    size_str = f"{file_size/1024:.1f} KB"
                else:
                    size_str = f"{file_size/(1024*1024):.1f} MB"
                    
                session["file_size"] = size_str
                
                # Set default language from user settings
                if "default_language" in user_settings:
                    session["selected_language"] = user_settings["default_language"]
                
                print(f"All data stored in session for: {file.filename}")
                
                # Add a new document notification to chat for the newly uploaded file
                try:
                    add_new_document_notification_to_chat(file.filename)
                except Exception as chat_error:
                    print(f"Error adding chat notification: {str(chat_error)}")
                
                # Commit the session to ensure it's saved
                session.modified = True
                
                # Check if the critical data was stored properly
                if "extracted_text" in session and "summary" in session and "questions" in session:
                    print("Session data check passed, redirecting to result")
                    # Double-check session has all required data before redirecting
                    return redirect(url_for('result'))
                else:
                    print("Session data check failed, some data is missing")
                    missing_keys = []
                    for key in ["extracted_text", "summary", "questions"]:
                        if key not in session:
                            missing_keys.append(key)
                    error_msg = f"Session storage error: Missing data for {', '.join(missing_keys)}"
                    print(error_msg)
                    # Try using the render_template approach instead of redirect
                    flash(error_msg, "error")
                    return render_template("result.html",
                                        extracted_text=extracted_text[:10000] if len(extracted_text) > 10000 else extracted_text,
                                        summary=summary,
                                        questions=questions_html,
                                        active_tab="summary",
                                        selected_language=user_settings.get("default_language", "en"),
                                        current_file=file.filename,
                                        csrf_token=generate_csrf())
            except Exception as session_error:
                print(f"Critical error storing data in session: {str(session_error)}")
                flash(f"Error saving analysis data: {str(session_error)}", "error")
                # Try to render the result directly instead of redirecting
                return render_template("result.html",
                                     extracted_text=extracted_text[:10000] if len(extracted_text) > 10000 else extracted_text,
                                     summary=summary,
                                     questions=questions_html,
                                     active_tab="summary",
                                     selected_language=user_settings.get("default_language", "en"),
                                     current_file=file.filename,
                                     csrf_token=generate_csrf())

        except Exception as e:
            print(f"Fatal error in file processing: {str(e)}")
            flash(f"An error occurred while processing the file: {str(e)}", "error")
            return render_template("upload.html", csrf_token=generate_csrf())

    # GET request
    return render_template("upload.html", csrf_token=generate_csrf())

@app.route("/result")
def result():
    """Default result page showing the summary tab"""
    try:
        # Check if any of the required data is missing
        missing_data = []
        for key in ["extracted_text", "summary", "questions"]:
            if key not in session:
                missing_data.append(key)
                
        if missing_data:
            error_message = f"Missing data in session: {', '.join(missing_data)}"
            print(f"Error in result route: {error_message}")
            flash("No analysis data found. Please upload a file first.", "error")
            return redirect(url_for("upload_file"))
        
        # Basic data validation
        extracted_text = get_from_session("extracted_text", "")
        summary = get_from_session("summary", "")
        questions = get_from_session("questions", "")
        
        # Print lengths for debugging
        print(f"Result route - Data lengths: extracted_text={len(extracted_text)}, summary={len(summary)}, questions={len(questions)}")
        
        # Additional validation - if any data is empty, show a more specific error
        if not extracted_text or not summary or not questions:
            empty_fields = []
            if not extracted_text: empty_fields.append("extracted text")
            if not summary: empty_fields.append("summary")
            if not questions: empty_fields.append("questions")
            
            error_message = f"Empty data in session: {', '.join(empty_fields)}"
            print(f"Error in result route: {error_message}")
            flash(f"Analysis produced empty results: {', '.join(empty_fields)}. Please try uploading again.", "error")
            return redirect(url_for("upload_file"))
        
        # Get current file name
        current_file = session.get("current_file", "Unknown file")
        
        # Use translated summary if available
        summary = get_from_session("translated_summary", summary)
        
        # Get the last selected language
        selected_language = session.get("selected_language", "en")
        
        # If we got this far, we should have valid data
        return render_template("result.html",
                            extracted_text=extracted_text,
                            summary=summary,
                            questions=questions,
                            active_tab="summary",
                            selected_language=selected_language,
                            current_file=current_file,
                            csrf_token=generate_csrf())
    except Exception as e:
        print(f"Unexpected error in result route: {str(e)}")
        flash(f"An error occurred loading the analysis results. Please try uploading again.", "error")
        return redirect(url_for("upload_file"))

@app.route("/result/questions")
def result_questions():
    """Result page with questions tab active"""
    if "extracted_text" not in session or "summary" not in session:
        flash("No analysis data found. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    # Get questions from session
    questions_html = get_from_session("questions", "")
    
    # If no questions found in session or it's empty, try to generate new ones
    if not questions_html:
        extracted_text = get_from_session("extracted_text", "")
        
        if extracted_text:
            print("No questions found in session, generating new questions")
            try:
                # Generate questions using the API
                new_questions = generate_questions(extracted_text)
                
                # Store the new questions in session
                store_in_session("questions", new_questions)
                
                # Use the new questions
                questions_html = new_questions
                print(f"Generated new questions: {len(questions_html)} characters")
            except Exception as e:
                print(f"Error generating questions: {str(e)}")
                # Use fallback questions if generation fails
                questions_html = format_questions_html(generate_fallback_questions(extracted_text))
                print("Using fallback questions")
        else:
            # If no extracted text, use empty container
            questions_html = "<div class='questions-container w-full'><p>No questions could be generated.</p></div>"
    
    # Check if we need to convert the HTML structure to add Show Answer buttons
    import re
    if "<div class=\"question-container\">" in questions_html and "show-answer-btn" not in questions_html:
        print("Converting question HTML to add Show Answer buttons")
        # This regular expression matches question containers with answers but no Show Answer buttons
        pattern = r'<div class="question-container">\s*<h3[^>]*>(.*?)</h3>\s*<div class="answer[^"]*"[^>]*>(.*?)</div>\s*</div>'
        
        # Replace with the proper structure that includes buttons and hidden answers
        replacement = r'<div class="question-container">\n    <h3 class="text-purple-700">\1</h3>\n    <button class="show-answer-btn">\n        <span>Show Answer</span>\n        <i class="fas fa-chevron-down"></i>\n    </button>\n    <div class="answer hidden">\2</div>\n</div>'
        
        # Perform the replacement
        questions_html = re.sub(pattern, replacement, questions_html, flags=re.DOTALL)
    
    # Force answers to be hidden if they aren't already
    if "</script>" not in questions_html:
        questions_html += """
<script>
document.addEventListener('DOMContentLoaded', function() {
    // Add FontAwesome if needed
    if (!document.querySelector('link[href*="font-awesome"]')) {
        var link = document.createElement('link');
        link.rel = 'stylesheet';
        link.href = 'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/all.min.css';
        document.head.appendChild(link);
    }
    
    // Force all answers to be hidden and setup button handlers
    var questionContainers = document.querySelectorAll('.question-container');
    console.log('Found ' + questionContainers.length + ' question containers');
    
    questionContainers.forEach(function(container) {
        var answer = container.querySelector('.answer');
        var button = container.querySelector('.show-answer-btn');
        
        // If there's no button but there is an answer, create a button
        if (!button && answer) {
            button = document.createElement('button');
            button.className = 'show-answer-btn';
            button.innerHTML = '<span>Show Answer</span><i class="fas fa-chevron-down"></i>';
            answer.parentNode.insertBefore(button, answer);
        }
        
        // Make sure the answer is hidden
        if (answer && !answer.classList.contains('hidden')) {
            answer.classList.add('hidden');
        }
        
        // Add event handler to the button
        if (button) {
            button.addEventListener('click', function() {
                var answer = this.nextElementSibling;
                var span = this.querySelector('span');
                var icon = this.querySelector('i');
                
                if (answer.classList.contains('hidden')) {
                    // Show answer
                    answer.classList.remove('hidden');
                    span.textContent = 'Hide Answer';
                    icon.className = 'fas fa-chevron-up';
                } else {
                    // Hide answer
                    answer.classList.add('hidden');
                    span.textContent = 'Show Answer';
                    icon.className = 'fas fa-chevron-down';
                }
            });
            
            // Style the button
            button.style.backgroundColor = '#f3e8ff';
            button.style.color = '#7c3aed';
            button.style.border = 'none';
            button.style.borderRadius = '6px';
            button.style.padding = '12px 16px';
            button.style.width = '100%';
            button.style.marginTop = '10px';
            button.style.marginBottom = '10px';
            button.style.cursor = 'pointer';
            button.style.display = 'flex';
            button.style.alignItems = 'center';
            button.style.justifyContent = 'space-between';
            button.style.fontWeight = '500';
        }
    });
});
</script>
"""
        
    return render_template("result.html",
                         extracted_text=get_from_session("extracted_text", ""),
                         summary=get_from_session("summary", ""),
                         questions=questions_html,
                         active_tab="questions",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/result/extracted")
def result_extracted():
    """Result page with extracted text tab active"""
    if "extracted_text" not in session or "summary" not in session or "questions" not in session:
        flash("No analysis data found. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Use translated extracted text if available
    extracted_text = get_from_session("translated_extracted_text", get_from_session("extracted_text", ""))
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("result.html",
                         extracted_text=extracted_text,
                         summary=get_from_session("summary", ""),
                         questions=get_from_session("questions", ""),
                         active_tab="extracted",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/pdf-to-text")
def pdf_to_text():
    return render_template("pdf-to-text.html")  # Flask will find it inside `templates/`

@app.route("/ai-summary")
def ai_summary():
    summary = session.get("summary", "No summary available.")  # Retrieve AI-generated summary
    return render_template("ai-summary.html", summary=summary)


@app.route("/free-tool")
def free_tool():
    return render_template("free-tool.html")
@app.route("/multilingual-support")
def multilingual_support():
    return render_template("multilingual-support.html")
@app.route("/download-summary")
def download_summary():
    summary_text = session.get("summary", "No summary available.")  # Retrieve from session if available
    return render_template("download-summary.html", summary_text=summary_text)
@app.route("/extract-text", methods=["POST"])
def extract_text():
    if "file" not in request.files:
        return jsonify({"error": "No file uploaded"}), 400

    pdf_file = request.files["file"]
    if not pdf_file.filename.lower().endswith(".pdf"):
        return jsonify({"error": "Invalid file type. Please upload a PDF."}), 400

    file_path = os.path.join(app.config["UPLOAD_FOLDER"], pdf_file.filename)
    pdf_file.save(file_path)

    # Extract text using PyMuPDF
    extracted_text = extract_text_from_pdf(file_path)

    return jsonify({"text": extracted_text})

# --- Translation Form Handler ---
@app.route("/translate_content", methods=["POST", "GET"])
def translate_content():
    """Handle form-based translation requests"""
    # Default values
    active_tab = "summary"
    target_lang = "en"
    
    try:
        # Check if this is a GET or POST request
        if request.method == "GET":
            # For GET requests, use query parameters
            active_tab = request.args.get("active_tab", "summary")
            target_lang = request.args.get("language", "en")
            
            print(f"Translation GET request - active_tab: {active_tab}, language: {target_lang}")
        else:
            # For POST requests, use form data
            # Get form data safely
            if request.form:
                if "active_tab" in request.form and request.form["active_tab"]:
                    active_tab = request.form["active_tab"]
                
                if "language" in request.form and request.form["language"]:
                    target_lang = request.form["language"]
            
            print(f"Translation POST request - active_tab: {active_tab}, language: {target_lang}")
        
        # Store the selected language in the session for future use
        session["selected_language"] = target_lang
        print(f"Stored language '{target_lang}' in session")
        
        # Determine which content to translate based on active tab
        content_to_translate = ""
        if active_tab == "summary":
            content_to_translate = get_from_session("summary", "")
        elif active_tab == "questions":
            content_to_translate = get_from_session("questions", "")
        elif active_tab == "extracted":
            content_to_translate = get_from_session("extracted_text", "")
        
        if not content_to_translate:
            flash("No content available to translate", "error")
            if active_tab == "questions":
                return redirect(url_for("result_questions"))
            elif active_tab == "extracted":
                return redirect(url_for("result_extracted"))
            else:
                return redirect(url_for("result"))
        
        # Special handling for questions HTML content
        if active_tab == "questions" and "<div class=\"question-container\">" in content_to_translate:
            try:
                import re
                
                # Extract all question texts
                question_pattern = r'<h3 class="font-bold text-lg mt-4 text-primary-700">Question: (.*?)</h3>'
                questions = re.findall(question_pattern, content_to_translate)
                
                # Extract all answer texts (preserving <strong> tags)
                answer_pattern = r'<div class="answer bg-gray-50 p-3 rounded-lg mt-2 mb-6"><strong>Answer:</strong> (.*?)</div>'
                answers = re.findall(answer_pattern, content_to_translate)
                
                # Translate each question and answer separately
                translated_questions = []
                translated_answers = []
                
                # Translate questions
                for q in questions:
                    try:
                        translated = GoogleTranslator(source="auto", target=target_lang).translate(q)
                        translated_questions.append(translated)
                    except Exception as e:
                        print(f"Error translating question: {str(e)}")
                        translated_questions.append(q)  # Keep original if translation fails
                
                # For answers, we need to handle the <strong> tags
                for a in answers:
                    # Extract parts to translate while preserving <strong> tags
                    strong_parts = []
                    text_parts = []
                    
                    # Replace <strong>text</strong> with placeholders
                    def replace_strong(match):
                        strong_parts.append(match.group(1))
                        return f"[STRONG_{len(strong_parts)-1}]"
                    
                    # Replace all <strong> tags with placeholders
                    text_with_placeholders = re.sub(r'<strong>(.*?)</strong>', replace_strong, a)
                    
                    try:
                        # Translate the text with placeholders
                        translated_text = GoogleTranslator(source="auto", target=target_lang).translate(text_with_placeholders)
                        
                        # Restore <strong> tags
                        for i, strong_text in enumerate(strong_parts):
                            # Also translate the text inside <strong> tags
                            translated_strong = GoogleTranslator(source="auto", target=target_lang).translate(strong_text)
                            translated_text = translated_text.replace(f"[STRONG_{i}]", f"<strong>{translated_strong}</strong>")
                        
                        translated_answers.append(translated_text)
                    except Exception as e:
                        print(f"Error translating answer: {str(e)}")
                        translated_answers.append(a)  # Keep original if translation fails
                
                # Reconstruct the HTML with translated content
                translated_html = ""
                for i in range(min(len(translated_questions), len(translated_answers))):
                    translated_html += f"""
<div class="question-container">
<h3 class="font-bold text-lg mt-4 text-primary-700">Question: {translated_questions[i]}</h3>
<div class="answer bg-gray-50 p-3 rounded-lg mt-2 mb-6"><strong>Answer:</strong> {translated_answers[i]}</div>
</div>
"""
                
                translated_text = translated_html
                print(f"Successfully translated structured questions content")
            except Exception as e:
                print(f"Error in structured questions translation: {str(e)}")
                # Fall back to regular translation if structured approach fails
                try:
                    # Limit content length for regular translation
                    if len(content_to_translate) > 2000:
                        content_to_translate = content_to_translate[:2000] + "... (content truncated for translation)"
                    
                    translated_text = GoogleTranslator(source="auto", target=target_lang).translate(content_to_translate)
                except Exception as fallback_error:
                    print(f"Translation fallback error: {str(fallback_error)}")
                    flash("Translation failed. Please try again with a different language.", "error")
                    if active_tab == "questions":
                        return redirect(url_for("result_questions"))
                    elif active_tab == "extracted":
                        return redirect(url_for("result_extracted"))
                    else:
                        return redirect(url_for("result"))
        else:
            # Regular translation for other content types
            # Perform translation - limit content length to avoid session cookie size issues
            if len(content_to_translate) > 2000:
                content_to_translate = content_to_translate[:2000] + "... (content truncated for translation)"
                
            try:
                # Use GoogleTranslator which seems more reliable
                translated_text = GoogleTranslator(source="auto", target=target_lang).translate(content_to_translate)
                print(f"Translation successful using GoogleTranslator")
            except Exception as translation_error:
                print(f"GoogleTranslator error: {str(translation_error)}")
                # Fallback to alternative translator if GoogleTranslator fails
                try:
                    translator = Translator(from_lang="auto", to_lang=target_lang)
                    translated_text = translator.translate(content_to_translate[:1000])  # Limit length further for fallback
                    print(f"Translation successful using fallback Translator")
                except Exception as fallback_error:
                    print(f"Fallback translator error: {str(fallback_error)}")
                    # If all translation attempts fail, return an error
                    flash(f"Translation failed. Please try again with a different language.", "error")
                    if active_tab == "questions":
                        return redirect(url_for("result_questions"))
                    elif active_tab == "extracted":
                        return redirect(url_for("result_extracted"))
                    else:
                        return redirect(url_for("result"))
        
        # Store result in session using compression
        if active_tab == "summary":
            store_in_session("translated_summary", translated_text)
            print(f"Stored translated summary in session, language: {target_lang}")
            return redirect(url_for("result"))
        elif active_tab == "questions":
            store_in_session("translated_questions", translated_text)
            print(f"Stored translated questions in session, language: {target_lang}")
            return redirect(url_for("result_questions"))
        elif active_tab == "extracted":
            store_in_session("translated_extracted_text", translated_text)
            print(f"Stored translated extracted text in session, language: {target_lang}")
            return redirect(url_for("result_extracted"))
        else:
            return redirect(url_for("result"))
            
    except Exception as e:
        print(f"Translation error: {str(e)}")
        flash(f"Translation error: {str(e)}", "error")
        return redirect(url_for("result"))

# --- Debug Route ---
@app.route("/debug_translate")
def debug_translate():
    """Debug route for testing translation"""
    try:
        # Use a safer translation approach for debugging
        test_text = "Hello world"
        target_lang = "es"
        
        # Try the primary translator
        try:
            translated = GoogleTranslator(source="en", target=target_lang).translate(test_text)
            translation_method = "GoogleTranslator"
        except Exception as e:
            # Fallback to alternative
            try:
                translator = Translator(from_lang="en", to_lang=target_lang)
                translated = translator.translate(test_text)
                translation_method = "Translator"
            except Exception as e2:
                return jsonify({
                    "error": f"Primary error: {str(e)}, Fallback error: {str(e2)}",
                    "original": test_text,
                    "language": target_lang
                }), 500
                
        return jsonify({
            "success": True,
            "original": test_text,
            "translated": translated,
            "language": target_lang,
            "method": translation_method
        })
    except Exception as e:
        return jsonify({
            "error": str(e),
            "original": "Hello world",
            "language": "es"
        }), 500

@app.route("/format_questions", methods=["POST"])
def format_questions():
    """Format raw questions and answers as HTML"""
    # Check if user is logged in but don't redirect if not
    is_logged_in = "user" in session
    
    # Get the raw questions from the request
    raw_questions = request.form.get("raw_questions", "")
    
    if not raw_questions:
        if is_logged_in:
            flash("No questions provided to format.", "error")
            return redirect(url_for("format_questions_page"))
        else:
            # For non-logged in users, just redirect to display with empty content
            return redirect(url_for("display_questions"))
    
    # Store the raw questions as-is first (with markdown) for the display page
    # Even if user is not logged in, we'll pass it directly to display_questions
    if is_logged_in:
        session["raw_questions"] = raw_questions
    
    # Ensure markdown bold is converted to HTML
    try:
        # First convert the markdown to HTML
        processed_questions = re.sub(r'\*\*(.*?)\*\*', r'<strong>\1</strong>', raw_questions)
        
        # Then format the questions as HTML
        formatted_html = format_questions_html(processed_questions)
        
        # Store the formatted questions in the session for logged in users
        if is_logged_in:
            session["questions"] = formatted_html
            flash("Questions formatted successfully!", "success")
    except Exception as e:
        if is_logged_in:
            flash(f"Error formatting questions: {str(e)}", "error")
            return redirect(url_for("format_questions_page"))
        else:
            # For non-logged in users, redirect to display with raw content
            return redirect(url_for("display_questions", raw_questions=raw_questions))
    
    # Always redirect to the display page
    # For non-logged in users, pass the raw questions as a parameter
    if not is_logged_in:
        return redirect(url_for("display_questions", raw_questions=raw_questions))
    
    # For logged in users, the content is stored in session
    return redirect(url_for("display_questions"))

@app.route("/format_questions_page")
def format_questions_page():
    """Display a page with a form to manually format questions and answers"""
    # Don't require login to access this page
    csrf_token = generate_csrf()
    return render_template("format_questions.html", csrf_token=csrf_token)

@app.route("/display_questions", methods=["GET", "POST"])
def display_questions():
    """Display questions with proper markdown formatting through client-side processing"""
    # Check if user is logged in but don't redirect if not
    is_logged_in = "user" in session
    
    raw_questions = ""
    
    if request.method == "POST":
        # Get questions from form submission
        raw_questions = request.form.get("raw_questions", "")
        
        # Store in session if user is logged in
        if is_logged_in:
            session["raw_questions"] = raw_questions
    elif is_logged_in and "raw_questions" in session:
        # Get questions from session if available
        raw_questions = session.get("raw_questions", "")
    elif is_logged_in and "questions" in session:
        # Get questions from session if available
        questions_html = session.get("questions", "")
        
        # If the session contains HTML, extract the questions and answers
        if questions_html and "<div" in questions_html:
            # Extract text from HTML for raw display - this is a simplistic approach
            import re
            question_matches = re.findall(r'Question: (.*?)</h3>', questions_html)
            answer_matches = re.findall(r'<strong>Answer:</strong> (.*?)</div>', questions_html)
            
            # Reconstruct raw questions
            raw_questions = ""
            for i in range(min(len(question_matches), len(answer_matches))):
                # Strip HTML tags from answers for raw display
                answer_text = re.sub(r'<[^>]+>', '', answer_matches[i])
                answer_text = re.sub(r'<strong>(.*?)</strong>', r'**\1**', answer_text)
                
                raw_questions += f"Question: {question_matches[i]}\nAnswer: {answer_text}\n\n"
        else:
            # If it's raw text, use it directly
            raw_questions = questions_html

    
    return render_template("display_questions.html", raw_questions=raw_questions)

@app.route("/clear_session")
def clear_session():
    """Clears all session data and redirects to the upload page"""
    session.clear()
    flash("All session data has been cleared successfully.", "success")
    return redirect(url_for("upload_file"))

# --- Security Functions ---
def check_csrf():
    """Check CSRF token for both form submissions and AJAX requests"""
    token_header = request.headers.get('X-CSRFToken')
    token_body = None
    
    # Also check for token in JSON body for AJAX requests
    if request.is_json:
        try:
            token_body = request.json.get('csrf_token')
        except Exception as e:
            print(f"Error extracting CSRF token from JSON: {str(e)}")
            
    token_form = request.form.get('csrf_token')
    session_token = session.get('csrf_token')
    
    token = token_header or token_form or token_body
    
    # Debug logging
    print(f"CSRF Check - Header token: {'Present' if token_header else 'Missing'}")
    print(f"CSRF Check - Form token: {'Present' if token_form else 'Missing'}")
    print(f"CSRF Check - JSON body token: {'Present' if token_body else 'Missing'}")
    print(f"CSRF Check - Session token: {'Present' if session_token else 'Missing'}")
    
    # For debugging CSRF issues
    if token and session_token:
        print(f"Token match: {token == session_token}")
        
    result = token and token == session_token
    print(f"CSRF Check result: {'Pass' if result else 'Fail'}")
    
    return result

# --- Route for Chatbot Q&A ---
@app.route("/chat", methods=["POST"])
def chat():
    """API endpoint to handle chat questions about the PDF"""
    try:
        print("\n----- Chat API Request -----")
        print(f"Request data: {request.get_data().decode('utf-8')[:200]}")
        print(f"Content-Type: {request.headers.get('Content-Type')}")
        print(f"X-CSRFToken: {request.headers.get('X-CSRFToken', 'Not provided')}")
        
        # Skip CSRF check for API requests with proper auth or if in debug mode
        # For production, consider a more robust CSRF validation approach
        if request.headers.get('Content-Type') == 'application/json':
            # For AJAX requests, we can be more lenient with CSRF in some cases
            # especially as we're also checking for user sessions
            pass
        else:
            print("Warning: Request doesn't have application/json Content-Type")
        
        # Check if there's extracted_text in session
        extracted_text = get_from_session("extracted_text", "")
        
        # Check if we need to retrieve full text from file if there was truncation
        full_text_path = session.get("full_text_path", "")
        current_file_path = session.get("file_path", "")
        current_file = session.get("current_file", "")
        
        # Verify that the full_text_path matches the current file to prevent using old text
        if full_text_path and os.path.exists(full_text_path):
            # Check if the full_text_path corresponds to the current file
            expected_text_path = f"{current_file_path}.txt" if current_file_path else ""
            
            if full_text_path != expected_text_path:
                print(f"WARNING: Full text path doesn't match current file!")
                print(f"Current file: {current_file}")
                print(f"Full text path: {full_text_path}")
                print(f"Expected text path: {expected_text_path}")
                
                # Don't use the full text path as it belongs to a different file
                full_text_path = ""
                
                # Update session to remove invalid full_text_path
                if "full_text_path" in session:
                    del session["full_text_path"]
                    print("Removed invalid full_text_path from session")
        
        # Now try to load the full text if the path is valid and corresponds to current file
        if full_text_path and os.path.exists(full_text_path):
            try:
                print(f"Loading full text from {full_text_path}")
                with open(full_text_path, 'r', encoding='utf-8') as f:
                    extracted_text = f.read()
                print(f"Loaded full text: {len(extracted_text)} characters")
            except Exception as e:
                print(f"Error loading full text from file: {str(e)}")
                # Continue with truncated text from session
                print(f"Using truncated text from session: {len(extracted_text)} characters")
        else:
            print(f"Using text from session: {len(extracted_text)} characters")
        
        if not extracted_text:
            print("No extracted text in session")
            return jsonify({"error": "No document loaded. Please upload a PDF first."}), 400
        
        # Get the query from the request
        if not request.is_json:
            print("Error: Request is not JSON")
            return jsonify({"error": "Request must be in JSON format"}), 400
            
        data = request.json
        query = data.get("query")
        
        # Check for special "full_text" request
        show_full_text = False
        if query and (query.lower().strip() == "show full text" or 
                     query.lower().strip() == "show all text" or
                     query.lower().strip() == "full text" or
                     query.lower().strip() == "all text"):
            show_full_text = True
            
        if not query:
            print("Error: Missing query parameter")
            return jsonify({"error": "No question provided"}), 400
            
        print(f"Processing query: '{query}'")
        
        # If it's a request for the full text
        if show_full_text:
            text_preview = f"Here is the full text of the document ({len(extracted_text)} characters):\n\n{extracted_text}"
            
            # Truncate if absolutely necessary to avoid UI issues
            if len(text_preview) > 100000:
                text_preview = text_preview[:100000] + "...\n\n[Text truncated due to display limitations]"
                
            chat_history = session.get("chat_history", [])
            chat_entry = {
                "question": query,
                "answer": text_preview,
                "info_not_found": False,
                "is_full_text": True
            }
            chat_history.append(chat_entry)
            
            # Limit history
            if len(chat_history) > 10:
                chat_history = chat_history[-10:]
                
            session["chat_history"] = chat_history
            
            return jsonify({
                "answer": text_preview, 
                "history": chat_history, 
                "info_not_found": False,
                "is_full_text": True
            })
        
        # Call AI API to get an answer
        try:
            api_url = "https://api.qewertyy.dev/models"
            
            # Create a more effective system prompt
            system_prompt = """You are an intelligent PDF document assistant. Answer questions based ONLY on the document content provided.
USER QUESTION: "{query}
Important guidelines:
1. Answer DIRECTLY and SPECIFICALLY to the user's question
2. If the user's question or any part of it isn't addressed in the document, explicitly state: "The document doesn't contain information about [topic]"
3. DO NOT include phrases like "Based on the document" or "According to the content" in your answers
4. Keep answers concise unless more detail is needed
5. Never make up information not found in the document
6. IMPORTANT: For queries like single words or phrases, check if they specifically appear in the document. If not, state "The term '[query]' is not mentioned in the document"
7. For any query that is clearly unrelated to the document content, respond with "The document doesn't discuss [topic]"
8. If the user asks about the document itself, provide information about the document's structure, content, and purpose.
9. If the user asks about the document's total words, characters, or pages, provide that information.

Focus on giving precise, accurate, and direct answers to the specific question asked.

"""
            
            # Provide the full document content in a user message - no truncation
            context_message = f"""DOCUMENT CONTENT:
{extracted_text}
ANSWER ONLY BASED ON THE ABOVE DOCUMENT."""

            # Log the document size
            print(f"Sending document of {len(extracted_text)} characters to API")
            
            payload = {
                "model_id": 5,
                "messages": [
                    {
                        "role": "system",
                        "content": system_prompt
                    },
                    {
                        "role": "user", 
                        "content": context_message
                    },
                    {
                        "role": "user",
                        "content": query
                    }
                ],
                "temperature": 0.5  # Lower temperature for more deterministic answers
            }
            headers = {"Content-Type": "application/json"}
            
            # print(f"Calling API with context of length {len(context_message)}")
            
            # # Debug: Print full system prompt and user message
            # print("\n===== FULL SYSTEM PROMPT =====")
            # print(system_prompt)
            # print("\n===== FULL USER MESSAGE (FIRST 1000 CHARS) =====")
            # print(context_message[:1000] + "..." if len(context_message) > 1000 else context_message)
            # print(f"\nTotal input length: {len(system_prompt) + len(context_message) + len(query)} characters")
            response = requests.post(api_url, json=payload, headers=headers, timeout=30)
            # print(f"API response status: {response.status_code}")
            
            if response.status_code == 200:
                response_data = response.json()
                answer = response_data.get("content", "")
                
                # Debug: Print full API response
                print("\n===== FULL API RESPONSE =====")
                print(answer)
                print("=============================\n")
                
                # Check if answer is valid
                if not answer or answer.strip() == "":
                    answer = "I don't see information about that in the document."
                
                # Clean the response from prefixes like "Based on the document,"
                common_prefixes = [
                    "Based on the document, ",
                    "According to the document, ",
                    "From the document, ",
                    "The document states that ",
                    "As per the document, "
                ]
                
                for prefix in common_prefixes:
                    if answer.startswith(prefix):
                        answer = answer[len(prefix):]
                        break
                
                # Add variation to responses about missing information
                not_found_phrases = [
                    "I don't see information about",
                    "I couldn't find information about",
                    "The document doesn't contain information about",
                    "The document doesn't mention",
                    "There's no information about",
                    "I don't see any details about"
                ]
                
                # Check if the response indicates information not found
                is_not_found_response = any(phrase in answer for phrase in not_found_phrases)
                
                # Create chat history or append to existing
                chat_history = session.get("chat_history", [])
                
                # Add metadata to help frontend distinguish between types of responses
                chat_entry = {
                    "question": query, 
                    "answer": answer,
                    "info_not_found": is_not_found_response
                }
                
                chat_history.append(chat_entry)
                
                # Limit history to last 10 conversations
                if len(chat_history) > 10:
                    chat_history = chat_history[-10:]
                    
                session["chat_history"] = chat_history
                
                return jsonify({"answer": answer, "history": chat_history, "info_not_found": is_not_found_response})
            else:
                error_message = f"API Error: Status code {response.status_code}"
                print(error_message)
                
                # Attempt to get more details from the response
                try:
                    error_details = response.text[:200]
                    print(f"API error details: {error_details}")
                except:
                    error_details = "No details available"
                
                # Create a fallback response - do NOT return error to client
                print("Generating fallback response instead of returning error")
                query_terms = query.lower().split()
                
                # Search for relevant paragraphs in the document
                relevant_sections = []
                paragraphs = extracted_text.split('\n\n')
                for para in paragraphs:
                    if para and any(term in para.lower() for term in query_terms):
                        relevant_sections.append(para)
                
                # Create a simplified fallback answer
                if relevant_sections:
                    # Simple but effective fallback
                    para = relevant_sections[0]
                    fallback_answer = f"I found this in the document that might be relevant: \"{para[:300]}...\""
                    is_not_found_response = False
                else:
                    fallback_answer = "I couldn't find information about that in the document."
                    is_not_found_response = True
                
                # Update chat history
                chat_history = session.get("chat_history", [])
                chat_entry = {
                    "question": query, 
                    "answer": fallback_answer,
                    "info_not_found": is_not_found_response
                }
                chat_history.append(chat_entry)
                
                # Limit history
                if len(chat_history) > 10:
                    chat_history = chat_history[-10:]
                session["chat_history"] = chat_history
                
                return jsonify({"answer": fallback_answer, "history": chat_history, "info_not_found": is_not_found_response})
        except Exception as api_error:
            print(f"API error: {str(api_error)}")
            
            # Create a fallback response if the API fails
            try:
                # Very simple keyword matching fallback
                fallback_answer = "I'm having trouble connecting to the AI service. "
                
                # Try to extract a relevant section from the document
                relevant_paragraphs = []
                query_terms = set(query.lower().split())
                paragraphs = extracted_text.split('\n\n')
                
                for para in paragraphs:
                    if any(term in para.lower() for term in query_terms):
                        relevant_paragraphs.append(para)
                
                is_not_found_response = False
                
                if relevant_paragraphs:
                    fallback_answer += "I found this in the document that might help: \"" + relevant_paragraphs[0] + "\""
                else:
                    # No matching paragraphs found, so mark as information not found
                    fallback_answer += "I couldn't find information about that in the document."
                    is_not_found_response = True
                
                # Create chat history or append to existing
                chat_history = session.get("chat_history", [])
                
                # Add metadata to help frontend distinguish between types of responses
                chat_entry = {
                    "question": query, 
                    "answer": fallback_answer,
                    "info_not_found": is_not_found_response
                }
                
                chat_history.append(chat_entry)
                
                # Limit history to last 10 conversations
                if len(chat_history) > 10:
                    chat_history = chat_history[-10:]
                    
                session["chat_history"] = chat_history
                
                return jsonify({"answer": fallback_answer, "history": chat_history, "info_not_found": is_not_found_response})
            except:
                return jsonify({"error": f"Error calling AI service: {str(api_error)}"}), 500
            
    except Exception as e:
        print(f"Chat error: {str(e)}")
        return jsonify({"error": f"An error occurred: {str(e)}"}), 500

@app.route("/result/chat")
def result_chat():
    """Result page with chat interface for the PDF"""
    try:
        # Make sure we have document data
        if "extracted_text" not in session:
            flash("No document loaded. Please upload a PDF first.", "error")
            return redirect(url_for("upload_file"))
        
        # Get extracted text and verify it's available
        extracted_text = get_from_session("extracted_text", "")
        if not extracted_text:
            print("ERROR: Empty extracted text in session")
            flash("No document content found. Please upload a PDF again.", "error")
            return redirect(url_for("upload_file"))
        
        # Debug - check what's in the session
        print("DEBUG - Session contains:")
        for key in session:
            if key == 'extracted_text':
                text_len = len(get_from_session('extracted_text', ''))
                print(f"  - extracted_text: {text_len} chars")
                
                # Log a sample of the text for debugging
                if text_len > 0:
                    sample = extracted_text[:100].replace('\n', ' ')
                    print(f"  - Text sample: {sample}...")
                else:
                    print("  - Text is empty!")
            else:
                print(f"  - {key}: present")
        
        # Get current file name
        current_file = session.get("current_file", "Unknown file")
        file_size = session.get("file_size", "Unknown size")
        
        # Verify the full_text_path matches the current file path to prevent using old text
        full_text_path = session.get("full_text_path", "")
        current_file_path = session.get("file_path", "")
        
        if full_text_path and current_file_path:
            expected_path = f"{current_file_path}.txt"
            if full_text_path != expected_path:
                print(f"WARNING: Chat using mismatched text file!")
                print(f"Current file: {current_file}")
                print(f"Full text path: {full_text_path}")
                print(f"Expected path: {expected_path}")
                # Remove invalid full_text_path
                if "full_text_path" in session:
                    del session["full_text_path"]
                    print("Removed invalid full_text_path from session")
        
        # Print document stats
        print(f"Document: {current_file}, Size: {file_size}, Text length: {len(extracted_text)} chars")
        
        # Get chat history or initialize empty list
        chat_history = session.get("chat_history", [])
        
        # Check if the first message in chat history refers to the current file
        # This ensures chat reflects the currently loaded document
        should_add_notification = True
        if chat_history:
            for entry in chat_history:
                # Check if we have a system notification about the current file
                if entry.get("system_message") and "newly uploaded document" in entry.get("answer", ""):
                    if current_file in entry.get("answer", ""):
                        should_add_notification = False
                        break
        
        # Add new document notification if needed
        if should_add_notification:
            print(f"Adding document notification for {current_file}")
            add_new_document_notification_to_chat(current_file)
            # Refresh chat history
        chat_history = session.get("chat_history", [])
        
        # Get the last selected language
        selected_language = session.get("selected_language", "en")
        
        # Generate a fresh CSRF token
        csrf_token = generate_csrf()
        print(f"Generated CSRF token for chat: {csrf_token[:10]}...")
        
        return render_template("chat.html",
                             extracted_text=extracted_text,
                             summary=get_from_session("summary", ""),
                             chat_history=chat_history,
                             active_tab="chat",
                             selected_language=selected_language,
                             current_file=current_file,
                             file_size=file_size,
                             csrf_token=csrf_token)
    except Exception as e:
        print(f"Error in result_chat: {str(e)}")
        flash("An error occurred while loading the chat interface.", "error")
        return redirect(url_for("upload_file"))

# --- Topic Extraction from PDF ---
def extract_topics(text, num_topics=5):
    """Extract main topics from the PDF text using TF-IDF and NLP"""
    if not text or text == "No text found.":
        return ["No text available for topic extraction"]
    
    try:
        # Use spaCy for better topic extraction if available
        if nlp:
            # Process the text with spaCy
            doc = nlp(text[:20000])  # Limit to prevent memory issues
            
            # Extract named entities
            entities = [ent.text for ent in doc.ents if ent.label_ in ["ORG", "PRODUCT", "GPE", "PERSON", "WORK_OF_ART"]]
            
            # Extract noun phrases
            noun_phrases = [chunk.text for chunk in doc.noun_chunks]
            
            # Combine entities and frequent noun phrases
            all_candidates = entities + noun_phrases
            
            # Count occurrences and get most frequent
            from collections import Counter
            topic_counter = Counter(all_candidates)
            topics = [topic for topic, count in topic_counter.most_common(num_topics+5)]
            
            # Remove duplicates and topics shorter than 3 characters
            clean_topics = []
            for topic in topics:
                topic = topic.strip()
                if len(topic) > 3 and topic not in clean_topics and not any(topic.lower() in t.lower() for t in clean_topics):
                    clean_topics.append(topic)
            
            return clean_topics[:num_topics]
        
        # Fallback to TF-IDF if spaCy is not available
        # Split text into sentences
        sentences = re.split(r'(?<=[.!?])\s+', text)
        
        # Create a TF-IDF vectorizer
        tfidf_vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, stop_words='english')
        
        # Fit and transform the text
        tfidf_matrix = tfidf_vectorizer.fit_transform(sentences)
        
        # Get feature names (terms)
        terms = tfidf_vectorizer.get_feature_names_out()
        
        # Get highest TF-IDF scores
        tfidf_scores = tfidf_matrix.sum(axis=0).A1
        
        # Get the indices of top terms
        top_indices = tfidf_scores.argsort()[-num_topics:][::-1]
        
        # Return the top terms
        return [terms[i] for i in top_indices]
    
    except Exception as e:
        print(f"Error extracting topics: {str(e)}")
        return ["Topic extraction failed", "Try uploading a different document"]

# --- Table Extraction from PDF ---
def extract_tables_from_pdf(pdf_path):
    """Extract tables from PDF file using PyMuPDF"""
    tables = []
    
    try:
        doc = fitz.open(pdf_path)
        
        for page_num, page in enumerate(doc):
            # Extract tables using PyMuPDF
            tab = page.find_tables()
            
            if tab.tables:
                for i, table in enumerate(tab.tables):
                    # Convert table to HTML
                    html_table = "<table class='min-w-full divide-y divide-gray-200'>"
                    
                    # Add header
                    html_table += "<thead class='bg-gray-50'><tr>"
                    for cell in table.header.cells:
                        html_table += f"<th scope='col' class='px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider'>{cell.text}</th>"
                    html_table += "</tr></thead>"
                    
                    # Add body
                    html_table += "<tbody class='bg-white divide-y divide-gray-200'>"
                    for row in table.rows:
                        html_table += "<tr>"
                        for cell in row.cells:
                            html_table += f"<td class='px-6 py-4 whitespace-nowrap text-sm text-gray-500'>{cell.text}</td>"
                        html_table += "</tr>"
                    html_table += "</tbody></table>"
                    
                    # Store table info
                    tables.append({
                        "page_num": page_num + 1,
                        "table_num": i + 1,
                        "html": html_table,
                        "rows": len(table.rows),
                        "cols": len(table.header.cells) if table.header else 0
                    })
        
        return tables
    except Exception as e:
        print(f"Error extracting tables: {str(e)}")
        return []

# --- Semantic Search in PDF ---
def semantic_search(query, text, num_results=5):
    """Perform semantic search within the document content"""
    if not query or not text:
        return []
    
    try:
        # Split text into paragraphs
        paragraphs = text.split('\n\n')
        
        # Filter out empty paragraphs and limit paragraph length
        paragraphs = [p.strip() for p in paragraphs if len(p.strip()) > 20]
        
        # Try using spaCy for better semantic matching if available
        if nlp:
            # Process the query
            query_doc = nlp(query)
            
            # Process each paragraph (limiting length to avoid memory issues)
            results = []
            for i, paragraph in enumerate(paragraphs):
                # Limit paragraph length for processing
                para_doc = nlp(paragraph[:1000])
                
                # Calculate similarity
                similarity = query_doc.similarity(para_doc)
                
                # Store result with similarity score
                results.append((paragraph, similarity, i))
            
            # Sort by similarity (descending)
            results.sort(key=lambda x: x[1], reverse=True)
            
            # Return top results with paragraph text, score and position
            return [(text, round(score, 2), pos) for text, score, pos in results[:num_results]]
        
        # Fallback to TF-IDF if spaCy is not available
        # Create TF-IDF vectorizer
        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
        
        # Add query to paragraphs for proper vectorization
        all_documents = paragraphs + [query]
        
        # Fit and transform all texts
        tfidf_matrix = tfidf_vectorizer.fit_transform(all_documents)
        
        # Get query vector (last item)
        query_vector = tfidf_matrix[-1]
        
        # Calculate cosine similarity
        from sklearn.metrics.pairwise import cosine_similarity
        cosine_similarities = cosine_similarity(query_vector, tfidf_matrix[:-1]).flatten()
        
        # Get top matches
        top_indices = cosine_similarities.argsort()[:-num_results-1:-1]
        
        # Return top results with paragraph text, score and position
        return [(paragraphs[i], round(float(cosine_similarities[i]), 2), i) for i in top_indices]
    
    except Exception as e:
        print(f"Error in semantic search: {str(e)}")
        return [("Search functionality encountered an error. Please try again.", 0, 0)]

# --- Text Highlighting ---
def generate_text_with_highlights(text, search_terms):
    """Highlight search terms in text"""
    if not text or not search_terms:
        return text
    
    try:
        # Create a safe HTML version of the text
        html_text = text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
        
        # Filter empty or too short terms
        valid_terms = [term for term in search_terms if term and len(term.strip()) > 1]
        
        if not valid_terms:
            return html_text.replace("\n", "<br>")
        
        # Sort terms by length (descending) to highlight longer phrases first
        valid_terms.sort(key=len, reverse=True)
        
        # Highlight each search term
        for term in valid_terms:
            term = term.strip()
            if not term:
                continue
                
            try:
                # Case-insensitive replacement with highlight
                pattern = re.compile(r'(\b' + re.escape(term) + r'\b)', re.IGNORECASE)
                replacement = r'<mark class="bg-yellow-200 px-1 rounded">\1</mark>'
                
                # First try with word boundaries
                new_html = pattern.sub(replacement, html_text)
                
                # If no replacements were made (e.g., for terms with special chars), try without word boundaries
                if new_html == html_text:
                    pattern = re.compile(re.escape(term), re.IGNORECASE)
                    html_text = pattern.sub(r'<mark class="bg-yellow-200 px-1 rounded">\g<0></mark>', html_text)
                else:
                    html_text = new_html
            except Exception as term_error:
                print(f"Error highlighting term '{term}': {str(term_error)}")
                # Continue with other terms if one fails
        
        # Convert newlines to <br> tags
        html_text = html_text.replace("\n", "<br>")
        
        return html_text
    except Exception as e:
        print(f"Error highlighting text: {str(e)}")
        # Return the text with newlines converted but without highlights
        return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\n", "<br>")

# --- New PDF AI Feature Routes ---

@app.route("/result/topics")
def result_topics():
    """Result page showing document topics"""
    if "extracted_text" not in session:
        flash("No analysis data found. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Extract topics if they don't exist in session
    if "topics" not in session:
        extracted_text = get_from_session("extracted_text", "")
        topics = extract_topics(extracted_text)
        session["topics"] = topics
    else:
        topics = session["topics"]
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("topics.html",
                         topics=topics,
                         active_tab="topics",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/tables")
def extract_tables():
    """Extract tables from the current PDF"""
    if "current_file" not in session:
        flash("No PDF uploaded. Please upload a PDF first.", "error")
        return redirect(url_for("upload_file"))
    
    try:
        # Get the current file path
        filename = session.get("current_file")
        filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
        
        # Extract tables
        tables = extract_tables_from_pdf(filepath)
        
        # Store in session
        session["tables"] = tables
        
        return render_template("tables.html",
                              tables=tables,
                              active_tab="tables",
                              current_file=filename,
                              csrf_token=generate_csrf())
    except Exception as e:
        flash(f"Error extracting tables: {str(e)}", "error")
        return redirect(url_for("result"))

@app.route("/search", methods=["GET", "POST"])
def semantic_search_route():
    """Semantic search within the PDF content"""
    if "extracted_text" not in session:
        flash("No document loaded. Please upload a PDF first.", "error")
        return redirect(url_for("upload_file"))
    
    results = []
    query = ""
    
    if request.method == "POST":
        # Get the search query
        query = request.form.get("query", "").strip()
        
        if query:
            # Get the extracted text
            extracted_text = get_from_session("extracted_text", "")
            
            # Perform semantic search
            results = semantic_search(query, extracted_text)
            
            # Format results for display with highlighting
            formatted_results = []
            for text, score, position in results:
                # Highlight the search query in the result
                highlighted_text = generate_text_with_highlights(text, [query] + query.split())
                formatted_results.append({
                    "text": highlighted_text,
                    "score": score,
                    "position": position
                })
            
            results = formatted_results
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("search.html",
                         results=results,
                         query=query,
                         active_tab="search",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/highlight", methods=["POST"])
def highlight_text():
    """Highlight text in the PDF"""
    print("\n----- Highlight Text API Request -----")
    
    if "extracted_text" not in session:
        print("Error: No extracted text in session")
        return jsonify({"error": "No document loaded"}), 400
    
    # Get the terms to highlight
    try:
        data = request.json
        terms = data.get("terms", [])
        
        print(f"Received highlight request with {len(terms)} terms")
        if len(terms) > 0:
            print(f"First term: '{terms[0][:30]}{'...' if len(terms[0]) > 30 else ''}'")
    
        if not terms:
            valid_terms = [term for term in terms if term and len(term.strip()) > 1]
    
            if not valid_terms:
                print("Error: No valid terms after filtering")
                return jsonify({"error": "No valid terms provided (too short or empty)"}), 400
                
        # Get the extracted text
            extracted_text = get_from_session("extracted_text", "")
            if not extracted_text:
                print("Error: Empty extracted text")
                return jsonify({"error": "Document text is empty"}), 400
                
            print(f"Retrieved extracted text: {len(extracted_text)} characters")
        
        # Generate highlighted HTML
            highlighted_html = generate_text_with_highlights(extracted_text, valid_terms)
            
            # Check if highlighting worked (should contain <mark> tags)
            if "<mark" not in highlighted_html:
                print("Warning: No highlights were applied, text unchanged")
                # Continue anyway as the text is still valid
            
            # Store in session with compression
            print(f"Storing highlighted text: {len(highlighted_html)} characters")
            store_in_session("highlighted_text", highlighted_html)
        
            # Verify storage was successful
            if "highlighted_text" not in session:
                print("Error: Failed to store highlighted_text in session")
                return jsonify({"error": "Failed to store highlighted text in session"}), 500
                
            print("Highlight operation successful")
        return jsonify({"success": True, "message": "Text highlighted successfully"})
    except Exception as e:
        error_message = f"Error in highlight_text: {str(e)}"
        print(error_message)
        import traceback
        traceback.print_exc()
        return jsonify({"error": error_message}), 500

@app.route("/result/highlighted")
def result_highlighted():
    """Show highlighted text"""
    print("\n----- Highlighting View Route -----")
    
    # Check for essential session data
    if "extracted_text" not in session:
        print("Error: No extracted text available")
        flash("No document loaded. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Check if highlighted text exists in session
    if "highlighted_text" not in session:
        print("Warning: No highlighted text in session, checking for extracted text")
        extracted_text = get_from_session("extracted_text", "")
        
        if not extracted_text:
            print("Error: Extracted text is empty")
            flash("No text content available to highlight.", "error")
            return redirect(url_for("upload_file"))
        
        # Display a flash message and generate default highlighting
        flash("No specific highlights found. Showing the full document text.", "warning")
        
        # Create a simple version of the text with no highlights
        basic_html = extracted_text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\n", "<br>")
        store_in_session("highlighted_text", basic_html)
        
        print("Created basic HTML version of extracted text as fallback")
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get the highlighted text
    highlighted_text = get_from_session("highlighted_text", "")
    
    # Final check - if still no highlighted text, use a placeholder
    if not highlighted_text:
        print("Error: Failed to retrieve highlighted text")
        highlighted_text = "<p>Unable to display highlighted text. The content may be too large or unavailable.</p>"
        flash("Error retrieving highlighted text. Showing placeholder.", "error")
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    print(f"Rendering highlighted view with {len(highlighted_text)} characters")
    
    return render_template("highlighted.html",
                         highlighted_text=highlighted_text,
                         active_tab="highlighted",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

# --- PDF Named Entity Recognition ---
def extract_named_entities(text):
    """Extract named entities from the document using spaCy"""
    if not text or text == "No text found.":
        return {}
    
    if not is_nlp_available():
        # Return basic fallback data when spaCy is not available
        return {
            "Note": ["spaCy model not available for entity extraction. Install with: python -m spacy download en_core_web_sm"]
        }
    
    try:
        # Process text with spaCy (limit length to prevent memory issues)
        doc = nlp(text[:50000])
        
        # Group entities by category
        entities = {}
        for ent in doc.ents:
            category = ent.label_
            if category not in entities:
                entities[category] = []
            
            # Avoid duplicates
            if ent.text not in entities[category]:
                entities[category].append(ent.text)
        
        # Map spaCy entity codes to readable names using the imported mapping
        readable_entities = {}
        for category, items in entities.items():
            readable_name = ENTITY_TYPE_MAP.get(category, category)
            readable_entities[readable_name] = items
        
        return readable_entities
    except Exception as e:
        print(f"Error extracting named entities: {str(e)}")
        return {"Error": [str(e)]}

# --- PDF Sentiment Analysis ---
def analyze_sentiment(text):
    """Analyze sentiment of PDF content using NLP"""
    if not text or text == "No text found.":
        return {"error": "No text available for sentiment analysis"}
    
    try:
        # Split text into paragraphs
        paragraphs = text.split('\n\n')
        paragraphs = [p for p in paragraphs if len(p.strip()) > 50]  # Only analyze substantial paragraphs
        
        # Limit number of paragraphs to analyze to avoid excessive processing
        max_paragraphs = 20
        if len(paragraphs) > max_paragraphs:
            # Take paragraphs from beginning, middle and end
            step = len(paragraphs) // max_paragraphs
            selected_indices = list(range(0, len(paragraphs), step))[:max_paragraphs]
            selected_paragraphs = [paragraphs[i] for i in selected_indices]
        else:
            selected_paragraphs = paragraphs
        
        results = []
        
        # Get positive and negative words from prompts module
        positive_words = SENTIMENT_WORDS["positive"]
        negative_words = SENTIMENT_WORDS["negative"]
        
        # Simple approach that works even without spaCy
        for i, para in enumerate(selected_paragraphs):
            para_lower = para.lower()
            pos_count = sum(1 for word in positive_words if word in para_lower)
            neg_count = sum(1 for word in negative_words if word in para_lower)
            
            total = pos_count + neg_count
            score = 0
            if total > 0:
                score = (pos_count - neg_count) / total
            
            sentiment = "neutral"
            if score > 0.2:
                sentiment = "positive"
            elif score < -0.2:
                sentiment = "negative"
            
            excerpt = para[:100] + "..." if len(para) > 100 else para
            
            results.append({
                "index": i,
                "excerpt": excerpt,
                "sentiment": sentiment,
                "score": round(score, 2),
                "positive_count": pos_count,
                "negative_count": neg_count
            })
        
        # Calculate overall document sentiment
        if results:
            avg_score = sum(r["score"] for r in results) / len(results)
            overall_sentiment = "neutral"
            if avg_score > 0.2:
                overall_sentiment = "positive"
            elif avg_score < -0.2:
                overall_sentiment = "negative"
            
            document_sentiment = {
                "overall_sentiment": overall_sentiment,
                "overall_score": round(avg_score, 2),
                "paragraph_sentiments": results
            }
            
            return document_sentiment
        else:
            return {"error": "No significant paragraphs found for sentiment analysis"}
    
    except Exception as e:
        print(f"Error in sentiment analysis: {str(e)}")
        return {"error": f"Sentiment analysis failed: {str(e)}"}

# --- PDF Document Classification ---
def classify_document(text, summary):
    """Classify the document into predefined categories based on content"""
    if not text or text == "No text found.":
        return {"category": "Unknown", "confidence": 0, "secondary_categories": []}
    
    try:
        # Use both the full text and summary for a more robust classification
        full_content = (text[:10000] + " " + summary).lower()
        
        # Calculate scores for each category
        category_scores = {}
        for category, keywords in DOCUMENT_CATEGORIES.items():
            # Count occurrences of each keyword
            score = sum(full_content.count(keyword.lower()) for keyword in keywords)
            # Normalize by the number of keywords in the category
            normalized_score = score / len(keywords) if len(keywords) > 0 else 0
            category_scores[category] = normalized_score
        
        # Get the primary category (highest score)
        if category_scores:
            sorted_categories = sorted(category_scores.items(), key=lambda x: x[1], reverse=True)
            primary_category = sorted_categories[0][0]
            highest_score = sorted_categories[0][1]
            
            # Convert score to confidence percentage (cap at 100%)
            confidence = min(highest_score / 2 * 100, 100)
            
            # Get secondary categories (at least 70% of the highest score)
            threshold = highest_score * 0.7
            secondary_categories = [
                {"category": cat, "confidence": min(score / 2 * 100, 100)} 
                for cat, score in sorted_categories[1:4] 
                if score >= threshold
            ]
            
            return {
                "category": primary_category,
                "confidence": round(confidence, 1),
                "secondary_categories": secondary_categories
            }
        else:
            return {"category": "Unknown", "confidence": 0, "secondary_categories": []}
    
    except Exception as e:
        print(f"Error in document classification: {str(e)}")
        return {"category": "Error", "confidence": 0, "error": str(e), "secondary_categories": []}

# --- PDF Citation Extraction ---
def extract_citations(text):
    """Extract potential citations from the document"""
    if not text or text == "No text found.":
        return []
    
    try:
        citations = []
        
        # Use citation patterns from the prompts module
        apa_pattern = re.compile(CITATION_PATTERNS["apa"])
        mla_pattern = re.compile(CITATION_PATTERNS["mla"])
        ieee_pattern = re.compile(CITATION_PATTERNS["ieee"])
        url_pattern = re.compile(CITATION_PATTERNS["url"])
        doi_pattern = re.compile(CITATION_PATTERNS["doi"])
        
        # Extract APA citations
        for match in apa_pattern.finditer(text):
            author, year = match.groups()
            citations.append({
                "type": "APA",
                "author": author.strip(),
                "year": year.strip(),
                "text": match.group(0)
            })
        
        # Extract MLA citations
        for match in mla_pattern.finditer(text):
            author, page = match.groups()
            citations.append({
                "type": "MLA",
                "author": author.strip(),
                "page": page.strip(),
                "text": match.group(0)
            })
        
        # Extract IEEE citations
        for match in ieee_pattern.finditer(text):
            ref_num = match.group(1)
            citations.append({
                "type": "IEEE",
                "reference_number": ref_num,
                "text": match.group(0)
            })
        
        # Extract URLs
        for match in url_pattern.finditer(text):
            url = match.group(0)
            citations.append({
                "type": "URL",
                "url": url,
                "text": url
            })
        
        # Extract DOIs
        for match in doi_pattern.finditer(text):
            doi = match.group(0)
            citations.append({
                "type": "DOI",
                "doi": doi,
                "text": doi
            })
        
        # Remove duplicates while preserving order
        unique_citations = []
        seen_texts = set()
        for citation in citations:
            if citation["text"] not in seen_texts:
                seen_texts.add(citation["text"])
                unique_citations.append(citation)
        
        return unique_citations[:100]  # Limit to 100 citations
    
    except Exception as e:
        print(f"Error extracting citations: {str(e)}")
        return []

@app.route("/result/entities")
def result_entities():
    """Display named entities extracted from the document"""
    if "extracted_text" not in session:
        flash("No document loaded. Please upload a PDF first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get extracted text
    extracted_text = get_from_session("extracted_text", "")
    if not extracted_text or extracted_text == "No text found.":
        # Create empty entities structure
        entities = {}
        session["entities"] = entities
    # Extract entities if they don't exist in session
    elif "entities" not in session:
        entities = extract_named_entities(extracted_text)
        session["entities"] = entities
    else:
        entities = session["entities"]
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("entities.html",
                         entities=entities,
                         active_tab="entities",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/result/sentiment")
def result_sentiment():
    """Display sentiment analysis of the document"""
    if "extracted_text" not in session:
        flash("No document loaded. Please upload a PDF first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get extracted text
    extracted_text = get_from_session("extracted_text", "")
    if not extracted_text or extracted_text == "No text found.":
        # Create error sentiment structure
        sentiment = {"error": "No text found in document or document is too short for sentiment analysis"}
        session["sentiment"] = sentiment
    # Perform sentiment analysis if it doesn't exist in session
    elif "sentiment" not in session:
        sentiment = analyze_sentiment(extracted_text)
        session["sentiment"] = sentiment
    else:
        sentiment = session["sentiment"]
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("sentiment.html",
                         sentiment=sentiment,
                         active_tab="sentiment",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/result/classify")
def result_classify():
    """Display document classification results"""
    if "extracted_text" not in session:
        flash("No document loaded. Please upload a PDF first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Get extracted text and summary
    extracted_text = get_from_session("extracted_text", "")
    summary = get_from_session("summary", "")
    
    if not extracted_text or extracted_text == "No text found.":
        # Create empty classification structure
        classification = {"category": "Unknown", "confidence": 0, "secondary_categories": []}
        session["classification"] = classification
    # Perform document classification if it doesn't exist in session
    elif "classification" not in session:
        classification = classify_document(extracted_text, summary)
        session["classification"] = classification
    else:
        classification = session["classification"]
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("classification.html",
                         classification=classification,
                         active_tab="classify",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/result/citations")
def result_citations():
    """Display citations extracted from the document"""
    if "extracted_text" not in session:
        flash("No analysis data found. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get current file name
    current_file = session.get("current_file", "Unknown file")
    
    # Extract citations if they don't exist in session
    if "citations" not in session:
        extracted_text = get_from_session("extracted_text", "")
        citations = extract_citations(extracted_text)
        session["citations"] = citations
    else:
        citations = session["citations"]
    
    # Get the last selected language
    selected_language = session.get("selected_language", "en")
    
    return render_template("citations.html",
                         citations=citations,
                         active_tab="citations",
                         selected_language=selected_language,
                         current_file=current_file,
                         csrf_token=generate_csrf())

@app.route("/debug_chat_api")
def debug_chat_api():
    """Debug route to check if we can connect to the chat API"""
    try:
        api_url = "https://api.qewertyy.dev/models"
        
        # Check if this is a character limit check
        is_limit_check = request.args.get('check_limits', 'false').lower() == 'true'
        
        # Create a simple test request
        if is_limit_check:
            # Use a specific prompt to ask about character limits
            payload = {
                "model_id": 5,
                "messages": [
                    {
                        "role": "system",
                        "content": "You are a technical assistant who provides specific information about API limits. Give exact numeric values and be precise."
                    },
                    {
                        "role": "user",
                        "content": "I am using the qewertyy.dev API at https://api.qewertyy.dev/models with model_id 5. What is the maximum number of characters or tokens I can send in a single request? Please provide specific numeric limits for both input and output. Do not ask for documentation - just provide your best information about this specific API's limits based on your knowledge."
                    }
                ],
                "temperature": 0.1
            }
        else:
            # Standard connection test
            payload = {
            "model_id": 5,
            "messages": [
                {
                    "role": "system",
                    "content": "You are a helpful assistant."
                },
                {
                    "role": "user",
                    "content": "Hello, this is a test message. Please respond with 'API connection successful'."
                }
                ],
                "temperature": 0.2
        }
        
        headers = {"Content-Type": "application/json"}
        
        # Try to connect with a timeout
        response = requests.post(api_url, json=payload, headers=headers, timeout=10)
        
        # Detailed response inspection
        response_text = response.text[:500] if response.text else "Empty response"
        try:
            response_json = response.json()
            parsed_json = True
        except:
            response_json = {"error": "Could not parse JSON"}
            parsed_json = False
        
        # Return detailed debug info
        result = {
            "status_code": response.status_code,
            "connection_successful": response.status_code == 200,
            "response_content": response_text,
            "parsed_json": parsed_json,
            "response_json": response_json,
            "content_type": response.headers.get('Content-Type', 'Unknown'),
            "response_time": f"{response.elapsed.total_seconds():.2f} seconds",
            "test_time": str(datetime.datetime.now()),
            "headers_sent": dict(headers),
            "payload_sample": str(payload)[:200] + "...",
            "server": request.host,
            "api_url": api_url,
            "client_ip": request.remote_addr,
            "check_type": "character_limits" if is_limit_check else "connection"
        }
        
        return jsonify(result)
    except Exception as e:
        return jsonify({
            "error": str(e),
            "error_type": type(e).__name__,
            "connection_successful": False,
            "test_time": str(datetime.datetime.now()),
            "server": request.host,
            "api_url": api_url if 'api_url' in locals() else "Not defined",
            "client_ip": request.remote_addr
        })

def add_new_document_notification_to_chat(filename):
    """Add a system message to the chat history when a new document is loaded"""
    chat_history = session.get("chat_history", [])
    
    # Check if we already have the notification at the start
    new_document_message = {
        "question": "New document loaded",
        "answer": f"Chat has been reset for the newly uploaded document: {filename}. You can now ask questions about this document.",
        "info_not_found": False,
        "system_message": True,
        "timestamp": str(datetime.datetime.now())
    }
    
    # Add the message to the beginning of the history
    chat_history.insert(0, new_document_message)
    
    # Limit history to last 10 entries
    if len(chat_history) > 10:
        chat_history = chat_history[:10]
    
    session["chat_history"] = chat_history
    print(f"Added new document notification to chat history for: {filename}")

# Add this new route after result_questions
@app.route("/regenerate_questions")
def regenerate_questions():
    """Force regeneration of questions from the document text"""
    if "extracted_text" not in session:
        flash("No document content found. Please upload a file first.", "error")
        return redirect(url_for("upload_file"))
    
    # Get extracted text and clear existing questions
    extracted_text = get_from_session("extracted_text", "")
    if "questions" in session:
        session.pop("questions", None)
        print("Cleared existing questions from session for regeneration")
    
    if extracted_text:
        try:
            # Generate fresh questions
            print("Manually regenerating questions")
            new_questions = generate_questions(extracted_text)
            
            # Store the new questions in session
            store_in_session("questions", new_questions)
            print(f"Successfully regenerated questions: {len(new_questions)} characters")
            
            # Add a success message
            flash("Questions have been regenerated based on your document content.", "success")
        except Exception as e:
            print(f"Error regenerating questions: {str(e)}")
            flash("Failed to regenerate questions. Please try again later.", "error")
    else:
        flash("No document content found to generate questions from.", "error")
    
    # Return to the questions tab
    return redirect(url_for("result_questions"))

if __name__ == "__main__":
    os.makedirs("uploads", exist_ok=True)
    port = int(os.environ.get("PORT", 5001))
    debug = os.environ.get("DEBUG", "False").lower() == "true"
    app.run(host="0.0.0.0", debug=debug, port=port)