first commit

.env.example

@@ -0,0 +1,81 @@
+# --- HF Spaces Deployment Settings ---
+
+# --- Application Settings ---
+APP_NAME=AI Universal Knowledge Ingestion System
+APP_VERSION=1.0.0
+PORT=7860
+HOST=0.0.0.0
+DEBUG=false
+MAX_FILE_SIZE_MB=100
+MAX_BATCH_FILES=5
+
+
+# LLM Provider Selection
+OLLAMA_ENABLED=false
+USE_OPENAI=true
+
+
+# OpenAI API Key (set this in HF Space Secrets tab)
+OPENAI_API_KEY=sk-your-actual-key-here
+OPENAI_MODEL=gpt-4o-mini
+
+
+# --- Generation Parameters ---
+DEFAULT_TEMPERATURE=0.1
+TOP_P=0.9
+MAX_TOKENS=1000
+CONTEXT_WINDOW=8192
+
+
+# --- Embedding Settings ---
+EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+EMBEDDING_DIMENSION=384
+EMBEDDING_DEVICE=cpu
+EMBEDDING_BATCH_SIZE=16
+
+
+# --- Chunking Settings ---
+FIXED_CHUNK_SIZE=384
+FIXED_CHUNK_OVERLAP=20
+FIXED_CHUNK_STRATEGY=fixed
+
+
+# --- Retrieval Settings ---
+TOP_K_RETRIEVE=10
+TOP_K_FINAL=5
+FAISS_NPROBE=16
+VECTOR_WEIGHT=0.6
+BM25_WEIGHT=0.4
+BM25_K1=1.5
+BM25_B=0.75
+ENABLE_RERANKING=true
+RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+
+
+# --- Storage Settings ---
+VECTOR_STORE_DIR=./data/vector_store
+METADATA_DB_PATH=./data/metadata.db
+AUTO_BACKUP=false
+BACKUP_DIR=./data/backups
+
+
+# --- Cache Settings ---
+ENABLE_CACHE=true
+CACHE_TYPE=memory
+CACHE_TTL=3600
+CACHE_MAX_SIZE=500
+
+
+# --- Logging Settings ---
+LOG_LEVEL=INFO
+LOG_DIR=./logs
+
+
+# --- Performance Settings ---
+MAX_WORKERS=2
+ASYNC_BATCH_SIZE=5
+
+
+# --- RAGAS Evaluation ---
+ENABLE_RAGAS=true
+RAGAS_ENABLE_GROUND_TRUTH=false

.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+venv/
+env/
+ENV/
+
+# Data
+data/
+*.db
+*.faiss
+*.pkl
+
+# DB
+sqlite/
+
+# Logs
+logs/
+*.log
+
+# Environment
+.env
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Test
+notebooks/.ipynb_checkpoints/
+__pycache__/

Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install minimal system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libmagic1 \
+    file \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create necessary directories
+RUN mkdir -p data/uploads data/vector_store data/backups logs
+
+# Set environment variables
+ENV HOST=0.0.0.0
+ENV PORT=7860
+ENV OLLAMA_ENABLED=false
+ENV PYTHONUNBUFFERED=1
+
+# Expose port for HF Spaces
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD python -c "import requests; requests.get('http://localhost:7860/api/health', timeout=5)"
+
+# Start application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--timeout-keep-alive", "120"]

README.md

@@ -0,0 +1,726 @@
+---
+title: QuerySphere
+emoji: 🧠
+colorFrom: indigo
+colorTo: purple
+sdk: docker
+pinned: false
+license: mit
+app_port: 7860
+short_description: RAG platform for document Q&A with zero API costs (local LLM) or cloud deployment with OpenAI integration.
+---
+
+<div align="center">
+
+# QuerySphere: RAG platform for document Q&A with Knowledge Ingestion
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![FastAPI](https://img.shields.io/badge/FastAPI-0.104+-green.svg)](https://fastapi.tiangolo.com/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+> **Enterprise-Grade RAG Platform with Multi-Format Document Ingestion, Hybrid Retrieval, and Zero API Costs**
+
+</div>
+
+---
+
+A MVP grade Retrieval-Augmented Generation (RAG) system that enables organizations to unlock knowledge trapped across documents and archives while maintaining complete data privacy and eliminating costly API dependencies.
+
+---
+
+## 📋 Table of Contents
+
+- [Overview](#-overview)
+- [Key Features](#-key-features)
+- [System Architecture](#-system-architecture)
+- [Technology Stack](#-technology-stack)
+- [Installation](#-installation)
+- [Quick Start](#-quick-start)
+- [Core Components](#-core-components)
+- [API Documentation](#-api-documentation)
+- [Configuration](#-configuration)
+- [RAGAS Evaluation](#-ragas-evaluation)
+- [Troubleshooting](#-troubleshooting)
+- [License](#-license)
+
+---
+
+## 🎯 Overview
+
+The AI Universal Knowledge Ingestion System addresses a critical enterprise pain point: **information silos that cost organizations 20% of employee productivity**. Unlike existing solutions (Humata AI, ChatPDF, NotebookLM) that charge $49/user/month and rely on expensive cloud LLM APIs, this system offers:
+
+### **Core Value Propositions**
+
+| Feature | Traditional Solutions | Our System |
+|---------|----------------------|------------|
+| **Privacy** | Cloud-based (data leaves premises) | 100% on-premise processing |
+| **Cost** | $49-99/user/month + API fees | Zero API costs (local inference) |
+| **Input Types** | PDF only | PDF, DOCX, TXT, ZIP archives |
+| **Quality Metrics** | Black box (no visibility) | RAGAS evaluation with detailed metrics |
+| **Retrieval** | Vector-only | Hybrid (Vector + BM25 + Reranking) |
+| **Chunking** | Fixed size | Adaptive (3 strategies) |
+
+### **Market Context**
+
+- **$8.5B** projected enterprise AI search market by 2027
+- **85%** of enterprises actively adopting AI-powered knowledge management
+- **Growing regulatory demands** for on-premise, privacy-compliant solutions
+
+---
+
+## ✨ Key Features
+
+### **1. Multi-Format Document Ingestion**
+- **Supported Formats**: PDF, DOCX, TXT
+- **Archive Processing**: ZIP files up to 2GB with recursive extraction
+- **Batch Upload**: Process multiple documents simultaneously
+- **OCR Support**: Extract text from scanned documents and images (PaddleOCR or EasyOCR)
+
+### **2. Intelligent Document Processing**
+- **Adaptive Chunking**: Automatically selects optimal strategy based on document size
+  - Fixed-size chunks (< 50K tokens): 512 tokens with 50 overlap
+  - Semantic chunks (50K-500K tokens): Section-aware splitting
+  - Hierarchical chunks (> 500K tokens): Parent-child structure
+- **Metadata Extraction**: Title, author, date, page numbers, section headers
+
+### **3. Hybrid Retrieval System**
+```mermaid
+graph LR
+    A[User Query] --> B[Query Embedding]
+    A --> C[Keyword Analysis]
+    B --> D[Vector Search<br/>FAISS]
+    C --> E[BM25 Search]
+    D --> F[Reciprocal Rank Fusion<br/>60% Vector + 40% BM25]
+    E --> F
+    F --> G[Cross-Encoder Reranking]
+    G --> H[Top-K Results]
+```
+
+- **Vector Search**: FAISS with BGE embeddings (384-dim)
+- **Keyword Search**: BM25 with optimized parameters (k1=1.5, b=0.75)
+- **Fusion Methods**: Weighted, Reciprocal Rank Fusion (RRF), CombSum
+- **Reranking**: Cross-encoder for precision boost
+
+### **4. Local LLM Generation**
+- **Ollama Integration**: Zero-cost inference with Mistral-7B or LLaMA-2
+- **Adaptive Temperature**: Context-aware generation parameters
+- **Citation Tracking**: Automatic source attribution with validation
+- **Streaming Support**: Token-by-token response generation
+
+### **5. RAGAS Quality Assurance**
+- **Real-Time Evaluation**: Answer relevancy, faithfulness, context precision/recall
+- **Automatic Metrics**: Computed for every query-response pair
+- **Analytics Dashboard**: Track quality trends over time
+- **Export Capability**: Download evaluation data for analysis
+- **Session Statistics**: Aggregate metrics across conversation sessions
+
+---
+
+## 🗂️ System Architecture
+
+### **High-Level Architecture**
+
+```mermaid
+graph TB
+    subgraph "Frontend Layer"
+        A[Web UI<br/>HTML/CSS/JS]
+    end
+    
+    subgraph "API Layer"
+        B[FastAPI Gateway<br/>REST Endpoints]
+    end
+    
+    subgraph "Ingestion Pipeline"
+        C[Document Parser<br/>PDF/DOCX/TXT]
+        D[Adaptive Chunker<br/>3 Strategies]
+        E[Embedding Generator<br/>BGE-small-en-v1.5]
+    end
+    
+    subgraph "Storage Layer"
+        F[FAISS Vector DB<br/>~10M vectors]
+        G[BM25 Keyword Index]
+        H[SQLite Metadata<br/>Documents & Chunks]
+        I[LRU Cache<br/>Embeddings]
+    end
+    
+    subgraph "Retrieval Engine"
+        J[Hybrid Retriever<br/>Vector + BM25]
+        K[Cross-Encoder<br/>Reranker]
+        L[Context Assembler]
+    end
+    
+    subgraph "Generation Engine"
+        M[Ollama LLM<br/>Mistral-7B]
+        N[Prompt Builder]
+        O[Citation Formatter]
+    end
+    
+    subgraph "Evaluation Engine"
+        P[RAGAS Evaluator<br/>Quality Metrics]
+    end
+    
+    A --> B
+    B --> C
+    C --> D
+    D --> E
+    E --> F
+    E --> G
+    D --> H
+    E --> I
+    
+    B --> J
+    J --> F
+    J --> G
+    J --> K
+    K --> L
+    
+    L --> N
+    N --> M
+    M --> O
+    O --> A
+    
+    M --> P
+    P --> H
+    P --> A
+```
+
+### **Why This Architecture?**
+
+#### **Modular Design**
+Each component is independent and replaceable:
+- **Parser**: Swap PDF libraries without affecting chunking
+- **Embedder**: Change from BGE to OpenAI embeddings with config update
+- **LLM**: Switch from Ollama to OpenAI API seamlessly
+
+#### **Separation of Concerns**
+```
+Ingestion → Storage → Retrieval → Generation → Evaluation
+```
+Each stage has clear inputs/outputs and single responsibility.
+
+#### **Performance Optimization**
+- **Async Processing**: Non-blocking I/O for uploads and LLM calls
+- **Batch Operations**: Embed 32 chunks simultaneously
+- **Local Caching**: LRU cache for query embeddings and frequent retrievals
+- **Indexing**: FAISS ANN for O(log n) search vs O(n) brute force
+
+---
+
+## 🔧 Technology Stack
+
+### **Core Technologies**
+
+| Component | Technology | Version | Why This Choice |
+|-----------|-----------|---------|-----------------|
+| **Backend** | FastAPI | 0.104+ | Async support, auto-docs, production-grade |
+| **LLM** | Ollama (Mistral-7B) | Latest | Zero API costs, on-premise, 20-30 tokens/sec |
+| **Embeddings** | BGE-small-en-v1.5 | 384-dim | SOTA quality, 10x faster than alternatives |
+| **Vector DB** | FAISS | Latest | Battle-tested, 10x faster than ChromaDB |
+| **Keyword Search** | BM25 (rank_bm25) | Latest | Fast probabilistic ranking |
+| **Document Parsing** | PyPDF2, python-docx | Latest | Industry standard, reliable |
+| **Chunking** | LlamaIndex | 0.9+ | Advanced semantic splitting |
+| **Reranking** | Cross-Encoder | Latest | +15% accuracy, minimal latency |
+| **Evaluation** | RAGAS | 0.1.9 | Automated RAG quality metrics |
+| **Frontend** | Alpine.js | 3.x | Lightweight reactivity, no build step |
+| **Database** | SQLite | 3.x | Zero-config, sufficient for metadata |
+| **Caching** | In-Memory LRU | Python functools | Fast, no external dependencies |
+
+### **Python Dependencies**
+
+```
+fastapi>=0.104.0
+uvicorn>=0.24.0
+ollama>=0.1.0
+sentence-transformers>=2.2.2
+faiss-cpu>=1.7.4
+llama-index>=0.9.0
+rank-bm25>=0.2.2
+PyPDF2>=3.0.0
+python-docx>=0.8.11
+pydantic>=2.0.0
+aiohttp>=3.9.0
+tiktoken>=0.5.0
+ragas==0.1.9
+datasets==2.14.6
+```
+
+---
+
+## 📦 Installation
+
+### **Prerequisites**
+
+- Python 3.10 or higher
+- 8GB RAM minimum (16GB recommended)
+- 10GB disk space for models and indexes
+- Ollama installed ([https://ollama.ai](https://ollama.ai))
+
+### **Step 1: Clone Repository**
+
+```bash
+git clone https://github.com/satyaki-mitra/docu-vault-ai.git
+cd docu-vault-ai
+```
+
+### **Step 2: Create Virtual Environment**
+
+```bash
+# Using conda (recommended)
+conda create -n rag_env python=3.10
+conda activate rag_env
+
+# Or using venv
+python -m venv rag_env
+source rag_env/bin/activate  # On Windows: rag_env\Scripts\activate
+```
+
+### **Step 3: Install Dependencies**
+
+```bash
+pip install -r requirements.txt
+```
+
+### **Step 4: Install Ollama and Model**
+
+```bash
+# Install Ollama (macOS)
+brew install ollama
+
+# Install Ollama (Linux)
+curl https://ollama.ai/install.sh | sh
+
+# Pull Mistral model
+ollama pull mistral:7b
+
+# Verify installation
+ollama list
+```
+
+### **Step 5: Configure Environment**
+
+```bash
+# Copy example config
+cp .env.example .env
+
+# Edit configuration (optional)
+nano .env
+```
+
+**Key Configuration Options:**
+
+```bash
+# LLM Settings
+OLLAMA_MODEL=mistral:7b
+DEFAULT_TEMPERATURE=0.1
+CONTEXT_WINDOW=8192
+
+# Retrieval Settings
+VECTOR_WEIGHT=0.6
+BM25_WEIGHT=0.4
+ENABLE_RERANKING=True
+TOP_K_RETRIEVE=10
+
+# RAGAS Evaluation
+ENABLE_RAGAS=True
+RAGAS_ENABLE_GROUND_TRUTH=False
+OPENAI_API_KEY=your_openai_api_key_here  # Required for RAGAS
+
+# Performance
+EMBEDDING_BATCH_SIZE=32
+MAX_WORKERS=4
+```
+
+---
+
+## 🚀 Quick Start
+
+### **1. Start Ollama Server**
+
+```bash
+# Terminal 1: Start Ollama
+ollama serve
+```
+
+### **2. Launch Application**
+
+```bash
+# Terminal 2: Start RAG system
+python app.py
+```
+
+Output:
+```
+INFO:     Started server process [12345]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:8000
+```
+
+### **3. Access Web Interface**
+
+Open browser to: **http://localhost:8000**
+
+### **4. Upload Documents**
+
+1. Click **"Upload Documents"**
+2. Select PDF/DOCX/TXT files (or ZIP archives)
+3. Click **"Start Building"**
+4. Wait for indexing to complete (progress bar shows status)
+
+### **5. Query Your Documents**
+
+```
+Query: "What are the key findings in the Q3 report?"
+
+Response: The Q3 report highlights three key findings: 
+[1] Revenue increased 23% year-over-year to $45.2M, 
+[2] Customer acquisition costs decreased 15%, and 
+[3] Net retention rate reached 118% [1].
+
+Sources:
+[1] Q3_Financial_Report.pdf (Page 3, Executive Summary)
+
+RAGAS Metrics:
+- Answer Relevancy: 0.89
+- Faithfulness: 0.94
+- Context Utilization: 0.87
+- Overall Score: 0.90
+```
+
+---
+
+## 🧩 Core Components
+
+### **1. Document Ingestion Pipeline**
+
+```python
+# High-level flow
+Document Upload → Parse → Clean → Chunk → Embed → Index
+```
+
+**Adaptive Chunking Logic:**
+
+```mermaid
+graph TD
+    A[Calculate Token Count] --> B{Tokens < 50K?}
+    B -->|Yes| C[Fixed Chunking<br/>512 tokens, 50 overlap]
+    B -->|No| D{Tokens < 500K?}
+    D -->|Yes| E[Semantic Chunking<br/>Section-aware]
+    D -->|No| F[Hierarchical Chunking<br/>Parent 2048, Child 512]
+```
+
+### **2. Hybrid Retrieval Engine**
+
+**Retrieval Flow:**
+
+```python
+# Pseudocode
+def hybrid_retrieve(query: str, top_k: int = 10):
+    # Dual retrieval
+    query_embedding = embedder.embed(query)
+    vector_results  = faiss_index.search(query_embedding, top_k * 2)
+    bm25_results    = bm25_index.search(query, top_k * 2)
+    
+    # Fusion (RRF)
+    fused_results   = reciprocal_rank_fusion(vector_results, 
+                                             bm25_results,
+                                             weights = (0.6, 0.4))
+    
+    # Reranking
+    reranked        = cross_encoder.rerank(query, fused_results, top_k)
+    
+    return reranked
+```
+
+### **3. Response Generation**
+
+**Temperature Control:**
+
+```mermaid
+graph LR
+    A[Query Type] --> B{Factual?}
+    B -->|Yes| C[Low Temp<br/>0.1-0.2]
+    B -->|No| D[Context Quality]
+    D -->|High| E[Medium Temp<br/>0.3-0.5]
+    D -->|Low| F[High Temp<br/>0.6-0.8]
+```
+
+### **4. RAGAS Evaluation Module**
+
+**Automatic Quality Assessment:**
+
+```python
+# After each query-response
+ragas_result = ragas_evaluator.evaluate_single(query              = user_query,
+                                               answer             = generated_answer,
+                                               contexts           = retrieved_chunks,
+                                               retrieval_time_ms  = retrieval_time,
+                                               generation_time_ms = generation_time,
+                                              )
+
+# Metrics computed:
+- Answer Relevancy (0-1)
+- Faithfulness (0-1)
+- Context Utilization (0-1)
+- Context Relevancy (0-1)
+- Overall Score (weighted average)
+```
+
+---
+
+## 📚 API Documentation
+
+### **Core Endpoints**
+
+#### **1. Health Check**
+
+```bash
+GET /api/health
+```
+
+**Response:**
+```json
+{
+  "status": "healthy",
+  "timestamp": "2024-11-27T03:00:00",
+  "components": {
+    "vector_store": true,
+    "llm": true,
+    "embeddings": true,
+    "retrieval": true
+  }
+}
+```
+
+#### **2. Upload Documents**
+
+```bash
+POST /api/upload
+Content-Type: multipart/form-data
+
+files: [file1.pdf, file2.docx]
+```
+
+#### **3. Start Processing**
+
+```bash
+POST /api/start-processing
+```
+
+#### **4. Query (Chat)**
+
+```bash
+POST /api/chat
+Content-Type: application/json
+
+{
+  "message": "What are the revenue figures?",
+  "session_id": "session_123"
+}
+```
+
+**Response includes RAGAS metrics:**
+```json
+{
+  "session_id": "session_123",
+  "response": "Revenue for Q3 was $45.2M [1]...",
+  "sources": [...],
+  "metrics": {
+    "retrieval_time": 245,
+    "generation_time": 3100,
+    "total_time": 3350
+  },
+  "ragas_metrics": {
+    "answer_relevancy": 0.89,
+    "faithfulness": 0.94,
+    "context_utilization": 0.87,
+    "context_relevancy": 0.91,
+    "overall_score": 0.90
+  }
+}
+```
+
+#### **5. RAGAS Endpoints**
+
+```bash
+# Get evaluation history
+GET /api/ragas/history
+
+# Get session statistics
+GET /api/ragas/statistics
+
+# Clear evaluation history
+POST /api/ragas/clear
+
+# Export evaluation data
+GET /api/ragas/export
+
+# Get RAGAS configuration
+GET /api/ragas/config
+```
+
+---
+
+## ⚙️ Configuration
+
+### **config/settings.py**
+
+**Key Configuration Sections:**
+
+#### **LLM Settings**
+```python
+OLLAMA_MODEL        = "mistral:7b"
+DEFAULT_TEMPERATURE = 0.1
+MAX_TOKENS          = 1000
+CONTEXT_WINDOW      = 8192
+```
+
+#### **RAGAS Settings**
+```python
+ENABLE_RAGAS              = True
+RAGAS_ENABLE_GROUND_TRUTH = False
+RAGAS_METRICS             = ["answer_relevancy",
+                             "faithfulness",
+                             "context_utilization",
+                             "context_relevancy"
+                            ]
+RAGAS_EVALUATION_TIMEOUT  = 60
+RAGAS_BATCH_SIZE          = 10
+```
+
+#### **Caching Settings**
+```python
+ENABLE_EMBEDDING_CACHE = True
+CACHE_MAX_SIZE         = 1000  # LRU cache size
+CACHE_TTL              = 3600  # Time to live in seconds
+```
+
+---
+
+## 📊 RAGAS Evaluation
+
+### **What is RAGAS?**
+
+RAGAS (Retrieval-Augmented Generation Assessment) is a framework for evaluating RAG systems using automated metrics. Our implementation provides real-time quality assessment for every query-response pair.
+
+### **Metrics Explained**
+
+| Metric | Definition | Target | Interpretation |
+|--------|-----------|--------|----------------|
+| **Answer Relevancy** | How well the answer addresses the question | > 0.85 | Measures usefulness to user |
+| **Faithfulness** | Is the answer grounded in retrieved context? | > 0.90 | Prevents hallucinations |
+| **Context Utilization** | How well the context is used in the answer | > 0.80 | Retrieval effectiveness |
+| **Context Relevancy** | Are retrieved chunks relevant to the query? | > 0.85 | Search quality |
+| **Overall Score** | Weighted average of all metrics | > 0.85 | System performance |
+
+### **Using the Analytics Dashboard**
+
+1. Navigate to **Analytics & Quality** section
+2. View real-time RAGAS metrics table
+3. Monitor session statistics (averages, trends)
+4. Export evaluation data for offline analysis
+
+### **Example Evaluation Output**
+
+```
+Query: "What were the Q3 revenue trends?"
+Answer: "Q3 revenue increased 23% YoY to $45.2M..."
+
+RAGAS Evaluation:
+├─ Answer Relevancy: 0.89 ✓ (Good)
+├─ Faithfulness: 0.94 ✓ (Excellent)
+├─ Context Utilization: 0.87 ✓ (Good)
+├─ Context Relevancy: 0.91 ✓ (Excellent)
+└─ Overall Score: 0.90 ✓ (Excellent)
+
+Performance:
+├─ Retrieval Time: 245ms
+├─ Generation Time: 3100ms
+└─ Total Time: 3345ms
+```
+
+---
+
+## 🔧 Troubleshooting
+
+### **Common Issues**
+
+#### **1. "RAGAS evaluation failed"**
+
+**Cause:** OpenAI API key not configured
+
+**Solution:**
+```bash
+# Add to .env file
+OPENAI_API_KEY=your_openai_api_key_here
+
+# Or disable RAGAS if not needed
+ENABLE_RAGAS=False
+```
+
+#### **2. "Context assembly returning 0 chunks"**
+
+**Cause:** Missing token counts in chunks
+
+**Solution:** Already fixed in `context_assembler.py`. Tokens calculated on-the-fly if missing.
+
+#### **3. "Slow query responses"**
+
+**Solutions:**
+- Enable embedding cache : `ENABLE_EMBEDDING_CACHE=True`
+- Reduce retrieval count : `TOP_K_RETRIEVE=5`
+- Disable reranking      : `ENABLE_RERANKING=False`
+
+- Use quantized model for faster inference
+
+#### **4. "RAGAS metrics not appearing"**
+
+**Symptoms:** Chat responses lack quality metrics
+
+**Solution:**
+```python
+# Verify RAGAS is enabled in settings
+ENABLE_RAGAS = True
+
+# Check OpenAI API key is valid
+# View logs for RAGAS evaluation errors
+tail -f logs/app.log | grep "RAGAS"
+```
+
+---
+
+## 📄 License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+---
+
+## 🙏 Acknowledgments
+
+**Open Source Technologies:**
+- [FastAPI](https://fastapi.tiangolo.com/) - Modern web framework
+- [Ollama](https://ollama.ai/) - Local LLM inference
+- [FAISS](https://github.com/facebookresearch/faiss) - Vector similarity search
+- [LlamaIndex](https://www.llamaindex.ai/) - Document chunking
+- [Sentence Transformers](https://www.sbert.net/) - Embedding models
+- [RAGAS](https://github.com/explodinggradients/ragas) - RAG evaluation
+
+**Research Papers:**
+- Karpukhin et al. (2020) - Dense Passage Retrieval
+- Robertson & Zaragoza (2009) - The Probabilistic Relevance Framework: BM25
+- Lewis et al. (2020) - Retrieval-Augmented Generation
+- Es et al. (2023) - RAGAS: Automated Evaluation of RAG
+
+---
+
+## 👤 Author
+
+Satyaki Mitra | Data Scientist | Generative-AI Enthusiast
+
+---
+
+<div align="center">
+
+**Built with ❤️ for the open-source community**
+
+</div>

app.py

@@ -0,0 +1,2224 @@
+# DEPENDENCIES
+import os
+import gc
+import io
+import csv
+import json
+import time
+import signal
+import atexit
+import shutil
+import asyncio
+import logging
+import uvicorn
+import tempfile
+import traceback
+import threading
+from typing import Set
+from typing import Any
+from typing import List
+from typing import Dict
+from pathlib import Path
+from typing import Tuple
+from fastapi import File
+from fastapi import Form
+from signal import SIGINT
+from signal import SIGTERM
+from pydantic import Field
+from fastapi import FastAPI
+from typing import Optional
+from datetime import datetime
+from datetime import timedelta
+from fastapi import UploadFile
+from pydantic import BaseModel
+from fastapi import HTTPException
+from config.models import PromptType
+from config.models import ChatRequest
+from config.models import LLMProvider
+from utils.helpers import IDGenerator
+from config.models import QueryRequest 
+from config.settings import get_settings
+from config.models import RAGASStatistics
+from config.models import RAGASExportData
+from config.models import DocumentMetadata
+from fastapi.responses import HTMLResponse
+from fastapi.responses import FileResponse
+from fastapi.responses import JSONResponse
+from contextlib import asynccontextmanager
+from utils.file_handler import FileHandler
+from utils.validators import FileValidator
+from fastapi.staticfiles import StaticFiles
+from utils.error_handler import RAGException
+from utils.error_handler import FileException
+from config.models import RAGASEvaluationResult
+from config.logging_config import setup_logging
+from generation.llm_client import get_llm_client
+from embeddings.bge_embedder import get_embedder
+from concurrent.futures import ThreadPoolExecutor
+from ingestion.router import get_ingestion_router
+from utils.validators import validate_upload_file
+from fastapi.middleware.cors import CORSMiddleware
+from vector_store.index_builder import get_index_builder
+from document_parser.parser_factory import ParserFactory
+from evaluation.ragas_evaluator import get_ragas_evaluator
+from vector_store.metadata_store import get_metadata_store
+from embeddings.embedding_cache import get_embedding_cache
+from ingestion.progress_tracker import get_progress_tracker
+from retrieval.hybrid_retriever import get_hybrid_retriever
+from chunking.adaptive_selector import get_adaptive_selector
+from retrieval.context_assembler import get_context_assembler
+from document_parser.parser_factory import get_parser_factory
+from chunking.adaptive_selector import AdaptiveChunkingSelector 
+from generation.response_generator import get_response_generator
+from config.models import ProcessingStatus as ProcessingStatusEnum
+
+
+# Setup logging and settings
+settings = get_settings()
+logger   = setup_logging(log_level      = settings.LOG_LEVEL,
+                         log_dir        = settings.LOG_DIR,
+                         enable_console = True,
+                         enable_file    = True,
+                        )
+
+
+# Global Cleanup Variables 
+_cleanup_registry : Set[str] = set()
+_cleanup_lock                = threading.RLock()
+_is_cleaning                 = False
+_cleanup_executor            = ThreadPoolExecutor(max_workers        = 2, 
+                                                  thread_name_prefix = "cleanup_",
+                                                 )
+
+
+# Analytics Cache Structure
+class AnalyticsCache:
+    """
+    Cache for analytics data to avoid recalculating on every request
+    """
+    def __init__(self, ttl_seconds: int = 30):
+        self.data            = None
+        self.last_calculated = None
+        self.ttl_seconds     = ttl_seconds
+        self.is_calculating  = False
+    
+
+    def is_valid(self) -> bool:
+        """
+        Check if cache is still valid
+        """
+        if self.data is None or self.last_calculated is None:
+            return False
+        
+        elapsed = (datetime.now() - self.last_calculated).total_seconds()
+        
+        return (elapsed < self.ttl_seconds)
+    
+
+    def update(self, data: Dict):
+        """
+        Update cache with new data
+        """
+        self.data            = data
+        self.last_calculated = datetime.now()
+    
+    
+    def get(self) -> Optional[Dict]:
+        """
+        Get cached data if valid
+        """
+        return self.data if self.is_valid() else None
+
+
+class CleanupManager:
+    """
+    Centralized cleanup manager with multiple redundancy layers
+    """
+    @staticmethod
+    def register_resource(resource_id: str, cleanup_func, *args, **kwargs):
+        """
+        Register a resource for cleanup
+        """
+        with _cleanup_lock:
+            _cleanup_registry.add(resource_id)
+        
+        # Register with atexit for process termination
+        atexit.register(cleanup_func, *args, **kwargs)
+        
+        return resource_id
+    
+
+    @staticmethod
+    def unregister_resource(resource_id: str):
+        """
+        Unregister a resource (if already cleaned up elsewhere)
+        """
+        with _cleanup_lock:
+            if resource_id in _cleanup_registry:
+                _cleanup_registry.remove(resource_id)
+    
+
+    @staticmethod
+    async def full_cleanup(state: Optional['AppState'] = None) -> bool:
+        """
+        Perform full system cleanup with redundancy
+        """
+        global _is_cleaning
+        
+        with _cleanup_lock:
+            if _is_cleaning:
+                logger.warning("Cleanup already in progress")
+                return False
+            
+            _is_cleaning = True
+        
+        try:
+            logger.info("Starting comprehensive system cleanup...")
+            
+            # Layer 1: Memory cleanup
+            success1 = await CleanupManager._cleanup_memory(state)
+            
+            # Layer 2: Disk cleanup (async to not block)
+            success2 = await CleanupManager._cleanup_disk_async()
+            
+            # Layer 3: Component cleanup
+            success3 = await CleanupManager._cleanup_components(state)
+            
+            # Layer 4: External resources
+            success4 = CleanupManager._cleanup_external_resources()
+            
+            # Clear registry
+            with _cleanup_lock:
+                _cleanup_registry.clear()
+            
+            overall_success = all([success1, success2, success3, success4])
+            
+            if overall_success:
+                logger.info("Comprehensive cleanup completed successfully")
+            
+            else:
+                logger.warning("Cleanup completed with some failures")
+            
+            return overall_success
+            
+        except Exception as e:
+            logger.error(f"Cleanup failed catastrophically: {e}", exc_info=True)
+            return False
+        
+        finally:
+            with _cleanup_lock:
+                _is_cleaning = False
+    
+    @staticmethod
+    async def _cleanup_memory(state: Optional['AppState']) -> bool:
+        """
+        Memory cleanup
+        """
+        try:
+            if not state:
+                logger.warning("No AppState provided for memory cleanup")
+                return True
+            
+            # Session cleanup
+            session_count = len(state.active_sessions)
+            state.active_sessions.clear()
+            state.config_overrides.clear()
+            logger.info(f"Cleared {session_count} sessions from memory")
+            
+            # Document data cleanup
+            doc_count   = len(state.processed_documents)
+            chunk_count = sum(len(chunks) for chunks in state.document_chunks.values())
+
+            state.processed_documents.clear()
+            state.document_chunks.clear()
+            state.uploaded_files.clear()
+            logger.info(f"Cleared {doc_count} documents ({chunk_count} chunks) from memory")
+            
+            # Performance data cleanup
+            state.query_timings.clear()
+            state.chunking_statistics.clear()
+            
+            # State reset
+            state.is_ready          = False
+            state.processing_status = "idle"
+            
+            # Cache cleanup
+            if hasattr(state, 'analytics_cache'):
+                state.analytics_cache.data = None
+            
+            # Force garbage collection
+            collected = gc.collect()
+            logger.debug(f"🧹 Garbage collection freed {collected} objects")
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Memory cleanup failed: {e}")
+            return False
+    
+    @staticmethod
+    async def _cleanup_disk_async() -> bool:
+        """
+        Asynchronous disk cleanup
+        """
+        try:
+            # Run in thread pool to avoid blocking
+            loop    = asyncio.get_event_loop()
+            success = await loop.run_in_executor(_cleanup_executor, CleanupManager._cleanup_disk_sync)
+
+            return success
+
+        except Exception as e:
+            logger.error(f"Async disk cleanup failed: {e}")
+            return False
+    
+
+    @staticmethod
+    def _cleanup_disk_sync() -> bool:
+        """
+        Synchronous disk cleanup
+        """
+        try:
+            logger.info("Starting disk cleanup...")
+            
+            # Track what we clean
+            cleaned_paths = list()
+            
+            # Vector store directory
+            if settings.VECTOR_STORE_DIR.exists():
+                vector_files = list(settings.VECTOR_STORE_DIR.glob("*"))
+                for file in vector_files:
+                    try:
+                        if file.is_file():
+                            file.unlink()
+                            cleaned_paths.append(str(file))
+                        
+                        elif file.is_dir():
+                            shutil.rmtree(file)
+                            cleaned_paths.append(str(file))
+                    
+                    except Exception as e:
+                        logger.warning(f"Failed to delete {file}: {e}")
+                
+                logger.info(f"Cleaned {len(cleaned_paths)} vector store files")
+            
+            # Upload directory (preserve directory structure)
+            if settings.UPLOAD_DIR.exists():
+                upload_files = list(settings.UPLOAD_DIR.glob("*"))
+                for file in upload_files:
+                    try:
+                        if file.is_file():
+                            file.unlink()
+                            cleaned_paths.append(str(file))
+                        
+                        elif file.is_dir():
+                            shutil.rmtree(file)
+                            cleaned_paths.append(str(file))
+                    
+                    except Exception as e:
+                        logger.warning(f"Failed to delete {file}: {e}")
+                
+                # Recreate empty directory
+                settings.UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+                logger.info(f"Cleaned {len(upload_files)} uploaded files")
+            
+            # Metadata database
+            metadata_path = Path(settings.METADATA_DB_PATH)
+            if metadata_path.exists():
+                try:
+                    metadata_path.unlink(missing_ok=True)
+                    cleaned_paths.append(str(metadata_path))
+                    logger.info("Cleaned metadata database")
+                
+                except Exception as e:
+                    logger.warning(f"Failed to delete metadata DB: {e}")
+            
+            # Backup directory
+            if settings.BACKUP_DIR.exists():
+                backup_files = list(settings.BACKUP_DIR.glob("*"))
+                for file in backup_files:
+                    try:
+                        if file.is_file():
+                            file.unlink()
+                        
+                        elif file.is_dir():
+                            shutil.rmtree(file)
+                    
+                    except:
+                        pass  # Silently fail for backups
+                logger.info(f"Cleaned {len(backup_files)} backup files")
+            
+            # Temp files cleanup
+            CleanupManager._cleanup_temp_files()
+            
+            logger.info(f"Disk cleanup completed: {len(cleaned_paths)} items cleaned")
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Disk cleanup failed: {e}")
+            return False
+    
+
+    @staticmethod
+    def _cleanup_temp_files():
+        """
+        Clean up temporary files
+        """
+        temp_dir = tempfile.gettempdir()
+        
+        # Clean our specific temp files (if any)
+        for pattern in ["rag_*", "faiss_*", "embedding_*"]:
+            for file in Path(temp_dir).glob(pattern):
+                try:
+                    file.unlink(missing_ok=True)
+                except:
+                    pass
+    
+
+    @staticmethod
+    async def _cleanup_components(state: Optional['AppState']) -> bool:
+        """
+        Component-specific cleanup
+        """
+        try:
+            if not state:
+                return True
+            
+            components_cleaned = 0
+            
+            # Vector store components
+            if state.index_builder:
+                try:
+                    state.index_builder.clear_indexes()
+                    components_cleaned += 1
+                
+                except Exception as e:
+                    logger.warning(f"Index builder cleanup failed: {e}")
+            
+            if state.metadata_store and hasattr(state.metadata_store, 'clear_all'):
+                try:
+                    state.metadata_store.clear_all()
+                    components_cleaned += 1
+                
+                except Exception as e:
+                    logger.warning(f"Metadata store cleanup failed: {e}")
+            
+            # RAGAS evaluator
+            if state.ragas_evaluator and hasattr(state.ragas_evaluator, 'clear_history'):
+                try:
+                    state.ragas_evaluator.clear_history()
+                    components_cleaned += 1
+                
+                except Exception as e:
+                    logger.warning(f"RAGAS evaluator cleanup failed: {e}")
+            
+            logger.info(f"Cleaned {components_cleaned} components")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Component cleanup failed: {e}")
+            return False
+    
+
+    @staticmethod
+    def _cleanup_external_resources() -> bool:
+        """
+        External resource cleanup
+        """
+        try:
+            # Close database connections
+            CleanupManager._close_db_connections()
+            
+            # Clean up thread pool
+            _cleanup_executor.shutdown(wait = False)
+            
+            logger.info("External resources cleaned")
+            return True
+            
+        except Exception as e:
+            logger.error(f"External resource cleanup failed: {e}")
+            return False
+    
+
+    @staticmethod
+    def _close_db_connections():
+        """
+        Close any open database connections
+        """
+        try:
+            # SQLite handles this automatically in most cases
+            pass
+        except:
+            pass
+
+    
+    @staticmethod
+    def handle_signal(signum, frame):
+        """
+        Signal handler for graceful shutdown
+        """
+        global _is_cleaning
+        
+        # If already cleaning up, don't raise KeyboardInterrupt
+        with _cleanup_lock:
+            if _is_cleaning:
+                logger.info(f"Signal {signum} received during cleanup - ignoring")
+                return
+            
+        if (signum == SIGINT):
+            logger.info("Ctrl+C received - shutdown initiated")
+            raise KeyboardInterrupt
+        
+        elif (signum == SIGTERM):
+            logger.info("SIGTERM received - shutdown initiated")
+            # Just log, not scheduling anything
+        
+        else:
+            logger.info(f"Signal {signum} received")
+
+
+# Global state manager
+class AppState:
+    """
+    Manages application state and components
+    """
+    def __init__(self):
+        self.is_ready            = False
+        self.processing_status   = "idle"
+        self.uploaded_files      = list()
+        self.active_sessions     = dict()
+        self.processed_documents = dict()
+        self.document_chunks     = dict()
+        
+        # Performance tracking
+        self.query_timings       = list()  
+        self.chunking_statistics = dict()
+        
+        # Core components
+        self.file_handler        = None
+        self.parser_factory      = None
+        self.chunking_selector   = None
+        
+        # Embeddings components
+        self.embedder            = None
+        self.embedding_cache     = None
+        
+        # Ingestion components
+        self.ingestion_router    = None
+        self.progress_tracker    = None
+        
+        # Vector store components
+        self.index_builder       = None
+        self.metadata_store      = None
+        
+        # Retrieval components
+        self.hybrid_retriever    = None
+        self.context_assembler   = None
+        
+        # Generation components
+        self.response_generator  = None
+        self.llm_client          = None
+        
+        # RAGAS component
+        self.ragas_evaluator     = None
+
+        # Processing tracking
+        self.current_processing  = None
+        self.processing_progress = {"status"       : "idle",
+                                    "current_step" : "Waiting",
+                                    "progress"     : 0,
+                                    "processed"    : 0,
+                                    "total"        : 0,
+                                    "details"      : {},
+                                   }
+
+        # Session-based configuration overrides
+        self.config_overrides    = dict()
+        
+        # Analytics cache
+        self.analytics_cache     = AnalyticsCache(ttl_seconds = 30)
+        
+        # System start time
+        self.start_time          = datetime.now()
+
+        # Add cleanup tracking 
+        self._cleanup_registered = False
+        self._cleanup_resources  = list()
+        
+        # Register with cleanup manager
+        self._register_for_cleanup()
+
+
+    def _register_for_cleanup(self):
+        """
+        Register this AppState instance for cleanup
+        """
+        if not self._cleanup_registered:
+            resource_id              = f"appstate_{id(self)}"
+
+            CleanupManager.register_resource(resource_id, self._emergency_cleanup)
+            self._cleanup_resources.append(resource_id)
+            
+            self._cleanup_registered = True
+    
+
+    def _emergency_cleanup(self):
+        """
+        Emergency cleanup if regular cleanup fails
+        """
+        try:
+            logger.warning("Performing emergency cleanup...")
+            
+            # Brutal but effective memory clearing
+            for attr in ['active_sessions', 'processed_documents', 'document_chunks', 'uploaded_files', 'query_timings', 'chunking_statistics']:
+                if hasattr(self, attr):
+                    getattr(self, attr).clear()
+            
+            # Nullify heavy objects
+            self.index_builder  = None
+            self.metadata_store = None
+            self.embedder       = None
+            
+            logger.warning("Emergency cleanup completed")
+        
+        except:
+            # Last resort - don't crash during emergency cleanup
+            pass  
+    
+
+    async def graceful_shutdown(self):
+        """
+        Graceful shutdown procedure
+        """
+        logger.info("Starting graceful shutdown...")
+        
+        # Notify clients (if any WebSocket connections)
+        await self._notify_clients()
+        
+        # Perform cleanup
+        await CleanupManager.full_cleanup(self)
+        
+        # Unregister from cleanup manager
+        for resource_id in self._cleanup_resources:
+            CleanupManager.unregister_resource(resource_id)
+        
+        logger.info("Graceful shutdown completed")
+    
+
+    async def _notify_clients(self):
+        """
+        Notify connected clients of shutdown
+        """
+        # Placeholder for WebSocket notifications
+        pass
+    
+
+    def add_query_timing(self, duration_ms: float):
+        """
+        Record query timing for analytics
+        """
+        self.query_timings.append((datetime.now(), duration_ms))
+        # Keep only last 1000 timings to prevent memory issues
+        if (len(self.query_timings) > 1000):
+            self.query_timings = self.query_timings[-1000:]
+    
+
+    def get_performance_metrics(self) -> Dict:
+        """
+        Calculate performance metrics from recorded timings
+        """
+        if not self.query_timings:
+            return {"avg_response_time" : 0,
+                    "min_response_time" : 0,
+                    "max_response_time" : 0,
+                    "total_queries"     : 0,
+                    "queries_last_hour" : 0,
+                   }
+        
+
+        # Get recent timings (last hour)
+        one_hour_ago   = datetime.now() - timedelta(hours = 1)
+        recent_timings = [t for t, _ in self.query_timings if (t > one_hour_ago)]
+        
+        # Calculate statistics
+        durations      = [duration for _, duration in self.query_timings]
+        
+        return {"avg_response_time" : int(sum(durations) / len(durations)),
+                "min_response_time" : int(min(durations)) if durations else 0,
+                "max_response_time" : int(max(durations)) if durations else 0,
+                "total_queries"     : len(self.query_timings),
+                "queries_last_hour" : len(recent_timings),
+                "p95_response_time" : int(sorted(durations)[int(len(durations) * 0.95)]) if (len(durations) > 10) else 0,
+               }
+
+    
+    def get_chunking_statistics(self) -> Dict:
+        """
+        Get statistics about chunking strategies used
+        """
+        if not self.chunking_statistics:
+            return {"primary_strategy" : "adaptive",
+                    "total_chunks"     : 0,
+                    "avg_chunk_size"   : 0,
+                    "strategies_used"  : {},
+                   }
+        
+        total_chunks = sum(self.chunking_statistics.values())
+        strategies   = {k: v for k, v in self.chunking_statistics.items() if (v > 0)}
+        
+        return {"primary_strategy" : max(strategies.items(), key=lambda x: x[1])[0] if strategies else "adaptive",
+                "total_chunks"     : total_chunks,
+                "strategies_used"  : strategies,
+               }
+
+    
+    def get_system_health(self) -> Dict:
+        """
+        Get comprehensive system health status
+        """
+        llm_healthy        = self.llm_client.check_health() if self.llm_client else False
+        vector_store_ready = self.is_ready
+        
+        # Check embedding model
+        embedding_ready    = self.embedder is not None
+        
+        # Check retrieval components
+        retrieval_ready    = (self.hybrid_retriever is not None and self.context_assembler is not None)
+        
+        # Determine overall status
+        if all([llm_healthy, vector_store_ready, embedding_ready, retrieval_ready]):
+            overall_status = "healthy"
+        
+        elif vector_store_ready and embedding_ready and retrieval_ready:
+            # LLM issues but RAG works
+            overall_status = "degraded"  
+        
+        else:
+            overall_status = "unhealthy"
+        
+        return {"overall"      : overall_status,
+                "llm"          : llm_healthy,
+                "vector_store" : vector_store_ready,
+                "embeddings"   : embedding_ready,
+                "retrieval"    : retrieval_ready,
+                "generation"   : self.response_generator is not None,
+               }
+
+    
+    def get_system_information(self) -> Dict:
+        """
+        Get current system information
+        """
+        # Get chunking strategy
+        chunking_strategy = "adaptive"
+        
+        if self.chunking_selector:
+            try:
+                # Try to get strategy from selector
+                if (hasattr(self.chunking_selector, 'get_current_strategy')):
+                    chunking_strategy = self.chunking_selector.get_current_strategy()
+                
+                elif (hasattr(self.chunking_selector, 'prefer_llamaindex')):
+                    chunking_strategy = "llama_index" if self.chunking_selector.prefer_llamaindex else "adaptive"
+            
+            except:
+                pass
+        
+        # Get vector store status
+        vector_store_status = "Not Ready"
+
+        if self.is_ready:
+            try:
+                index_stats  = self.index_builder.get_index_stats() if self.index_builder else {}
+                total_chunks = index_stats.get('total_chunks_indexed', 0)
+                
+                if (total_chunks > 0):
+                    vector_store_status = f"Ready ({total_chunks} chunks)"
+                
+                else:
+                    vector_store_status = "Empty"
+            
+            except:
+                vector_store_status = "Ready"
+        
+        # Get model info
+        current_model   = settings.OLLAMA_MODEL
+        embedding_model = settings.EMBEDDING_MODEL
+        
+        # Uptime
+        uptime_seconds  = (datetime.now() - self.start_time).total_seconds()
+        
+        return {"vector_store_status"   : vector_store_status,
+                "current_model"         : current_model,
+                "embedding_model"       : embedding_model,
+                "chunking_strategy"     : chunking_strategy,
+                "system_uptime_seconds" : int(uptime_seconds),
+                "last_updated"          : datetime.now().isoformat(),
+               }
+    
+
+    def calculate_quality_metrics(self) -> Dict:
+        """
+        Calculate quality metrics for the system
+        """
+        total_queries = 0
+        total_sources = 0
+        source_counts = list()
+        
+        # Analyze session data
+        for session_id, messages in self.active_sessions.items():
+            total_queries += len(messages)
+            
+            for msg in messages:
+                sources        = len(msg.get('sources', []))
+                total_sources += sources
+
+                source_counts.append(sources)
+        
+        # Calculate averages
+        avg_sources_per_query = total_sources / total_queries if total_queries > 0 else 0
+        
+        # Calculate metrics based on heuristics
+        # These are simplified - for production, use RAGAS or similar framework
+        
+        if (total_queries == 0):
+            return {"answer_relevancy"  : 0.0,
+                    "faithfulness"      : 0.0,
+                    "context_precision" : 0.0,
+                    "context_recall"    : None,
+                    "overall_score"     : 0.0,
+                    "confidence"        : "low",
+                    "metrics_available" : False
+                   }
+        
+        # Heuristic calculations
+        answer_relevancy  = min(0.9, 0.7 + (avg_sources_per_query * 0.1))
+        faithfulness      = min(0.95, 0.8 + (avg_sources_per_query * 0.05))
+        context_precision = min(0.85, 0.6 + (avg_sources_per_query * 0.1))
+        
+        # Overall score weighted average
+        overall_score     = (answer_relevancy * 0.4 + faithfulness * 0.3 + context_precision * 0.3)
+        
+        confidence        = "high" if total_queries > 10 else ("medium" if (total_queries > 3) else "low")
+        
+        return {"answer_relevancy"      : round(answer_relevancy, 3),
+                "faithfulness"          : round(faithfulness, 3),
+                "context_precision"     : round(context_precision, 3),
+                "context_recall"        : None,  # Requires ground truth
+                "overall_score"         : round(overall_score, 3),
+                "avg_sources_per_query" : round(avg_sources_per_query, 2),
+                "queries_with_sources"  : sum(1 for count in source_counts if count > 0),
+                "confidence"            : confidence,
+                "metrics_available"     : True,
+                "evaluation_note"       : "Metrics are heuristic estimates. For accurate evaluation, use RAGAS framework.",
+               }
+
+    
+    def calculate_comprehensive_analytics(self) -> Dict:
+        """
+        Calculate comprehensive analytics data
+        """
+        # Performance metrics
+        performance     = self.get_performance_metrics()
+        
+        # System information
+        system_info     = self.get_system_information()
+        
+        # Quality metrics
+        quality_metrics = self.calculate_quality_metrics()
+        
+        # Health status
+        health_status   = self.get_system_health()
+        
+        # Chunking statistics
+        chunking_stats  = self.get_chunking_statistics()
+        
+        # Document statistics
+        total_docs      = len(self.processed_documents)
+        total_chunks    = sum(len(chunks) for chunks in self.document_chunks.values())
+        
+        # Session statistics
+        total_sessions  = len(self.active_sessions)
+        total_messages  = sum(len(msgs) for msgs in self.active_sessions.values())
+        
+        # File statistics
+        uploaded_files  = len(self.uploaded_files)
+        total_file_size = sum(f.get('size', 0) for f in self.uploaded_files)
+        
+        # Index statistics
+        index_stats     = dict()
+
+        if self.index_builder:
+            try:
+                index_stats = self.index_builder.get_index_stats()
+            
+            except:
+                index_stats = {"error": "Could not retrieve index stats"}
+        
+        return {"performance_metrics" : performance,
+                "quality_metrics"     : quality_metrics,
+                "system_information"  : system_info,
+                "health_status"       : health_status,
+                "chunking_statistics" : chunking_stats,
+                "document_statistics" : {"total_documents"         : total_docs,
+                                         "total_chunks"            : total_chunks,
+                                         "uploaded_files"          : uploaded_files,
+                                         "total_file_size_bytes"   : total_file_size,
+                                         "total_file_size_mb"      : round(total_file_size / (1024 * 1024), 2) if (total_file_size > 0) else 0,
+                                         "avg_chunks_per_document" : round(total_chunks / total_docs, 2) if (total_docs > 0) else 0,
+                                        },
+                "session_statistics"  : {"total_sessions"           : total_sessions,
+                                         "total_messages"           : total_messages,
+                                         "avg_messages_per_session" : round(total_messages / total_sessions, 2) if (total_sessions > 0) else 0
+                                        },
+                "index_statistics"    : index_stats,
+                "calculated_at"       : datetime.now().isoformat(),
+                "cache_info"          : {"from_cache"      : False,
+                                         "next_refresh_in" : self.analytics_cache.ttl_seconds,
+                                        }
+               }
+
+
+def _setup_signal_handlers():
+    """
+    Setup signal handlers for graceful shutdown
+    """
+    try:
+        signal.signal(signal.SIGINT, CleanupManager.handle_signal)
+        signal.signal(signal.SIGTERM, CleanupManager.handle_signal)
+        logger.debug("Signal handlers registered")
+    
+    except Exception as e:
+        logger.warning(f"Failed to setup signal handlers: {e}")
+
+
+def _atexit_cleanup():
+    """
+    Atexit handler as last resort
+    """
+    logger.info("Atexit cleanup triggered")
+    
+    # Check if it's already in a cleanup process
+    with _cleanup_lock:
+        if _is_cleaning:
+            logger.info("Cleanup already in progress, skipping atexit cleanup")
+            return
+
+    try:
+        # Check if app exists
+        if (('app' in globals()) and (hasattr(app.state, 'app'))):
+            # Run cleanup in background thread
+            cleanup_thread = threading.Thread(target  = lambda: asyncio.run(CleanupManager.full_cleanup(app.state.app)),
+                                              name    = "atexit_cleanup",
+                                              daemon  = True,
+                                             )
+            cleanup_thread.start()
+            cleanup_thread.join(timeout = 5.0)
+    
+    except Exception as e:
+        logger.error(f"Atexit cleanup error: {e}")
+        # Don't crash during atexit
+
+
+async def _brute_force_cleanup_app_state(state: AppState):
+    """
+    Brute force AppState cleanup
+    """
+    try:
+        # Clear all collections
+        for attr_name in dir(state):
+            if not attr_name.startswith('_'):
+                attr = getattr(state, attr_name)
+                
+                if isinstance(attr, (list, dict, set)):
+                    attr.clear()
+        
+        # Nullify heavy components
+        for attr_name in ['index_builder', 'metadata_store', 'embedder', 'llm_client', 'ragas_evaluator']:
+            if hasattr(state, attr_name):
+                setattr(state, attr_name, None)
+        
+    except:
+        pass
+
+
+# Application lifespan manager
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Manage application startup and shutdown with multiple cleanup guarantees
+    """
+    # Setup signal handlers FIRST
+    _setup_signal_handlers()
+    
+    # Register atexit cleanup
+    atexit.register(_atexit_cleanup)
+
+    logger.info("Starting AI Universal Knowledge Ingestion System...")
+    
+    try:
+        # Initialize application state
+        app.state.app = AppState()
+        
+        # Initialize core components
+        await initialize_components(app.state.app)
+        
+        logger.info("Application startup complete. System ready.")
+        
+        # Yield control to FastAPI
+        yield
+    
+    except Exception as e:
+        logger.error(f"Application runtime error: {e}", exc_info = True)
+        raise
+    
+    finally:
+        # GUARANTEED cleanup (even on crash)
+        logger.info("Beginning guaranteed cleanup sequence...")
+        
+        # Set the cleaning flag
+        with _cleanup_lock:
+            _is_cleaning = True
+        
+        try:
+            # Simple cleanup
+            if (hasattr(app.state, 'app')):
+                # Just clear the state, don't run full cleanup again
+                await _brute_force_cleanup_app_state(app.state.app)
+                
+                # Clean up disk resources
+                await CleanupManager._cleanup_disk_async()
+                
+                # Shutdown the executor
+                _cleanup_executor.shutdown(wait = True)
+        
+        except Exception as e:
+            logger.error(f"Cleanup error in lifespan finally: {e}")
+    
+
+
+# Create FastAPI application
+app = FastAPI(title       = "AI Universal Knowledge Ingestion System",
+              description = "Enterprise RAG Platform with Multi-Source Ingestion",
+              version     = "1.0.0",
+              lifespan    = lifespan,
+             )
+
+
+# Add CORS middleware
+app.add_middleware(CORSMiddleware,
+                   allow_origins     = ["*"],
+                   allow_credentials = True,
+                   allow_methods     = ["*"],
+                   allow_headers     = ["*"],
+                  )
+
+
+# ============================================================================
+# INITIALIZATION FUNCTIONS
+# ============================================================================
+async def initialize_components(state: AppState):
+    """
+    Initialize all application components
+    """
+    try:
+        logger.info("Initializing components...")
+        
+        # Create necessary directories
+        create_directories()
+        
+        # Initialize utilities
+        state.file_handler      = FileHandler()
+        logger.info("FileHandler initialized")
+        
+        # Initialize document parsing
+        state.parser_factory    = get_parser_factory()
+        logger.info(f"ParserFactory initialized with support for: {', '.join(state.parser_factory.get_supported_extensions())}")
+        
+        # Initialize chunking
+        state.chunking_selector = get_adaptive_selector()
+        logger.info("AdaptiveChunkingSelector initialized")
+        
+        # Initialize embeddings
+        state.embedder          = get_embedder()
+        state.embedding_cache   = get_embedding_cache()
+        logger.info(f"Embedder initialized: {state.embedder.get_model_info()}")
+        
+        # Initialize ingestion
+        state.ingestion_router  = get_ingestion_router()
+        state.progress_tracker  = get_progress_tracker()
+        logger.info("Ingestion components initialized")
+        
+        # Initialize vector store
+        state.index_builder     = get_index_builder()
+        state.metadata_store    = get_metadata_store()
+        logger.info("Vector store components initialized")
+        
+        # Check if indexes exist and load them
+        if state.index_builder.is_index_built():
+            logger.info("Existing indexes found - loading...")
+            index_stats    = state.index_builder.get_index_stats()
+
+            logger.info(f"Indexes loaded: {index_stats}")
+            state.is_ready = True
+        
+        # Initialize retrieval
+        state.hybrid_retriever  = get_hybrid_retriever()
+        state.context_assembler = get_context_assembler()
+        logger.info("Retrieval components initialized")
+        
+        # Initialize generation components
+        state.response_generator = get_response_generator(provider   = LLMProvider.OLLAMA,
+                                                          model_name = settings.OLLAMA_MODEL,
+                                                         )
+
+        state.llm_client         = get_llm_client(provider = LLMProvider.OLLAMA)
+
+        logger.info(f"Generation components initialized: model={settings.OLLAMA_MODEL}")
+
+        # Check LLM health
+        if state.llm_client.check_health():
+            logger.info("LLM provider health check: PASSED")
+        
+        else:
+            logger.warning("LLM provider health check: FAILED - Ensure Ollama is running")
+            logger.warning("- Run: ollama serve (in a separate terminal)")
+            logger.warning("- Run: ollama pull mistral (if model not downloaded)")
+
+        # Initialize RAGAS evaluator
+        if settings.ENABLE_RAGAS:
+            state.ragas_evaluator = get_ragas_evaluator(enable_ground_truth_metrics = settings.RAGAS_ENABLE_GROUND_TRUTH)
+
+            logger.info("RAGAS evaluator initialized")
+
+        else:
+            logger.info("RAGAS evaluation disabled in settings")
+
+        
+        logger.info("All components initialized successfully")
+        
+    except Exception as e:
+        logger.error(f"Component initialization failed: {e}", exc_info = True)
+        raise
+
+
+async def cleanup_components(state: AppState):
+    """
+    Cleanup components on shutdown
+    """
+    try:
+        logger.info("Starting component cleanup...")
+    
+        # Use the cleanup manager
+        await CleanupManager.full_cleanup(state)
+        
+        logger.info("Component cleanup complete")
+        
+    except Exception as e:
+        logger.error(f"Component cleanup error: {e}", exc_info = True)
+        
+        # Last-ditch effort
+        await _brute_force_cleanup_app_state(state)
+
+
+def create_directories():
+    """
+    Create necessary directories
+    """
+    directories = [settings.UPLOAD_DIR,
+                   settings.VECTOR_STORE_DIR,
+                   settings.BACKUP_DIR,
+                   Path(settings.METADATA_DB_PATH).parent,
+                   settings.LOG_DIR,
+                  ]
+    
+    for directory in directories:
+        Path(directory).mkdir(parents = True, exist_ok = True)
+    
+    logger.info("Directories created/verified")
+
+
+# ============================================================================
+# API ENDPOINTS
+# ============================================================================
+@app.get("/", response_class = HTMLResponse)
+async def serve_frontend():
+    """
+    Serve the main frontend HTML
+    """
+    frontend_path = Path("frontend/index.html")
+    if frontend_path.exists():
+        return FileResponse(frontend_path)
+    
+    raise HTTPException(status_code = 404, 
+                        detail      = "Frontend not found",
+                       )
+
+
+@app.get("/api/health")
+async def health_check():
+    """
+    Health check endpoint
+    """
+    state         = app.state.app
+    
+    health_status = state.get_system_health()
+    
+    return {"status"     : health_status["overall"],
+            "timestamp"  : datetime.now().isoformat(),
+            "version"    : "1.0.0",
+            "components" : {"vector_store"     : health_status["vector_store"],
+                            "llm"              : health_status["llm"],
+                            "embeddings"       : health_status["embeddings"],
+                            "retrieval"        : health_status["retrieval"],
+                            "generation"       : health_status["generation"],
+                            "hybrid_retriever" : health_status["retrieval"],
+                           },
+            "details"    : health_status
+           }
+
+
+@app.get("/api/system-info")
+async def get_system_info():
+    """
+    Get system information and status
+    """
+    state       = app.state.app
+    
+    # Get system information
+    system_info = state.get_system_information()
+    
+    # Get LLM provider info
+    llm_info    = dict()
+
+    if state.llm_client:
+        llm_info = state.llm_client.get_provider_info()
+    
+    # Get current configuration
+    current_config = {"inference_model"  : settings.OLLAMA_MODEL,
+                      "embedding_model"  : settings.EMBEDDING_MODEL,
+                      "vector_weight"    : settings.VECTOR_WEIGHT,
+                      "bm25_weight"      : settings.BM25_WEIGHT,
+                      "temperature"      : settings.DEFAULT_TEMPERATURE,
+                      "max_tokens"       : settings.MAX_TOKENS,
+                      "chunk_size"       : settings.FIXED_CHUNK_SIZE,
+                      "chunk_overlap"    : settings.FIXED_CHUNK_OVERLAP,
+                      "top_k_retrieve"   : settings.TOP_K_RETRIEVE,
+                      "enable_reranking" : settings.ENABLE_RERANKING,
+                     }
+                    
+    return {"system_state"       : {"is_ready"          : state.is_ready,
+                                    "processing_status" : state.processing_status,
+                                    "total_documents"   : len(state.uploaded_files),
+                                    "active_sessions"   : len(state.active_sessions),
+                                   },
+            "configuration"      : current_config,
+            "llm_provider"       : llm_info,
+            "system_information" : system_info,
+            "timestamp"          : datetime.now().isoformat()
+           }
+
+
+@app.post("/api/upload")
+async def upload_files(files: List[UploadFile] = File(...)):
+    """
+    Upload multiple files
+    """
+    state = app.state.app
+    
+    try:
+        logger.info(f"Received {len(files)} files for upload")
+        uploaded_info = list()
+        
+        for file in files:
+            try:
+                # Validate file type
+                if not state.parser_factory.is_supported(Path(file.filename)):
+                    logger.warning(f"Unsupported file type: {file.filename}")
+                    continue
+                
+                # Save file to upload directory
+                file_path     = settings.UPLOAD_DIR / FileHandler.generate_unique_filename(file.filename, settings.UPLOAD_DIR)
+                
+                # Write file content
+                content       = await file.read()
+
+                with open(file_path, 'wb') as f:
+                    f.write(content)
+                
+                # Get file metadata
+                file_metadata = FileHandler.get_file_metadata(file_path)
+                
+                file_info     = {"filename"      : file_path.name,
+                                 "original_name" : file.filename,
+                                 "size"          : file_metadata["size_bytes"],
+                                 "upload_time"   : datetime.now().isoformat(),
+                                 "file_path"     : str(file_path),
+                                 "status"        : "uploaded",
+                                }
+                
+                uploaded_info.append(file_info)
+                state.uploaded_files.append(file_info)
+                
+                logger.info(f"Uploaded: {file.filename} -> {file_path.name}")
+                
+            except Exception as e:
+                logger.error(f"Failed to upload {file.filename}: {e}")
+                continue
+        
+        # Clear analytics cache since we have new data
+        state.analytics_cache.data = None
+        
+        return {"success" : True,
+                "message" : f"Successfully uploaded {len(uploaded_info)} files",
+                "files"   : uploaded_info,
+               }
+        
+    except Exception as e:
+        logger.error(f"Upload error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.post("/api/start-processing")
+async def start_processing():
+    """
+    Start processing uploaded documents
+    """
+    state = app.state.app
+    
+    if not state.uploaded_files:
+        raise HTTPException(status_code = 400, 
+                            detail      = "No files uploaded",
+                           )
+
+     
+    if (state.processing_status == "processing"):
+        raise HTTPException(status_code = 400, 
+                            detail      = "Processing already in progress",
+                           )
+    
+    try:
+        state.processing_status   = "processing"
+        state.processing_progress = {"status"       : "processing",
+                                     "current_step" : "Starting document processing...",
+                                     "progress"     : 0,
+                                     "processed"    : 0,
+                                     "total"        : len(state.uploaded_files),
+                                     "details"      : {},
+                                    }
+        
+        logger.info("Starting document processing pipeline...")
+        
+        all_chunks     = list()
+        chunking_stats = dict()
+        
+        # Process each file
+        for idx, file_info in enumerate(state.uploaded_files):
+            try:
+                file_path                                 = Path(file_info["file_path"])
+                
+                # Update progress - Parsing
+                state.processing_progress["current_step"] = f"Parsing {file_info['original_name']}..."
+                state.processing_progress["progress"]     = int((idx / len(state.uploaded_files)) * 20)
+                
+                # Parse document
+                logger.info(f"Parsing document: {file_path}")
+                text, metadata                            = state.parser_factory.parse(file_path,
+                                                                                       extract_metadata = True,
+                                                                                       clean_text       = True,
+                                                                                      )
+                
+                if not text:
+                    logger.warning(f"No text extracted from {file_path}")
+                    continue
+                
+                logger.info(f"Extracted {len(text)} characters from {file_path}")
+                
+                # Update progress - Chunking
+                state.processing_progress["current_step"] = f"Chunking {file_info['original_name']}..."
+                state.processing_progress["progress"]     = int((idx / len(state.uploaded_files)) * 40) + 20
+                
+                # Chunk document
+                logger.info(f"Chunking document: {metadata.document_id}")
+                chunks                                    = state.chunking_selector.chunk_text(text     = text,
+                                                                                               metadata = metadata,
+                                                                                              )
+                
+                # Get strategy used from metadata or selector
+                strategy_used = "adaptive"  # Default
+
+                if (metadata and hasattr(metadata, 'chunking_strategy')):
+                    strategy_used = metadata.chunking_strategy.value if metadata.chunking_strategy else "adaptive"
+                
+                elif (hasattr(state.chunking_selector, 'last_strategy_used')):
+                    strategy_used = state.chunking_selector.last_strategy_used
+
+                # Track chunking strategy usage
+                if strategy_used not in chunking_stats:
+                    chunking_stats[strategy_used] = 0
+
+                chunking_stats[strategy_used] += len(chunks)
+                
+                logger.info(f"Created {len(chunks)} chunks for {metadata.document_id} using {strategy_used}")
+                
+                # Update progress - Embedding
+                state.processing_progress["current_step"] = f"Generating embeddings for {file_info['original_name']}..."
+                state.processing_progress["progress"]     = int((idx / len(state.uploaded_files)) * 60) + 40
+                
+                # Generate embeddings for chunks
+                logger.info(f"Generating embeddings for {len(chunks)} chunks...")
+                chunks_with_embeddings                    = state.embedder.embed_chunks(chunks     = chunks,
+                                                                                        batch_size = settings.EMBEDDING_BATCH_SIZE,
+                                                                                        normalize  = True,
+                                                                                       )
+                
+                logger.info(f"Generated embeddings for {len(chunks_with_embeddings)} chunks")
+                
+                # Store chunks
+                all_chunks.extend(chunks_with_embeddings)
+                
+                # Store processed document and chunks
+                state.processed_documents[metadata.document_id] = {"metadata"          : metadata,
+                                                                   "text"              : text,
+                                                                   "file_info"         : file_info,
+                                                                   "chunks_count"      : len(chunks_with_embeddings),
+                                                                   "processed_time"    : datetime.now().isoformat(),
+                                                                   "chunking_strategy" : strategy_used,
+                                                                  }
+                
+                state.document_chunks[metadata.document_id]     = chunks_with_embeddings
+                
+                # Update progress
+                state.processing_progress["processed"]          = idx + 1
+                
+            except Exception as e:
+                logger.error(f"Failed to process {file_info['original_name']}: {e}", exc_info=True)
+                continue
+        
+        # Update chunking statistics
+        state.chunking_statistics = chunking_stats
+        
+        if not all_chunks:
+            raise Exception("No chunks were successfully processed")
+        
+        # Update progress - Building indexes
+        state.processing_progress["current_step"] = "Building vector and keyword indexes..."
+        state.processing_progress["progress"]     = 80
+        
+        # Build indexes (FAISS + BM25 + Metadata)
+        logger.info(f"Building indexes for {len(all_chunks)} chunks...")
+        index_stats                               = state.index_builder.build_indexes(chunks  = all_chunks,
+                                                                                      rebuild = True,
+                                                                                     )
+        
+        logger.info(f"Indexes built: {index_stats}")
+        
+        # Update progress - Indexing for hybrid retrieval
+        state.processing_progress["current_step"] = "Indexing for hybrid retrieval..."
+        state.processing_progress["progress"]     = 95
+        
+        # Mark as ready
+        state.processing_status                   = "ready"
+        state.is_ready                            = True
+        state.processing_progress["status"]       = "ready"
+        state.processing_progress["current_step"] = "Processing complete"
+        state.processing_progress["progress"]     = 100
+        
+        # Clear analytics cache
+        state.analytics_cache.data                = None
+        
+        logger.info(f"Processing complete. Processed {len(state.processed_documents)} documents with {len(all_chunks)} total chunks.")
+        
+        return {"success"             : True,
+                "message"             : "Processing completed successfully",
+                "status"              : "ready",
+                "documents_processed" : len(state.processed_documents),
+                "total_chunks"        : len(all_chunks),
+                "chunking_statistics" : chunking_stats,
+                "index_stats"         : index_stats,
+               }
+        
+    except Exception as e:
+        state.processing_status             = "error"
+        state.processing_progress["status"] = "error"
+
+        logger.error(f"Processing error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/processing-status")
+async def get_processing_status():
+    """
+    Get current processing status
+    """
+    state = app.state.app
+    
+    return {"status"              : state.processing_progress["status"],
+            "progress"            : state.processing_progress["progress"],
+            "current_step"        : state.processing_progress["current_step"],
+            "processed_documents" : state.processing_progress["processed"],
+            "total_documents"     : state.processing_progress["total"],
+            "details"             : state.processing_progress["details"],
+           }
+
+
+@app.get("/api/ragas/history")
+async def get_ragas_history():
+    """
+    Get RAGAS evaluation history for current session
+    """
+    state = app.state.app
+    
+    if not settings.ENABLE_RAGAS or not state.ragas_evaluator:
+
+        raise HTTPException(status_code = 400,
+                            detail      = "RAGAS evaluation is not enabled. Set ENABLE_RAGAS=True in settings.",
+                           )
+    
+    try:
+        history = state.ragas_evaluator.get_evaluation_history()
+        stats   = state.ragas_evaluator.get_session_statistics()
+        
+        return {"success"     : True,
+                "total_count" : len(history),
+                "statistics"  : stats.model_dump(), 
+                "history"     : history
+               }
+        
+    except Exception as e:
+        logger.error(f"RAGAS history retrieval error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.post("/api/ragas/clear")
+async def clear_ragas_history():
+    """
+    Clear RAGAS evaluation history
+    """
+    state = app.state.app
+    
+    if not settings.ENABLE_RAGAS or not state.ragas_evaluator:
+
+        raise HTTPException(status_code = 400,
+                            detail      = "RAGAS evaluation is not enabled.",
+                           )
+    
+    try:
+        state.ragas_evaluator.clear_history()
+        
+        return {"success" : True,
+                "message" : "RAGAS evaluation history cleared, new session started",
+               }
+        
+    except Exception as e:
+        logger.error(f"RAGAS history clear error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/ragas/statistics")
+async def get_ragas_statistics():
+    """
+    Get aggregate RAGAS statistics for current session
+    """
+    state = app.state.app
+    
+    if not settings.ENABLE_RAGAS or not state.ragas_evaluator:
+
+        raise HTTPException(status_code = 400,
+                            detail      = "RAGAS evaluation is not enabled.",
+                           )
+    
+    try:
+        stats = state.ragas_evaluator.get_session_statistics()
+        
+        return {"success"    : True,
+                "statistics" : stats.model_dump(),
+               }
+        
+    except Exception as e:
+        logger.error(f"RAGAS statistics error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/ragas/export")
+async def export_ragas_data():
+    """
+    Export all RAGAS evaluation data
+    """
+    state = app.state.app
+    
+    if not settings.ENABLE_RAGAS or not state.ragas_evaluator:
+
+        raise HTTPException(status_code = 400,
+                            detail      = "RAGAS evaluation is not enabled.",
+                           )
+    
+    try:
+        export_data = state.ragas_evaluator.export_to_dict()
+        
+        return JSONResponse(content = json.loads(export_data.model_dump_json()))
+        
+    except Exception as e:
+        logger.error(f"RAGAS export error: {e}", exc_info = True)
+
+        raise HTTPException(status_code = 500,
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/ragas/config")
+async def get_ragas_config():
+    """
+    Get current RAGAS configuration
+    """
+    return {"enabled"              : settings.ENABLE_RAGAS,
+            "ground_truth_enabled" : settings.RAGAS_ENABLE_GROUND_TRUTH,
+            "base_metrics"         : settings.RAGAS_METRICS,
+            "ground_truth_metrics" : settings.RAGAS_GROUND_TRUTH_METRICS,
+            "evaluation_timeout"   : settings.RAGAS_EVALUATION_TIMEOUT,
+            "batch_size"           : settings.RAGAS_BATCH_SIZE,
+           }
+
+
+@app.post("/api/chat")
+async def chat(request: ChatRequest):
+    """
+    Handle chat queries with LLM-based intelligent routing (generic vs RAG)
+    Supports both conversational queries and document-based queries
+    """
+    state      = app.state.app
+    
+    message    = request.message
+    session_id = request.session_id
+    
+    # Check LLM health (required for both general and RAG queries)
+    if not state.llm_client.check_health():
+        raise HTTPException(status_code = 503,
+                            detail      = "LLM service unavailable. Please ensure Ollama is running.",
+                           )
+    
+    try:
+        logger.info(f"Chat query received: {message}")
+        
+        # Check if documents are available
+        has_documents = state.is_ready and (len(state.processed_documents) > 0)
+        
+        logger.debug(f"System state - Documents available: {has_documents}, Processed docs: {len(state.processed_documents)}, System ready: {state.is_ready}")
+        
+        # Get conversation history for this session (for general queries)
+        conversation_history = None
+        
+        if (session_id and (session_id in state.active_sessions)):
+            # Convert to format expected by general_responder
+            conversation_history = list()
+            
+            # Last 10 messages for context
+            for msg in state.active_sessions[session_id][-10:]:
+                conversation_history.append({"role"    : "user",
+                                             "content" : msg.get("query", ""),
+                                           })
+
+                conversation_history.append({"role"    : "assistant",
+                                             "content" : msg.get("response", ""),
+                                           })
+        
+        # Create QueryRequest object
+        query_request     = QueryRequest(query            = message,
+                                         top_k            = settings.TOP_K_RETRIEVE,
+                                         enable_reranking = settings.ENABLE_RERANKING,
+                                         temperature      = settings.DEFAULT_TEMPERATURE,
+                                         top_p            = settings.TOP_P,
+                                         max_tokens       = settings.MAX_TOKENS,
+                                         include_sources  = True,
+                                         include_metrics  = False,
+                                         stream           = False,
+                                        )
+        
+        # Generate response using response generator (with LLM-based routing)
+        start_time        = time.time()
+
+        query_response    = await state.response_generator.generate_response(request              = query_request,
+                                                                             conversation_history = conversation_history,
+                                                                             has_documents        = has_documents,  # Pass document availability
+                                                                            )
+        
+        # Convert to ms
+        total_time        = (time.time() - start_time) * 1000
+        
+        # Record timing for analytics
+        state.add_query_timing(total_time)
+
+        # Determine query type using response metadata
+        is_general_query  = False
+
+        # Default to rag
+        actual_query_type = "rag"  
+
+        # Check if response has metadata
+        if (hasattr(query_response, 'query_type')):
+            actual_query_type = query_response.query_type
+            is_general_query  = (actual_query_type == "general")
+
+        elif (hasattr(query_response, 'is_general_query')):
+            is_general_query  = query_response.is_general_query
+            actual_query_type = "general" if is_general_query else "rag"
+
+        else:
+            # Method 2: Check sources (fallback)
+            has_sources       = query_response.sources and len(query_response.sources) > 0
+            is_general_query  = not has_sources
+            actual_query_type = "general" if is_general_query else "rag"
+        
+        logger.debug(f"Query classification: actual_query_type={actual_query_type}, has_sources={query_response.sources and len(query_response.sources) > 0}")
+    
+        # Extract contexts for RAGAS evaluation (only if RAG was used)
+        contexts          = list()
+
+        if query_response.sources:
+            contexts = [chunk.chunk.text for chunk in query_response.sources]
+        
+        # Run RAGAS evaluation (only if RAGAS enabled)
+        ragas_result   = None
+
+        if (settings.ENABLE_RAGAS and state.ragas_evaluator):
+            try:
+                logger.info("Running RAGAS evaluation...")
+                
+                ragas_result = state.ragas_evaluator.evaluate_single(query              = message,
+                                                                     answer             = query_response.answer,
+                                                                     contexts           = contexts,
+                                                                     ground_truth       = None,
+                                                                     retrieval_time_ms  = int(query_response.retrieval_time_ms),
+                                                                     generation_time_ms = int(query_response.generation_time_ms),
+                                                                     total_time_ms      = int(query_response.total_time_ms),
+                                                                     chunks_retrieved   = len(query_response.sources),
+                                                                     query_type         = actual_query_type,
+                                                                    )
+                
+                logger.info(f"RAGAS evaluation complete: type={actual_query_type.upper()}, relevancy={ragas_result.answer_relevancy:.3f}, faithfulness={ragas_result.faithfulness:.3f}, overall={ragas_result.overall_score:.3f}")
+            
+            except Exception as e:
+                logger.error(f"RAGAS evaluation failed: {e}", exc_info = True)
+                # Continue without RAGAS metrics - don't fail the request
+        
+        # Format sources for response
+        sources = list()
+
+        for i, chunk_with_score in enumerate(query_response.sources[:5], 1):
+            chunk  = chunk_with_score.chunk
+
+            source = {"rank"             : i,
+                      "score"            : chunk_with_score.score,
+                      "document_id"      : chunk.document_id,
+                      "chunk_id"         : chunk.chunk_id,
+                      "text_preview"     : chunk.text[:500] + "..." if len(chunk.text) > 500 else chunk.text,
+                      "page_number"      : chunk.page_number,
+                      "section_title"    : chunk.section_title,
+                      "retrieval_method" : chunk_with_score.retrieval_method,
+                     }
+
+            sources.append(source)
+        
+        # Generate session ID if not provided
+        if not session_id:
+            session_id = f"session_{datetime.now().timestamp()}"
+        
+        # Determine query type for response metadata
+        is_general_query = (actual_query_type == "general")
+        
+        # Prepare response
+        response         = {"session_id"       : session_id,
+                            "response"         : query_response.answer,
+                            "sources"          : sources,
+                            "is_general_query" : is_general_query,
+                            "metrics"          : {"retrieval_time"    : int(query_response.retrieval_time_ms),
+                                                  "generation_time"   : int(query_response.generation_time_ms),
+                                                  "total_time"        : int(query_response.total_time_ms),
+                                                  "chunks_retrieved"  : len(query_response.sources),
+                                                  "chunks_used"       : len(sources),
+                                                  "tokens_used"       : query_response.tokens_used.get("total", 0) if query_response.tokens_used else 0,
+                                                  "actual_total_time" : int(total_time),
+                                                  "query_type"        : actual_query_type,
+                                                  "llm_classified"    : True,  # Now using LLM for classification
+                                                 },
+                           }
+        
+        # Add RAGAS metrics if evaluation succeeded
+        if ragas_result:
+            response["ragas_metrics"] = {"answer_relevancy"   : round(ragas_result.answer_relevancy, 3),
+                                         "faithfulness"       : round(ragas_result.faithfulness, 3),
+                                         "context_precision"  : round(ragas_result.context_precision, 3) if ragas_result.context_precision else None,
+                                         "context_relevancy"  : round(ragas_result.context_relevancy, 3),
+                                         "overall_score"      : round(ragas_result.overall_score, 3),
+                                         "context_recall"     : round(ragas_result.context_recall, 3) if ragas_result.context_recall else None,
+                                         "answer_similarity"  : round(ragas_result.answer_similarity, 3) if ragas_result.answer_similarity else None,
+                                         "answer_correctness" : round(ragas_result.answer_correctness, 3) if ragas_result.answer_correctness else None,
+                                         "query_type"         : ragas_result.query_type, 
+                                        }
+        else:
+            response["ragas_metrics"] = None
+        
+        # Store in session
+        if session_id not in state.active_sessions:
+            state.active_sessions[session_id] = list()
+        
+        state.active_sessions[session_id].append({"query"            : message,
+                                                  "response"         : query_response.answer,
+                                                  "sources"          : sources,
+                                                  "timestamp"        : datetime.now().isoformat(),
+                                                  "metrics"          : response["metrics"],
+                                                  "ragas_metrics"    : response.get("ragas_metrics", {}),
+                                                  "is_general_query" : is_general_query,
+                                                })
+        
+        # Clear analytics cache when new data is available
+        state.analytics_cache.data = None
+        
+        logger.info(f"Chat response generated successfully in {int(total_time)}ms | (type: {actual_query_type.upper()})")
+        
+        return response
+        
+    except Exception as e:
+        logger.error(f"Chat error: {e}", exc_info = True)
+        
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/configuration")
+async def get_configuration():
+    """
+    Get current configuration
+    """
+    state        = app.state.app
+    
+    # Get system health
+    health_status = state.get_system_health()
+    
+    return {"configuration" : {"inference_model"   : settings.OLLAMA_MODEL,
+                               "embedding_model"   : settings.EMBEDDING_MODEL,
+                               "chunking_strategy" : "adaptive",
+                               "chunk_size"        : settings.FIXED_CHUNK_SIZE,
+                               "chunk_overlap"     : settings.FIXED_CHUNK_OVERLAP,
+                               "retrieval_top_k"   : settings.TOP_K_RETRIEVE,
+                               "vector_weight"     : settings.VECTOR_WEIGHT,
+                               "bm25_weight"       : settings.BM25_WEIGHT,
+                               "temperature"       : settings.DEFAULT_TEMPERATURE,
+                               "max_tokens"        : settings.MAX_TOKENS,
+                               "enable_reranking"  : settings.ENABLE_RERANKING,
+                               "is_ready"          : state.is_ready,
+                               "llm_healthy"       : health_status["llm"],
+                              },
+            "health"        : health_status,
+           }
+
+
+@app.post("/api/configuration")
+async def update_configuration(temperature: float = Form(None), max_tokens: int = Form(None), retrieval_top_k: int = Form(None),
+                               vector_weight: float = Form(None), bm25_weight: float = Form(None), enable_reranking: bool = Form(None),
+                               session_id: str = Form(None)):
+    """
+    Update system configuration (runtime parameters only)
+    """
+    state = app.state.app
+    
+    try:
+        updates = dict()
+        
+        # Runtime parameters (no rebuild required)
+        if (temperature is not None):
+            updates["temperature"] = temperature
+        
+        if (max_tokens and (max_tokens != settings.MAX_TOKENS)):
+            updates["max_tokens"] = max_tokens
+        
+        if (retrieval_top_k and (retrieval_top_k != settings.TOP_K_RETRIEVE)):
+            updates["retrieval_top_k"] = retrieval_top_k
+        
+        if ((vector_weight is not None) and (vector_weight != settings.VECTOR_WEIGHT)):
+            updates["vector_weight"] = vector_weight
+            
+            # Update hybrid retriever weights
+            if bm25_weight is not None:
+                state.hybrid_retriever.update_weights(vector_weight, bm25_weight)
+        
+        if ((bm25_weight is not None) and (bm25_weight != settings.BM25_WEIGHT)):
+            updates["bm25_weight"] = bm25_weight
+        
+        if (enable_reranking is not None):
+            updates["enable_reranking"] = enable_reranking
+        
+        # Store session-based config overrides
+        if session_id:
+            if session_id not in state.config_overrides:
+                state.config_overrides[session_id] = {}
+            
+            state.config_overrides[session_id].update(updates)
+        
+        logger.info(f"Configuration updated: {updates}")
+        
+        # Clear analytics cache since configuration changed
+        state.analytics_cache.data = None
+        
+        return {"success" : True,
+                "message" : "Configuration updated successfully",
+                "updates" : updates,
+               }
+        
+    except Exception as e:
+        logger.error(f"Configuration update error: {e}", exc_info = True)
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/analytics")
+async def get_analytics():
+    """
+    Get comprehensive system analytics and metrics with caching
+    """
+    state = app.state.app
+    
+    try:
+        # Check cache first
+        cached_data = state.analytics_cache.get()
+        
+        if cached_data:
+            cached_data["cache_info"]["from_cache"] = True
+            
+            return cached_data
+        
+        # Calculate fresh analytics
+        analytics_data = state.calculate_comprehensive_analytics()
+        
+        # Update cache
+        state.analytics_cache.update(analytics_data)
+        
+        return analytics_data
+        
+    except Exception as e:
+        logger.error(f"Analytics calculation error: {e}", exc_info = True)
+        
+        # Return basic analytics even if calculation fails
+        return {"performance_metrics" : {"avg_response_time" : 0,
+                                         "total_queries"     : 0,
+                                         "queries_last_hour" : 0,
+                                         "error"             : "Could not calculate performance metrics"
+                                        },
+                "quality_metrics"     : {"answer_relevancy"  : 0.0,
+                                         "faithfulness"      : 0.0,
+                                         "context_precision" : 0.0,
+                                         "overall_score"     : 0.0,
+                                         "confidence"        : "low",
+                                         "metrics_available" : False,
+                                         "error"             : "Could not calculate quality metrics"
+                                        },
+                "system_information"  : state.get_system_information() if hasattr(state, 'get_system_information') else {},
+                "health_status"       : {"overall" : "unknown"},
+                "document_statistics" : {"total_documents" : len(state.processed_documents),
+                                         "total_chunks"    : sum(len(chunks) for chunks in state.document_chunks.values()),
+                                         "uploaded_files"  : len(state.uploaded_files)
+                                        },
+                "session_statistics"  : {"total_sessions" : len(state.active_sessions),
+                                         "total_messages" : sum(len(msgs) for msgs in state.active_sessions.values())
+                                        },
+                "calculated_at"       : datetime.now().isoformat(),
+                "error"               : str(e)
+               }
+
+
+@app.get("/api/analytics/refresh")
+async def refresh_analytics():
+    """
+    Force refresh analytics cache
+    """
+    state = app.state.app
+    
+    try:
+        # Clear cache
+        state.analytics_cache.data = None
+        
+        # Calculate fresh analytics
+        analytics_data             = state.calculate_comprehensive_analytics()
+        
+        return {"success" : True,
+                "message" : "Analytics cache refreshed successfully",
+                "data"    : analytics_data,
+               }
+        
+    except Exception as e:
+        logger.error(f"Analytics refresh error: {e}", exc_info = True)
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/analytics/detailed")
+async def get_detailed_analytics():
+    """
+    Get detailed analytics including query history and component performance
+    """
+    state = app.state.app
+    
+    try:
+        # Get basic analytics
+        analytics         = await get_analytics()
+        
+        # Add detailed session information
+        detailed_sessions = list()
+
+        for session_id, messages in state.active_sessions.items():
+            session_info = {"session_id"            : session_id,
+                            "message_count"         : len(messages),
+                            "first_message"         : messages[0]["timestamp"] if messages else None,
+                            "last_message"          : messages[-1]["timestamp"] if messages else None,
+                            "total_response_time"   : sum(msg.get("metrics", {}).get("total_time", 0) for msg in messages),
+                            "avg_sources_per_query" : sum(len(msg.get("sources", [])) for msg in messages) / len(messages) if messages else 0,
+                           }
+
+            detailed_sessions.append(session_info)
+        
+        # Add component performance if available
+        component_performance = dict()
+
+        if state.hybrid_retriever:
+            try:
+                retrieval_stats                    = state.hybrid_retriever.get_retrieval_stats()
+                component_performance["retrieval"] = retrieval_stats
+
+            except:
+                component_performance["retrieval"] = {"error": "Could not retrieve stats"}
+        
+        if state.embedder:
+            try:
+                embedder_info                       = state.embedder.get_model_info()
+                component_performance["embeddings"] = {"model"     : embedder_info.get("model_name", "unknown"),
+                                                       "dimension" : embedder_info.get("embedding_dim", 0),
+                                                       "device"    : embedder_info.get("device", "cpu"),
+                                                      }
+            except:
+                component_performance["embeddings"] = {"error": "Could not retrieve stats"}
+        
+        analytics["detailed_sessions"]     = detailed_sessions
+        analytics["component_performance"] = component_performance
+        
+        return analytics
+        
+    except Exception as e:
+        logger.error(f"Detailed analytics error: {e}", exc_info = True)
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/export-chat/{session_id}")
+async def export_chat(session_id: str, format: str = "json"):
+    """
+    Export chat history
+    """
+    state = app.state.app
+    if session_id not in state.active_sessions:
+        raise HTTPException(status_code = 404, 
+                            detail      = "Session not found",
+                           )
+
+    try:
+        chat_history = state.active_sessions[session_id]
+        
+        if (format == "json"):
+            return JSONResponse(content = {"session_id"     : session_id,
+                                           "export_time"    : datetime.now().isoformat(),
+                                           "total_messages" : len(chat_history),
+                                           "history"        : chat_history,
+                                          }
+                               )
+
+        elif (format == "csv"):
+            output = io.StringIO()
+            
+            if chat_history:
+                fieldnames = ["timestamp", "query", "response", "sources_count", "response_time_ms"]
+                writer     = csv.DictWriter(output, fieldnames = fieldnames)
+                writer.writeheader()
+                
+                for entry in chat_history:
+                    writer.writerow({"timestamp"        : entry.get("timestamp", ""),
+                                     "query"            : entry.get("query", ""),
+                                     "response"         : entry.get("response", ""),
+                                     "sources_count"    : len(entry.get("sources", [])),
+                                     "response_time_ms" : entry.get("metrics", {}).get("total_time", 0),
+                                   })
+            
+            return JSONResponse(content = {"csv"        : output.getvalue(),
+                                           "session_id" : session_id,
+                                           "format"     : "csv",
+                                          }
+                               )
+
+        else:
+            raise HTTPException(status_code = 400, 
+                                detail      = "Unsupported format. Use 'json' or 'csv'",
+                               )
+            
+    except Exception as e:
+        logger.error(f"Export error: {e}", exc_info = True)
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+                           
+
+@app.post("/api/cleanup/session/{session_id}")
+async def cleanup_session(session_id: str):
+    """
+    Clean up specific session
+    """
+    state = app.state.app
+    
+    if session_id in state.active_sessions:
+        del state.active_sessions[session_id]
+        
+        if session_id in state.config_overrides:
+            del state.config_overrides[session_id]
+        
+        # Check if no sessions left
+        if not state.active_sessions:
+            logger.info("No active sessions, suggesting vector store cleanup")
+            
+            return {"success"    : True, 
+                    "message"    : f"Session {session_id} cleaned up",
+                    "suggestion" : "No active sessions remaining. Consider cleaning vector store.",
+                   }
+        
+        return {"success" : True, 
+                "message" : f"Session {session_id} cleaned up",
+               }
+    
+    return {"success" : False, 
+            "message" : "Session not found",
+           }
+
+
+@app.post("/api/cleanup/vector-store")
+async def cleanup_vector_store():
+    """
+    Manual vector store cleanup
+    """
+    state = app.state.app
+    
+    try:
+        # Use cleanup manager
+        success = await CleanupManager.full_cleanup(state)
+        
+        if success:
+            return {"success" : True, 
+                    "message" : "Vector store and all data cleaned up",
+                   }
+        
+        else:
+            return {"success" : False, 
+                    "message" : "Cleanup completed with errors",
+                   }
+            
+    except Exception as e:
+        logger.error(f"Manual cleanup error: {e}")
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.post("/api/cleanup/full")
+async def full_cleanup_endpoint():
+    """
+    Full system cleanup endpoint
+    """
+    state = app.state.app
+    
+    try:
+        # Also clean up frontend sessions
+        state.active_sessions.clear()
+        state.config_overrides.clear()
+        
+        # Full cleanup
+        success = await CleanupManager.full_cleanup(state)
+        
+        return {"success" : success,
+                "message" : "Full system cleanup completed",
+                "details" : {"sessions_cleaned" : 0,  # Already cleared above
+                             "memory_freed"     : "All application state",
+                             "disk_space_freed" : "All vector store and uploaded files",
+                            }
+               }
+        
+    except Exception as e:
+        logger.error(f"Full cleanup endpoint error: {e}")
+        raise HTTPException(status_code = 500, 
+                            detail      = str(e),
+                           )
+
+
+@app.get("/api/cleanup/status")
+async def get_cleanup_status():
+    """
+    Get cleanup status and statistics
+    """
+    state = app.state.app
+    
+    return {"sessions_active"       : len(state.active_sessions),
+            "documents_processed"   : len(state.processed_documents),
+            "total_chunks"          : sum(len(chunks) for chunks in state.document_chunks.values()),
+            "vector_store_ready"    : state.is_ready,
+            "cleanup_registry_size" : len(_cleanup_registry),
+            "suggested_action"      : "cleanup_vector_store" if state.is_ready else "upload_documents",
+           }
+
+
+# ============================================================================
+# MAIN ENTRY POINT
+# ============================================================================
+if __name__ == "__main__":
+    try:
+        # Run the app
+        uvicorn.run("app:app",
+                    host                      = settings.HOST,
+                    port                      = settings.PORT,
+                    reload                    = settings.DEBUG,
+                    log_level                 = "info",
+                    timeout_graceful_shutdown = 10.0,
+                    access_log                = False,
+                   )
+        
+    except KeyboardInterrupt:
+        logger.info("Keyboard interrupt received - normal shutdown")
+    
+    except Exception as e:
+        logger.error(f"Application crashed: {e}", exc_info = True)
+    
+    finally:
+        # Simple final cleanup
+        logger.info("Application stopping, final cleanup...")
+        try:
+            # Shutdown executor if it exists
+            if '_cleanup_executor' in globals():
+                _cleanup_executor.shutdown(wait = True)
+
+        except:
+            pass
+        
+        logger.info("Application stopped")

chunking/adaptive_selector.py

@@ -0,0 +1,465 @@
+# DEPENDENCIES
+import re
+from typing import List
+from typing import Optional
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.base_chunker import BaseChunker
+from chunking.base_chunker import ChunkerConfig
+from chunking.token_counter import TokenCounter
+from chunking.fixed_chunker import FixedChunker
+from chunking.semantic_chunker import SemanticChunker
+from chunking.llamaindex_chunker import LlamaIndexChunker
+from chunking.hierarchical_chunker import HierarchicalChunker
+
+
+
+# Setup Settings and Logging
+logger = get_logger(__name__)
+settings = get_settings()
+
+
+class AdaptiveChunkingSelector:
+    """
+    Intelligent chunking strategy selector with structure detection:
+    - Analyzes document characteristics (size, structure, content type)
+    - Detects structured documents (projects, sections, hierarchies)
+    - Automatically selects optimal chunking strategy
+    - Prioritizes section-aware chunking for structured content
+    
+    Strategy Selection Logic (UPDATED):
+    - Small docs (< 1K tokens) → Fixed chunking
+    - Medium structured docs → Semantic (section-aware)
+    - Medium unstructured docs → LlamaIndex or basic semantic
+    - Large docs (>500K tokens) → Hierarchical chunking
+    """
+    def __init__(self, prefer_llamaindex: bool = True):
+        """
+        Initialize adaptive selector with all chunking strategies
+        
+        Arguments:
+        ----------
+            prefer_llamaindex { bool } : Prefer LlamaIndex over custom semantic chunking when available
+        """
+        self.logger               = logger
+        self.token_counter        = TokenCounter()
+        self.prefer_llamaindex    = prefer_llamaindex
+        
+        # Initialize all chunking strategies
+        self.fixed_chunker        = FixedChunker()
+        self.semantic_chunker     = SemanticChunker(respect_section_boundaries = True)
+        self.hierarchical_chunker = HierarchicalChunker()
+        self.llamaindex_chunker   = LlamaIndexChunker()
+        
+        # Strategy thresholds (from settings)
+        self.small_doc_threshold  = settings.SMALL_DOC_THRESHOLD     
+        self.large_doc_threshold  = settings.LARGE_DOC_THRESHOLD    
+        
+        # Check LlamaIndex availability
+        self.llamaindex_available = self.llamaindex_chunker._initialized
+        
+        self.logger.info(f"Initialized AdaptiveChunkingSelector: LlamaIndex available={self.llamaindex_available}, prefer_llamaindex={self.prefer_llamaindex}, section_aware_semantic=True")
+    
+
+    def select_chunking_strategy(self, text: str, metadata: Optional[DocumentMetadata] = None) -> tuple[ChunkingStrategy, dict]:
+        """
+        Analyze document and select optimal chunking strategy: Detects structured documents and prioritizes section-aware chunking
+        
+        Arguments:
+        ----------
+            text     { str }                : Document text
+
+            metadata { DocumentMetadata }   : Document metadata
+        
+        Returns:
+        --------
+                     { tuple }              : Tuple of (selected_strategy, analysis_results)
+        """
+        analysis        = self._analyze_document(text     = text, 
+                                                 metadata = metadata,
+                                                )
+        
+        # Check if document has clear structure (projects, sections)
+        has_structure   = analysis.get("has_structure", False)
+        structure_score = analysis.get("structure_score", 0)
+        
+        # Strategy selection logic
+        if (analysis["total_tokens"] <= self.small_doc_threshold):
+            strategy = ChunkingStrategy.FIXED
+            reason   = f"Small document ({analysis['total_tokens']} tokens) - fixed chunking for simplicity"
+        
+        elif (analysis["total_tokens"] <= self.large_doc_threshold):
+            # Medium documents: check for structure
+            if (has_structure and (structure_score > 0.3)):
+                # Structured document detected - use section-aware semantic chunking
+                strategy = ChunkingStrategy.SEMANTIC
+                reason  = (f"Medium structured document ({analysis['total_tokens']} tokens, structure_score={structure_score:.2f}) - section-aware semantic chunking")
+            
+            elif self.llamaindex_available and self.prefer_llamaindex:
+                strategy = ChunkingStrategy.SEMANTIC
+                reason   = f"Medium document ({analysis['total_tokens']} tokens) - LlamaIndex semantic chunking"
+            
+            else:
+                strategy = ChunkingStrategy.SEMANTIC
+                reason   = f"Medium document ({analysis['total_tokens']} tokens) - semantic chunking"
+        
+        else:
+            strategy = ChunkingStrategy.HIERARCHICAL
+            reason   = f"Large document ({analysis['total_tokens']} tokens) - hierarchical chunking"
+        
+        # Override based on document structure if available
+        if (metadata and self._has_clear_structure(metadata)):
+            if (strategy == ChunkingStrategy.FIXED):
+                # Upgrade to semantic for structured documents
+                strategy = ChunkingStrategy.SEMANTIC
+                reason   = "Document has clear structure - section-aware semantic chunking preferred"
+        
+        analysis["selected_strategy"] = strategy
+        analysis["selection_reason"]  = reason
+        analysis["llamaindex_used"]   = ((strategy == ChunkingStrategy.SEMANTIC) and self.llamaindex_available and self.prefer_llamaindex and not has_structure)
+        
+        self.logger.info(f"Selected {strategy.value}: {reason}")
+        
+        return strategy, analysis
+    
+
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None, force_strategy: Optional[ChunkingStrategy] = None) -> List[DocumentChunk]:
+        """
+        Automatically select strategy and chunk text
+        
+        Arguments:
+        ----------
+            text           { str }                : Document text
+            
+            metadata       { DocumentMetadata }   : Document metadata
+            
+            force_strategy { ChunkingStrategy }   : Force specific strategy (optional)
+        
+        Returns:
+        --------
+                           { list }               : List of DocumentChunk objects
+        """
+        if not text or not text.strip():
+            return []
+        
+        # Select strategy (or use forced strategy)
+        if force_strategy:
+            strategy        = force_strategy
+            analysis        = self._analyze_document(text     = text, 
+                                                     metadata = metadata,
+                                                    )
+
+            reason          = f"Forced strategy: {force_strategy.value}"
+            llamaindex_used = False
+        else:
+            strategy, analysis = self.select_chunking_strategy(text     = text,
+                                                               metadata = metadata,
+                                                              )
+            reason             = analysis["selection_reason"]
+            llamaindex_used    = analysis["llamaindex_used"]
+        
+        # Get appropriate chunker
+        if ((strategy == ChunkingStrategy.SEMANTIC) and llamaindex_used):
+            chunker      = self.llamaindex_chunker
+            chunker_name = "LlamaIndex Semantic"
+
+        else:
+            chunker      = self._get_chunker_for_strategy(strategy = strategy)
+            chunker_name = strategy.value
+        
+        # Update metadata with strategy information
+        if metadata:
+            metadata.chunking_strategy          = strategy
+            metadata.extra["chunking_analysis"] = {"strategy"         : strategy.value,
+                                                   "chunker_used"     : chunker_name,
+                                                   "reason"           : reason,
+                                                   "total_tokens"     : analysis["total_tokens"],
+                                                   "estimated_chunks" : analysis[f"estimated_{strategy.value.lower()}_chunks"],
+                                                   "llamaindex_used"  : llamaindex_used,
+                                                   "has_structure"    : analysis.get("has_structure", False),
+                                                   "structure_score"  : analysis.get("structure_score", 0),
+                                                  }
+        
+        self.logger.info(f"Using {chunker_name} chunker for document")
+        
+        # Perform chunking
+        try:
+            chunks = chunker.chunk_text(text     = text, 
+                                        metadata = metadata,
+                                       )
+            
+            # Add strategy metadata to chunks
+            for chunk in chunks:
+                chunk.metadata["chunking_strategy"] = strategy.value
+                chunk.metadata["chunker_used"]      = chunker_name
+                
+                if llamaindex_used:
+                    chunk.metadata["llamaindex_splitter"] = self.llamaindex_chunker.splitter_type
+            
+            self.logger.info(f"Successfully created {len(chunks)} chunks using {chunker_name}")
+            
+            # Log section coverage statistics
+            chunks_with_sections = sum(1 for c in chunks if c.section_title)
+            if (chunks_with_sections > 0):
+                self.logger.info(f"Section coverage: {chunks_with_sections}/{len(chunks)} chunks ({chunks_with_sections/len(chunks)*100:.1f}%) have section titles")
+            
+            return chunks
+        
+        except Exception as e:
+            self.logger.error(f"{chunker_name} chunking failed: {repr(e)}, falling back to fixed chunking")
+            
+            # Fallback to fixed chunking
+            return self.fixed_chunker.chunk_text(text     = text, 
+                                                 metadata = metadata,
+                                                )
+    
+
+    def _analyze_document(self, text: str, metadata: Optional[DocumentMetadata] = None) -> dict:
+        """
+        Analyze document characteristics for strategy selection: Includes structure detection
+        
+        Arguments:
+        ----------
+            text     { str }                : Document text
+
+            metadata { DocumentMetadata }   : Document metadata
+        
+        Returns:
+        --------
+                     { dict }               : Analysis results
+        """
+        # Basic token analysis
+        total_tokens                   = self.token_counter.count_tokens(text = text)
+        total_chars                    = len(text)
+        total_words                    = len(text.split())
+        
+        # Estimate chunks for each strategy
+        estimated_fixed_chunks         = max(1, total_tokens // settings.FIXED_CHUNK_SIZE)
+        estimated_semantic_chunks      = max(1, total_tokens // (settings.FIXED_CHUNK_SIZE * 2)) 
+        estimated_hierarchical_chunks  = max(1, total_tokens // settings.CHILD_CHUNK_SIZE)  
+        estimated_llamaindex_chunks    = max(1, total_tokens // (settings.FIXED_CHUNK_SIZE * 1.5))
+        
+        # Structure analysis (simple heuristics)
+        sentence_count                 = len(self.token_counter._split_into_sentences(text = text))
+        avg_sentence_length            = total_words / sentence_count if (sentence_count > 0) else 0
+        
+        # Paragraph detection (rough)
+        paragraphs                     = [p for p in text.split('\n\n') if p.strip()]
+        paragraph_count                = len(paragraphs)
+        
+        # NEW: Detect document structure
+        has_structure, structure_score = self._detect_document_structure(text)
+        
+        analysis                       = {"total_tokens"                  : total_tokens,
+                                          "total_chars"                   : total_chars,
+                                          "total_words"                   : total_words,
+                                          "sentence_count"                : sentence_count,
+                                          "paragraph_count"               : paragraph_count,
+                                          "avg_sentence_length"           : avg_sentence_length,
+                                          "estimated_fixed_chunks"        : estimated_fixed_chunks,
+                                          "estimated_semantic_chunks"     : estimated_semantic_chunks,
+                                          "estimated_llamaindex_chunks"   : estimated_llamaindex_chunks,
+                                          "estimated_hierarchical_chunks" : estimated_hierarchical_chunks,
+                                          "document_size_category"        : self._get_size_category(total_tokens),
+                                          "llamaindex_available"          : self.llamaindex_available,
+                                          "has_structure"                 : has_structure,
+                                          "structure_score"               : structure_score,
+                                         }
+        
+        # Add metadata-based insights if available
+        if metadata:
+            analysis.update({"document_type"       : metadata.document_type.value,
+                             "file_size_mb"        : metadata.file_size_mb,
+                             "num_pages"           : metadata.num_pages,
+                             "has_clear_structure" : self._has_clear_structure(metadata),
+                           })
+        
+        return analysis
+    
+
+    def _detect_document_structure(self, text: str) -> tuple[bool, float]:
+        """
+        Analyzes text for structural patterns and detect if document has clear structural elements (projects, sections, etc.) 
+        & returns: (has_structure, structure_score)
+        """
+        structure_indicators = 0
+        max_indicators       = 5
+        
+        # Check for project-style headers: "a) Project Name", "b) Project Name"
+        project_headers      = len(re.findall(r'^[a-z]\)\s+[A-Z]', text, re.MULTILINE))
+        
+        if (project_headers > 2):
+            structure_indicators += 1
+        
+        # Check for bullet point lists: "●" or "❖"
+        bullet_points = text.count('●') + text.count('❖')
+        
+        if (bullet_points > 5):
+            structure_indicators += 1
+        
+        # Check for numbered sections: "1.", "2.", etc.
+        numbered_sections = len(re.findall(r'^\d+\.\s+[A-Z]', text, re.MULTILINE))
+        
+        if (numbered_sections > 2):
+            structure_indicators += 1
+        
+        # Check for subsection markers ending with ":"
+        subsection_markers = len(re.findall(r'^●\s+\w+.*:', text, re.MULTILINE))
+        
+        if (subsection_markers > 3):
+            structure_indicators += 1
+        
+        # Check for consistent indentation patterns
+        lines          = text.split('\n')
+        indented_lines = sum(1 for line in lines if line.startswith('   ') or line.startswith('\t'))
+        
+        # >20% indented
+        if (indented_lines > len(lines) * 0.2): 
+            structure_indicators += 1
+        
+        has_structure   = (structure_indicators >= 2)
+        structure_score = structure_indicators / max_indicators
+        
+        if has_structure:
+            self.logger.info(f"Document structure detected: score={structure_score:.2f} (project_headers={project_headers}, bullets={bullet_points}, "
+                             f"numbered_sections={numbered_sections}, subsections={subsection_markers})")
+        
+        return has_structure, structure_score
+    
+
+    def _get_chunker_for_strategy(self, strategy: ChunkingStrategy) -> BaseChunker:
+        """
+        Get chunker instance for specified strategy
+        
+        Arguments:
+        ----------
+            strategy { ChunkingStrategy } : Chunking strategy
+        
+        Returns:
+        --------
+            { BaseChunker }               : Chunker instance
+        """
+        chunkers = {ChunkingStrategy.FIXED        : self.fixed_chunker,
+                    ChunkingStrategy.SEMANTIC     : self.semantic_chunker,
+                    ChunkingStrategy.HIERARCHICAL : self.hierarchical_chunker,
+                   }
+        
+        return chunkers.get(strategy, self.fixed_chunker)
+    
+
+    def _get_size_category(self, total_tokens: int) -> str:
+        """
+        Categorize document by size
+        """
+        if (total_tokens <= self.small_doc_threshold):
+            return "small"
+
+        elif (total_tokens <= self.large_doc_threshold):
+            return "medium"
+        
+        else:
+            return "large"
+    
+
+    def _has_clear_structure(self, metadata: DocumentMetadata) -> bool:
+        """
+        Check if document has clear structural elements
+        """
+        if metadata.extra:
+            # DOCX with multiple sections/headings
+            if (metadata.document_type.value == "docx"):
+                if (metadata.extra.get("num_sections", 0) > 1):
+                    return True
+
+                if (metadata.extra.get("num_paragraphs", 0) > 50):
+                    return True
+            
+            # PDF with multiple pages and likely structure
+            if (metadata.document_type.value == "pdf"):
+                if metadata.num_pages and metadata.num_pages > 10:
+                    return True
+        
+        return False
+    
+
+    def get_strategy_recommendations(self, text: str, metadata: Optional[DocumentMetadata] = None) -> dict:
+        """
+        Get detailed strategy recommendations with pros/cons
+        """
+        analysis                              = self._analyze_document(text, metadata)
+        
+        # LlamaIndex recommendation
+        llamaindex_recommendation             = {"recommended_for"  : ["Medium documents", "Structured content", "Superior semantic analysis"],
+                                                 "pros"             : ["Best semantic boundary detection", "LlamaIndex ecosystem integration", "Advanced embedding-based splitting"],
+                                                 "cons"             : ["Additional dependency", "Slower initialization", "More complex setup"],
+                                                 "estimated_chunks" : analysis["estimated_llamaindex_chunks"],
+                                                 "available"        : self.llamaindex_available,
+                                                }
+        
+        recommendations                       = {"fixed"        : {"recommended_for"  : ["Small documents", "Homogeneous content", "Simple processing"],
+                                                                   "pros"             : ["Fast", "Reliable", "Predictable chunk sizes"],
+                                                                   "cons"             : ["May break semantic boundaries", "Ignores document structure"],
+                                                                   "estimated_chunks" : analysis["estimated_fixed_chunks"],
+                                                                  },
+                                                 "semantic"     : {"recommended_for"  : ["Medium documents", "Structured content", "When coherence matters"],
+                                                                   "pros"             : ["Preserves topic boundaries", "Respects section structure", "Better context coherence"],
+                                                                   "cons"             : ["Slower (requires embeddings)", "Less predictable chunk sizes"],
+                                                                   "estimated_chunks" : analysis["estimated_semantic_chunks"],
+                                                                   "section_aware"    : True,
+                                                                  },
+                                                 "llamaindex"   : llamaindex_recommendation,
+                                                 "hierarchical" : {"recommended_for"  : ["Large documents", "Complex structure", "Granular search needs"],
+                                                                   "pros"             : ["Best for large docs", "Granular + context search", "Scalable"],
+                                                                   "cons"             : ["Complex implementation", "More chunks to manage", "Higher storage"],
+                                                                   "estimated_chunks" : analysis["estimated_hierarchical_chunks"],
+                                                                  }
+                                                }
+        
+        # Add selected strategy
+        selected_strategy, analysis_result    = self.select_chunking_strategy(text     = text, 
+                                                                              metadata = metadata,
+                                                                             )
+        
+        recommendations["selected_strategy"]  = selected_strategy.value
+        recommendations["selection_reason"]   = analysis_result["selection_reason"]
+        recommendations["llamaindex_used"]    = analysis_result["llamaindex_used"]
+        recommendations["structure_detected"] = analysis_result.get("has_structure", False)
+        
+        return recommendations
+
+
+# Global adaptive selector instance
+_adaptive_selector = None
+
+
+def get_adaptive_selector() -> AdaptiveChunkingSelector:
+    """
+    Get global adaptive selector instance (singleton)
+    """
+    global _adaptive_selector
+
+    if _adaptive_selector is None:
+        _adaptive_selector = AdaptiveChunkingSelector()
+
+    return _adaptive_selector
+
+
+def adaptive_chunk_text(text: str, metadata: Optional[DocumentMetadata] = None, force_strategy: Optional[ChunkingStrategy] = None) -> List[DocumentChunk]:
+    """
+    Convenience function for adaptive chunking
+    """
+    selector = get_adaptive_selector()
+
+    return selector.chunk_text(text, metadata, force_strategy)
+
+
+def analyze_document(text: str, metadata: Optional[DocumentMetadata] = None) -> dict:
+    """
+    Analyze document without chunking
+    """
+    selector = get_adaptive_selector()
+    
+    return selector._analyze_document(text, metadata)

chunking/base_chunker.py

@@ -0,0 +1,405 @@
+# DEPENDENCIES
+import re
+from abc import ABC
+from typing import List
+from pathlib import Path
+from typing import Optional
+from abc import abstractmethod
+from config.models import DocumentChunk
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.token_counter import count_tokens
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class BaseChunker(ABC):
+    """
+    Abstract base class for all chunking strategies: Implements Template Method pattern for consistent chunking pipeline
+    """
+    def __init__(self, strategy_name: ChunkingStrategy):
+        """
+        Initialize base chunker
+        
+        Arguments:
+        ----------
+            strategy_name { ChunkingStrategy } : Chunking strategy enum
+        """
+        self.strategy_name = strategy_name
+        self.logger        = logger
+    
+
+    @abstractmethod
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Chunk text into smaller pieces - must be implemented by subclasses
+        
+        Arguments:
+        ----------
+            text     { str }                : Input text to chunk
+
+            metadata { DocumentMetadata }   : Document metadata
+        
+        Returns:
+        --------
+                     { list }               : List of DocumentChunk objects
+        """
+        pass
+
+    
+    def chunk_document(self, text: str, metadata: DocumentMetadata) -> List[DocumentChunk]:
+        """
+        Chunk document with full metadata: Template method that calls chunk_text and adds metadata
+        
+        Arguments:
+        ----------
+            text     { str }                : Document text
+
+            metadata { DocumentMetadata }   : Document metadata
+        
+        Returns:
+        --------
+                     { list }               : List of DocumentChunk objects with metadata
+        """
+        try:
+            self.logger.info(f"Chunking document {metadata.document_id} using {self.strategy_name.value}")
+            
+            # Validate input
+            if not text or not text.strip():
+                self.logger.warning(f"Empty text for document {metadata.document_id}")
+                return []
+            
+            # Perform chunking
+            chunks                     = self.chunk_text(text     = text,
+                                                         metadata = metadata,
+                                                        )
+            
+            # Update metadata
+            metadata.num_chunks        = len(chunks)
+            metadata.chunking_strategy = self.strategy_name
+            
+            # Validate chunks
+            if not self.validate_chunks(chunks):
+                self.logger.warning(f"Chunk validation failed for {metadata.document_id}")
+            
+            self.logger.info(f"Created {len(chunks)} chunks for {metadata.document_id}")
+            
+            return chunks
+            
+        except Exception as e:
+            self.logger.error(f"Chunking failed for {metadata.document_id}: {repr(e)}")
+            raise
+
+    
+    def _create_chunk(self, text: str, chunk_index: int, document_id: str, start_char: int, end_char: int, page_number: Optional[int] = None, 
+                      section_title: Optional[str] = None, metadata: Optional[dict] = None) -> DocumentChunk:
+        """
+        Create a DocumentChunk object with proper formatting
+        
+        Arguments:
+        ----------
+            text          { str }  : Chunk text
+
+            chunk_index   { int }  : Index of chunk in document
+            
+            document_id   { str }  : Parent document ID
+            
+            start_char    { int }  : Start character position
+            
+            end_char      { int }  : End character position
+            
+            page_number   { int }  : Page number (if applicable)
+            
+            section_title { str }  : Section heading (CRITICAL for retrieval)
+            
+            metadata      { dict } : Additional metadata
+        
+        Returns:
+        --------
+            { DocumentChunk }      : DocumentChunk object
+        """
+        # Generate unique chunk ID
+        chunk_id    = f"chunk_{document_id}_{chunk_index}"
+        
+        # Count tokens
+        token_count = count_tokens(text)
+        
+        # Create chunk with section context
+        chunk       = DocumentChunk(chunk_id      = chunk_id,
+                                    document_id   = document_id,
+                                    text          = text,
+                                    chunk_index   = chunk_index,
+                                    start_char    = start_char,
+                                    end_char      = end_char,
+                                    page_number   = page_number,
+                                    section_title = section_title,  
+                                    token_count   = token_count,
+                                    metadata      = metadata or {},
+                                   )
+        
+        return chunk
+    
+
+    def _extract_page_number(self, text: str, full_text: str) -> Optional[int]:
+        """
+        Try to extract page number from text: Looks for [PAGE N] markers inserted during parsing
+        """
+        # Look for page markers in current chunk
+        page_match = re.search(r'\[PAGE (\d+)\]', text)
+        
+        if page_match:
+            return int(page_match.group(1))
+        
+        # Alternative: try to determine from position in full text
+        if full_text:
+            chunk_start = full_text.find(text[:min(200, len(text))])
+            
+            if (chunk_start >= 0):
+                text_before  = full_text[:chunk_start]
+                page_matches = re.findall(r'\[PAGE (\d+)\]', text_before)
+                
+                if page_matches:
+                    return int(page_matches[-1])
+        
+        return None
+    
+
+    def _clean_chunk_text(self, text: str) -> str:
+        """
+        Clean chunk text by removing markers and extra whitespace
+        
+        Arguments:
+        ----------
+            text { str } : Raw chunk text
+        
+        Returns:
+        --------
+             { str }     : Cleaned text
+        """
+        # Remove page markers
+        text = re.sub(r'\[PAGE \d+\]', '', text)
+        
+        # Remove other common markers
+        text = re.sub(r'\[HEADER\]|\[FOOTER\]|\[TABLE \d+\]', '', text)
+        
+        # Normalize whitespace
+        text = re.sub(r'\s+', ' ', text)
+        text = text.strip()
+        
+        return text
+    
+
+    def validate_chunks(self, chunks: List[DocumentChunk]) -> bool:
+        """
+        Validate chunk list for consistency
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks to validate
+        
+        Returns:
+        --------
+              { bool }      : True if valid
+        """
+        if not chunks:
+            return True
+        
+        # Check all chunks have the same document_id
+        doc_ids = {chunk.document_id for chunk in chunks}
+        
+        if (len(doc_ids) > 1):
+            self.logger.error(f"Chunks have multiple document IDs: {doc_ids}")
+            return False
+        
+        # Check chunk indices are sequential
+        indices          = [chunk.chunk_index for chunk in chunks]
+        expected_indices = list(range(len(chunks)))
+        
+        if (indices != expected_indices):
+            self.logger.warning(f"Non-sequential chunk indices: {indices}")
+        
+        # Check for empty chunks
+        empty_chunks = [c.chunk_index for c in chunks if not c.text.strip()]
+        
+        if empty_chunks:
+            self.logger.warning(f"Empty chunks at indices: {empty_chunks}")
+        
+        # Check token counts
+        zero_token_chunks = [c.chunk_index for c in chunks if (c.token_count == 0)]
+        
+        if zero_token_chunks:
+            self.logger.warning(f"Zero-token chunks at indices: {zero_token_chunks}")
+        
+        # NEW: Check section_title preservation (important for structured documents)
+        chunks_with_sections = [c for c in chunks if c.section_title]
+
+        if chunks_with_sections:
+            self.logger.info(f"{len(chunks_with_sections)}/{len(chunks)} chunks have section titles preserved")
+        
+        return True
+
+    
+    def get_chunk_statistics(self, chunks: List[DocumentChunk]) -> dict:
+        """
+        Calculate statistics for chunk list
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks
+        
+        Returns:
+        --------
+              { dict }      : Dictionary with statistics
+        """
+        if not chunks:
+            return {"num_chunks"           : 0,
+                    "total_tokens"         : 0,
+                    "avg_tokens_per_chunk" : 0,
+                    "min_tokens"           : 0,
+                    "max_tokens"           : 0,
+                    "total_chars"          : 0,
+                    "avg_chars_per_chunk"  : 0,
+                    "chunks_with_sections" : 0,
+                   }
+            
+        token_counts         = [c.token_count for c in chunks]
+        char_counts          = [len(c.text) for c in chunks]
+        chunks_with_sections = sum(1 for c in chunks if c.section_title)
+        
+        stats                = {"num_chunks"           : len(chunks),
+                                "total_tokens"         : sum(token_counts),
+                                "avg_tokens_per_chunk" : sum(token_counts) / len(chunks),
+                                "min_tokens"           : min(token_counts),
+                                "max_tokens"           : max(token_counts),
+                                "total_chars"          : sum(char_counts),
+                                "avg_chars_per_chunk"  : sum(char_counts) / len(chunks),
+                                "strategy"             : self.strategy_name.value,
+                                "chunks_with_sections" : chunks_with_sections,
+                                "section_coverage_pct" : (chunks_with_sections / len(chunks)) * 100,
+                               }
+        
+        return stats
+
+    
+    def merge_chunks(self, chunks: List[DocumentChunk], max_tokens: int) -> List[DocumentChunk]:
+        """
+        Merge small chunks up to max_tokens: Useful for optimizing chunk sizes
+        
+        Arguments:
+        ----------
+            chunks     { list } : List of chunks to merge
+
+            max_tokens { int }  : Maximum tokens per merged chunk
+        
+        Returns:
+        --------
+                { list }        : List of merged chunks
+        """
+        if not chunks:
+            return []
+        
+        merged         = list()
+        current_chunks = list()
+        current_tokens = 0
+        document_id    = chunks[0].document_id
+        
+        for chunk in chunks:
+            if ((current_tokens + chunk.token_count) <= max_tokens):
+                current_chunks.append(chunk)
+                current_tokens += chunk.token_count
+
+            else:
+                # Save current merged chunk
+                if current_chunks:
+                    merged_text  = " ".join(c.text for c in current_chunks)
+                    merged_chunk = self._create_chunk(text          = merged_text,
+                                                      chunk_index   = len(merged),
+                                                      document_id   = document_id,
+                                                      start_char    = current_chunks[0].start_char,
+                                                      end_char      = current_chunks[-1].end_char,
+                                                      page_number   = current_chunks[0].page_number,
+                                                      section_title = current_chunks[0].section_title,
+                                                     )
+                    merged.append(merged_chunk)
+                
+                # Start new chunk
+                current_chunks = [chunk]
+                current_tokens = chunk.token_count
+        
+        # Add final merged chunk
+        if current_chunks:
+            merged_text  = " ".join(c.text for c in current_chunks)
+            merged_chunk = self._create_chunk(text          = merged_text,
+                                              chunk_index   = len(merged),
+                                              document_id   = document_id,
+                                              start_char    = current_chunks[0].start_char,
+                                              end_char      = current_chunks[-1].end_char,
+                                              page_number   = current_chunks[0].page_number,
+                                              section_title = current_chunks[0].section_title,
+                                             )
+            merged.append(merged_chunk)
+        
+        self.logger.info(f"Merged {len(chunks)} chunks into {len(merged)}")
+        
+        return merged
+
+
+    def __str__(self) -> str:
+        """
+        String representation
+        """
+        return f"{self.__class__.__name__}(strategy={self.strategy_name.value})"
+    
+
+    def __repr__(self) -> str:
+        """
+        Detailed representation
+        """
+        return self.__str__()
+
+
+
+class ChunkerConfig:
+    """
+    Configuration for chunking strategies: Provides a way to pass parameters to chunkers
+    """
+    def __init__(self, chunk_size: int = 512, overlap: int = 50, respect_boundaries: bool = True, min_chunk_size: int = 100, **kwargs):
+        """
+        Initialize chunker configuration
+        
+        Arguments:
+        ----------
+            chunk_size         { int }  : Target chunk size in tokens
+
+            overlap            { int }  : Overlap between chunks in tokens
+            
+            respect_boundaries { bool } : Respect sentence/paragraph/section boundaries
+            
+            min_chunk_size     { int }  : Minimum chunk size in tokens
+            
+            **kwargs                    : Additional strategy-specific parameters
+        """
+        self.chunk_size         = chunk_size
+        self.overlap            = overlap
+        self.respect_boundaries = respect_boundaries
+        self.min_chunk_size     = min_chunk_size
+        self.extra              = kwargs
+
+    
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary
+        """
+        return {"chunk_size"         : self.chunk_size,
+                "overlap"            : self.overlap,
+                "respect_boundaries" : self.respect_boundaries,
+                "min_chunk_size"     : self.min_chunk_size,
+                **self.extra
+               }
+    
+
+    def __repr__(self) -> str:
+        return f"ChunkerConfig({self.to_dict()})"

chunking/fixed_chunker.py

@@ -0,0 +1,376 @@
+# DEPENDENCIES
+import re
+from typing import List
+from typing import Optional
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.base_chunker import BaseChunker
+from chunking.base_chunker import ChunkerConfig
+from chunking.token_counter import TokenCounter
+from chunking.overlap_manager import OverlapManager
+
+
+# Setup Settings and Logging
+logger   = get_logger(__name__)
+settings = get_settings()
+
+
+class FixedChunker(BaseChunker):
+    """
+    Fixed-size chunking strategy : Splits text into chunks of approximately equal token count with overlap
+    
+    Best for:
+    - Small to medium documents (<50K tokens)
+    - Homogeneous content
+    - When simplicity is preferred
+    """
+    def __init__(self, chunk_size: int = None, overlap: int = None, respect_sentence_boundaries: bool = True, min_chunk_size: int = 100):
+        """
+        Initialize fixed chunker
+        
+        Arguments:
+        ----------
+            chunk_size                  { int }  : Target tokens per chunk (default from settings)
+            
+            overlap                     { int }  : Overlap tokens between chunks (default from settings)
+            
+            respect_sentence_boundaries { bool } : Try to break at sentence boundaries
+            
+            min_chunk_size              { int }  : Minimum chunk size in tokens
+        """
+        super().__init__(ChunkingStrategy.FIXED)
+        
+        self.chunk_size                  = chunk_size or settings.FIXED_CHUNK_SIZE
+        self.overlap                     = overlap or settings.FIXED_CHUNK_OVERLAP
+        self.respect_sentence_boundaries = respect_sentence_boundaries
+        self.min_chunk_size              = min_chunk_size
+        
+        # Initialize token counter and overlap manager
+        self.token_counter               = TokenCounter()
+        self.overlap_manager             = OverlapManager(overlap_tokens = self.overlap)
+        
+        # Validate parameters
+        if (self.overlap >= self.chunk_size):
+            raise ValueError(f"Overlap ({self.overlap}) must be less than chunk_size ({self.chunk_size})")
+        
+        self.logger.info(f"Initialized FixedChunker: chunk_size={self.chunk_size}, overlap={self.overlap}, respect_boundaries={self.respect_sentence_boundaries}")
+    
+
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Chunk text into fixed-size pieces
+        
+        Arguments:
+        ----------
+            text            { str }       : Input text
+
+            metadata { DocumentMetaData } : Document metadata
+        
+        Returns:
+        --------
+                     { list }             : List of DocumentChunk objects
+        """
+        if not text or not text.strip():
+            return []
+        
+        document_id = metadata.document_id if metadata else "unknown"
+        
+        # Split into sentences if respecting boundaries
+        if self.respect_sentence_boundaries:
+            chunks = self._chunk_with_sentence_boundaries(text        = text, 
+                                                          document_id = document_id,
+                                                         )
+        
+        else:
+            chunks = self._chunk_without_boundaries(text        = text, 
+                                                    document_id = document_id,
+                                                   )
+        
+        # Clean and validate
+        chunks = [c for c in chunks if (c.token_count >= self.min_chunk_size)]
+        
+        # Use OverlapManager to add proper overlap
+        if ((len(chunks) > 1) and (self.overlap > 0)):
+            chunks = self.overlap_manager.add_overlap(chunks         = chunks, 
+                                                      overlap_tokens = self.overlap,
+                                                     )
+        
+        self.logger.debug(f"Created {len(chunks)} fixed-size chunks")
+        
+        return chunks
+    
+
+    def _chunk_with_sentence_boundaries(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Chunk text respecting sentence boundaries
+        
+        Arguments:
+        ----------
+            text        { str } : Input text
+
+            document_id { str } : Document ID
+        
+        Returns:
+        --------
+                { list }        : List of chunks without overlap (overlap added later)
+        """
+        # Split into sentences
+        sentences         = self._split_sentences(text = text)
+        
+        chunks            = list()
+        current_sentences = list()
+        current_tokens    = 0
+        start_char        = 0
+        
+        for sentence in sentences:
+            sentence_tokens = self.token_counter.count_tokens(text = sentence)
+            
+            # If single sentence exceeds chunk_size, split it
+            if (sentence_tokens > self.chunk_size):
+                # Save current chunk if any
+                if current_sentences:
+                    chunk_text        = " ".join(current_sentences)
+                    chunk             = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                                           chunk_index = len(chunks),
+                                                           document_id = document_id,
+                                                           start_char  = start_char,
+                                                           end_char    = start_char + len(chunk_text),
+                                                          )
+                    chunks.append(chunk)
+
+                    current_sentences = list()
+                    current_tokens    = 0
+                    start_char       += len(chunk_text)
+                
+                # Split long sentence and add as separate chunks
+                long_sentence_chunks = self._split_long_sentence(sentence    = sentence,
+                                                                 document_id = document_id,
+                                                                 start_index = len(chunks),
+                                                                 start_char  = start_char,
+                                                                )
+                chunks.extend(long_sentence_chunks)
+                start_char          += len(sentence)
+                
+                continue
+            
+            # Check if adding this sentence exceeds chunk_size
+            if (((current_tokens + sentence_tokens) > self.chunk_size) and current_sentences):
+                # Save current chunk WITHOUT overlap (overlap added later)
+                chunk_text = " ".join(current_sentences)
+                chunk      = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                                chunk_index = len(chunks),
+                                                document_id = document_id,
+                                                start_char  = start_char,
+                                                end_char    = start_char + len(chunk_text),
+                                               )
+                chunks.append(chunk)
+                
+                # OverlapManager will handle the overlap here
+                current_sentences = [sentence]
+                current_tokens    = sentence_tokens
+                start_char       += len(chunk_text)
+
+            else:
+                # Add sentence to current chunk
+                current_sentences.append(sentence)
+                current_tokens += sentence_tokens
+        
+        # Add final chunk
+        if current_sentences:
+            chunk_text = " ".join(current_sentences)
+            chunk      = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                            chunk_index = len(chunks),
+                                            document_id = document_id,
+                                            start_char  = start_char,
+                                            end_char    = start_char + len(chunk_text),
+                                           )
+            chunks.append(chunk)
+        
+        return chunks
+    
+
+    def _chunk_without_boundaries(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Chunk text without respecting boundaries (pure token-based)
+        
+        Arguments:
+        ----------
+            text        { str } : Input text
+
+            document_id { str } : Document ID
+        
+        Returns:
+        --------
+                  { list }      : List of chunks WITHOUT overlap
+        """
+        # Use token counter's split method
+        chunk_texts = self.token_counter.split_into_token_chunks(text,
+                                                                 chunk_size = self.chunk_size,
+                                                                 overlap    = 0,
+                                                                )
+        
+        chunks      = list()
+        current_pos = 0
+        
+        for i, chunk_text in enumerate(chunk_texts):
+            chunk        = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                              chunk_index = i,
+                                              document_id = document_id,
+                                              start_char  = current_pos,
+                                              end_char    = current_pos + len(chunk_text),
+                                             )
+
+            chunks.append(chunk)
+            current_pos += len(chunk_text)
+        
+        return chunks
+    
+
+    def _split_sentences(self, text: str) -> List[str]:
+        """
+        Split text into sentences
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+            { list }     : List of sentences
+        """
+        # Handle common abbreviations: Protect them temporarily
+        protected     = text
+        abbreviations = ['Dr.', 'Mr.', 'Mrs.', 'Ms.', 'Jr.', 'Sr.', 'Prof.', 'Inc.', 'Ltd.', 'Corp.', 'Co.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'Ph.D.', 'M.D.', 'B.A.', 'M.A.', 'U.S.', 'U.K.']
+        
+        for abbr in abbreviations:
+            protected = protected.replace(abbr, abbr.replace('.', '<DOT>'))
+        
+        # Split on sentence boundaries
+        # - Pattern: period/question/exclamation followed by space and capital letter
+        sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])'
+        sentences        = re.split(sentence_pattern, protected)
+        
+        # Restore abbreviations
+        sentences        = [s.replace('<DOT>', '.').strip() for s in sentences]
+        
+        # Filter empty
+        sentences        = [s for s in sentences if s]
+        
+        return sentences
+
+    
+    def _split_long_sentence(self, sentence: str, document_id: str, start_index: int, start_char: int) -> List[DocumentChunk]:
+        """
+        Split a sentence that's longer than chunk_size
+        
+        Arguments:
+        ----------
+            sentence    { str } : Long sentence
+            
+            document_id { str } : Document ID
+            
+            start_index { str } : Starting chunk index
+            
+            start_char  { int } : Starting character position
+        
+        Returns:
+        --------
+                { list }        : List of chunks
+        """
+        # Split by commas, semicolons, or just by tokens
+        parts          = re.split(r'[,;]', sentence)
+        
+        chunks         = list()
+        current_text   = list()
+        current_tokens = 0
+        
+        for part in parts:
+            part = part.strip()
+            if not part:
+                continue
+            
+            part_tokens = self.token_counter.count_tokens(part)
+            
+            if (((current_tokens + part_tokens) > self.chunk_size) and current_text):
+                # Save current chunk
+                chunk_text     = " ".join(current_text)
+                chunk          = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                                    chunk_index = start_index + len(chunks),
+                                                    document_id = document_id,
+                                                    start_char  = start_char,
+                                                    end_char    = start_char + len(chunk_text),
+                                                   )
+                chunks.append(chunk)
+                start_char    += len(chunk_text)
+                current_text   = []
+                current_tokens = 0
+            
+            current_text.append(part)
+            current_tokens += part_tokens
+        
+        # Add final part
+        if current_text:
+            chunk_text = " ".join(current_text)
+            chunk      = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                            chunk_index = start_index + len(chunks),
+                                            document_id = document_id,
+                                            start_char  = start_char,
+                                            end_char    = start_char + len(chunk_text),
+                                           )
+            chunks.append(chunk)
+        
+        return chunks
+    
+
+    def _get_overlap_sentences(self, sentences: List[str], overlap_tokens: int) -> List[str]:
+        """
+        Get last few sentences that fit in overlap window
+        
+        Arguments:
+        ----------
+            sentences      { list } : List of sentences
+
+            overlap_tokens { int }  : Target overlap tokens
+        
+        Returns:
+        --------
+                  { list }          : List of overlap sentences
+        """
+        overlap = list()
+        tokens  = 0
+        
+        # Add sentences from the end until we reach overlap size
+        for sentence in reversed(sentences):
+            sentence_tokens = self.token_counter.count_tokens(sentence)
+            
+            if ((tokens + sentence_tokens) <= overlap_tokens):
+                overlap.insert(0, sentence)
+                tokens += sentence_tokens
+            
+            else:
+                break
+        
+        return overlap
+
+    
+    @classmethod
+    def from_config(cls, config: ChunkerConfig) -> 'FixedChunker':
+        """
+        Create FixedChunker from configuration
+        
+        Arguments:
+        ----------
+            config { ChunkerConfig } : ChunkerConfig object
+        
+        Returns:
+        --------
+            FixedChunker instance
+        """
+        return cls(chunk_size                  = config.chunk_size,
+                   overlap                     = config.overlap,
+                   respect_sentence_boundaries = config.respect_boundaries,
+                   min_chunk_size              = config.min_chunk_size,
+                  )

chunking/hierarchical_chunker.py

@@ -0,0 +1,280 @@
+# DEPENDENCIES
+from typing import List
+from typing import Optional
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.base_chunker import BaseChunker
+from chunking.base_chunker import ChunkerConfig
+from chunking.token_counter import TokenCounter
+from chunking.overlap_manager import OverlapManager
+from chunking.fixed_chunker import FixedChunker
+
+
+# Setup Settings and Logging
+logger   = get_logger(__name__)
+settings = get_settings()
+
+
+class HierarchicalChunker(BaseChunker):
+    """
+    Hierarchical chunking strategy:
+    - Creates parent chunks (large) and child chunks (small)
+    - Child chunks for granular search, parent chunks for context
+    - Maintains parent-child relationships for context expansion
+    
+    Best for:
+    - Large documents (>500K tokens)
+    - Complex documents with nested structure
+    - When both granular search and context preservation are needed
+    """
+    def __init__(self, parent_chunk_size: int = None, child_chunk_size: int = None, overlap: int = None, min_chunk_size: int = 100):
+        """
+        Initialize hierarchical chunker
+        
+        Arguments:
+        ----------
+            parent_chunk_size { int } : Size of parent chunks in tokens
+            
+            child_chunk_size  { int } : Size of child chunks in tokens
+            
+            overlap           { int } : Overlap between child chunks
+            
+            min_chunk_size    { int } : Minimum chunk size in tokens
+        """
+        super().__init__(ChunkingStrategy.HIERARCHICAL)
+        
+        self.parent_chunk_size = parent_chunk_size or settings.PARENT_CHUNK_SIZE
+        self.child_chunk_size  = child_chunk_size or settings.CHILD_CHUNK_SIZE
+        self.overlap           = overlap or settings.FIXED_CHUNK_OVERLAP
+        self.min_chunk_size    = min_chunk_size
+        
+        # Validate parameters
+        if (self.child_chunk_size >= self.parent_chunk_size):
+            raise ValueError(f"Child chunk size ({self.child_chunk_size}) must be smaller than parent chunk size ({self.parent_chunk_size})")
+        
+        # Initialize dependencies
+        self.token_counter     = TokenCounter()
+        self.overlap_manager   = OverlapManager(overlap_tokens = self.overlap)
+        self.child_chunker     = FixedChunker(chunk_size                  = self.child_chunk_size,
+                                              overlap                     = self.overlap,
+                                              respect_sentence_boundaries = True,
+                                             )
+        
+        self.logger.info(f"Initialized HierarchicalChunker: parent_size={self.parent_chunk_size}, child_size={self.child_chunk_size}, overlap={self.overlap}")
+    
+
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Create hierarchical chunks with parent-child relationships
+        
+        Arguments:
+        ----------
+            text            { str }       : Input text
+
+            metadata { DocumentMetaData } : Document metadata
+        
+        Returns:
+        --------
+                     { list }             : List of DocumentChunk objects (children with parent references)
+        """
+        if not text or not text.strip():
+            return []
+        
+        document_id      = metadata.document_id if metadata else "unknown"
+        
+        # Create parent chunks (large context windows)
+        parent_chunks    = self._create_parent_chunks(text, document_id)
+        
+        # For each parent chunk, create child chunks (granular search)
+        all_child_chunks = list()
+        
+        for parent_chunk in parent_chunks:
+            child_chunks = self._create_child_chunks(parent_chunk    = parent_chunk,
+                                                     parent_text     = text,
+                                                     document_id     = document_id,
+                                                    )
+            all_child_chunks.extend(child_chunks)
+        
+        # Step 3: Filter small chunks
+        all_child_chunks = [c for c in all_child_chunks if (c.token_count >= self.min_chunk_size)]
+        
+        self.logger.info(f"Created {len(all_child_chunks)} child chunks from {len(parent_chunks)} parent chunks")
+        
+        return all_child_chunks
+    
+
+    def _create_parent_chunks(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Create large parent chunks for context preservation
+        
+        Arguments:
+        ----------
+            text        { str } : Input text
+
+            document_id { str } : Document ID
+        
+        Returns:
+        --------
+                { list }        : List of parent chunks WITHOUT overlap
+        """
+        # Use fixed chunking for parents (no overlap between parents)
+        parent_chunker = FixedChunker(chunk_size                  = self.parent_chunk_size,
+                                      overlap                     = 0,  # No overlap between parents
+                                      respect_sentence_boundaries = True,
+                                     )
+        
+        # Create parent chunks
+        parent_chunks  = parent_chunker._chunk_with_sentence_boundaries(text        = text,
+                                                                        document_id = document_id,
+                                                                       )
+        
+        # Add parent metadata
+        for i, chunk in enumerate(parent_chunks):
+            chunk.metadata["chunk_type"]      = "parent"
+            chunk.metadata["parent_chunk_id"] = chunk.chunk_id
+        
+        return parent_chunks
+    
+
+    def _create_child_chunks(self, parent_chunk: DocumentChunk, parent_text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Create child chunks within a parent chunk
+        
+        Arguments:
+        ----------
+            parent_chunk { DocumentChunk } : Parent chunk object
+
+            parent_text  { str }           : Full parent text (for position reference)
+            
+            document_id  { str }           : Document ID
+        
+        Returns:
+        --------
+                   { list }                : List of child chunks with parent references
+        """
+        # Extract the actual text segment from parent_text using parent chunk positions
+        parent_segment = parent_text[parent_chunk.start_char:parent_chunk.end_char]
+        
+        # Create child chunks within this parent segment
+        child_chunker  = FixedChunker(chunk_size                  = self.child_chunk_size,
+                                      overlap                     = self.overlap,
+                                      respect_sentence_boundaries = True,
+                                     )
+        
+        # Create child chunks with proper positioning
+        child_chunks   = child_chunker._chunk_with_sentence_boundaries(text        = parent_segment,
+                                                                       document_id = document_id,
+                                                                      )
+        
+        # Update child chunks with parent relationship and correct positions
+        for i, child_chunk in enumerate(child_chunks):
+            # Adjust positions to be relative to full document
+            child_chunk.start_char                 += parent_chunk.start_char
+            child_chunk.end_char                   += parent_chunk.start_char
+            
+            # Add parent relationship metadata
+            child_chunk.metadata["chunk_type"]      = "child"
+            child_chunk.metadata["parent_chunk_id"] = parent_chunk.chunk_id
+            child_chunk.metadata["parent_index"]    = i
+            
+            # Update chunk ID to reflect hierarchy
+            child_chunk.chunk_id                    = f"{parent_chunk.chunk_id}_child_{i}"
+        
+        return child_chunks
+    
+
+    def expand_to_parent_context(self, child_chunk: DocumentChunk, all_chunks: List[DocumentChunk]) -> DocumentChunk:
+        """
+        Expand a child chunk to include full parent context for generation
+        
+        Arguments:
+        ----------
+            child_chunk { DocumentChunk } : Child chunk to expand
+
+            all_chunks  { list }          : All chunks from the document
+        
+        Returns:
+        --------
+                   { DocumentChunk }      : Expanded chunk with parent context
+        """
+        # Find the parent chunk
+        parent_chunk_id = child_chunk.metadata.get("parent_chunk_id")
+        
+        if not parent_chunk_id:
+            return child_chunk
+        
+        parent_chunk = next((c for c in all_chunks if c.chunk_id == parent_chunk_id), None)
+        
+        if not parent_chunk:
+            return child_chunk
+        
+        # Create expanded chunk with parent context
+        expanded_text  = f"[PARENT_CONTEXT]\n{parent_chunk.text}\n\n[CHILD_CONTEXT]\n{child_chunk.text}"
+        
+        expanded_chunk = DocumentChunk(chunk_id        = f"{child_chunk.chunk_id}_expanded",
+                                       document_id     = child_chunk.document_id,
+                                       text            = expanded_text,
+                                       chunk_index     = child_chunk.chunk_index,
+                                       start_char      = child_chunk.start_char,
+                                       end_char        = child_chunk.end_char,
+                                       page_number     = child_chunk.page_number,
+                                       section_title   = child_chunk.section_title,
+                                       token_count     = self.token_counter.count_tokens(expanded_text),
+                                       parent_chunk_id = parent_chunk_id,
+                                       child_chunk_ids = [child_chunk.chunk_id],
+                                       metadata        = {**child_chunk.metadata, "expanded": True},
+                                      )
+        
+        return expanded_chunk
+    
+
+    def get_parent_child_relationships(self, chunks: List[DocumentChunk]) -> dict:
+        """
+        Extract parent-child relationships from chunks
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks
+        
+        Returns:
+        --------
+              { dict }      : Dictionary mapping parent IDs to child chunks
+        """
+        relationships = dict()
+        
+        for chunk in chunks:
+            if (chunk.metadata.get("chunk_type") == "parent"):
+                relationships[chunk.chunk_id] = {"parent"   : chunk,
+                                                 "children" : [],
+                                                }
+        
+        for chunk in chunks:
+            parent_id = chunk.metadata.get("parent_chunk_id")
+            
+            if parent_id and parent_id in relationships:
+                relationships[parent_id]["children"].append(chunk)
+        
+        return relationships
+    
+
+    @classmethod
+    def from_config(cls, config: ChunkerConfig) -> 'HierarchicalChunker':
+        """
+        Create HierarchicalChunker from configuration
+        
+        Arguments:
+        ----------
+            config { ChunkerConfig } : ChunkerConfig object
+        
+        Returns:
+        --------
+            { HierarchicalChunker }  : HierarchicalChunker instance
+        """
+        return cls(parent_chunk_size = config.extra.get('parent_size', settings.PARENT_CHUNK_SIZE),
+                   child_chunk_size  = config.extra.get('child_size', settings.CHILD_CHUNK_SIZE),
+                   overlap           = config.overlap,
+                   min_chunk_size    = config.min_chunk_size,
+                  )

chunking/llamaindex_chunker.py

@@ -0,0 +1,240 @@
+# DEPENDENCIES
+from typing import List
+from typing import Optional
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.base_chunker import BaseChunker
+from chunking.base_chunker import ChunkerConfig
+from chunking.token_counter import TokenCounter
+from chunking.semantic_chunker import SemanticChunker
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.node_parser import TokenTextSplitter
+from llama_index.core.schema import Document as LlamaDocument
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core.node_parser import SemanticSplitterNodeParser
+
+
+# Setup Settings and Logging
+logger   = get_logger(__name__)
+settings = get_settings()
+
+
+class LlamaIndexChunker(BaseChunker):
+    """
+    LlamaIndex-based semantic chunking strategy:
+    - Uses LlamaIndex's advanced semantic splitting algorithms
+    - Provides superior boundary detection using embeddings
+    - Supports multiple LlamaIndex splitter types
+    
+    Best for:
+    - Documents requiring sophisticated semantic analysis
+    - When LlamaIndex ecosystem integration is needed
+    - Advanced chunking with embedding-based boundaries
+    """
+    def __init__(self, chunk_size: int = None, overlap: int = None, splitter_type: str = "semantic", min_chunk_size: int = 100):
+        """
+        Initialize LlamaIndex chunker
+        
+        Arguments:
+        ----------
+            chunk_size     { int }  : Target tokens per chunk
+            
+            overlap        { int }  : Overlap tokens between chunks
+            
+            splitter_type  { str }  : Type of LlamaIndex splitter ("semantic", "sentence", "token")
+            
+            min_chunk_size { int }  : Minimum chunk size in tokens
+        """
+        # Use SEMANTIC since it's semantic-based
+        super().__init__(ChunkingStrategy.SEMANTIC)  
+        
+        self.chunk_size     = chunk_size or settings.FIXED_CHUNK_SIZE
+        self.overlap        = overlap or settings.FIXED_CHUNK_OVERLAP
+        self.splitter_type  = splitter_type
+        self.min_chunk_size = min_chunk_size
+        
+        # Initialize token counter
+        self.token_counter  = TokenCounter()
+        
+        # Initialize LlamaIndex components
+        self._splitter      = None
+        self._initialized   = False
+        
+        self._initialize_llamaindex()
+        
+        self.logger.info(f"Initialized LlamaIndexChunker: chunk_size={self.chunk_size}, overlap={self.overlap}, splitter_type={self.splitter_type}")
+    
+
+    def _initialize_llamaindex(self):
+        """
+        Initialize LlamaIndex splitter with proper error handling
+        """
+        try:
+            # Initialize embedding model
+            embed_model = HuggingFaceEmbedding(model_name = settings.EMBEDDING_MODEL)
+            
+            # Initialize appropriate splitter based on type
+            if (self.splitter_type == "semantic"):
+                self._splitter = SemanticSplitterNodeParser(buffer_size         = 1,
+                                                            breakpoint_percentile_threshold = 95,
+                                                            embed_model        = embed_model,
+                                                           )
+            
+            elif (self.splitter_type == "sentence"):
+                self._splitter = SentenceSplitter(chunk_size       = self.chunk_size,
+                                                  chunk_overlap    = self.overlap,
+                                                 )
+            
+            elif (self.splitter_type == "token"):
+                self._splitter = TokenTextSplitter(chunk_size       = self.chunk_size,
+                                                   chunk_overlap    = self.overlap,
+                                                  )
+            
+            else:
+                self.logger.warning(f"Unknown splitter type: {self.splitter_type}, using semantic")
+                self._splitter = SemanticSplitterNodeParser(buffer_size                     = 1,
+                                                            breakpoint_percentile_threshold = 95,
+                                                            embed_model                     = embed_model,
+                                                           )
+            
+            self._initialized = True
+            self.logger.info(f"Successfully initialized LlamaIndex {self.splitter_type} splitter")
+            
+        except ImportError as e:
+            self.logger.error(f"LlamaIndex not available: {repr(e)}")
+            self._initialized = False
+        
+        except Exception as e:
+            self.logger.error(f"Failed to initialize LlamaIndex: {repr(e)}")
+            self._initialized = False
+    
+
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Chunk text using LlamaIndex semantic splitting
+        
+        Arguments:
+        ----------
+            text            { str }       : Input text
+
+            metadata { DocumentMetaData } : Document metadata
+        
+        Returns:
+        --------
+                     { list }             : List of DocumentChunk objects
+        """
+        if not text or not text.strip():
+            return []
+        
+        # Fallback if LlamaIndex not available
+        if not self._initialized:
+            self.logger.warning("LlamaIndex not available, falling back to simple semantic chunking")
+            return self._fallback_chunking(text        = text, 
+                                           metadata    = metadata,
+                                          )
+        
+        document_id = metadata.document_id if metadata else "unknown"
+        
+        try:
+            # Create LlamaIndex document
+            llama_doc = LlamaDocument(text = text)
+            
+            # Get nodes from splitter
+            nodes     = self._splitter.get_nodes_from_documents([llama_doc])
+            
+            # Convert nodes to our DocumentChunk format
+            chunks    = list()
+            start_pos = 0
+            
+            for i, node in enumerate(nodes):
+                chunk_text = node.text
+                
+                # Create chunk
+                chunk      = self._create_chunk(text          = self._clean_chunk_text(chunk_text),
+                                                chunk_index   = i,
+                                                document_id   = document_id,
+                                                start_char    = start_pos,
+                                                end_char      = start_pos + len(chunk_text),
+                                                metadata      = {"llamaindex_splitter" : self.splitter_type,
+                                                                 "node_id"             : node.node_id,
+                                                                 "chunk_type"          : "llamaindex_semantic",
+                                                                }
+                                               )
+                
+                chunks.append(chunk)
+                start_pos += len(chunk_text)
+            
+            # Filter out chunks that are too small
+            chunks = [c for c in chunks if (c.token_count >= self.min_chunk_size)]
+            
+            self.logger.debug(f"Created {len(chunks)} chunks using LlamaIndex {self.splitter_type} splitter")
+            
+            return chunks
+            
+        except Exception as e:
+            self.logger.error(f"LlamaIndex chunking failed: {repr(e)}")
+            return self._fallback_chunking(text     = text, 
+                                           metadata = metadata,
+                                          )
+    
+
+    def _fallback_chunking(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Fallback to basic semantic chunking when LlamaIndex fails
+        
+        Arguments:
+        ----------
+            text            { str }       : Input text
+
+            metadata { DocumentMetaData } : Document metadata
+        
+        Returns:
+        --------
+                     { list }             : List of chunks
+        """
+        fallback_chunker = SemanticChunker(chunk_size           = self.chunk_size,
+                                           overlap              = self.overlap,
+                                           similarity_threshold = 0.95,
+                                           min_chunk_size       = self.min_chunk_size,
+                                          )
+        
+        return fallback_chunker.chunk_text(text, metadata)
+    
+
+    def get_splitter_info(self) -> dict:
+        """
+        Get information about the LlamaIndex splitter configuration
+        
+        Returns:
+        --------
+            { dict }    : Splitter information
+        """
+        return {"splitter_type"   : self.splitter_type,
+                "chunk_size"      : self.chunk_size,
+                "overlap"         : self.overlap,
+                "initialized"     : self._initialized,
+                "min_chunk_size"  : self.min_chunk_size,
+               }
+    
+
+    @classmethod
+    def from_config(cls, config: ChunkerConfig) -> 'LlamaIndexChunker':
+        """
+        Create LlamaIndexChunker from configuration
+        
+        Arguments:
+        ----------
+            config { ChunkerConfig } : ChunkerConfig object
+        
+        Returns:
+        --------
+            { LlamaIndexChunker }    : LlamaIndexChunker instance
+        """
+        return cls(chunk_size    = config.chunk_size,
+                   overlap       = config.overlap,
+                   splitter_type = config.extra.get('llamaindex_splitter', 'semantic'),
+                   min_chunk_size = config.min_chunk_size,
+                  )

chunking/overlap_manager.py

@@ -0,0 +1,421 @@
+# DEPENDENCIES
+from typing import List
+from typing import Tuple
+from typing import Optional
+from config.models import DocumentChunk
+from config.logging_config import get_logger
+from chunking.token_counter import TokenCounter
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class OverlapManager:
+    """
+    Manages overlapping regions between chunks : ensures smooth context transitions and optimal retrieval
+    """
+    def __init__(self, overlap_tokens: int = 50):
+        """
+        Initialize overlap manager
+        
+        Arguments:
+        ----------
+            overlap_tokens { int } : Target overlap in tokens
+        """
+        self.overlap_tokens = overlap_tokens
+        self.token_counter  = TokenCounter()
+        self.logger         = logger
+    
+
+    def add_overlap(self, chunks: List[DocumentChunk], overlap_tokens: Optional[int] = None) -> List[DocumentChunk]:
+        """
+        Add overlap to existing chunks
+        
+        Arguments:
+        ----------
+            chunks         { list } : List of chunks without overlap
+
+            overlap_tokens { int }  : Override default overlap
+        
+        Returns:
+        --------
+                  { list }          : List of chunks with overlap
+        """
+        if (not chunks or (len(chunks) < 2)):
+            return chunks
+        
+        overlap           = overlap_tokens or self.overlap_tokens
+        overlapped_chunks = list()
+        
+        for i, chunk in enumerate(chunks):
+            if (i == 0):
+                # First chunk: no prefix, add suffix from next
+                new_text = chunk.text
+                if (i + 1 < len(chunks)):
+                    suffix   = self._get_overlap_text(text           = chunks[i + 1].text,
+                                                      overlap_tokens = overlap,
+                                                      from_start     = True,
+                                                     )
+
+                    new_text = new_text + " " + suffix
+            
+            elif (i == len(chunks) - 1):
+                # Last chunk: add prefix from previous, no suffix
+                prefix   = self._get_overlap_text(text           = chunks[i - 1].text,
+                                                  overlap_tokens = overlap,
+                                                  from_start     = False,
+                                                 )
+
+                new_text = prefix + " " + chunk.text
+            
+            else:
+                # Middle chunk: add both prefix and suffix
+                prefix   = self._get_overlap_text(text           = chunks[i - 1].text,
+                                                  overlap_tokens = overlap,
+                                                  from_start     = False,
+                                                 )
+
+                suffix   = self._get_overlap_text(text           = chunks[i + 1].text,
+                                                  overlap_tokens = overlap,
+                                                  from_start     = True,
+                                                 )
+
+                new_text = prefix + " " + chunk.text + " " + suffix
+            
+            # Create new chunk with overlapped text
+            overlapped_chunk = DocumentChunk(chunk_id      = chunk.chunk_id,
+                                             document_id   = chunk.document_id,
+                                             text          = new_text,
+                                             chunk_index   = chunk.chunk_index,
+                                             start_char    = chunk.start_char,
+                                             end_char      = chunk.end_char,
+                                             page_number   = chunk.page_number,
+                                             section_title = chunk.section_title,
+                                             token_count   = self.token_counter.count_tokens(new_text),
+                                             metadata      = chunk.metadata,
+                                            )
+
+            overlapped_chunks.append(overlapped_chunk)
+        
+        self.logger.debug(f"Added overlap to {len(chunks)} chunks")
+        return overlapped_chunks
+    
+
+    def _get_overlap_text(self, text: str, overlap_tokens: int, from_start: bool) -> str:
+        """
+        Extract overlap text from beginning or end
+        
+        Arguments:
+        ----------
+            text            { str } : Source text
+            
+            overlap_tokens  { int } : Number of tokens to extract
+            
+            from_start     { bool } : True for start, False for end
+        
+        Returns:
+        --------
+                  { str }           : Overlap text
+        """
+        total_tokens = self.token_counter.count_tokens(text)
+    
+        if (total_tokens <= overlap_tokens):
+            return text
+        
+        if from_start:
+            # Get first N tokens
+            return self.token_counter.truncate_to_tokens(text       = text, 
+                                                         max_tokens = overlap_tokens, 
+                                                         suffix     = "",
+                                                        )
+        
+        else:
+            # Get last N tokens using token counter's boundary finding
+            char_pos, overlap_text = self.token_counter.find_token_boundaries(text          = text, 
+                                                                              target_tokens = overlap_tokens,
+                                                                             )
+            
+            # Take from the end instead of beginning
+            if (char_pos < len(text)):
+                return text[-char_pos:] if (char_pos > 0) else text
+
+            return overlap_text
+    
+
+    def remove_overlap(self, chunks: List[DocumentChunk]) -> List[DocumentChunk]:
+        """
+        Remove overlap from chunks (get core content only)
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks with overlap
+        
+        Returns:
+        --------
+              { list }      : List of chunks without overlap
+        """
+        if (not chunks or (len(chunks) < 2)):
+            return chunks
+        
+        core_chunks = list()
+        
+        for i, chunk in enumerate(chunks):
+            if (i == 0):
+                # First chunk: remove suffix
+                core_text = self._remove_suffix_overlap(text      = chunk.text,
+                                                        next_text = chunks[i + 1].text if i + 1 < len(chunks) else "",
+                                                       )
+            elif (i == len(chunks) - 1):
+                # Last chunk: remove prefix
+                core_text = self._remove_prefix_overlap(text = chunk.text,
+                                                        previous_text = chunks[i - 1].text,
+                                                       )
+            else:
+                # Middle chunk: remove both
+                temp_text = self._remove_prefix_overlap(text = chunk.text,
+                                                        previous_text = chunks[i - 1].text,
+                                                       )
+
+                core_text = self._remove_suffix_overlap(text      = temp_text,
+                                                        next_text = chunks[i + 1].text,
+                                                       )
+            
+            core_chunk = DocumentChunk(chunk_id      = chunk.chunk_id,
+                                       document_id   = chunk.document_id,
+                                       text          = core_text,
+                                       chunk_index   = chunk.chunk_index,
+                                       start_char    = chunk.start_char,
+                                       end_char      = chunk.end_char,
+                                       page_number   = chunk.page_number,
+                                       section_title = chunk.section_title,
+                                       token_count   = self.token_counter.count_tokens(core_text),
+                                       metadata      = chunk.metadata,
+                                      )
+
+            core_chunks.append(core_chunk)
+        
+        return core_chunks
+
+    
+    def _remove_prefix_overlap(self, text: str, previous_text: str) -> str:
+        """
+        Remove overlap with previous chunk
+        """
+        if not text or not previous_text:
+            return text
+        
+        words       = text.split()
+        prev_words  = previous_text.split()
+        
+        # Find longest common suffix-prefix match
+        max_overlap = 0
+
+        for overlap_size in range(1, min(len(words), len(prev_words)) + 1):
+            if (words[:overlap_size] == prev_words[-overlap_size:]):
+                max_overlap = overlap_size
+        
+        if (max_overlap > 0):
+            return " ".join(words[max_overlap:])
+        
+        return text
+    
+
+    def _remove_suffix_overlap(self, text: str, next_text: str) -> str:
+        """
+        Remove overlap with next chunk
+        """
+        # Find common suffix
+        words         = text.split()
+        next_words    = next_text.split()
+        
+        common_length = 0
+
+        for i in range(1, min(len(words), len(next_words)) + 1):
+            if (words[-i] == next_words[i - 1]):
+                common_length += 1
+
+            else:
+                break
+        
+        if (common_length > 0):
+            return " ".join(words[:-common_length])
+
+        return text
+    
+
+    def calculate_overlap_percentage(self, chunks: List[DocumentChunk]) -> float:
+        """
+        Calculate average overlap percentage
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks
+        
+        Returns:
+        --------
+              { float }     : Average overlap percentage
+        """
+        if (len(chunks) < 2):
+            return 0.0
+        
+        overlaps = list()
+
+        for i in range(len(chunks) - 1):
+            overlap = self._measure_overlap(chunks[i].text, chunks[i + 1].text)
+            
+            overlaps.append(overlap)
+        
+        return sum(overlaps) / len(overlaps) if overlaps else 0.0
+    
+
+    def _measure_overlap(self, text1: str, text2: str) -> float:
+        """
+        Measure overlap between two texts
+        
+        Arguments:
+        ----------
+            text1 { str } : First text
+
+            text2 { str } : Second text
+        
+        Returns:
+        --------
+             { float }    : Overlap percentage (0-100)
+        """
+        words1      = set(text1.lower().split())
+        words2      = set(text2.lower().split())
+        
+        if (not words1 or not words2):
+            return 0.0
+        
+        common      = words1 & words2
+        overlap_pct = (len(common) / min(len(words1), len(words2))) * 100
+        
+        return overlap_pct
+    
+
+    def optimize_overlaps(self, chunks: List[DocumentChunk], target_overlap: int, tolerance: int = 10) -> List[DocumentChunk]:
+        """
+        Optimize overlap sizes to target
+        
+        Arguments:
+        ----------
+            chunks        { list } : List of chunks
+            
+            target_overlap { int } : Target overlap in tokens
+            
+            tolerance      { int } : Acceptable deviation in tokens
+        
+        Returns:
+        --------
+                  { list }         : Optimized chunks
+        """
+        if (len(chunks) < 2):
+            return chunks
+
+        # Validate target_overlap is reasonable
+        if (target_overlap <= 0):
+            self.logger.warning("Target overlap must be positive, using default")
+            target_overlap = self.overlap_tokens
+        
+        optimized = list()
+        
+        for i in range(len(chunks)):
+            chunk = chunks[i]
+            
+            # Check current overlap with next chunk
+            if (i < len(chunks) - 1):
+                current_overlap = self._count_overlap_tokens(text1 = chunk.text,
+                                                             text2 = chunks[i + 1].text,
+                                                            )
+                
+                # Adjust if outside tolerance
+                if (abs(current_overlap - target_overlap) > tolerance):
+                    # Add or remove text to reach target
+                    if (current_overlap < target_overlap):
+                        # Need more overlap
+                        additional = self._get_overlap_text(text           = chunks[i + 1].text,
+                                                            overlap_tokens = target_overlap - current_overlap,
+                                                            from_start     = True,
+                                                           )
+
+                        new_text   = chunk.text + " " + additional
+
+                    else:
+                        # Need less overlap
+                        new_text = self.token_counter.truncate_to_tokens(text       = chunk.text,
+                                                                         max_tokens = self.token_counter.count_tokens(chunk.text) - (current_overlap - target_overlap),
+                                                                        )
+                    
+                    chunk = DocumentChunk(chunk_id      = chunk.chunk_id,
+                                          document_id   = chunk.document_id,
+                                          text          = new_text,
+                                          chunk_index   = chunk.chunk_index,
+                                          start_char    = chunk.start_char,
+                                          end_char      = chunk.end_char,
+                                          page_number   = chunk.page_number,
+                                          section_title = chunk.section_title,
+                                          token_count   = self.token_counter.count_tokens(new_text),
+                                          metadata      = chunk.metadata,
+                                         )
+            
+            optimized.append(chunk)
+        
+        return optimized
+
+    
+    def _count_overlap_tokens(self, text1: str, text2: str) -> int:
+        """
+        Count overlapping tokens between two texts
+        """
+        # Find longest common substring at the boundary
+        words1      = text1.split()
+        words2      = text2.split()
+        
+        max_overlap = 0
+
+        for i in range(1, min(len(words1), len(words2)) + 1):
+            if (words1[-i:] == words2[:i]):
+                overlap_text = " ".join(words1[-i:])
+                max_overlap  = self.token_counter.count_tokens(overlap_text)
+        
+        return max_overlap
+    
+
+    def get_overlap_statistics(self, chunks: List[DocumentChunk]) -> dict:
+        """
+        Get statistics about overlaps
+        
+        Arguments:
+        ----------
+            chunks { list } : List of chunks
+        
+        Returns:
+        --------
+              { dict }      : Statistics dictionary
+        """
+        if (len(chunks) < 2):
+            return {"num_chunks"             : len(chunks),
+                    "num_overlaps"           : 0,
+                    "avg_overlap_tokens"     : 0,
+                    "avg_overlap_percentage" : 0,
+                   }
+        
+        overlap_tokens      = list()
+        overlap_percentages = list()
+        
+        for i in range(len(chunks) - 1):
+            tokens = self._count_overlap_tokens(chunks[i].text, chunks[i + 1].text)
+            pct    = self._measure_overlap(chunks[i].text, chunks[i + 1].text)
+            
+            overlap_tokens.append(tokens)
+            overlap_percentages.append(pct)
+        
+        return {"num_chunks"             : len(chunks),
+                "num_overlaps"           : len(overlap_tokens),
+                "avg_overlap_tokens"     : sum(overlap_tokens) / len(overlap_tokens) if overlap_tokens else 0,
+                "min_overlap_tokens"     : min(overlap_tokens) if overlap_tokens else 0,
+                "max_overlap_tokens"     : max(overlap_tokens) if overlap_tokens else 0,
+                "avg_overlap_percentage" : sum(overlap_percentages) / len(overlap_percentages) if overlap_percentages else 0,
+               }

chunking/semantic_chunker.py

@@ -0,0 +1,607 @@
+# DEPENDENCIES
+import re
+import numpy as np
+from typing import List
+from typing import Tuple
+from typing import Optional
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import DocumentMetadata
+from config.models import ChunkingStrategy
+from config.logging_config import get_logger
+from chunking.base_chunker import BaseChunker
+from chunking.base_chunker import ChunkerConfig
+from chunking.token_counter import TokenCounter
+from chunking.fixed_chunker import FixedChunker
+from chunking.overlap_manager import OverlapManager
+from sentence_transformers import SentenceTransformer
+
+
+# Setup Settings and Logging
+logger   = get_logger(__name__)
+settings = get_settings()
+
+
+class SemanticChunker(BaseChunker):
+    """
+    Semantic chunking strategy with section-aware splitting:
+    - Detects section boundaries and NEVER crosses them
+    - Creates chunks based on semantic similarity within sections
+    - Preserves hierarchical structure (sections → subsections → content)
+    
+    Best for:
+    - Medium documents (50K-500K tokens)
+    - Documents with clear topics/sections
+    - When context coherence is critical
+    """
+    def __init__(self, chunk_size: int = None, overlap: int = None, similarity_threshold: float = None, min_chunk_size: int = 100, 
+                 embedding_model: Optional[SentenceTransformer] = None, respect_section_boundaries: bool = True):
+        """
+        Initialize semantic chunker
+        
+        Arguments:
+        ----------
+            chunk_size                  { int }                  : Target tokens per chunk (soft limit)
+
+            overlap                     { int }                  : Overlap tokens between chunks
+            
+            similarity_threshold        { float }                : Threshold for semantic breakpoints (0-1)
+            
+            min_chunk_size              { int }                  : Minimum chunk size in tokens
+            
+            embedding_model             { SentenceTransformer }  : Pre-loaded embedding model (optional)
+            
+            respect_section_boundaries  { bool }                 : Detect and respect section headers
+        """
+        super().__init__(ChunkingStrategy.SEMANTIC)
+        
+        self.chunk_size                  = chunk_size or settings.FIXED_CHUNK_SIZE
+        self.overlap                     = overlap or settings.FIXED_CHUNK_OVERLAP
+        self.similarity_threshold        = similarity_threshold or settings.SEMANTIC_BREAKPOINT_THRESHOLD
+        self.min_chunk_size              = min_chunk_size
+        self.respect_section_boundaries  = respect_section_boundaries
+        
+        # Initialize token counter and overlap manager
+        self.token_counter               = TokenCounter()
+        self.overlap_manager             = OverlapManager(overlap_tokens = self.overlap)
+        
+        # Initialize or use provided embedding model
+        if embedding_model is not None:
+            self.embedding_model = embedding_model
+
+        else:
+            try:
+                self.logger.info(f"Loading embedding model: {settings.EMBEDDING_MODEL}")
+                self.embedding_model = SentenceTransformer(settings.EMBEDDING_MODEL)
+                self.logger.info("Embedding model loaded successfully")
+           
+            except Exception as e:
+                self.logger.error(f"Failed to load embedding model: {repr(e)}")
+                self.embedding_model = None
+        
+        self.logger.info(f"Initialized SemanticChunker: chunk_size={self.chunk_size}, threshold={self.similarity_threshold}, "
+                         f"model_loaded={self.embedding_model is not None}, section_aware={self.respect_section_boundaries}")
+    
+
+    def chunk_text(self, text: str, metadata: Optional[DocumentMetadata] = None) -> List[DocumentChunk]:
+        """
+        Chunk text based on semantic similarity AND section structure
+        
+        Arguments:
+        ----------
+            text            { str }                : Input text
+
+            metadata        { DocumentMetadata }   : Document metadata
+        
+        Returns:
+        --------
+                            { list }               : List of DocumentChunk objects
+        """
+        if not text or not text.strip():
+            return []
+        
+        document_id = metadata.document_id if metadata else "unknown"
+        
+        # If embedding model not available, fall back to fixed chunking
+        if self.embedding_model is None:
+            self.logger.warning("Embedding model not available, using sentence-based chunking")
+            return self._fallback_chunking(text=text, document_id=document_id)
+        
+        # Detect section headers if enabled
+        if self.respect_section_boundaries:
+            headers = self._detect_section_headers(text)
+            
+            if headers:
+                self.logger.info(f"Detected {len(headers)} section headers - using section-aware chunking")
+                chunks = self._chunk_by_sections(text        = text,
+                                                 headers     = headers, 
+                                                 document_id = document_id,
+                                                )
+            
+            else:
+                self.logger.info("No section headers detected - using standard semantic chunking")
+                chunks = self._chunk_semantic(text        = text, 
+                                              document_id = document_id,
+                                             )
+
+        else:
+            chunks = self._chunk_semantic(text        = text, 
+                                          document_id = document_id,
+                                         )
+        
+        # Filter out chunks that are too small
+        chunks = [c for c in chunks if (c.token_count >= self.min_chunk_size)]
+        
+        # Use OverlapManager to add proper overlap between semantic chunks
+        if ((len(chunks) > 1) and (self.overlap > 0)):
+            chunks = self.overlap_manager.add_overlap(chunks         = chunks, 
+                                                      overlap_tokens = self.overlap,
+                                                     )
+        
+        self.logger.debug(f"Created {len(chunks)} semantic chunks")
+        
+        return chunks
+    
+
+    def _detect_section_headers(self, text: str) -> List[Tuple[int, str, str, int]]:
+        """
+        Detect section headers in text to preserve document structure and returns a list of (line_index, header_type, header_text, char_position)
+        
+        Detects:
+        - Project headers 
+        - Subsection headers
+        - Major section headers
+        """
+        headers       = list()
+        lines         = text.split('\n')
+        char_position = 0
+        
+        for i, line in enumerate(lines):
+            line_stripped = line.strip()
+            
+            # Pattern 1: Headers - "a) Name" or "b) Name"
+            if (re.match(r'^[a-z]\)\s+[A-Z]', line_stripped)):
+                headers.append((i, 'section', line_stripped, char_position))
+                self.logger.debug(f"Detected section header at line {i}: {line_stripped[:60]}")
+            
+            # Pattern 2: Subsection headers - "● Subsection:" (bullet with colon)
+            elif ((line_stripped.startswith('●')) and (':' in line_stripped)):
+                headers.append((i, 'subsection', line_stripped, char_position))
+                self.logger.debug(f"Detected subsection header at line {i}: {line_stripped[:60]}")
+            
+            # Pattern 3: Major section headers - "1. SECTION NAME" or all caps with numbers
+            elif (re.match(r'^\d+\.\s+[A-Z\s&]+:', line_stripped)):
+                headers.append((i, 'section', line_stripped, char_position))
+                self.logger.debug(f"Detected major section at line {i}: {line_stripped[:60]}")
+            
+            # Pattern 4: All caps headers (must be substantial)
+            elif (line_stripped.isupper() and (len(line_stripped) > 15) and (not line_stripped.startswith('●'))):
+                headers.append((i, 'category', line_stripped, char_position))
+                self.logger.debug(f"Detected category header at line {i}: {line_stripped[:60]}")
+            
+            # +1 for newline
+            char_position += len(line) + 1  
+        
+        return headers
+    
+
+    def _chunk_by_sections(self, text: str, headers: List[Tuple], document_id: str) -> List[DocumentChunk]:
+        """
+        Create chunks that never cross section boundaries: Each chunk preserves its parent section in metadata
+        """
+        lines                     = text.split('\n')
+        chunks                    = list()
+        
+        # Group lines by their parent section
+        current_section_lines     = list()
+        current_section_header    = None
+        current_subsection_header = None
+        start_char                = 0
+        
+        for line_idx, line in enumerate(lines):
+            # Check if this line is a header
+            matching_headers = [h for h in headers if (h[0] == line_idx)]
+            
+            if matching_headers:
+                header_info = matching_headers[0]
+                header_type = header_info[1]
+                header_text = header_info[2]
+                
+                # If we hit a Header, save previous section
+                if (header_type == 'section'):
+                    if current_section_lines:
+                        # Create chunks from previous section
+                        section_text   = '\n'.join(current_section_lines)
+                        section_chunks = self._split_section_if_large(text              = section_text,
+                                                                      document_id       = document_id,
+                                                                      start_index       = len(chunks),
+                                                                      start_char        = start_char,
+                                                                      section_header    = current_section_header,
+                                                                      subsection_header = current_subsection_header,
+                                                                     )
+                        chunks.extend(section_chunks)
+                        start_char    += len(section_text) + 1
+                    
+                    # Start new section
+                    current_section_header    = header_text
+                    current_subsection_header = None
+                    current_section_lines     = [line]
+                
+                # If we hit a SUBSECTION header within a section
+                elif (header_type == 'subsection'):
+                    if (current_section_lines and current_subsection_header):
+                        # Save previous subsection
+                        section_text          = '\n'.join(current_section_lines)
+                        section_chunks        = self._split_section_if_large(text              = section_text,
+                                                                             document_id       = document_id,
+                                                                             start_index       = len(chunks),
+                                                                             start_char        = start_char,
+                                                                             section_header    = current_section_header,
+                                                                             subsection_header = current_subsection_header,
+                                                                            )
+                        chunks.extend(section_chunks)
+                        start_char           += len(section_text) + 1
+                        current_section_lines = list()
+                    
+                    # Update subsection
+                    current_subsection_header = header_text
+                    current_section_lines.append(line)
+                
+                else:
+                    current_section_lines.append(line)
+            
+            else:
+                current_section_lines.append(line)
+        
+        # Process final section
+        if current_section_lines:
+            section_text   = '\n'.join(current_section_lines)
+
+            section_chunks = self._split_section_if_large(text              = section_text,
+                                                          document_id       = document_id,
+                                                          start_index       = len(chunks),
+                                                          start_char        = start_char,
+                                                          section_header    = current_section_header,
+                                                          subsection_header = current_subsection_header,
+                                                         )
+            chunks.extend(section_chunks)
+        
+        return chunks
+    
+
+    def _split_section_if_large(self, text: str, document_id: str, start_index: int, start_char: int, section_header: Optional[str],
+                                subsection_header: Optional[str]) -> List[DocumentChunk]:
+        """
+        Split a section if it's too large, while preserving section context: Always stores section info in metadata
+        """
+        token_count   = self.token_counter.count_tokens(text)
+        
+        # Build section title for metadata
+        section_parts = list()
+
+        if section_header:
+            section_parts.append(section_header)
+
+        if subsection_header:
+            section_parts.append(subsection_header)
+            
+        section_title = " | ".join(section_parts) if section_parts else None
+        
+        # If section fits in one chunk, keep it whole
+        if (token_count <= self.chunk_size * 1.5):
+            chunk = self._create_chunk(text          = self._clean_chunk_text(text),
+                                       chunk_index   = start_index,
+                                       document_id   = document_id,
+                                       start_char    = start_char,
+                                       end_char      = start_char + len(text),
+                                       section_title = section_title, 
+                                       metadata      = {"section_header"    : section_header,
+                                                        "subsection_header" : subsection_header,
+                                                        "semantic_chunk"    : True,
+                                                        "section_aware"     : True,
+                                                       }
+                                      )
+            return [chunk]
+        
+        # Section too large - split by bullet points or sentences: But always keep section context in metadata
+        if '❖' in text or '●' in text:
+            # Split by bullet points (Interactive Demo Features style)
+            parts = re.split(r'(❖[^\n]+)', text)
+            parts = [p for p in parts if p.strip()]
+
+        else:
+            # Split by sentences within this section
+            parts = self._split_sentences(text)
+        
+        sub_chunks  = []
+        current_pos = start_char
+        
+        for part in parts:
+            if not part.strip():
+                continue
+            
+            part_tokens = self.token_counter.count_tokens(part)
+            
+            # Create chunk with preserved section context
+            chunk       = self._create_chunk(text          = self._clean_chunk_text(part),
+                                             chunk_index   = start_index + len(sub_chunks),
+                                             document_id   = document_id,
+                                             start_char    = current_pos,
+                                             end_char      = current_pos + len(part),
+                                             section_title = section_title, 
+                                             metadata      = {"section_header"     : section_header,
+                                                              "subsection_header"  : subsection_header,
+                                                              "parent_section"     : section_title,
+                                                              "semantic_chunk"     : True,
+                                                              "section_aware"      : True,
+                                                              "is_subsection_part" : True,
+                                                             }
+                                            )
+            sub_chunks.append(chunk)
+            current_pos += len(part)
+        
+        if sub_chunks:
+            return sub_chunks
+
+        else:
+            chunks_list = [self._create_chunk(text          = self._clean_chunk_text(text),
+                                              chunk_index   = start_index,
+                                              document_id   = document_id,
+                                              start_char    = start_char,
+                                              end_char      = start_char + len(text),
+                                              section_title = section_title,
+                                              metadata      = {"section_header"    : section_header,
+                                                               "subsection_header" : subsection_header,
+                                                               "semantic_chunk"    : True,
+                                                              }
+                                             )
+                          ]
+
+            return chunks_list
+    
+
+    def _chunk_semantic(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Standard semantic chunking (when no headers detected)
+        """
+        # Split into sentences
+        sentences = self._split_sentences(text = text)
+        
+        if (len(sentences) < 2):
+            return self._create_single_chunk(text=text, document_id=document_id)
+        
+        # Calculate semantic similarities
+        similarities = self._calculate_similarities(sentences=sentences)
+        
+        # Find breakpoints
+        breakpoints  = self._find_breakpoints(similarities=similarities)
+        
+        # Create chunks WITHOUT overlap
+        chunks       = self._create_chunks_from_breakpoints(sentences   = sentences,
+                                                            breakpoints = breakpoints,
+                                                            document_id = document_id,
+                                                           )
+        
+        return chunks
+    
+
+    def _split_sentences(self, text: str) -> List[str]:
+        """
+        Split text into sentences
+        """
+        # Protect abbreviations
+        protected        = text
+        abbreviations    = ['Dr.', 'Mr.', 'Mrs.', 'Ms.', 'Jr.', 'Sr.', 'Prof.', 'Inc.', 'Ltd.', 'Corp.', 'Co.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'Ph.D.', 'M.D.', 'B.A.', 'M.A.', 'U.S.', 'U.K.']
+        
+        for abbr in abbreviations:
+            protected = protected.replace(abbr, abbr.replace('.', '<DOT>'))
+        
+        # Split on sentence boundaries
+        sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])'
+        sentences        = re.split(sentence_pattern, protected)
+        
+        # Restore abbreviations
+        sentences        = [s.replace('<DOT>', '.').strip() for s in sentences]
+        
+        # Filter empty
+        sentences        = [s for s in sentences if s]
+        
+        return sentences
+    
+
+    def _calculate_similarities(self, sentences: List[str]) -> List[float]:
+        """
+        Calculate cosine similarity between adjacent sentences
+        """
+        if (len(sentences) < 2):
+            return []
+        
+        self.logger.debug(f"Generating embeddings for {len(sentences)} sentences")
+        
+        embeddings   = self.embedding_model.encode(sentences,
+                                                   show_progress_bar = False,
+                                                   convert_to_numpy  = True,
+                                                  )
+        
+        similarities = list()
+
+        for i in range(len(embeddings) - 1):
+            similarity = self._cosine_similarity(vec1 = embeddings[i], 
+                                                 vec2 = embeddings[i + 1],
+                                                )
+            similarities.append(similarity)
+        
+        return similarities
+    
+
+    @staticmethod
+    def _cosine_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
+        """
+        Calculate cosine similarity between two vectors
+        """
+        dot_product = np.dot(vec1, vec2)
+        norm1       = np.linalg.norm(vec1)
+        norm2       = np.linalg.norm(vec2)
+        
+        if ((norm1 == 0) or (norm2 == 0)):
+            return 0.0
+        
+        return dot_product / (norm1 * norm2)
+    
+
+    def _find_breakpoints(self, similarities: List[float]) -> List[int]:
+        """
+        Find breakpoints where semantic similarity drops significantly
+        """
+        if not similarities:
+            return []
+        
+        similarities_array = np.array(similarities)
+        threshold          = np.percentile(similarities_array, (1 - self.similarity_threshold) * 100)
+        
+        breakpoints        = [0]
+
+        for i, sim in enumerate(similarities):
+            if (sim < threshold):
+                breakpoints.append(i + 1)
+        
+        self.logger.debug(f"Found {len(breakpoints)} breakpoints with threshold {threshold:.3f}")
+        
+        return breakpoints
+    
+
+    def _create_chunks_from_breakpoints(self, sentences: List[str], breakpoints: List[int], document_id: str) -> List[DocumentChunk]:
+        """
+        Create chunks from sentences and breakpoints WITHOUT overlap
+        """
+        chunks      = list()
+        breakpoints = sorted(set(breakpoints))
+        
+        if (breakpoints[-1] != len(sentences)):
+            breakpoints.append(len(sentences))
+        
+        current_pos = 0
+        
+        for i in range(len(breakpoints) - 1):
+            start_idx = breakpoints[i]
+            end_idx   = breakpoints[i + 1]
+            
+            chunk_sentences = sentences[start_idx:end_idx]
+            
+            if not chunk_sentences:
+                continue
+            
+            chunk_text  = " ".join(chunk_sentences)
+            token_count = self.token_counter.count_tokens(chunk_text)
+            
+            if (token_count > self.chunk_size * 1.5):
+                sub_chunks = self._split_large_chunk_simple(chunk_sentences = chunk_sentences,
+                                                            document_id     = document_id,
+                                                            start_index     = len(chunks),
+                                                            start_char      = current_pos,
+                                                           )
+                chunks.extend(sub_chunks)
+           
+            else:
+                chunk = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                           chunk_index = len(chunks),
+                                           document_id = document_id,
+                                           start_char  = current_pos,
+                                           end_char    = current_pos + len(chunk_text),
+                                           metadata    = {"sentences"      : len(chunk_sentences),
+                                                          "semantic_chunk" : True,
+                                                         }
+                                          )
+
+                chunks.append(chunk)
+            
+            current_pos += len(chunk_text)
+        
+        return chunks
+    
+
+    def _split_large_chunk_simple(self, chunk_sentences: List[str], document_id: str, start_index: int, start_char: int) -> List[DocumentChunk]:
+        """
+        Split a large chunk into smaller pieces without overlap
+        """
+        sub_chunks        = list()
+        current_sentences = list()
+        current_tokens    = 0
+        current_pos       = start_char
+        
+        for sentence in chunk_sentences:
+            sentence_tokens = self.token_counter.count_tokens(sentence)
+            
+            if (((current_tokens + sentence_tokens) > self.chunk_size) and current_sentences):
+                chunk_text        = " ".join(current_sentences)
+                chunk             = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                                       chunk_index = start_index + len(sub_chunks),
+                                                       document_id = document_id,
+                                                       start_char  = current_pos,
+                                                       end_char    = current_pos + len(chunk_text),
+                                                      )
+                sub_chunks.append(chunk)
+                
+                current_sentences = [sentence]
+                current_tokens    = sentence_tokens
+                current_pos      += len(chunk_text)
+
+            else:
+                current_sentences.append(sentence)
+                current_tokens += sentence_tokens
+        
+        if current_sentences:
+            chunk_text = " ".join(current_sentences)
+            chunk      = self._create_chunk(text        = self._clean_chunk_text(chunk_text),
+                                            chunk_index = start_index + len(sub_chunks),
+                                            document_id = document_id,
+                                            start_char  = current_pos,
+                                            end_char    = current_pos + len(chunk_text),
+                                           )
+            sub_chunks.append(chunk)
+        
+        return sub_chunks
+    
+
+    def _create_single_chunk(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Create a single chunk for short text
+        """
+        chunk = self._create_chunk(text        = self._clean_chunk_text(text),
+                                   chunk_index = 0,
+                                   document_id = document_id,
+                                   start_char  = 0,
+                                   end_char    = len(text),
+                                  )
+        return [chunk]
+    
+
+    def _fallback_chunking(self, text: str, document_id: str) -> List[DocumentChunk]:
+        """
+        Fallback to sentence-based chunking when embeddings unavailable
+        """
+        fallback_chunker = FixedChunker(chunk_size                  = self.chunk_size,
+                                        overlap                     = self.overlap,
+                                        respect_sentence_boundaries = True,
+                                       )
+        
+        metadata         = DocumentMetadata(document_id     = document_id,
+                                            filename        = "fallback",
+                                            document_type   = "txt",
+                                            file_size_bytes = len(text),
+                                           )
+        
+        return fallback_chunker.chunk_text(text, metadata)
+    
+
+    @classmethod
+    def from_config(cls, config: ChunkerConfig) -> 'SemanticChunker':
+        """
+        Create SemanticChunker from configuration
+        """
+        return cls(chunk_size                 = config.chunk_size,
+                   overlap                    = config.overlap,
+                   similarity_threshold       = config.extra.get('semantic_threshold', settings.SEMANTIC_BREAKPOINT_THRESHOLD),
+                   min_chunk_size             = config.min_chunk_size,
+                   respect_section_boundaries = config.extra.get('respect_section_boundaries', True),
+                  )

chunking/token_counter.py

@@ -0,0 +1,520 @@
+# DEPENDENCIES
+import re
+import tiktoken
+from typing import List
+from typing import Optional
+from config.models import TokenizerType
+from config.settings import get_settings
+from config.logging_config import get_logger
+
+
+# Setup Logger and settings
+logger   = get_logger(__name__)
+settings = get_settings()
+
+
+class TokenCounter:
+    """
+    Token counting utility with support for multiple tokenizers: Provides accurate token counts for chunking and context management
+    """
+    def __init__(self, tokenizer_type: str = "cl100k_base"):
+        """
+        Initialize token counter
+        
+        Arguments:
+        ----------
+            tokenizer_type { str } : Type of tokenizer to use
+        """
+        self.tokenizer_type = tokenizer_type
+        self.logger         = logger
+
+        # Validate tokenizer type
+        valid_tokenizers    = [t.value for t in TokenizerType]
+        
+        if tokenizer_type not in valid_tokenizers:
+            self.logger.warning(f"Invalid tokenizer type: {tokenizer_type}, using approximate")
+            self.tokenizer_type = TokenizerType.APPROXIMATE
+            self.tokenizer      = None
+            
+            return
+        
+        # Initialize tokenizer
+        if (tokenizer_type != TokenizerType.APPROXIMATE):
+            try:
+                self.tokenizer = tiktoken.get_encoding(tokenizer_type)
+                self.logger.debug(f"Initialized tiktoken tokenizer: {tokenizer_type}")
+            
+            except Exception as e:
+                self.logger.warning(f"Failed to load tiktoken: {repr(e)}, using approximation")
+                
+                self.tokenizer      = None
+                self.tokenizer_type = TokenizerType.APPROXIMATE
+        
+        else:
+            self.tokenizer = None
+        
+
+    def count_tokens(self, text: str) -> int:
+        """
+        Count tokens in text
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+             { int }     : Number of tokens
+        """
+        if not text:
+            return 0
+        
+        if self.tokenizer is not None:
+            # Use tiktoken for accurate counting
+            try:
+                tokens = self.tokenizer.encode(text)
+                return len(tokens)
+            
+            except Exception as e:
+                self.logger.warning(f"Tokenizer error: {e}, falling back to approximation")
+                return self._approximate_token_count(text)
+        
+        else:
+            # Use approximation
+            return self._approximate_token_count(text = text)
+    
+
+    def _approximate_token_count(self, text: str) -> int:
+        """
+        Approximate token count using multiple heuristics
+        """
+        if not text:
+            return 0
+        
+        # Method 1: Word-based estimation (accounts for subword tokenization)
+        words      = text.split()
+        word_count = len(words)
+        
+        # Method 2: Character-based estimation
+        char_count = len(text)
+        
+        # Method 3: Hybrid approach with weighting
+        # - Short texts: more word-based (better for code/short docs)
+        # - Long texts: more character-based (better for prose)
+        if (char_count < 1000):
+            # Prefer word-based for short texts : Slightly higher for short texts
+            estimate = word_count * 1.33  
+        
+        else:
+            # Balanced approach for longer texts
+            word_estimate = word_count * 1.3
+            char_estimate = char_count / 4.0
+            estimate      = (word_estimate + char_estimate) / 2
+        
+        # Ensure reasonable bounds
+        min_tokens = max(1, word_count)  # At least 1 token per word
+        max_tokens = char_count // 2     # At most 1 token per 2 chars
+        
+        return max(min_tokens, min(int(estimate), max_tokens))
+    
+
+    def encode(self, text: str) -> List[int]:
+        """
+        Encode text to token IDs
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+             { list }    : List of token IDs
+        """
+        if self.tokenizer is None:
+            raise ValueError("Cannot encode with approximate tokenizer")
+        
+        return self.tokenizer.encode(text)
+    
+
+    def decode(self, tokens: List[int]) -> str:
+        """
+        Decode token IDs to text
+        
+        Arguments:
+        ----------
+            tokens { list } : List of token IDs
+        
+        Returns:
+        --------
+              { str }       : Decoded text
+        """
+        if self.tokenizer is None:
+            raise ValueError("Cannot decode with approximate tokenizer")
+        
+        return self.tokenizer.decode(tokens)
+    
+
+    def truncate_to_tokens(self, text: str, max_tokens: int, suffix: str = "") -> str:
+        """
+        Truncate text to maximum token count
+        
+        Arguments:
+        ----------
+            text       { str } : Input text
+
+            max_tokens { int } : Maximum number of tokens
+            
+            suffix     { str } : Suffix to add (e.g., "...")
+        
+        Returns:
+        --------
+                { str }        : Truncated text
+        """
+        if self.tokenizer is not None:
+            # Use precise token-based truncation
+            tokens = self.encode(text)
+            
+            if (len(tokens) <= max_tokens):
+                return text
+            
+            # Account for suffix tokens
+            suffix_tokens    = len(self.encode(suffix)) if suffix else 0
+            truncate_at      = max_tokens - suffix_tokens
+            
+            truncated_tokens = tokens[:truncate_at]
+            truncated_text   = self.decode(truncated_tokens)
+            
+            return truncated_text + suffix
+
+        else:
+            # Use character-based approximation
+            current_tokens = self.count_tokens(text = text)
+            
+            if (current_tokens <= max_tokens):
+                return text
+            
+            # Estimate character position
+            ratio         = max_tokens / current_tokens
+            char_position = int(len(text) * ratio)
+            
+            # Find nearest word boundary
+            truncated     = text[:char_position]
+            last_space    = truncated.rfind(' ')
+            
+            if (last_space > 0):
+                truncated = truncated[:last_space]
+            
+            return truncated + suffix
+    
+
+    def split_into_token_chunks(self, text: str, chunk_size: int, overlap: int = 0) -> List[str]:
+        """
+        Split text into chunks of approximately equal token count
+        
+        Arguments:
+        ----------
+            text       { str } : Input text
+
+            chunk_size { int } : Target tokens per chunk
+            
+            overlap    { int } : Number of overlapping tokens between chunks
+        
+        Returns:
+        --------
+               { list }        : List of text chunks
+        """
+        if (overlap >= chunk_size):
+            raise ValueError("Overlap must be less than chunk_size")
+        
+        if self.tokenizer is not None:
+            precise_chunks = self._split_precise(text       = text, 
+                                                 chunk_size = chunk_size, 
+                                                 overlap    = overlap,
+                                                )
+
+            return precise_chunks
+        
+        else:
+            approximate_chunks = self._split_approximate(text       = text, 
+                                                         chunk_size = chunk_size, 
+                                                         overlap    = overlap,
+                                                        )
+
+            return approximate_chunks
+
+    
+    def _split_precise(self, text: str, chunk_size: int, overlap: int) -> List[str]:
+        """
+        Split using precise token counts
+        """
+        tokens = self.encode(text)
+        chunks = list()
+        
+        start  = 0
+
+        while (start < len(tokens)):
+            # Get chunk tokens
+            end          = min(start + chunk_size, len(tokens))
+            chunk_tokens = tokens[start:end]
+            
+            # Decode to text
+            chunk_text   = self.decode(chunk_tokens)
+
+            chunks.append(chunk_text)
+            
+            # Move to next chunk with overlap
+            start        = end - overlap
+            
+            # Avoid infinite loop
+            if ((start >= len(tokens)) or ((end == len(tokens)))):
+                break
+        
+        return chunks
+    
+
+    def _split_approximate(self, text: str, chunk_size: int, overlap: int) -> List[str]:
+        """
+        Split using approximate token counts
+        """
+        # Estimate characters per chunk : Rule = ~4 chars per token
+        chars_per_chunk = chunk_size * 4
+        overlap_chars   = overlap * 4
+        
+        chunks          = list()
+        sentences       = self._split_into_sentences(text = text)
+        
+        current_chunk   = list()
+        current_tokens  = 0
+        
+        for sentence in sentences:
+            sentence_tokens = self.count_tokens(text = sentence)
+            
+            if (((current_tokens + sentence_tokens) > chunk_size) and current_chunk):
+                # Save current chunk
+                chunk_text = " ".join(current_chunk)
+                
+                chunks.append(chunk_text)
+                
+                # Start new chunk with overlap
+                if (overlap > 0):
+                    # Keep last few sentences for overlap
+                    overlap_text   = chunk_text[-overlap_chars:] if len(chunk_text) > overlap_chars else chunk_text
+                    current_chunk  = [overlap_text, sentence]
+                    current_tokens = self.count_tokens(text = " ".join(current_chunk))
+                
+                else:
+                    current_chunk  = [sentence]
+                    current_tokens = sentence_tokens
+            
+            else:
+                current_chunk.append(sentence)
+                
+                current_tokens += sentence_tokens
+        
+        # Add final chunk
+        if current_chunk:
+            chunks.append(" ".join(current_chunk))
+        
+        return chunks
+    
+
+    @staticmethod
+    def _split_into_sentences(text: str) -> List[str]:
+        """
+        Simple sentence splitter with better edge case handling
+        """
+        if not text.strip():
+            return []
+        
+        # Split on sentence boundaries
+        sentences = re.split(r'(?<=[.!?])\s+', text)
+        
+        # Filter and clean
+        final_sentences = list()
+
+        for sentence in sentences:
+            sentence = sentence.strip()
+
+            if sentence:
+                # Handle abbreviations (basic)
+                if not any(sentence.endswith(abbr) for abbr in ['Dr.', 'Mr.', 'Mrs.', 'Ms.', 'etc.']):
+                    final_sentences.append(sentence)
+                
+                else:
+                    # For abbreviations, keep with next sentence if possible
+                    if final_sentences:
+                        final_sentences[-1] += " " + sentence
+                    
+                    else:
+                        final_sentences.append(sentence)
+        
+        return final_sentences
+    
+
+    def get_token_stats(self, text: str) -> dict:
+        """
+        Get comprehensive token statistics
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+            { dict }     : Dictionary with statistics
+        """
+        token_count = self.count_tokens(text = text)
+        char_count  = len(text)
+        word_count  = len(text.split())
+        
+        stats       = {"tokens"          : token_count,
+                       "characters"      : char_count,
+                       "words"           : word_count,
+                       "chars_per_token" : char_count / token_count if (token_count > 0) else 0,
+                       "tokens_per_word" : token_count / word_count if (word_count > 0) else 0,
+                       "tokenizer"       : self.tokenizer_type,
+                      }
+        
+        return stats
+    
+
+    def estimate_cost(self, text: str, cost_per_1k_tokens: float = 0.002) -> float:
+        """
+        Estimate API cost for text.
+        
+        Arguments:
+        ----------
+            text                { str }  : Input text
+
+            cost_per_1k_tokens { float } : Cost per 1000 tokens (default: GPT-4 input)
+        
+        Returns:
+        --------
+                    { float }            : Estimated cost in dollars
+        """
+        tokens = self.count_tokens(text = text)
+        cost   = (tokens / 1000) * cost_per_1k_tokens
+
+        return round(cost, 6)
+    
+
+    def batch_count_tokens(self, texts: List[str]) -> List[int]:
+        """
+        Count tokens for multiple texts efficiently
+        
+        Arguments:
+        ----------
+            texts { list } : List of texts
+        
+        Returns:
+        --------
+             { list }      : List of token counts
+        """
+        token_counts = [self.count_tokens(text = text) for text in texts]
+        
+        return token_counts 
+    
+
+    def find_token_boundaries(self, text: str, target_tokens: int) -> tuple[int, str]:
+        """
+        Find character position that gives approximately target tokens
+        
+        Arguments:
+        ----------
+            text          { str } : Input text
+
+            target_tokens { int } : Target number of tokens
+        
+        Returns:
+        --------
+                 { tuple }        : Tuple of (character_position, text_up_to_position)
+        """
+        if self.tokenizer is not None:
+            tokens = self.encode(text)
+
+            if (len(tokens) <= target_tokens):
+                return len(text), text
+            
+            target_tokens_subset = tokens[:target_tokens]
+            result_text          = self.decode(target_tokens_subset)
+            
+            return len(result_text), result_text
+        
+        else:
+            # Approximate
+            total_tokens = self.count_tokens(text = text)
+            
+            if (total_tokens <= target_tokens):
+                return len(text), text
+            
+            ratio    = target_tokens / total_tokens
+            char_pos = int(len(text) * ratio)
+            
+            return char_pos, text[:char_pos]
+
+
+# Global counter instance
+_counter = None
+
+
+def get_token_counter(tokenizer_type: str = "cl100k_base") -> TokenCounter:
+    """
+    Get global token counter instance
+    
+    Arguments:
+    ----------
+        tokenizer_type { str } : Tokenizer type
+    
+    Returns:
+    --------
+         { TokenCounter }      : TokenCounter instance
+    """
+    global _counter
+
+    if _counter is None or _counter.tokenizer_type != tokenizer_type:
+        _counter = TokenCounter(tokenizer_type)
+    
+    return _counter
+
+
+# Convenience functions
+def count_tokens(text: str, tokenizer_type: str = "cl100k_base") -> int:
+    """
+    Quick token count
+    
+    Arguments:
+    ----------
+        text           { str } : Input text
+
+        tokenizer_type { str } : Tokenizer type
+    
+    Returns:
+    --------
+              { int }          : Token count
+    """
+    counter = get_token_counter(tokenizer_type)
+
+    return counter.count_tokens(text)
+
+
+def truncate_to_tokens(text: str, max_tokens: int, suffix: str = "...", tokenizer_type: str = "cl100k_base") -> str:
+    """
+    Truncate text to max tokens
+    
+    Arguments:
+    ----------
+        text           { str } : Input text
+        
+        max_tokens     { int } : Maximum tokens
+        
+        suffix         { str } : Suffix to add
+        
+        tokenizer_type { str } : Tokenizer type
+    
+    Returns:
+    ---------
+              { str }          : Truncated text
+    """
+    counter = get_token_counter(tokenizer_type)
+    
+    return counter.truncate_to_tokens(text, max_tokens, suffix)

config/logging_config.py

@@ -0,0 +1,301 @@
+# DEPENDENCIES
+import sys
+import logging
+from pathlib import Path
+from typing import Optional
+from datetime import datetime
+from logging.handlers import RotatingFileHandler
+
+
+class ColoredFormatter(logging.Formatter):
+    """
+    Custom formatter with color support for console output
+    """
+    # ANSI color codes
+    COLORS = {'DEBUG'    : '\033[36m',   # Cyan
+              'INFO'     : '\033[32m',   # Green
+              'WARNING'  : '\033[33m',   # Yellow
+              'ERROR'    : '\033[31m',   # Red
+              'CRITICAL' : '\033[35m',   # Magenta
+              'RESET'    : '\033[0m',    # Reset
+             }
+    
+
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        Format log record with color
+        """
+        # Add color to level name
+        levelname = record.levelname
+        
+        if levelname in self.COLORS:
+            record.levelname = (f"{self.COLORS[levelname]}{levelname}{self.COLORS['RESET']}")
+        
+        # Add color to logger name
+        record.name = f"\033[34m{record.name}\033[0m"  # Blue
+        
+        return super().format(record)
+
+
+class StructuredFormatter(logging.Formatter):
+    """
+    Structured JSON-like formatter for file logging
+    """
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        Format log record as structured data
+        """
+        log_data = {"timestamp" : datetime.fromtimestamp(record.created).isoformat(),
+                    "level"     : record.levelname,
+                    "logger"    : record.name,
+                    "message"   : record.getMessage(),
+                    "module"    : record.module,
+                    "function"  : record.funcName,
+                    "line"      : record.lineno,
+                   }
+        
+        # Add exception info if present
+        if record.exc_info:
+            log_data["exception"] = self.formatException(record.exc_info)
+        
+        # Add extra fields
+        if hasattr(record, "extra"):
+            log_data.update(record.extra)
+        
+        # Format as key=value pairs (easier to read than JSON)
+        parts = [f"{k}={v}" for k, v in log_data.items()]
+        
+        return " | ".join(parts)
+
+
+def setup_logging(log_level: str = "INFO", log_dir: Optional[Path] = None, log_format: Optional[str] = None, enable_console: bool = True,  
+                  enable_file: bool = True, max_bytes: int = 10 * 1024 * 1024, backup_count: int = 5) -> logging.Logger:
+    """
+    Setup comprehensive logging configuration
+    
+    Arguments:
+    ----------
+        log_level      { str }  : Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        
+        log_dir        { Path } : Directory for log files
+        
+        log_format     { str }  : Custom log format string
+        
+        enable_console { bool } : Enable console output
+        
+        enable_file    { bool } : Enable file output
+        
+        max_bytes      { int }  : Max file size before rotation
+        
+        backup_count   { int }  : Number of backup files to keep
+    
+    Returns:
+    --------
+        { logging.Logger }      : Configured root logger
+    """
+    
+    # Get root logger
+    logger = logging.getLogger()
+    logger.setLevel(getattr(logging, log_level.upper()))
+    
+    # Clear existing handlers
+    logger.handlers.clear()
+    
+    # Default format
+    if log_format is None:
+        log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    
+    # Console handler with colors
+    if enable_console:
+        console_handler   = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(logging.DEBUG)
+        console_formatter = ColoredFormatter(log_format, datefmt = "%Y-%m-%d %H:%M:%S")
+        console_handler.setFormatter(console_formatter)
+        logger.addHandler(console_handler)
+    
+    # File handler with rotation
+    if enable_file and log_dir:
+        log_dir = Path(log_dir)
+        log_dir.mkdir(parents = True, exist_ok = True)
+        
+        # Main log file
+        log_file       = log_dir / f"app_{datetime.now().strftime('%Y%m%d')}.log"
+        file_handler   = RotatingFileHandler(log_file, maxBytes = max_bytes, backupCount = backup_count, encoding = "utf-8")
+        file_handler.setLevel(logging.DEBUG)
+        
+        # Use structured formatter for files
+        file_formatter = StructuredFormatter()
+        file_handler.setFormatter(file_formatter)
+        logger.addHandler(file_handler)
+        
+        # Separate error log
+        error_file     = log_dir / f"error_{datetime.now().strftime('%Y%m%d')}.log"
+        error_handler  = RotatingFileHandler(error_file, maxBytes = max_bytes, backupCount = backup_count, encoding = "utf-8")
+
+        error_handler.setLevel(logging.ERROR)
+        error_handler.setFormatter(file_formatter)
+        logger.addHandler(error_handler)
+    
+    # Suppress noisy third-party loggers
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+    logging.getLogger("sentence_transformers").setLevel(logging.WARNING)
+    logging.getLogger("transformers").setLevel(logging.WARNING)
+    logging.getLogger("torch").setLevel(logging.WARNING)
+    logging.getLogger("playwright").setLevel(logging.WARNING)
+    logging.getLogger("faiss").setLevel(logging.WARNING)
+    logging.getLogger("llama_index").setLevel(logging.WARNING)
+    logging.getLogger("langchain").setLevel(logging.WARNING)
+    
+    logger.info(f"Logging configured: level={log_level}, console={enable_console}, file={enable_file}")
+    
+    return logger
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance for a specific module
+    
+    Arguments:
+    ----------
+        name       { str }    : Logger name (typically __name__)
+    
+    Returns:
+    --------
+        { logging.Logger }    : Logger instance
+    """
+    return logging.getLogger(name)
+
+
+class LoggerAdapter(logging.LoggerAdapter):
+    """
+    Custom logger adapter that adds contextual information: Useful for tracking request IDs, user IDs, etc
+    """
+    def process(self, msg: str, kwargs: dict) -> tuple[str, dict]:
+        """
+        Add extra context to log messages
+        """
+        extra = self.extra.copy()
+        
+        # Add to structured logging
+        if 'extra' not in kwargs:
+            kwargs['extra'] = {}
+        
+        kwargs['extra'].update(extra)
+        
+        # Add to message
+        context_parts = [f"{k}={v}" for k, v in extra.items()]
+        
+        if context_parts:
+            msg = f"[{', '.join(context_parts)}] {msg}"
+        
+        return msg, kwargs
+
+
+def get_context_logger(name: str, **context) -> LoggerAdapter:
+    """
+    Get a logger with contextual information
+    
+    Arguments:
+    ----------
+        name      { str } : Logger name
+
+        **context         : Context key-value pairs
+    
+    Returns:
+    --------
+        { LoggerAdapter } : Logger adapter with context
+    """
+    base_logger = get_logger(name)
+    
+    return LoggerAdapter(base_logger, context)
+
+
+# Performance logging utilities
+class TimedLogger:
+    """
+    Context manager for timing operations and logging
+    """
+    def __init__(self, logger: logging.Logger, operation: str, level: int = logging.INFO, log_start: bool = False):
+        self.logger     = logger
+        self.operation  = operation
+        self.level      = level
+        self.log_start  = log_start
+        self.start_time = None
+    
+
+    def __enter__(self):
+        if self.log_start:
+            self.logger.log(self.level, f"{self.operation} started")
+        
+        self.start_time = datetime.now()
+        
+        return self
+    
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        duration = (datetime.now() - self.start_time).total_seconds()
+        
+        if exc_type is None:
+            self.logger.log(self.level, f"{self.operation} completed in {duration:.2f}s")
+
+        else:
+            self.logger.error(f"{self.operation} failed after {duration:.2f}s: {exc_val}")
+
+
+# Logging decorators
+def log_execution(logger: Optional[logging.Logger] = None, level: int = logging.INFO):
+    """
+    Decorator to log function execution time
+    """
+    def decorator(func):
+        nonlocal logger
+        
+        if logger is None:
+            logger = get_logger(func.__module__)
+        
+        def wrapper(*args, **kwargs):
+            func_name = f"{func.__module__}.{func.__name__}"
+            
+            with TimedLogger(logger, func_name, level=level):
+                return func(*args, **kwargs)
+        
+        return wrapper
+    return decorator
+
+
+def log_exceptions(logger: Optional[logging.Logger] = None, reraise: bool = True):
+    """
+    Decorator to log exceptions with full traceback
+    """
+    def decorator(func):
+        nonlocal logger
+        
+        if logger is None:
+            logger = get_logger(func.__module__)
+        
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            
+            except Exception as e:
+                func_name = f"{func.__module__}.{func.__name__}"
+                
+                logger.exception(f"Exception in {func_name}: {str(e)}")
+                
+                if reraise:
+                    raise
+                
+                return None
+        
+        return wrapper
+    return decorator
+
+
+# Initialize logging on module import (with defaults) : This will be overridden by app.py with actual settings
+_default_logger = setup_logging(log_level      = "DEBUG",
+                                enable_console = True,
+                                enable_file    = True, 
+                               )

config/models.py

@@ -0,0 +1,697 @@
+# DEPENDENCIES
+import re
+from enum import Enum
+from typing import Any
+from typing import List
+from typing import Dict
+from pathlib import Path
+from typing import Literal
+from pydantic import Field
+from typing import Optional
+from datetime import datetime
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import field_validator
+
+
+# Enums
+class DocumentType(str, Enum):
+    """
+    Supported document types
+    """
+    PDF     = "pdf"
+    DOCX    = "docx"
+    TXT     = "txt"
+    URL     = "url"
+    IMAGE   = "image"    
+    ARCHIVE = "archive"
+
+
+class IngestionInputType(str, Enum):
+    """
+    Supported input types for ingestion
+    """
+    FILE    = "file"
+    URL     = "url"
+    ARCHIVE = "archive"
+    TEXT    = "text"
+
+
+class ProcessingStatus(str, Enum):
+    """
+    Document processing status
+    """
+    PENDING    = "pending"
+    PROCESSING = "processing"
+    COMPLETED  = "completed"
+    FAILED     = "failed"
+
+
+class TokenizerType(str, Enum):
+    """
+    Supported tokenizer types
+    """
+    CL100K      = "cl100k_base"  # GPT-4, GPT-3.5-turbo
+    P50K        = "p50k_base"    # Codex, text-davinci-002/003
+    R50K        = "r50k_base"    # GPT-3, text-davinci-001
+    GPT2        = "gpt2"         # GPT-2
+    APPROXIMATE = "approximate"  # Fast approximation
+
+
+class ChunkingStrategy(str, Enum):
+    """
+    Available chunking strategies
+    """
+    FIXED        = "fixed"
+    SEMANTIC     = "semantic"
+    HIERARCHICAL = "hierarchical"
+
+
+class LLMProvider(str, Enum):
+    """
+    Supported LLM providers
+    """
+    OLLAMA    = "ollama"
+    OPENAI    = "openai"
+
+
+class TemperatureStrategy(str, Enum):
+    """
+    Temperature control strategies
+    """
+    FIXED       = "fixed"
+    ADAPTIVE    = "adaptive"
+    CONFIDENCE  = "confidence"
+    PROGRESSIVE = "progressive"
+
+
+class CitationStyle(str, Enum):
+    """
+    Supported citation styles
+    """
+    NUMERIC  = "numeric"
+    VERBOSE  = "verbose"
+    MINIMAL  = "minimal"
+    ACADEMIC = "academic"
+    LEGAL    = "legal"
+
+
+class PromptType(str, Enum):
+    """
+    Supported prompt types
+    """
+    QA             = "qa"
+    SUMMARY        = "summary"
+    ANALYTICAL     = "analytical"
+    COMPARISON     = "comparison"
+    EXTRACTION     = "extraction"
+    CREATIVE       = "creative"
+    CONVERSATIONAL = "conversational"
+
+
+# Document Models
+class DocumentMetadata(BaseModel):
+    """
+    Metadata extracted from documents
+    """
+    model_config                                         = ConfigDict(arbitrary_types_allowed = True)
+    
+    document_id             : str                        = Field(..., description = "Unique document identifier")
+    filename                : str                        = Field(..., description = "Original filename")
+    file_path               : Optional[Path]             = Field(None, description = "Path to uploaded file")
+    document_type           : DocumentType               = Field(..., description = "Type of document")
+    
+    # Content metadata
+    title                   : Optional[str]              = Field(None, description = "Document title")
+    author                  : Optional[str]              = Field(None, description = "Document author")
+    created_date            : Optional[datetime]         = Field(None, description = "Document creation date")
+    modified_date           : Optional[datetime]         = Field(None, description = "Last modification date")
+    
+    # Processing metadata
+    upload_date             : datetime                   = Field(default_factory = datetime.now)
+    processed_date          : Optional[datetime]         = Field(None)
+    status                  : ProcessingStatus           = Field(default = ProcessingStatus.PENDING)
+    
+    # Size metrics
+    file_size_bytes         : int                        = Field(..., gt = 0, description = "File size in bytes")
+    num_pages               : Optional[int]              = Field(None, ge = 1, description = "Number of pages (PDFs)")
+    num_tokens              : Optional[int]              = Field(None, ge = 0, description = "Total tokens")
+    num_chunks              : Optional[int]              = Field(None, ge = 0, description = "Number of chunks")
+    
+    # Processing info
+    chunking_strategy       : Optional[ChunkingStrategy] = Field(None)
+    processing_time_seconds : Optional[float]            = Field(None, ge = 0.0)
+    error_message           : Optional[str]              = Field(None)
+    
+    # Additional metadata
+    extra                   : Dict[str, Any]             = Field(default_factory = dict)
+    
+
+    @field_validator("file_size_bytes")
+    @classmethod
+    def validate_file_size(cls, v: int) -> int:
+        """
+        Ensure file size is reasonable
+        """
+        max_size = 2 * 1024 * 1024 * 1024  # 2GB
+        
+        if (v > max_size):
+            raise ValueError(f"File size {v} exceeds maximum {max_size}")
+
+        return v
+    
+    @property
+    def file_size_mb(self) -> float:
+        """
+        File size in megabytes
+        """
+        return self.file_size_bytes / (1024 * 1024)
+
+
+
+class DocumentChunk(BaseModel):
+    """
+    A single chunk of text from a document
+    """
+    chunk_id        : str                   = Field(..., description = "Unique chunk identifier")
+    document_id     : str                   = Field(..., description = "Parent document ID")
+    
+    # Content
+    text            : str                   = Field(..., min_length = 1, description = "Chunk text content")
+    embedding       : Optional[List[float]] = Field(None, description = "Vector embedding")
+    
+    # Position metadata
+    chunk_index     : int                   = Field(..., ge = 0, description = "Chunk position in document")
+    start_char      : int                   = Field(..., ge = 0, description = "Start character position")
+    end_char        : int                   = Field(..., ge = 0, description = "End character position")
+    
+    # Page/section info
+    page_number     : Optional[int]         = Field(None, ge = 1, description = "Page number (if applicable)")
+    section_title   : Optional[str]         = Field(None, description = "Section heading")
+    
+    # Hierarchical info (for hierarchical chunking)
+    parent_chunk_id : Optional[str]         = Field(None)
+    child_chunk_ids : List[str]             = Field(default_factory = list)
+    
+    # Token info
+    token_count     : int                   = Field(..., gt = 0, description = "Number of tokens")
+    
+    # Metadata
+    metadata        : Dict[str, Any]        = Field(default_factory = dict)
+    
+    
+    @property
+    def char_count(self) -> int:
+        """
+        Number of characters in chunk
+        """
+        return self.end_char - self.start_char
+
+
+class ChunkWithScore(BaseModel):
+    """
+    Chunk with retrieval score
+    """
+    chunk            : DocumentChunk
+    score            : float          = Field(..., description = "Relevance score (can be any real number)")
+    rank             : int            = Field(..., ge = 1, description = "Rank in results")
+    retrieval_method : str            = Field('vector', description = "Retrieval method used")
+    
+
+    @property
+    def citation(self) -> str:
+        parts = [self.chunk.document_id]
+        
+        # Add source filename if available
+        if ((hasattr(self.chunk, 'metadata')) and ('filename' in self.chunk.metadata)):
+            parts.append(f"file: {self.chunk.metadata['filename']}")
+        
+        if self.chunk.page_number:
+            parts.append(f"page {self.chunk.page_number}")
+        
+        if self.chunk.section_title:
+            parts.append(f"section: {self.chunk.section_title}")
+        
+        return ", ".join(parts)
+
+
+# Embedding Request 
+class EmbeddingRequest(BaseModel):
+    texts      : List[str]
+    normalize  : bool          = True
+    device     : Optional[str] = None
+    batch_size : Optional[int] = None
+
+
+# Query Models 
+class QueryRequest(BaseModel):
+    """
+    User query request
+    """
+    model_config                           = ConfigDict(protected_namespaces = ())
+
+    query            : str                 = Field(..., min_length = 1, max_length = 1000, description = "User question")
+    
+    # Retrieval parameters
+    top_k            : Optional[int]       = Field(5, ge = 1, le = 20, description = "Number of chunks to retrieve")
+    enable_reranking : Optional[bool]      = Field(False)
+    
+    # Generation parameters
+    temperature      : Optional[float]     = Field(0.1, ge = 0.0, le = 1.0)
+    top_p            : Optional[float]     = Field(0.9, ge = 0.0, le = 1.0)
+    max_tokens       : Optional[int]       = Field(1000, ge = 50, le = 4000)
+    
+    # Filters
+    document_ids     : Optional[List[str]] = Field(None, description = "Filter by specific documents")
+    date_from        : Optional[datetime]  = Field(None)
+    date_to          : Optional[datetime]  = Field(None)
+    
+    # Response preferences
+    include_sources  : bool                = Field(True, description = "Include source citations")
+    include_metrics  : bool                = Field(False, description = "Include quality metrics")
+    stream           : bool                = Field(False, description = "Stream response tokens")
+
+
+class QueryResponse(BaseModel):
+    """
+    Response to user query
+    """
+    query              : str                        = Field(..., description = "Original query")
+    answer             : str                        = Field(..., description = "Generated answer")
+    
+    # Retrieved context
+    sources            : List[ChunkWithScore]       = Field(default_factory = list)
+    
+    # Metrics
+    retrieval_time_ms  : float                      = Field(..., ge = 0.0)
+    generation_time_ms : float                      = Field(..., ge = 0.0)
+    total_time_ms      : float                      = Field(..., ge = 0.0)
+    
+    tokens_used        : Optional[Dict[str, int]]   = Field(None)  # {input: X, output: Y}
+    
+    # Quality metrics (if enabled)
+    metrics            : Optional[Dict[str, float]] = Field(None)
+    
+    # Metadata
+    timestamp          : datetime                   = Field(default_factory = datetime.now)
+    model_used         : str                        = Field(...)
+    
+    model_config                                    = ConfigDict(protected_namespaces = ())
+
+    @property
+    def citation_text(self) -> str:
+        """
+        Format citations as text
+        """
+        if not self.sources:
+            return ""
+        
+        citations = list()
+
+        for i, source in enumerate(self.sources, 1):
+            citations.append(f"[{i}] {source.citation}")
+        
+        return "\n".join(citations)
+
+
+# Upload Models
+class UploadRequest(BaseModel):
+    """
+    File upload request metadata
+    """
+    filename        : str           = Field(..., min_length = 1)
+    file_size_bytes : int           = Field(..., gt = 0)
+    content_type    : Optional[str] = Field(None)
+    
+
+    @field_validator("filename")
+    @classmethod
+    def validate_filename(cls, v: str) -> str:
+        """
+        Ensure filename is safe
+        """
+        # Remove path traversal attempts
+        v = Path(v).name
+        
+        if not v or v.startswith("."):
+            raise ValueError("Invalid filename")
+        
+        return v
+
+
+class UploadResponse(BaseModel):
+    """
+    File upload response
+    """
+    document_id : str              = Field(..., description = "Generated document ID")
+    filename    : str              = Field(...)
+    status      : ProcessingStatus = Field(...)
+    message     : str              = Field(...)
+    upload_date : datetime         = Field(default_factory = datetime.now)
+
+
+class ProcessingProgress(BaseModel):
+    """
+    Real-time processing progress
+    """
+    document_id                 : str              = Field(...)
+    status                      : ProcessingStatus = Field(...)
+    
+    # Progress tracking
+    progress_percentage         : float            = Field(0.0, ge = 0.0, le = 100.0)
+    current_step                : str              = Field(..., description = "Current processing step")
+    
+    # Stats
+    chunks_processed            : int              = Field(0, ge = 0)
+    total_chunks                : Optional[int]    = Field(None)
+    
+    # Timing
+    start_time                  : datetime         = Field(...)
+    elapsed_seconds             : float            = Field(0.0, ge = 0.0)
+    estimated_remaining_seconds : Optional[float]  = Field(None)
+    
+    # Messages
+    log_messages                : List[str]        = Field(default_factory = list)
+    error_message               : Optional[str]    = Field(None)
+
+
+# Embedding Models
+class EmbeddingRequest(BaseModel):
+    """
+    Request to generate embeddings
+    """
+    texts      : List[str]     = Field(..., min_length = 1, max_length = 1000)
+    batch_size : Optional[int] = Field(32, ge = 1, le = 128)
+    normalize  : bool          = Field(True, description = "Normalize embeddings to unit length")
+
+
+class EmbeddingResponse(BaseModel):
+    """
+    Embedding generation response
+    """
+    embeddings         : List[List[float]] = Field(...)
+    dimension          : int               = Field(..., gt = 0)
+    num_embeddings     : int               = Field(..., gt = 0)
+    processing_time_ms : float             = Field(..., ge = 0.0)
+
+
+# Retrieval Models
+class RetrievalRequest(BaseModel):
+    """
+    Request for document retrieval
+    """
+    query         : str                 = Field(..., min_length = 1)
+    top_k         : int                 = Field(10, ge = 1, le = 100)
+    
+    # Retrieval method
+    use_vector    : bool                = Field(True)
+    use_bm25      : bool                = Field(True)
+    vector_weight : Optional[float]     = Field(0.6, ge = 0.0, le = 1.0)
+    
+    # Filters
+    document_ids  : Optional[List[str]] = Field(None)
+    min_score     : Optional[float]     = Field(None, ge = 0.0, le = 1.0)
+
+
+class RetrievalResponse(BaseModel):
+    """
+    Document retrieval response
+    """
+    chunks            : List[ChunkWithScore] = Field(...)
+    retrieval_time_ms : float                = Field(..., ge = 0.0)
+    num_candidates    : int                  = Field(..., ge = 0)
+
+
+# System Models
+class HealthCheck(BaseModel):
+    """
+    System health check response
+    """
+    status                    : Literal["healthy", "degraded", "unhealthy"] = Field(...)
+    timestamp                 : datetime                                    = Field(default_factory = datetime.now)
+    
+    # Component status
+    ollama_available          : bool                                        = Field(...)
+    vector_store_available    : bool                                        = Field(...)
+    embedding_model_available : bool                                        = Field(...)
+    
+    # Stats
+    total_documents           : int                                         = Field(0, ge = 0)
+    total_chunks              : int                                         = Field(0, ge = 0)
+    
+    # Version info
+    version                   : str                                         = Field(...)
+    
+    # Issues
+    warnings                  : List[str]                                   = Field(default_factory = list)
+    errors                    : List[str]                                   = Field(default_factory = list)
+
+
+class SystemStats(BaseModel):
+    """
+    System statistics
+    """
+    # Document stats
+    total_documents     : int            = Field(0, ge = 0)
+    documents_by_type   : Dict[str, int] = Field(default_factory = dict)
+    total_file_size_mb  : float          = Field(0.0, ge = 0.0)
+    
+    # Chunk stats
+    total_chunks        : int            = Field(0, ge = 0)
+    avg_chunk_size      : float          = Field(0.0, ge = 0.0)
+    
+    # Query stats
+    total_queries       : int            = Field(0, ge = 0)
+    avg_query_time_ms   : float          = Field(0.0, ge = 0.0)
+    avg_retrieval_score : float          = Field(0.0, ge = 0.0)
+    
+    # Timestamp
+    generated_at        : datetime       = Field(default_factory = datetime.now)
+
+
+class ErrorResponse(BaseModel):
+    """
+    Standard error response
+    """
+    error      : str                      = Field(..., description = "Error type")
+    message    : str                      = Field(..., description = "Human-readable error message")
+    detail     : Optional[Dict[str, Any]] = Field(None, description = "Additional error details")
+    timestamp  : datetime                 = Field(default_factory = datetime.now)
+    request_id : Optional[str]            = Field(None)
+
+
+# Configuration Models
+class ChunkingConfig(BaseModel):
+    """
+    Chunking configuration
+    """
+    strategy           : ChunkingStrategy = Field(...)
+    chunk_size         : int              = Field(..., gt = 0)
+    overlap            : int              = Field(..., ge = 0)
+    
+    # Strategy-specific params
+    semantic_threshold : Optional[float]  = Field(None, ge = 0.0, le = 1.0)
+    parent_size        : Optional[int]    = Field(None, gt = 0)
+    child_size         : Optional[int]    = Field(None, gt = 0)
+
+
+class RetrievalConfig(BaseModel):
+    """
+    Retrieval configuration
+    """
+    top_k            : int   = Field(10, ge = 1, le = 100)
+    vector_weight    : float = Field(0.6, ge = 0.0, le = 1.0)
+    bm25_weight      : float = Field(0.4, ge = 0.0, le = 1.0)
+    enable_reranking : bool  = Field(False)
+    faiss_nprobe     : int   = Field(10, ge = 1, le = 100)
+    
+
+    @field_validator("bm25_weight")
+    @classmethod
+    def validate_weights(cls, v: float, info) -> float:
+        """
+        Ensure weights sum to 1.0
+        """
+        if ("vector_weight" in info.data):
+            vector_weight = info.data["vector_weight"]
+
+            if (abs(vector_weight + v - 1.0) > 0.01):
+                raise ValueError("vector_weight + bm25_weight must equal 1.0")
+
+        return v
+
+
+# Chat Response
+class ChatRequest(BaseModel):
+    message    : str
+    session_id : Optional[str] = None
+
+
+# Validation Utilities
+def validate_document_id(document_id: str) -> bool:
+    """
+    Validate document ID format
+    """
+    # Format: doc_<timestamp>_<hash>
+    pattern = r'^doc_\d{10,}_[a-f0-9]{8}$'
+
+    return bool(re.match(pattern, document_id))
+
+
+def validate_chunk_id(chunk_id: str) -> bool:
+    """
+    Validate chunk ID format
+    """
+    # Format: chunk_<doc_id>_<index>
+    pattern = r'^chunk_doc_\d+_[a-f0-9]{8}_\d+$'
+
+    return bool(re.match(pattern, chunk_id))
+
+
+# RAGAS Evaluation Models
+class RAGASEvaluationResult(BaseModel):
+    """
+    Single RAGAS evaluation result
+    """
+    model_config                         = ConfigDict(arbitrary_types_allowed = True)
+    
+    # Input data
+    query              : str             = Field(..., description = "User query")
+    answer             : str             = Field(..., description = "Generated answer")
+    contexts           : List[str]       = Field(..., description = "Retrieved context chunks")
+    ground_truth       : Optional[str]   = Field(None, description = "Reference answer (if available)")
+    timestamp          : str             = Field(..., description = "Evaluation timestamp")
+    
+    # RAGAS metrics (without ground truth)
+    answer_relevancy   : float           = Field(..., ge = 0.0, le = 1.0, description = "How well answer addresses question")
+    faithfulness       : float           = Field(..., ge = 0.0, le = 1.0, description = "Is answer grounded in context")
+    context_precision  : Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Are relevant chunks ranked high (requires ground truth)")
+    context_utilization: Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Context utilization score (without ground truth)") 
+    context_relevancy  : float           = Field(..., ge = 0.0, le = 1.0, description = "How relevant are retrieved chunks")
+    
+    # RAGAS metrics (requiring ground truth)
+    context_recall     : Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Coverage of ground truth")
+    answer_similarity  : Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Similarity to reference")
+    answer_correctness : Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Correctness vs reference")
+    
+    # Performance metadata
+    retrieval_time_ms  : int             = Field(..., ge = 0, description = "Retrieval time in milliseconds")
+    generation_time_ms : int             = Field(..., ge = 0, description = "Generation time in milliseconds")
+    total_time_ms      : int             = Field(..., ge = 0, description = "Total time in milliseconds")
+    chunks_retrieved   : int             = Field(..., ge = 0, description = "Number of chunks retrieved")
+    query_type         : str             = Field("rag", description = "Type of query: 'rag' or 'general'")
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert to dictionary
+        """
+        return self.model_dump()
+    
+
+    @property
+    def has_ground_truth_metrics(self) -> bool:
+        """
+        Check if ground truth metrics are available
+        """
+        return any([self.context_recall is not None,
+                    self.answer_similarity is not None,
+                    self.answer_correctness is not None
+                  ])
+    
+    @property
+    def overall_score(self) -> float:
+        """
+        Calculate weighted overall score
+        """
+        scores  = list()
+        weights = list()
+        
+        # Always include these metrics
+        scores.append(self.answer_relevancy)
+        weights.append(0.4)
+        
+        scores.append(self.faithfulness)
+        weights.append(0.3)
+        
+        scores.append(self.context_relevancy)
+        weights.append(0.1)
+        
+        # Include context_precision OR context_utilization (but not both)
+        if self.context_precision is not None:
+            scores.append(self.context_precision)
+            weights.append(0.2)
+        
+        elif self.context_utilization is not None:
+            scores.append(self.context_utilization)
+            weights.append(0.2)
+        
+        else:
+            # If neither is available, adjust weights
+            weights = [w * 1.25 for w in weights]  # Scale existing weights
+        
+        # Calculate weighted average
+        if (sum(weights) > 0):
+            weighted_sum = sum(s * w for s, w in zip(scores, weights))
+            score        = weighted_sum / sum(weights)
+        
+        else:
+            score = sum(scores) / len(scores) if scores else 0.0
+        
+        return round(score, 3)
+
+
+class RAGASStatistics(BaseModel):
+    """
+    Aggregate RAGAS statistics for a session
+    """
+    total_evaluations      : int             = Field(..., ge = 0, description = "Total number of evaluations")
+    
+    # Average metrics
+    avg_answer_relevancy   : float           = Field(..., ge = 0.0, le = 1.0)
+    avg_faithfulness       : float           = Field(..., ge = 0.0, le = 1.0)
+    avg_context_precision  : Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Average context precision (requires ground truth)")
+    avg_context_utilization: Optional[float] = Field(None, ge = 0.0, le = 1.0, description = "Average context utilization (without ground truth)") 
+    avg_context_relevancy  : float           = Field(..., ge = 0.0, le = 1.0)
+    avg_overall_score      : float           = Field(..., ge = 0.0, le = 1.0)
+    
+    # Ground truth metrics (if available)
+    avg_context_recall     : Optional[float] = Field(None, ge = 0.0, le = 1.0)
+    avg_answer_similarity  : Optional[float] = Field(None, ge = 0.0, le = 1.0)
+    avg_answer_correctness : Optional[float] = Field(None, ge = 0.0, le = 1.0)
+    
+    # Performance metrics
+    avg_retrieval_time_ms  : float           = Field(..., ge = 0.0)
+    avg_generation_time_ms : float           = Field(..., ge = 0.0)
+    avg_total_time_ms      : float           = Field(..., ge = 0.0)
+    
+    # Quality indicators
+    min_score              : float           = Field(..., ge = 0.0, le = 1.0, description = "Lowest overall score")
+    max_score              : float           = Field(..., ge = 0.0, le = 1.0, description = "Highest overall score")
+    std_dev                : float           = Field(..., ge = 0.0, description = "Standard deviation of scores")
+    
+    # Session info
+    session_start          : datetime        = Field(..., description = "When evaluation session started")
+    last_updated           : datetime         = Field(..., description = "Last evaluation timestamp")
+
+
+class RAGASExportData(BaseModel):
+    """
+    Complete RAGAS evaluation export data
+    """
+    export_timestamp     : datetime                    = Field(default_factory = datetime.now)
+    total_evaluations    : int                         = Field(..., ge = 0)
+    statistics           : RAGASStatistics
+    evaluations          : List[RAGASEvaluationResult]
+    
+    # Configuration info
+    ground_truth_enabled : bool                        = Field(...)
+    ragas_version        : str                         = Field(default = "0.1.9")
+    
+    @property
+    def export_filename(self) -> str:
+        """
+        Generate export filename
+        """
+        timestamp = self.export_timestamp.strftime("%Y%m%d_%H%M%S")
+
+        return f"ragas_evaluation_{timestamp}.json"

config/settings.py

@@ -0,0 +1,247 @@
+# DEPENDENCIES
+import os
+import time
+import torch
+from pathlib import Path
+from pydantic import Field
+from typing import Literal
+from typing import Optional
+from pydantic import field_validator
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    """
+    Application configuration with environment variable support
+    
+    Environment variables take precedence over defaults
+    """
+    # Huggingface Space Deployment mode detection
+    IS_HF_SPACE                   : bool                                                     = Field(default = os.getenv("SPACE_ID") is not None, description = "Running in HF Space")
+    
+    # Application Settings
+    APP_NAME                      : str                                                      = "AI Universal Knowledge Ingestion System"
+    APP_VERSION                   : str                                                      = "1.0.0"
+    DEBUG                         : bool                                                     = Field(default = False, description = "Enable debug mode")
+    HOST                          : str                                                      = Field(default = "0.0.0.0", description = "API host")
+    PORT                          : int                                                      = Field(default = int(os.getenv("PORT", 8000)), description = "API port (7860 for HF Spaces)")
+    
+    # LLM Provider Selection (ADD THESE)
+    OLLAMA_ENABLED                : bool                                                     = Field(default = os.getenv("OLLAMA_ENABLED", "true").lower() == "true", description = "Enable Ollama (set false for HF Spaces)")
+    USE_OPENAI                    : bool                                                     = Field(default = os.getenv("USE_OPENAI", "false").lower() == "true", description = "Use OpenAI API instead of local LLM")
+    
+    
+    # File Upload Settings
+    MAX_FILE_SIZE_MB              : int                                                      = Field(default = 100, description = "Max file size in MB")
+    MAX_BATCH_FILES               : int                                                      = Field(default = 10, description = "Max files per upload")
+    ALLOWED_EXTENSIONS            : list[str]                                                = Field(default = ["pdf", "docx", "txt"], description = "Allowed file extensions")
+    UPLOAD_DIR                    : Path                                                     = Field(default = Path("data/uploads"), description = "Directory for uploaded files")
+    
+    # Ollama LLM Settings
+    OLLAMA_BASE_URL               : str                                                      = Field(default = "http://localhost:11434", description = "Ollama API endpoint")
+    OLLAMA_MODEL                  : str                                                      = Field(default = "mistral:7b", description = "Ollama model name")
+    OLLAMA_TIMEOUT                : int                                                      = Field(default = 120, description = "Ollama request timeout (seconds)")
+    
+    # Generation parameters
+    DEFAULT_TEMPERATURE           : float                                                    = Field(default = 0.1, ge = 0.0, le = 1.0, description = "LLM temperature (0=deterministic, 1=creative)")
+    TOP_P                         : float                                                    = Field(default = 0.9, ge = 0.0, le = 1.0, description = "Nucleus sampling threshold")
+    MAX_TOKENS                    : int                                                      = Field(default = 1000, description = "Max output tokens")
+    CONTEXT_WINDOW                : int                                                      = Field(default = 8192, description = "Model context window size")
+    
+    # OpenAI Settings
+    OPENAI_API_KEY                : Optional[str]                                            = Field(default = os.getenv("OPENAI_API_KEY"), description = "Open AI API secret key")
+    OPENAI_MODEL                  : str                                                      = Field(default = "gpt-3.5-turbo", description = "Ollama model name")
+    
+    # Embedding Settings
+    EMBEDDING_MODEL               : str                                                      = Field(default = "BAAI/bge-small-en-v1.5", description = "HuggingFace embedding model")
+    EMBEDDING_DIMENSION           : int                                                      = Field(default = 384, description = "Embedding vector dimension")
+    EMBEDDING_DEVICE              : Literal["cpu", "cuda", "mps"]                            = Field(default = "cpu", description = "Device for embedding generation")
+    EMBEDDING_BATCH_SIZE          : int                                                      = Field(default = 32, description = "Batch size for embedding generation")
+    
+    # Chunking Settings
+    # Fixed chunking
+    FIXED_CHUNK_SIZE              : int                                                      = Field(default = 512, description = "Fixed chunk size in tokens")
+    FIXED_CHUNK_OVERLAP           : int                                                      = Field(default = 25, description = "Overlap between chunks")
+    
+    # Semantic chunking
+    SEMANTIC_BREAKPOINT_THRESHOLD : float                                                    = Field(default = 0.80, description = "Percentile for semantic breakpoints")
+
+    # Hierarchical chunking
+    PARENT_CHUNK_SIZE             : int                                                      = Field(default = 2048, description = "Parent chunk size")
+    CHILD_CHUNK_SIZE              : int                                                      = Field(default = 512, description = "Child chunk size")
+    
+    # Adaptive thresholds
+    SMALL_DOC_THRESHOLD           : int                                                      = Field(default = 1000, description = "Token threshold for fixed chunking")
+    LARGE_DOC_THRESHOLD           : int                                                      = Field(default = 500000, description = "Token threshold for hierarchical chunking")
+    
+    # Retrieval Settings
+    # Vector search
+    TOP_K_RETRIEVE                : int                                                      = Field(default = 10, description = "Top chunks to retrieve")
+    TOP_K_FINAL                   : int                                                      = Field(default = 5, description = "Final chunks after reranking")
+    FAISS_NPROBE                  : int                                                      = Field(default = 10, description = "FAISS search probes")
+    
+    # Hybrid search weights
+    VECTOR_WEIGHT                 : float                                                    = Field(default = 0.6, description = "Vector search weight")
+    BM25_WEIGHT                   : float                                                    = Field(default = 0.4, description = "BM25 search weight")
+    
+    # BM25 parameters
+    BM25_K1                       : float                                                    = Field(default = 1.5, description = "BM25 term saturation")
+    BM25_B                        : float                                                    = Field(default = 0.75, description = "BM25 length normalization")
+    
+    # Reranking
+    ENABLE_RERANKING              : bool                                                     = Field(default = True, description = "Enable cross-encoder reranking")
+    RERANKER_MODEL                : str                                                      = Field(default = "cross-encoder/ms-marco-MiniLM-L-6-v2", description = "Reranker model")
+    
+    # Storage Settings
+    VECTOR_STORE_DIR              : Path                                                     = Field(default = Path("data/vector_store"), description = "FAISS index storage")
+    METADATA_DB_PATH              : Path                                                     = Field(default = Path("data/metadata.db"), description = "SQLite metadata database")
+    
+    # Backup
+    AUTO_BACKUP                   : bool                                                     = Field(default = True, description = "Enable auto-backup")
+    BACKUP_INTERVAL               : int                                                      = Field(default = 1000, description = "Backup every N documents")
+    BACKUP_DIR                    : Path                                                     = Field(default = Path("data/backups"), description = "Backup directory")
+    
+    # Cache Settings
+    ENABLE_CACHE                  : bool                                                     = Field(default = True, description = "Enable embedding cache")
+    CACHE_TYPE                    : Literal["memory", "redis"]                               = Field(default = "memory", description = "Cache backend")
+    CACHE_TTL                     : int                                                      = Field(default = 3600, description = "Cache TTL in seconds")
+    CACHE_MAX_SIZE                : int                                                      = Field(default = 1000, description = "Max cached items")
+    
+    # Redis (if used)
+    REDIS_HOST                    : str                                                      = Field(default = "localhost", description = "Redis host")
+    REDIS_PORT                    : int                                                      = Field(default = 6379, description = "Redis port")
+    REDIS_DB                      : int                                                      = Field(default = 0, description = "Redis database number")
+    
+    # Logging Settings
+    LOG_LEVEL                     : Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(default = "INFO", description = "Logging level")
+    LOG_DIR                       : Path                                                     = Field(default = Path("logs"), description = "Log file directory")
+    LOG_FORMAT                    : str                                                      = Field(default = "%(asctime)s - %(name)s - %(levelname)s - %(message)s", description = "Log format string")
+    LOG_ROTATION                  : str                                                      = Field(default = "500 MB", description = "Log rotation size")
+    LOG_RETENTION                 : str                                                      = Field(default = "30 days", description = "Log retention period")
+    
+    # Evaluation Settings
+    ENABLE_RAGAS                  : bool                                                     = Field(default = True, description = "Enable Ragas evaluation")
+    RAGAS_ENABLE_GROUND_TRUTH     : bool                                                     = Field(default = False, description = "Enable RAGAS metrics requiring ground truth")
+    RAGAS_METRICS                 : list[str]                                                = Field(default = ["answer_relevancy", "faithfulness", "context_utilization", "context_relevancy"], description = "Ragas metrics to compute (base metrics without ground truth)")
+    RAGAS_GROUND_TRUTH_METRICS    : list[str]                                                = Field(default = ["context_precision", "context_recall", "answer_similarity", "answer_correctness"], description = "Ragas metrics requiring ground truth")
+    RAGAS_EVALUATION_TIMEOUT      : int                                                      = Field(default = 60, description = "RAGAS evaluation timeout in seconds")
+    RAGAS_BATCH_SIZE              : int                                                      = Field(default = 10, description = "Batch size for RAGAS evaluations")
+
+    # Web Scraping Settings (for future)
+    SCRAPING_ENABLED              : bool                                                     = Field(default = False, description = "Enable web scraping")
+    USER_AGENT                    : str                                                      = Field(default = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", description = "User agent for scraping")
+    REQUEST_DELAY                 : float                                                    = Field(default = 2.0, description = "Delay between requests (seconds)")
+    MAX_RETRIES                   : int                                                      = Field(default = 3, description = "Max scraping retries")
+    
+    # Performance Settings
+    MAX_WORKERS                   : int                                                      = Field(default = 4, description = "Max parallel workers")
+    ASYNC_BATCH_SIZE              : int                                                      = Field(default = 10, description = "Async batch size")
+    
+    # Security Settings
+    ENABLE_AUTH                   : bool                                                     = Field(default = False, description = "Enable authentication")
+    SECRET_KEY                    : str                                                      = Field(default = os.getenv("SECRET_KEY", "dev-key-change-in-production"))
+    
+    FIXED_CHUNK_STRATEGY          : str                                                      = Field(default = "fixed", description = "Default chunking strategy")
+    
+
+    class Config:
+        env_file          = ".env"
+        env_file_encoding = "utf-8"
+        case_sensitive    = True
+    
+
+    @field_validator("UPLOAD_DIR", "VECTOR_STORE_DIR", "LOG_DIR", "BACKUP_DIR", "METADATA_DB_PATH")
+    @classmethod
+    def create_directories(cls, v: Path) -> Path:
+        """
+        Ensure directories exist
+        """
+        if v.suffix:  # It's a file path (like metadata.db)
+            v.parent.mkdir(parents = True, exist_ok = True)
+
+        else:  # It's a directory
+            v.mkdir(parents = True, exist_ok = True)
+
+        return v
+
+    
+    @field_validator("VECTOR_WEIGHT", "BM25_WEIGHT")
+    @classmethod
+    def validate_weights_sum(cls, v: float, info) -> float:
+        """
+        Ensure vector and BM25 weights are valid
+        """
+        if ((info.field_name == "BM25_WEIGHT") and ("VECTOR_WEIGHT" in info.data)):
+            vector_weight = info.data["VECTOR_WEIGHT"]
+            
+            if (abs(vector_weight + v - 1.0) > 0.01):
+                raise ValueError(f"VECTOR_WEIGHT ({vector_weight}) + BM25_WEIGHT ({v}) must sum to 1.0")
+        
+        return v
+    
+
+    @property
+    def max_file_size_bytes(self) -> int:
+        """
+        Convert MB to bytes
+        """
+        return self.MAX_FILE_SIZE_MB * 1024 * 1024
+    
+
+    @property
+    def is_cuda_available(self) -> bool:
+        """
+        Check if CUDA device is requested and available
+        """
+        if self.EMBEDDING_DEVICE == "cuda":
+            try:
+                return torch.cuda.is_available()
+
+            except ImportError:
+                return False
+
+        return False
+    
+
+    def get_ollama_url(self, endpoint: str) -> str:
+        """
+        Construct full Ollama API URL
+        """
+        return f"{self.OLLAMA_BASE_URL.rstrip('/')}/{endpoint.lstrip('/')}"
+    
+
+    @classmethod
+    def get_timestamp_ms(cls) -> int:
+        """
+        Get current timestamp in milliseconds
+        """
+        return int(time.time() * 1000)
+
+
+    def summary(self) -> dict:
+        """
+        Get configuration summary (excluding sensitive data)
+        """
+        return {"app_name"           : self.APP_NAME,
+                "version"            : self.APP_VERSION,
+                "ollama_model"       : self.OLLAMA_MODEL,
+                "embedding_model"    : self.EMBEDDING_MODEL,
+                "embedding_device"   : self.EMBEDDING_DEVICE,
+                "max_file_size_mb"   : self.MAX_FILE_SIZE_MB,
+                "allowed_extensions" : self.ALLOWED_EXTENSIONS,
+                "chunking_strategy"  : {"small_threshold" : self.SMALL_DOC_THRESHOLD, "large_threshold" : self.LARGE_DOC_THRESHOLD},
+                "retrieval"          : {"top_k" : self.TOP_K_RETRIEVE, "hybrid_weights" : {"vector" : self.VECTOR_WEIGHT, "bm25" : self.BM25_WEIGHT}},
+                "evaluation"         : {"ragas_enabled" : self.ENABLE_RAGAS, "ragas_ground_truth" : self.RAGAS_ENABLE_GROUND_TRUTH, "ragas_metrics" : self.RAGAS_METRICS},
+               }
+
+
+# Global settings instance
+settings = Settings()
+
+
+# Convenience function for getting settings
+def get_settings() -> Settings:
+    """
+    Get global settings instance
+    """
+    return settings

docs/API.md

@@ -0,0 +1,904 @@
+# AI Universal Knowledge Ingestion System - API Documentation
+
+## Overview
+The AI Universal Knowledge Ingestion System is a production-grade RAG (Retrieval-Augmented Generation) platform that enables organizations to unlock knowledge from multiple document sources while maintaining complete data privacy and eliminating API costs.
+
+**Base URL:** http://localhost:8000 (or your deployed domain)
+
+**API Version:** v1.0.0
+
+---
+
+## Authentication
+Currently, the API operates without authentication for local development. For production deployments, consider implementing:
+
+- API Key Authentication
+- JWT Tokens
+- OAuth2
+
+---
+
+## Rate Limiting
+- Default: 100 requests per minute per IP
+- File Uploads: 10MB max per file, 50MB total per request
+- Chat Endpoints: 30 requests per minute per session
+
+---
+
+## Response Format
+
+All API responses follow this standard format:
+
+```json
+{
+  "success": true,
+  "data": {...},
+  "message": "Operation completed successfully",
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+Error responses:
+
+```json
+{
+  "success": false,
+  "error": "Error Type",
+  "message": "Human-readable error message",
+  "detail": {...},
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+---
+
+## System Management Endpoints
+
+### Get System Health
+
+**GET** `/api/health`
+
+Check system health and component status.
+
+**Response:**
+```json
+{
+  "status": "healthy",
+  "timestamp": "2024-01-15T10:30:00Z",
+  "version": "1.0.0",
+  "components": {
+    "vector_store": true,
+    "llm": true,
+    "embeddings": true,
+    "retrieval": true,
+    "generation": true
+  },
+  "details": {
+    "overall": "healthy",
+    "vector_store": true,
+    "llm": true,
+    "embeddings": true,
+    "retrieval": true,
+    "generation": true
+  }
+}
+```
+
+### Get System Information
+
+**GET** `/api/system-info`
+
+Get comprehensive system status and statistics.
+
+**Response:**
+```json
+{
+  "system_state": {
+    "is_ready": true,
+    "processing_status": "ready",
+    "total_documents": 15,
+    "active_sessions": 3
+  },
+  "configuration": {
+    "inference_model": "mistral:7b",
+    "embedding_model": "BAAI/bge-small-en-v1.5",
+    "retrieval_top_k": 10,
+    "vector_weight": 0.6,
+    "bm25_weight": 0.4,
+    "temperature": 0.1,
+    "enable_reranking": true
+  },
+  "llm_provider": {
+    "provider": "ollama",
+    "model": "mistral:7b",
+    "status": "healthy"
+  },
+  "system_information": {
+    "vector_store_status": "Ready (145 chunks)",
+    "current_model": "mistral:7b",
+    "embedding_model": "BAAI/bge-small-en-v1.5",
+    "chunking_strategy": "adaptive",
+    "system_uptime_seconds": 3600
+  },
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+---
+
+## Document Management Endpoints
+
+### Upload Files
+
+**POST** `/api/upload`
+
+Upload multiple documents for processing.
+
+**Form Data:**
+- `files`: List of files (PDF, DOCX, TXT, ZIP) - max 2GB total
+
+**Supported Formats:**
+- PDF Documents (.pdf)
+- Microsoft Word (.docx, .doc)
+- Text Files (.txt, .md)
+- ZIP Archives (.zip) - automatic extraction
+
+**Response:**
+```json
+{
+  "success": true,
+  "message": "Successfully uploaded 3 files",
+  "files": [
+    {
+      "filename": "document_20240115_103000.pdf",
+      "original_name": "quarterly_report.pdf",
+      "size": 1542890,
+      "upload_time": "2024-01-15T10:30:00Z",
+      "file_path": "/uploads/document_20240115_103000.pdf",
+      "status": "uploaded"
+    }
+  ]
+}
+```
+
+### Start Processing
+
+**POST** `/api/start-processing`
+
+Start processing uploaded documents through the RAG pipeline.
+
+**Pipeline Stages:**
+1. Document parsing and text extraction
+2. Adaptive chunking (fixed/semantic/hierarchical)
+3. Embedding generation with BGE model
+4. Vector indexing (FAISS + BM25)
+5. Knowledge base compilation
+
+**Response:**
+```json
+{
+  "success": true,
+  "message": "Processing completed successfully",
+  "status": "ready",
+  "documents_processed": 3,
+  "total_chunks": 245,
+  "chunking_statistics": {
+    "adaptive": 120,
+    "semantic": 80,
+    "hierarchical": 45
+  },
+  "index_stats": {
+    "total_chunks_indexed": 245,
+    "vector_index_size": 245,
+    "bm25_indexed": true,
+    "metadata_stored": true
+  }
+}
+```
+
+### Get Processing Status
+
+**GET** `/api/processing-status`
+
+Monitor real-time processing progress.
+
+**Response:**
+```json
+{
+  "status": "processing",
+  "progress": 65,
+  "current_step": "Generating embeddings for quarterly_report.pdf...",
+  "processed": 2,
+  "total": 3,
+  "details": {
+    "chunks_processed": 156,
+    "embeddings_generated": 156
+  }
+}
+```
+
+---
+
+## Chat & Query Endpoints
+
+### Chat with Documents
+
+**POST** `/api/chat`
+
+Query your knowledge base with natural language questions. Includes automatic RAGAS evaluation if enabled.
+
+**Request Body (JSON):**
+```json
+{
+  "message": "What were the Q3 revenue trends?",
+  "session_id": "session_1705314600"
+}
+```
+
+**Response:**
+```json
+{
+  "session_id": "session_1705314600",
+  "response": "Based on the Q3 financial report, revenue increased by 15% quarter-over-quarter, reaching $45 million. The growth was primarily driven by enterprise sales and new market expansion. [1][2]",
+  "sources": [
+    {
+      "rank": 1,
+      "score": 0.894,
+      "document_id": "doc_1705300000_abc123",
+      "chunk_id": "chunk_doc_1705300000_abc123_0",
+      "text_preview": "Q3 Financial Highlights: Revenue growth of 15% QoQ reaching $45M...",
+      "page_number": 7,
+      "section_title": "Financial Performance",
+      "retrieval_method": "hybrid"
+    }
+  ],
+  "metrics": {
+    "retrieval_time": 245,
+    "generation_time": 3100,
+    "total_time": 3345,
+    "chunks_retrieved": 8,
+    "chunks_used": 3,
+    "tokens_used": 487
+  },
+  "ragas_metrics": {
+    "answer_relevancy": 0.89,
+    "faithfulness": 0.94,
+    "context_utilization": 0.87,
+    "context_relevancy": 0.91,
+    "overall_score": 0.90,
+    "context_precision": null,
+    "context_recall": null,
+    "answer_similarity": null,
+    "answer_correctness": null
+  }
+}
+```
+
+**Note:** Ground truth metrics (context_precision, context_recall, answer_similarity, answer_correctness) are null unless ground truth is provided and `RAGAS_ENABLE_GROUND_TRUTH=True`.
+
+### Export Chat History
+
+**GET** `/api/export-chat/{session_id}`
+
+Export conversation history for analysis or reporting.
+
+**Parameters:**
+- `session_id`: string (required) - Session identifier
+- `format`: string (optional) - Export format: `json` (default) or `csv`
+
+**Response (JSON):**
+```json
+{
+  "session_id": "session_1705314600",
+  "export_time": "2024-01-15T11:00:00Z",
+  "total_messages": 5,
+  "history": [
+    {
+      "query": "What was the Q3 revenue growth?",
+      "response": "Revenue increased by 15% quarter-over-quarter...",
+      "sources": [...],
+      "timestamp": "2024-01-15T10:30:00Z",
+      "metrics": {
+        "total_time": 3345
+      },
+      "ragas_metrics": {
+        "answer_relevancy": 0.89,
+        "faithfulness": 0.94,
+        "overall_score": 0.90
+      }
+    }
+  ]
+}
+```
+
+---
+
+## RAGAS Evaluation Endpoints
+
+### Get RAGAS History
+
+**GET** `/api/ragas/history`
+
+Get complete RAGAS evaluation history for the current session.
+
+**Response:**
+```json
+{
+  "success": true,
+  "total_count": 25,
+  "statistics": {
+    "total_evaluations": 25,
+    "avg_answer_relevancy": 0.876,
+    "avg_faithfulness": 0.912,
+    "avg_context_utilization": 0.845,
+    "avg_context_relevancy": 0.889,
+    "avg_overall_score": 0.881,
+    "avg_retrieval_time_ms": 235,
+    "avg_generation_time_ms": 3250,
+    "avg_total_time_ms": 3485,
+    "min_score": 0.723,
+    "max_score": 0.967,
+    "std_dev": 0.089,
+    "session_start": "2024-01-15T09:00:00Z",
+    "last_updated": "2024-01-15T11:00:00Z"
+  },
+  "history": [
+    {
+      "query": "What were the Q3 revenue trends?",
+      "answer": "Revenue increased by 15%...",
+      "contexts": ["Q3 Financial Highlights...", "Revenue breakdown..."],
+      "timestamp": "2024-01-15T10:30:00Z",
+      "answer_relevancy": 0.89,
+      "faithfulness": 0.94,
+      "context_utilization": 0.87,
+      "context_relevancy": 0.91,
+      "overall_score": 0.90,
+      "retrieval_time_ms": 245,
+      "generation_time_ms": 3100,
+      "total_time_ms": 3345,
+      "chunks_retrieved": 8
+    }
+  ]
+}
+```
+
+### Get RAGAS Statistics
+
+**GET** `/api/ragas/statistics`
+
+Get aggregate RAGAS statistics for the current session.
+
+**Response:**
+```json
+{
+  "success": true,
+  "statistics": {
+    "total_evaluations": 25,
+    "avg_answer_relevancy": 0.876,
+    "avg_faithfulness": 0.912,
+    "avg_context_utilization": 0.845,
+    "avg_context_relevancy": 0.889,
+    "avg_overall_score": 0.881,
+    "avg_retrieval_time_ms": 235,
+    "avg_generation_time_ms": 3250,
+    "avg_total_time_ms": 3485,
+    "min_score": 0.723,
+    "max_score": 0.967,
+    "std_dev": 0.089,
+    "session_start": "2024-01-15T09:00:00Z",
+    "last_updated": "2024-01-15T11:00:00Z"
+  }
+}
+```
+
+### Clear RAGAS History
+
+**POST** `/api/ragas/clear`
+
+Clear all RAGAS evaluation history and start a new session.
+
+**Response:**
+```json
+{
+  "success": true,
+  "message": "RAGAS evaluation history cleared, new session started"
+}
+```
+
+### Export RAGAS Data
+
+**GET** `/api/ragas/export`
+
+Export all RAGAS evaluation data as JSON.
+
+**Response:** JSON file download containing:
+```json
+{
+  "export_timestamp": "2024-01-15T11:00:00Z",
+  "total_evaluations": 25,
+  "statistics": {...},
+  "evaluations": [...],
+  "ground_truth_enabled": false
+}
+```
+
+### Get RAGAS Configuration
+
+**GET** `/api/ragas/config`
+
+Get current RAGAS configuration settings.
+
+**Response:**
+```json
+{
+  "enabled": true,
+  "ground_truth_enabled": false,
+  "base_metrics": [
+    "answer_relevancy",
+    "faithfulness",
+    "context_utilization",
+    "context_relevancy"
+  ],
+  "ground_truth_metrics": [
+    "context_precision",
+    "context_recall",
+    "answer_similarity",
+    "answer_correctness"
+  ],
+  "evaluation_timeout": 60,
+  "batch_size": 10
+}
+```
+
+---
+
+## Analytics Endpoints
+
+### Get System Analytics
+
+**GET** `/api/analytics`
+
+Get comprehensive system analytics and performance metrics with caching.
+
+**Response:**
+```json
+{
+  "performance_metrics": {
+    "avg_response_time": 3485,
+    "min_response_time": 2100,
+    "max_response_time": 8900,
+    "total_queries": 127,
+    "queries_last_hour": 23,
+    "p95_response_time": 7200
+  },
+  "quality_metrics": {
+    "answer_relevancy": 0.876,
+    "faithfulness": 0.912,
+    "context_precision": 0.845,
+    "context_recall": null,
+    "overall_score": 0.878,
+    "avg_sources_per_query": 4.2,
+    "queries_with_sources": 125,
+    "confidence": "high",
+    "metrics_available": true
+  },
+  "system_information": {
+    "vector_store_status": "Ready (245 chunks)",
+    "current_model": "mistral:7b",
+    "embedding_model": "BAAI/bge-small-en-v1.5",
+    "chunking_strategy": "adaptive",
+    "system_uptime_seconds": 7200,
+    "last_updated": "2024-01-15T11:00:00Z"
+  },
+  "health_status": {
+    "overall": "healthy",
+    "llm": true,
+    "vector_store": true,
+    "embeddings": true,
+    "retrieval": true,
+    "generation": true
+  },
+  "chunking_statistics": {
+    "primary_strategy": "semantic",
+    "total_chunks": 245,
+    "strategies_used": {
+      "fixed": 98,
+      "semantic": 112,
+      "hierarchical": 35
+    }
+  },
+  "document_statistics": {
+    "total_documents": 15,
+    "total_chunks": 245,
+    "uploaded_files": 15,
+    "total_file_size_bytes": 52428800,
+    "total_file_size_mb": 50.0,
+    "avg_chunks_per_document": 16.3
+  },
+  "session_statistics": {
+    "total_sessions": 8,
+    "total_messages": 127,
+    "avg_messages_per_session": 15.9
+  },
+  "index_statistics": {
+    "total_chunks_indexed": 245,
+    "vector_index_size": 245,
+    "bm25_indexed": true
+  },
+  "calculated_at": "2024-01-15T11:00:00Z",
+  "cache_info": {
+    "from_cache": false,
+    "next_refresh_in": 30
+  }
+}
+```
+
+### Refresh Analytics Cache
+
+**GET** `/api/analytics/refresh`
+
+Force refresh analytics cache and get fresh data.
+
+**Response:**
+```json
+{
+  "success": true,
+  "message": "Analytics cache refreshed successfully",
+  "data": {
+    // Same structure as /api/analytics
+  }
+}
+```
+
+### Get Detailed Analytics
+
+**GET** `/api/analytics/detailed`
+
+Get detailed analytics including session breakdowns and component performance.
+
+**Response:**
+```json
+{
+  // All fields from /api/analytics, plus:
+  "detailed_sessions": [
+    {
+      "session_id": "session_1705314600",
+      "message_count": 12,
+      "first_message": "2024-01-15T09:00:00Z",
+      "last_message": "2024-01-15T10:45:00Z",
+      "total_response_time": 38500,
+      "avg_sources_per_query": 3.8
+    }
+  ],
+  "component_performance": {
+    "retrieval": {
+      "avg_time_ms": 245,
+      "cache_hit_rate": 0.23
+    },
+    "embeddings": {
+      "model": "BAAI/bge-small-en-v1.5",
+      "dimension": 384,
+      "device": "cpu"
+    }
+  }
+}
+```
+
+---
+
+## Configuration Endpoints
+
+### Get Current Configuration
+
+**GET** `/api/configuration`
+
+Retrieve current system configuration.
+
+**Response:**
+```json
+{
+  "configuration": {
+    "inference_model": "mistral:7b",
+    "embedding_model": "BAAI/bge-small-en-v1.5",
+    "vector_weight": 0.6,
+    "bm25_weight": 0.4,
+    "temperature": 0.1,
+    "max_tokens": 1000,
+    "chunk_size": 512,
+    "chunk_overlap": 50,
+    "top_k_retrieve": 10,
+    "enable_reranking": true,
+    "is_ready": true,
+    "llm_healthy": true
+  },
+  "health": {
+    "overall": "healthy",
+    "llm": true,
+    "vector_store": true,
+    "embeddings": true,
+    "retrieval": true,
+    "generation": true
+  }
+}
+```
+
+### Update Configuration
+
+**POST** `/api/configuration`
+
+Update system configuration parameters.
+
+**Form Data:**
+- `temperature`: float (0.0-1.0) - Generation temperature
+- `max_tokens`: integer (100-4000) - Maximum response tokens
+- `retrieval_top_k`: integer (1-50) - Number of chunks to retrieve
+- `vector_weight`: float (0.0-1.0) - Weight for vector search
+- `bm25_weight`: float (0.0-1.0) - Weight for keyword search
+- `enable_reranking`: boolean - Enable cross-encoder reranking
+- `session_id`: string (optional) - Session identifier for overrides
+
+**Response:**
+```json
+{
+  "success": true,
+  "message": "Configuration updated successfully",
+  "updates": {
+    "temperature": 0.2,
+    "retrieval_top_k": 15
+  }
+}
+```
+
+---
+
+## Error Handling
+
+### Common HTTP Status Codes
+
+- **200** - Success
+- **400** - Bad Request (invalid parameters)
+- **404** - Resource Not Found
+- **500** - Internal Server Error
+- **503** - Service Unavailable (component not ready)
+
+### Error Response Examples
+
+#### RAGAS Evaluation Disabled:
+```json
+{
+  "success": false,
+  "error": "RAGASDisabled",
+  "message": "RAGAS evaluation is not enabled. Set ENABLE_RAGAS=True in settings.",
+  "detail": {
+    "current_setting": "ENABLE_RAGAS=False"
+  },
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+#### System Not Ready:
+```json
+{
+  "success": false,
+  "error": "SystemNotReady",
+  "message": "System not ready. Please upload and process documents first.",
+  "detail": {
+    "is_ready": false,
+    "documents_processed": 0
+  },
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+#### LLM Service Unavailable:
+```json
+{
+  "success": false,
+  "error": "LLMUnavailable",
+  "message": "LLM service unavailable. Please ensure Ollama is running.",
+  "detail": {
+    "llm_healthy": false,
+    "suggestion": "Run 'ollama serve' in a separate terminal"
+  },
+  "timestamp": "2024-01-15T10:30:00Z"
+}
+```
+
+---
+
+## Best Practices
+
+### 1. File Upload
+
+- Use chunked upload for large files (>100MB)
+- Compress documents into ZIP archives for multiple files
+- Ensure documents are text-extractable (not scanned images without OCR)
+
+### 2. Query Optimization
+
+- Be specific and contextual in questions
+- Use natural language - no special syntax required
+- Break complex questions into multiple simpler queries
+
+### 3. Session Management
+
+- Reuse `session_id` for conversation continuity
+- Sessions automatically expire after 24 hours of inactivity
+- Export important conversations for long-term storage
+
+### 4. RAGAS Evaluation
+
+- Ensure OpenAI API key is configured for RAGAS to work
+- Monitor evaluation metrics to track system quality
+- Use analytics endpoints to identify quality trends
+- Export evaluation data regularly for offline analysis
+
+### 5. Performance Monitoring
+
+- Monitor response times and token usage
+- Use analytics endpoint for system health checks
+- Set up alerts for quality metric degradation
+- Enable caching for frequently accessed embeddings
+
+### 6. Configuration Management
+
+- Test configuration changes with a few queries first
+- Monitor RAGAS metrics after configuration updates
+- Use session-based overrides for experimentation
+- Document optimal configurations for different use cases
+
+---
+
+## SDK Examples
+
+### Python Client
+
+```python
+import requests
+
+class KnowledgeBaseClient:
+    def __init__(self, base_url="http://localhost:8000"):
+        self.base_url = base_url
+        self.session_id = None
+        
+    def upload_documents(self, file_paths):
+        files = [('files', open(fpath, 'rb')) for fpath in file_paths]
+        response = requests.post(f"{self.base_url}/api/upload", files=files)
+        return response.json()
+    
+    def start_processing(self):
+        response = requests.post(f"{self.base_url}/api/start-processing")
+        return response.json()
+    
+    def query(self, question):
+        data = {'message': question}
+        if self.session_id:
+            data['session_id'] = self.session_id
+        response = requests.post(f"{self.base_url}/api/chat", json=data)
+        result = response.json()
+        if not self.session_id:
+            self.session_id = result.get('session_id')
+        return result
+    
+    def get_ragas_history(self):
+        response = requests.get(f"{self.base_url}/api/ragas/history")
+        return response.json()
+    
+    def get_analytics(self):
+        response = requests.get(f"{self.base_url}/api/analytics")
+        return response.json()
+
+# Usage
+client = KnowledgeBaseClient()
+
+# Upload and process
+client.upload_documents(['report.pdf', 'contract.docx'])
+client.start_processing()
+
+# Query
+result = client.query("What are the key findings?")
+print(result['response'])
+print(f"Quality Score: {result['ragas_metrics']['overall_score']}")
+
+# Get analytics
+analytics = client.get_analytics()
+print(f"Avg Response Time: {analytics['performance_metrics']['avg_response_time']}ms")
+```
+
+### JavaScript Client
+
+```javascript
+class KnowledgeBaseClient {
+    constructor(baseUrl = 'http://localhost:8000') {
+        this.baseUrl = baseUrl;
+        this.sessionId = null;
+    }
+    
+    async uploadDocuments(files) {
+        const formData = new FormData();
+        files.forEach(file => formData.append('files', file));
+        
+        const response = await fetch(`${this.baseUrl}/api/upload`, {
+            method: 'POST',
+            body: formData
+        });
+        return await response.json();
+    }
+    
+    async startProcessing() {
+        const response = await fetch(`${this.baseUrl}/api/start-processing`, {
+            method: 'POST'
+        });
+        return await response.json();
+    }
+    
+    async query(question) {
+        const body = { message: question };
+        if (this.sessionId) body.session_id = this.sessionId;
+        
+        const response = await fetch(`${this.baseUrl}/api/chat`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(body)
+        });
+        
+        const result = await response.json();
+        if (!this.sessionId) this.sessionId = result.session_id;
+        return result;
+    }
+    
+    async getRagasHistory() {
+        const response = await fetch(`${this.baseUrl}/api/ragas/history`);
+        return await response.json();
+    }
+    
+    async getAnalytics() {
+        const response = await fetch(`${this.baseUrl}/api/analytics`);
+        return await response.json();
+    }
+}
+
+// Usage
+const client = new KnowledgeBaseClient();
+
+// Query
+const result = await client.query("What are the revenue trends?");
+console.log(result.response);
+console.log(`Quality: ${result.ragas_metrics.overall_score}`);
+
+// Get RAGAS history
+const history = await client.getRagasHistory();
+console.log(`Total evaluations: ${history.total_count}`);
+console.log(`Avg relevancy: ${history.statistics.avg_answer_relevancy}`);
+```
+
+---
+
+## Support & Troubleshooting
+
+### For API issues:
+
+- Check system health endpoint first
+- Verify document processing status
+- Review error messages and suggested actions
+- Check component readiness flags
+
+### For RAGAS issues:
+
+- Ensure OpenAI API key is configured
+- Check RAGAS is enabled in settings
+- Monitor evaluation timeout settings
+- Review logs for detailed error messages
+
+### For quality issues:
+
+- Monitor RAGAS evaluation metrics
+- Adjust retrieval and generation parameters
+- Review source citations for context relevance
+- Consider document preprocessing improvements
+
+---
+
+> **This API provides a complete RAG solution with multi-format document ingestion, intelligent retrieval, local LLM generation, and comprehensive RAGAS-based quality evaluation.**
+
+---

docs/ARCHITECTURE.md

@@ -0,0 +1,882 @@
+# AI Universal Knowledge Ingestion System - Technical Architecture Document
+
+## 1. System Overview
+
+### 1.1 High-Level Architecture
+
+```mermaid
+graph TB
+    subgraph "Frontend Layer"
+        A[Web UI<br/>HTML/CSS/JS]
+        B[File Upload<br/>Drag & Drop]
+        C[Chat Interface<br/>Real-time]
+        D[Analytics Dashboard<br/>RAGAS Metrics]
+    end
+    
+    subgraph "API Gateway"
+        E[FastAPI Server<br/>Python 3.11+]
+    end
+    
+    subgraph "Core Processing Engine"
+        F[Ingestion Module]
+        G[Processing Module]
+        H[Retrieval Module]
+        I[Generation Module]
+        J[Evaluation Module]
+    end
+    
+    subgraph "AI/ML Layer"
+        K[Ollama LLM<br/>Mistral-7B]
+        L[Embedding Model<br/>BGE-small-en]
+        M[FAISS Vector DB]
+    end
+    
+    subgraph "Quality Assurance"
+        N[RAGAS Evaluator<br/>Real-time Metrics]
+    end
+    
+    A --> E
+    E --> F
+    F --> G
+    G --> H
+    H --> I
+    I --> K
+    G --> L
+    L --> M
+    H --> M
+    I --> N
+    N --> E
+```
+
+### 1.2 System Characteristics
+
+| Aspect | Specification |
+|--------|---------------|
+| **Architecture Style** | Modular Microservices-inspired |
+| **Deployment** | Docker Containerized |
+| **Processing Model** | Async/Event-driven |
+| **Data Flow** | Pipeline-based with Checkpoints |
+| **Scalability** | Horizontal (Stateless API) + Vertical (GPU) |
+| **Caching** | In-Memory LRU Cache |
+| **Evaluation** | Real-time RAGAS Metrics |
+
+---
+
+## 2. Component Architecture
+
+### 2.1 Ingestion Module
+
+```mermaid
+flowchart TD
+    A[User Input] --> B{Input Type Detection}
+    
+    B -->|PDF/DOCX| D[Document Parser]
+    B -->|ZIP| E[Archive Extractor]
+    
+    subgraph D [Document Processing]
+        D1[PyPDF2<br/>PDF Text]
+        D2[python-docx<br/>Word Docs]
+        D3[EasyOCR<br/>Scanned PDFs]
+    end
+    
+    subgraph E [Archive Handling]
+        E1[zipfile<br/>Extraction]
+        E2[Recursive Processing]
+        E3[Size Validation<br/>2GB Max]
+    end
+    
+    D --> F[Text Cleaning]
+    E --> F
+    
+    F --> G[Encoding Normalization]
+    G --> H[Structure Preservation]
+    H --> I[Output: Cleaned Text<br/>+ Metadata]
+```
+
+#### Ingestion Specifications:
+
+| Component | Technology | Configuration | Limits |
+|-----------|------------|---------------|---------|
+| **PDF Parser** | PyPDF2 + EasyOCR | OCR: English+Multilingual | 1000 pages max |
+| **Document Parser** | python-docx | Preserve formatting | 50MB per file |
+| **Archive Handler** | zipfile | Recursion depth: 5 | 2GB total, 10k files |
+
+### 2.2 Processing Module
+
+#### 2.2.1 Adaptive Chunking Strategy
+
+```mermaid
+flowchart TD
+    A[Input Text] --> B[Token Count Analysis]
+    B --> C{Document Size}
+    
+    C -->|<50K tokens| D[Fixed-Size Chunking]
+    C -->|50K-500K tokens| E[Semantic Chunking]
+    C -->|>500K tokens| F[Hierarchical Chunking]
+    
+    subgraph D [Strategy 1: Fixed]
+        D1[Chunk Size: 512 tokens]
+        D2[Overlap: 50 tokens]
+        D3[Method: Simple sliding window]
+    end
+    
+    subgraph E [Strategy 2: Semantic]
+        E1[Breakpoint: 95th percentile similarity]
+        E2[Method: LlamaIndex SemanticSplitter]
+        E3[Preserve: Section boundaries]
+    end
+    
+    subgraph F [Strategy 3: Hierarchical]
+        F1[Parent: 2048 tokens]
+        F2[Child: 512 tokens]
+        F3[Retrieval: Child → Parent expansion]
+    end
+    
+    D --> G[Chunk Metadata]
+    E --> G
+    F --> G
+    
+    G --> H[Embedding Generation]
+```
+
+#### 2.2.2 Embedding Pipeline
+
+```python
+# Embedding Configuration
+EMBEDDING_CONFIG = {
+    "model": "BAAI/bge-small-en-v1.5",
+    "dimensions": 384,
+    "batch_size": 32,
+    "normalize": True,
+    "device": "cuda" if torch.cuda.is_available() else "cpu",
+    "max_sequence_length": 512
+}
+```
+
+| Parameter | Value | Rationale |
+|-----------|-------|-----------|
+| **Model** | BAAI/bge-small-en-v1.5 | SOTA quality, 62.17 MTEB score |
+| **Dimensions** | 384 | Optimal speed/accuracy balance |
+| **Batch Size** | 32 | Memory efficiency on GPU/CPU |
+| **Normalization** | L2 | Required for cosine similarity |
+| **Speed** | 1000 docs/sec (CPU) | 10x faster than alternatives |
+
+---
+
+### 2.3 Storage Module Architecture
+
+```mermaid
+graph TB
+    subgraph "Storage Layer"
+        A[FAISS Vector Store]
+        B[BM25 Keyword Index]
+        C[SQLite Metadata]
+        D[LRU Cache<br/>In-Memory]
+    end
+    
+    subgraph A [Vector Storage Architecture]
+        A1[IndexHNSW<br/>Large datasets]
+        A2[IndexIVFFlat<br/>Medium datasets]
+        A3[IndexFlatL2<br/>Small datasets]
+    end
+    
+    subgraph B [Keyword Index]
+        B1[rank_bm25 Library]
+        B2[TF-IDF Weights]
+        B3[In-memory Index]
+    end
+    
+    subgraph C [Metadata Management]
+        C1[Document Metadata]
+        C2[Chunk Relationships]
+        C3[User Sessions]
+        C4[RAGAS Evaluations]
+    end
+    
+    subgraph D [Cache Layer]
+        D1[Query Embeddings]
+        D2[Frequent Results]
+        D3[LRU Eviction]
+    end
+    
+    A --> E[Hybrid Retrieval]
+    B --> E
+    C --> E
+    D --> E
+```
+
+#### Vector Store Configuration
+
+| Index Type | Use Case | Parameters | Performance |
+|------------|----------|------------|-------------|
+| **IndexFlatL2** | < 100K vectors | Exact search | O(n), High accuracy |
+| **IndexIVFFlat** | 100K-1M vectors | nprobe: 10-20 | O(log n), Balanced |
+| **IndexHNSW** | > 1M vectors | M: 16, efConstruction: 40 | O(log n), Fastest |
+
+#### Caching Strategy
+
+```python
+# LRU Cache Configuration
+CACHE_CONFIG = {
+    "max_size": 1000,        # Maximum cached items
+    "ttl": 3600,             # Time to live (seconds)
+    "eviction": "LRU",       # Least Recently Used
+    "cache_embeddings": True,
+    "cache_results": True
+}
+```
+
+**Benefits:**
+- **Reduced latency**: 80% reduction for repeat queries
+- **Resource efficiency**: Avoid re-computing embeddings
+- **No external dependencies**: Pure Python implementation
+- **Memory efficient**: LRU eviction prevents unbounded growth
+
+---
+
+### 2.4 Retrieval Module
+
+#### 2.4.1 Hybrid Retrieval Pipeline
+
+```mermaid
+flowchart TD
+    A[User Query] --> B[Query Processing]
+    
+    B --> C[Vector Embedding]
+    B --> D[Keyword Extraction]
+    
+    C --> E[FAISS Search<br/>Top-K: 10]
+    D --> F[BM25 Search<br/>Top-K: 10]
+    
+    E --> G[Reciprocal Rank Fusion]
+    F --> G
+    
+    G --> H{Reranking Enabled?}
+    
+    H -->|Yes| I[Cross-Encoder Reranking]
+    H -->|No| J[Final Top-5 Selection]
+    
+    I --> J
+    
+    J --> K[Context Assembly]
+    K --> L[Citation Formatting]
+    L --> M[Output: Context + Sources]
+```
+
+#### 2.4.2 Retrieval Algorithms
+
+**Hybrid Fusion Formula:**
+
+```text
+RRF_score(doc) = vector_weight * (1 / (60 + vector_rank)) + bm25_weight * (1 / (60 + bm25_rank))
+```
+
+**Default Weights:**
+- Vector Similarity: 60%
+- BM25 Keyword: 40%
+
+**BM25 Parameters:**
+
+```python
+BM25_CONFIG = {
+    "k1": 1.5,      # Term frequency saturation
+    "b": 0.75,      # Length normalization
+    "epsilon": 0.25  # Smoothing factor
+}
+```
+
+---
+
+### 2.5 Generation Module
+
+#### 2.5.1 LLM Integration Architecture
+
+```mermaid
+graph TB
+    subgraph "Ollama Integration"
+        A[Ollama Server]
+        B[Mistral-7B-Instruct]
+        C[LLaMA-2-13B-Chat]
+    end
+    
+    subgraph "Prompt Engineering"
+        D[System Prompt Template]
+        E[Context Formatting]
+        F[Citation Injection]
+    end
+    
+    subgraph "Generation Control"
+        G[Temperature Controller]
+        H[Token Manager]
+        I[Streaming Handler]
+    end
+    
+    A --> J[API Client]
+    B --> A
+    C --> A
+    
+    D --> K[Prompt Assembly]
+    E --> K
+    F --> K
+    
+    G --> L[Generation Parameters]
+    H --> L
+    I --> L
+    
+    K --> M[LLM Request]
+    L --> M
+    M --> J
+    J --> N[Response Processing]
+```
+
+#### 2.5.2 LLM Configuration
+
+| Parameter | Default Value | Range | Description |
+|-----------|---------------|-------|-------------|
+| **Model** | Mistral-7B-Instruct | - | Primary inference model |
+| **Temperature** | 0.1 | 0.0-1.0 | Response creativity |
+| **Max Tokens** | 1000 | 100-4000 | Response length limit |
+| **Top-P** | 0.9 | 0.1-1.0 | Nucleus sampling |
+| **Context Window** | 32K | - | Mistral model capacity |
+
+---
+
+### 2.6 RAGAS Evaluation Module
+
+#### 2.6.1 RAGAS Evaluation Pipeline
+
+```mermaid
+flowchart LR
+    A[Query] --> B[Generated Answer]
+    C[Retrieved Context] --> B
+    
+    B --> D[RAGAS Evaluator]
+    C --> D
+    
+    D --> E[Answer Relevancy]
+    D --> F[Faithfulness]
+    D --> G[Context Utilization]
+    D --> H[Context Relevancy]
+    
+    E --> I[Metrics Aggregation]
+    F --> I
+    G --> I
+    H --> I
+    
+    I --> J[Analytics Dashboard]
+    I --> K[SQLite Storage]
+    I --> L[Session Statistics]
+```
+
+#### 2.6.2 Evaluation Metrics
+
+| Metric | Target | Measurement Method | Importance |
+|--------|--------|-------------------|------------|
+| **Answer Relevancy** | > 0.85 | LLM-based evaluation | Core user satisfaction |
+| **Faithfulness** | > 0.90 | Grounded in context check | Prevents hallucinations |
+| **Context Utilization** | > 0.80 | How well context is used | Generation effectiveness |
+| **Context Relevancy** | > 0.85 | Retrieved chunks relevance | Retrieval quality |
+
+**Implementation Details:**
+
+```python
+# RAGAS Configuration
+RAGAS_CONFIG = {
+    "enable_ragas": True,
+    "enable_ground_truth": False,
+    "base_metrics": [
+        "answer_relevancy",
+        "faithfulness",
+        "context_utilization",
+        "context_relevancy"
+    ],
+    "ground_truth_metrics": [
+        "context_precision",
+        "context_recall",
+        "answer_similarity",
+        "answer_correctness"
+    ],
+    "evaluation_timeout": 60,
+    "batch_size": 10
+}
+```
+
+**Evaluation Flow:**
+
+1. **Automatic Trigger**: Every query-response pair is evaluated
+2. **Async Processing**: Evaluation runs in background (non-blocking)
+3. **Storage**: Results stored in SQLite for analytics
+4. **Aggregation**: Session-level statistics computed on-demand
+5. **Export**: Full evaluation data available for download
+
+---
+
+## 3. Data Flow & Workflows
+
+### 3.1 End-to-End Processing Pipeline
+
+```mermaid
+sequenceDiagram
+    participant U as User
+    participant F as Frontend
+    participant A as API Gateway
+    participant I as Ingestion
+    participant P as Processing
+    participant S as Storage
+    participant R as Retrieval
+    participant G as Generation
+    participant E as RAGAS Evaluator
+    
+    U->>F: Upload Documents
+    F->>A: POST /api/upload
+    A->>I: Process Input Sources
+    
+    Note over I: Parallel Processing
+    I->>I: Document Parsing
+    I->>P: Extracted Text + Metadata
+    
+    P->>P: Adaptive Chunking
+    P->>P: Embedding Generation
+    P->>S: Store Vectors + Indexes
+    
+    S->>F: Processing Complete
+    
+    U->>F: Send Query
+    F->>A: POST /api/chat
+    
+    A->>R: Hybrid Retrieval
+    R->>S: Vector + BM25 Search
+    S->>R: Top-K Chunks
+    
+    R->>G: Context + Query
+    G->>G: LLM Generation
+    G->>F: Response + Citations
+    
+    G->>E: Auto-evaluation (async)
+    E->>E: Compute RAGAS Metrics
+    E->>S: Store Evaluation Results
+    E->>F: Return Metrics
+```
+
+### 3.2 Real-time Query Processing
+
+```mermaid
+flowchart TD
+    A[User Query] --> B[Query Understanding]
+    B --> C[Check Cache]
+    
+    C --> D{Cache Hit?}
+    D -->|Yes| E[Return Cached Embedding]
+    D -->|No| F[Generate Embedding]
+    
+    F --> G[Store in Cache]
+    E --> H[FAISS Vector Search]
+    G --> H
+    
+    B --> I[Keyword Extraction]
+    I --> J[BM25 Keyword Search]
+    
+    H --> K[Reciprocal Rank Fusion]
+    J --> K
+    
+    K --> L[Top-20 Candidates]
+    L --> M{Reranking Enabled?}
+    
+    M -->|Yes| N[Cross-Encoder Reranking]
+    M -->|No| O[Select Top-5]
+    
+    N --> O
+    O --> P[Context Assembly]
+    P --> Q[LLM Prompt Construction]
+    Q --> R[Ollama Generation]
+    R --> S[Citation Formatting]
+    S --> T[Response Streaming]
+    T --> U[User Display]
+    
+    R --> V[Async RAGAS Evaluation]
+    V --> W[Compute Metrics]
+    W --> X[Store Results]
+    X --> Y[Update Dashboard]
+```
+
+---
+
+## 4. Infrastructure & Deployment
+
+### 4.1 Container Architecture
+
+```mermaid
+graph TB
+    subgraph "Docker Compose Stack"
+        A[Frontend Container<br/>nginx:alpine]
+        B[Backend Container<br/>python:3.11]
+        C[Ollama Container<br/>ollama/ollama]
+    end
+    
+    subgraph "External Services"
+        D[FAISS Indices<br/>Persistent Volume]
+        E[SQLite Database<br/>Persistent Volume]
+        F[Log Files<br/>Persistent Volume]
+    end
+    
+    A --> B
+    B --> C
+    B --> D
+    B --> E
+    B --> F
+```
+
+### 4.2 Resource Requirements
+
+#### 4.2.1 Minimum Deployment
+
+| Resource | Specification | Purpose |
+|----------|---------------|---------|
+| **CPU** | 4 cores | Document processing, embeddings |
+| **RAM** | 8GB | Model loading, FAISS indices, cache |
+| **Storage** | 20GB | Models, indices, documents |
+| **GPU** | Optional | 2-3x speedup for inference |
+
+#### 4.2.2 Production Deployment
+
+| Resource | Specification | Purpose |
+|----------|---------------|---------|
+| **CPU** | 8+ cores | Concurrent processing |
+| **RAM** | 16GB+ | Larger datasets, caching |
+| **GPU** | RTX 3090/4090 | 20-30 tokens/sec inference |
+| **Storage** | 100GB+ SSD | Fast vector search |
+
+---
+
+## 5. API Architecture
+
+### 5.1 REST API Endpoints
+
+```mermaid
+graph TB
+    subgraph "System Management"
+        A[GET /api/health]
+        B[GET /api/system-info]
+        C[GET /api/configuration]
+        D[POST /api/configuration]
+    end
+    
+    subgraph "Document Management"
+        E[POST /api/upload]
+        F[POST /api/start-processing]
+        G[GET /api/processing-status]
+    end
+    
+    subgraph "Query & Chat"
+        H[POST /api/chat]
+        I[GET /api/export-chat/:session_id]
+    end
+    
+    subgraph "RAGAS Evaluation"
+        J[GET /api/ragas/history]
+        K[GET /api/ragas/statistics]
+        L[POST /api/ragas/clear]
+        M[GET /api/ragas/export]
+        N[GET /api/ragas/config]
+    end
+    
+    subgraph "Analytics"
+        O[GET /api/analytics]
+        P[GET /api/analytics/refresh]
+        Q[GET /api/analytics/detailed]
+    end
+```
+
+### 5.2 Request/Response Flow
+
+```python
+# Typical Chat Request Flow with RAGAS
+REQUEST_FLOW = {
+    "authentication": "None (local deployment)",
+    "rate_limiting": "100 requests/minute per IP",
+    "validation": "Query length, session ID format",
+    "processing": "Async with progress tracking",
+    "response": "JSON with citations + metrics + RAGAS scores",
+    "caching": "LRU cache for embeddings",
+    "evaluation": "Automatic RAGAS metrics (async)"
+}
+```
+
+---
+
+## 6. Monitoring & Quality Assurance
+
+### 6.1 RAGAS Integration
+
+```mermaid
+graph LR
+    A[API Gateway] --> B[Query Processing]
+    C[Retrieval Module] --> B
+    D[Generation Module] --> B
+    
+    B --> E[RAGAS Evaluator]
+    
+    E --> F[Analytics Dashboard]
+    
+    F --> G[Answer Relevancy]
+    F --> H[Faithfulness]
+    F --> I[Context Utilization]
+    F --> J[Context Relevancy]
+    F --> K[Session Statistics]
+```
+
+### 6.2 Key Performance Indicators
+
+| Category | Metric | Target | Alert Threshold |
+|----------|--------|--------|-----------------|
+| **Performance** | Query Latency (p95) | < 5s | > 10s |
+| **Quality** | Answer Relevancy | > 0.85 | < 0.70 |
+| **Quality** | Faithfulness | > 0.90 | < 0.80 |
+| **Quality** | Context Utilization | > 0.80 | < 0.65 |
+| **Quality** | Overall Score | > 0.85 | < 0.70 |
+| **Reliability** | Uptime | > 99.5% | < 95% |
+
+### 6.3 Analytics Dashboard Features
+
+**Real-Time Metrics:**
+- RAGAS evaluation table with all query-response pairs
+- Session-level aggregate statistics
+- Performance metrics (latency, throughput)
+- Component health status
+
+**Historical Analysis:**
+- Quality trend over time
+- Performance degradation detection
+- Cache hit rate monitoring
+- Resource utilization tracking
+
+**Export Capabilities:**
+- JSON export of all evaluation data
+- CSV export for external analysis
+- Session-based filtering
+- Time-range queries
+
+---
+
+## 7. Technology Stack Details
+
+### Complete Technology Matrix
+
+| Layer | Component | Technology | Version | Purpose |
+|-------|-----------|------------|---------|----------|
+| **Frontend** | UI Framework | HTML5/CSS3/JS | - | Responsive interface |
+| **Frontend** | Styling | Tailwind CSS | 3.3+ | Utility-first CSS |
+| **Frontend** | Icons | Font Awesome | 6.0+ | Icon library |
+| **Backend** | API Framework | FastAPI | 0.104+ | Async REST API |
+| **Backend** | Python Version | Python | 3.11+ | Runtime |
+| **AI/ML** | LLM Engine | Ollama | 0.1.20+ | Local LLM inference |
+| **AI/ML** | Primary Model | Mistral-7B-Instruct | v0.2 | Text generation |
+| **AI/ML** | Embeddings | sentence-transformers | 2.2.2+ | Vector embeddings |
+| **AI/ML** | Embedding Model | BAAI/bge-small-en | v1.5 | Semantic search |
+| **Vector DB** | Storage | FAISS | 1.7.4+ | Vector similarity |
+| **Search** | Keyword | rank-bm25 | 0.2.1 | BM25 implementation |
+| **Evaluation** | Quality | Ragas | 0.1.9 | RAG evaluation |
+| **Document** | PDF | PyPDF2 | 3.0+ | PDF text extraction |
+| **Document** | Word | python-docx | 1.1+ | DOCX processing |
+| **OCR** | Text Recognition | EasyOCR | 1.7+ | Scanned documents |
+| **Database** | Metadata | SQLite | 3.35+ | Local storage |
+| **Cache** | In-memory | Python functools | - | LRU caching |
+| **Deployment** | Container | Docker | 24.0+ | Containerization |
+| **Deployment** | Orchestration | Docker Compose | 2.20+ | Multi-container |
+
+---
+
+## 8. Key Architectural Decisions
+
+### 8.1 Why Local Caching Instead of Redis?
+
+**Decision:** Use in-memory LRU cache with Python's `functools.lru_cache`
+
+**Rationale:**
+- **Simplicity**: No external service to manage
+- **Performance**: Faster access (no network overhead)
+- **MVP Focus**: Adequate for initial deployment
+- **Resource Efficient**: No additional memory footprint
+- **Easy Migration**: Can upgrade to Redis later if needed
+
+**Trade-offs:**
+- Cache doesn't persist across restarts
+- Can't share cache across multiple instances
+- Limited by single-process memory
+
+### 8.2 Why RAGAS for Evaluation?
+
+**Decision:** Integrate RAGAS for real-time quality assessment
+
+**Rationale:**
+- **Automated Metrics**: No manual annotation required
+- **Production-Ready**: Quantifiable quality scores
+- **Real-Time**: Evaluate every query-response pair
+- **Comprehensive**: Multiple dimensions of quality
+- **Research-Backed**: Based on academic research
+
+**Implementation Details:**
+- OpenAI API key required for LLM-based metrics
+- Async evaluation to avoid blocking responses
+- SQLite storage for historical analysis
+- Export capability for offline processing
+
+### 8.3 Why No Web Scraping?
+
+**Decision:** Removed web scraping from MVP
+
+**Rationale:**
+- **Complexity**: Anti-scraping mechanisms require maintenance
+- **Reliability**: Website changes break scrapers
+- **Legal**: Potential legal/ethical issues
+- **Scope**: Focus on core RAG functionality first
+
+**Alternative:**
+- Users can save web pages as PDFs
+- Future enhancement if market demands it
+
+---
+
+## 9. Performance Optimization Strategies
+
+### 9.1 Embedding Cache Strategy
+
+```python
+# Cache Implementation
+from functools import lru_cache
+
+@lru_cache(maxsize=1000)
+def get_query_embedding(query: str) -> np.ndarray:
+    """Cache query embeddings for repeat queries"""
+    return embedder.embed(query)
+
+# Benefits:
+# - 80% reduction in latency for repeat queries
+# - No re-computation of identical queries
+# - Automatic LRU eviction
+```
+
+### 9.2 Batch Processing
+
+```python
+# Batch Embedding Generation
+BATCH_SIZE = 32
+
+def embed_chunks_batch(chunks: List[str]) -> List[np.ndarray]:
+    embeddings = []
+    for i in range(0, len(chunks), BATCH_SIZE):
+        batch = chunks[i:i+BATCH_SIZE]
+        batch_embeddings = embedder.embed_batch(batch)
+        embeddings.extend(batch_embeddings)
+    return embeddings
+```
+
+### 9.3 Async Processing
+
+```python
+# Async Document Processing
+import asyncio
+
+async def process_documents_async(documents: List[Path]):
+    tasks = [process_single_document(doc) for doc in documents]
+    results = await asyncio.gather(*tasks)
+    return results
+```
+
+---
+
+## 10. Security Considerations
+
+### 10.1 Data Privacy
+
+- **On-Premise Processing**: All data stays local
+- **No External APIs**: Except OpenAI for RAGAS (configurable)
+- **Local LLM**: Ollama runs entirely on-premise
+- **Encrypted Storage**: Optional SQLite encryption
+
+### 10.2 Input Validation
+
+```python
+# File Upload Validation
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100MB
+ALLOWED_EXTENSIONS = {'.pdf', '.docx', '.txt', '.zip'}
+
+def validate_upload(file: UploadFile):
+    # Check extension
+    if Path(file.filename).suffix not in ALLOWED_EXTENSIONS:
+        raise ValueError("Unsupported file type")
+    
+    # Check size
+    if file.size > MAX_FILE_SIZE:
+        raise ValueError("File too large")
+    
+    # Scan for malicious content (optional)
+    # scan_for_malware(file)
+```
+
+### 10.3 Rate Limiting
+
+```python
+# Simple rate limiting
+from fastapi import Request
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+rate_limits = defaultdict(list)
+
+def check_rate_limit(request: Request, limit: int = 100):
+    ip = request.client.host
+    now = datetime.now()
+    
+    # Clean old requests
+    rate_limits[ip] = [
+        ts for ts in rate_limits[ip] 
+        if now - ts < timedelta(minutes=1)
+    ]
+    
+    # Check limit
+    if len(rate_limits[ip]) >= limit:
+        raise HTTPException(429, "Rate limit exceeded")
+    
+    rate_limits[ip].append(now)
+```
+
+---
+
+## Conclusion
+
+This architecture document provides a comprehensive technical blueprint for the AI Universal Knowledge Ingestion System. The modular design, clear separation of concerns, and production-ready considerations make this system suitable for enterprise deployment while maintaining flexibility for future enhancements.
+
+### Key Architectural Strengths
+
+1. **Modularity**: Each component is independent and replaceable
+2. **Scalability**: Horizontal scaling through stateless API design
+3. **Performance**: Intelligent caching and batch processing
+4. **Quality**: Real-time RAGAS evaluation for continuous monitoring
+5. **Privacy**: Complete on-premise processing with local LLM
+6. **Simplicity**: Minimal external dependencies (no Redis, no web scraping)
+
+### Future Enhancements
+
+**Short-term:**
+- Redis cache for multi-instance deployments
+- Advanced monitoring dashboard
+- User authentication and authorization
+- API rate limiting enhancements
+
+**Long-term:**
+- Distributed processing with Celery
+- Web scraping module (optional)
+- Fine-tuned domain-specific embeddings
+- Multi-tenant support
+- Advanced analytics and reporting
+
+---
+
+Document Version: 1.0
+Last Updated: November 2025
+Author: Satyaki Mitra
+
+---
+
+> This document is part of the AI Universal Knowledge Ingestion System technical documentation suite.

document_parser/docx_parser.py

@@ -0,0 +1,425 @@
+# DEPENDENCIES
+import docx
+import hashlib
+from typing import List
+from pathlib import Path
+from typing import Optional
+from docx.table import Table
+from datetime import datetime
+from docx.document import Document
+from config.models import DocumentType
+from docx.text.paragraph import Paragraph
+from utils.text_cleaner import TextCleaner
+from config.models import DocumentMetadata
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import DOCXParseError
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class DOCXParser:
+    """
+    Comprehensive DOCX parsing with structure preservation: Handles paragraphs, tables, headers, and footers
+    """
+    def __init__(self):
+        self.logger = logger
+    
+
+    @handle_errors(error_type = DOCXParseError, log_error = True, reraise = True)
+    def parse(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, include_tables: bool = True, include_headers_footers: bool = False) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse DOCX and extract text and metadata
+        
+        Arguments:
+        ----------
+            file_path               { Path } : Path to DOCX file
+            
+            extract_metadata        { bool } : Extract document metadata
+            
+            clean_text              { bool } : Clean extracted text
+            
+            include_tables          { bool } : Include table content
+            
+            include_headers_footers { bool } : Include headers and footers
+        
+        Returns:
+        --------
+                      { tuple }              : Tuple of (extracted_text, metadata)
+        
+        Raises:
+        -------
+            DOCXParseError                   : If parsing fails
+        """
+        file_path = Path(file_path)
+        
+        if not file_path.exists():
+            raise DOCXParseError(str(file_path), original_error = FileNotFoundError(f"DOCX file not found: {file_path}"))
+        
+        self.logger.info(f"Parsing DOCX: {file_path}")
+        
+        try:
+            # Open document
+            doc            = docx.Document(file_path)
+            
+            # Extract text content
+            text_parts     = list()
+            
+            # Extract paragraphs
+            paragraph_text = self._extract_paragraphs(doc = doc)
+
+            text_parts.append(paragraph_text)
+            
+            # Extract tables
+            if include_tables:
+                table_text = self._extract_tables(doc)
+                
+                if table_text:
+                    text_parts.append("\n[TABLES]\n" + table_text)
+            
+            # Extract headers and footers
+            if include_headers_footers:
+                header_footer_text = self._extract_headers_footers(doc)
+                
+                if header_footer_text:
+                    text_parts.append("\n[HEADERS/FOOTERS]\n" + header_footer_text)
+            
+            # Combine all text
+            text_content = "\n".join(text_parts)
+            
+            # Extract metadata
+            metadata     = None
+            
+            if extract_metadata:
+                metadata = self._extract_metadata(doc, file_path)
+            
+            # Clean text
+            if clean_text:
+                text_content = TextCleaner.clean(text_content,
+                                                 remove_html          = False, 
+                                                 normalize_whitespace = True,
+                                                 preserve_structure   = True,
+                                                )
+            
+            self.logger.info(f"Successfully parsed DOCX: {len(text_content)} characters, {len(doc.paragraphs)} paragraphs")
+            
+            return text_content, metadata
+        
+        except Exception as e:
+            self.logger.error(f"Failed to parse DOCX {file_path}: {str(e)}")
+
+            raise DOCXParseError(str(file_path), original_error = e)
+    
+
+    def _extract_paragraphs(self, doc: Document) -> str:
+        """
+        Extract text from paragraphs, preserving structure
+        
+        Arguments:
+        ----------
+            doc { Document } : Document object
+        
+        Returns:
+        --------
+              { str }        : Combined paragraph text
+        """
+        text_parts = list()
+        
+        for i, para in enumerate(doc.paragraphs):
+            text = para.text.strip()
+            
+            if not text:
+                continue
+            
+            # Detect headings
+            if para.style.name.startswith('Heading'):
+                heading_level = para.style.name.replace('Heading', '').strip()
+                text_parts.append(f"\n[HEADING {heading_level}] {text}\n")
+            
+            else:
+                text_parts.append(text)
+        
+        return "\n".join(text_parts)
+    
+
+    def _extract_tables(self, doc: Document) -> str:
+        """
+        Extract text from tables
+        
+        Arguments:
+        ----------
+            doc { Document } : Document object
+        
+        Returns:
+        --------
+               { str }       : Combined table text
+        """
+        if not doc.tables:
+            return ""
+        
+        table_parts = list()
+        
+        for table_idx, table in enumerate(doc.tables):
+            table_text = self._parse_table(table)
+            
+            if table_text:
+                table_parts.append(f"\n[TABLE {table_idx + 1}]\n{table_text}")
+        
+        return "\n".join(table_parts)
+    
+
+    def _parse_table(self, table: Table) -> str:
+        """
+        Parse a single table into text
+        
+        Arguments:
+        ----------
+            table { Table } : Table object
+        
+        Returns:
+        --------
+              { str }       : Table text
+        """
+        rows_text = list()
+        
+        for row in table.rows:
+            cells_text = list()
+            for cell in row.cells:
+                cell_text = cell.text.strip()
+
+                cells_text.append(cell_text)
+            
+            # Join cells with pipe separator for readability
+            rows_text.append(" | ".join(cells_text))
+        
+        return "\n".join(rows_text)
+    
+
+    def _extract_headers_footers(self, doc: Document) -> str:
+        """
+        Extract headers and footers
+        
+        Arguments:
+        ----------
+            doc { Document } : Document object
+        
+        Returns:
+        --------
+               { str }       : Headers and footers text
+        """
+        parts = list()
+        
+        # Extract from each section
+        for section in doc.sections:
+            # Header
+            if section.header:
+                header_text = self._extract_paragraphs_from_element(element = section.header)
+                
+                if header_text:
+                    parts.append(f"[HEADER]\n{header_text}")
+            
+            # Footer
+            if section.footer:
+                footer_text = self._extract_paragraphs_from_element(element = section.footer)
+                
+                if footer_text:
+                    parts.append(f"[FOOTER]\n{footer_text}")
+        
+        return "\n".join(parts)
+    
+
+    @staticmethod
+    def _extract_paragraphs_from_element(element) -> str:
+        """
+        Extract paragraphs from header/footer element
+        """
+        parts = list()
+
+        for para in element.paragraphs:
+            text = para.text.strip()
+            if text:
+                parts.append(text)
+
+        return "\n".join(parts)
+    
+
+    def _extract_metadata(self, doc: Document, file_path: Path) -> DocumentMetadata:
+        """
+        Extract metadata from DOCX
+        
+        Arguments:
+        ----------
+            doc       { Document} : Document object
+
+            file_path   { Path }  : Path to DOCX file
+        
+        Returns:
+        --------
+            { DocumentMetadata }  : DocumentMetadata object
+        """
+        # Get core properties
+        core_props      = doc.core_properties
+        
+        # Extract fields
+        title           = core_props.title or file_path.stem
+        author          = core_props.author
+        created_date    = core_props.created
+        modified_date   = core_props.modified
+        
+        # Get file size
+        file_size       = file_path.stat().st_size
+        
+        # Generate document ID
+        doc_hash        = hashlib.md5(str(file_path).encode()).hexdigest()
+        doc_id          = f"doc_{int(datetime.now().timestamp())}_{doc_hash}"
+        
+        # Count paragraphs and estimate pages
+        num_paragraphs  = len(doc.paragraphs)
+        
+        # Rough estimate: 500 words per page, 5-10 words per paragraph
+        estimated_pages = max(1, num_paragraphs // 50)
+        
+        # Create metadata object
+        metadata        = DocumentMetadata(document_id     = doc_id,
+                                           filename        = file_path.name,
+                                           file_path       = file_path,
+                                           document_type   = DocumentType.DOCX,
+                                           title           = title,
+                                           author          = author,
+                                           created_date    = created_date,
+                                           modified_date   = modified_date,
+                                           file_size_bytes = file_size,
+                                           num_pages       = estimated_pages,
+                                           extra           = {"num_paragraphs" : num_paragraphs,
+                                                              "num_tables"     : len(doc.tables),
+                                                              "num_sections"   : len(doc.sections),
+                                                              "category"       : core_props.category,
+                                                              "comments"       : core_props.comments,
+                                                              "keywords"       : core_props.keywords,
+                                                              "subject"        : core_props.subject,
+                                                             }
+                                          )
+        
+        return metadata
+    
+
+    def get_paragraph_count(self, file_path: Path) -> int:
+        """
+        Get number of paragraphs in document
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to DOCX file
+        
+        Returns:
+        --------
+                { int }        : Number of paragraphs
+        """
+        try:
+            doc = docx.Document(file_path)
+            return len(doc.paragraphs)
+
+        except Exception as e:
+            self.logger.error(f"Failed to get paragraph count: {repr(e)}")
+            raise DOCXParseError(str(file_path), original_error = e)
+    
+
+    def extract_section(self, file_path: Path, section_index: int, clean_text: bool = True) -> str:
+        """
+        Extract text from a specific section
+        
+        Arguments:
+        ----------
+            file_path     { Path } : Path to DOCX file
+            
+            section_index { int }  : Section index (0-indexed)
+            
+            clean_text    { bool } : Clean extracted text
+        
+        Returns:
+        --------
+                    { str }        : Section text
+        """
+        try:
+            doc = docx.Document(file_path)
+            
+            if ((section_index < 0) or (section_index >= len(doc.sections))):
+                raise ValueError(f"Section index {section_index} out of range (0-{len(doc.sections)-1})")
+            
+            # Note: Extracting text by section is not straightforward in python-docx
+            section = doc.sections[section_index]
+            
+            # For now, we'll extract the entire document
+            text    = "\n".join([para.text for para in doc.paragraphs if para.text.strip()])
+            
+            if clean_text:
+                text = TextCleaner.clean(text)
+            
+            return text
+        
+        except Exception as e:
+            self.logger.error(f"Failed to extract section: {repr(e)}")
+            raise DOCXParseError(str(file_path), original_error = e)
+    
+
+    def extract_heading_sections(self, file_path: Path, clean_text: bool = True) -> dict[str, str]:
+        """
+        Extract text organized by headings
+        
+        Arguments:
+        ----------
+            file_path  { Path } : Path to DOCX file
+
+            clean_text { bool } : Clean extracted text
+        
+        Returns:
+        --------
+                { dict }        : Dictionary mapping heading text to content
+        """
+        try:
+            doc             = docx.Document(file_path)
+            sections        = dict()
+            current_content = list()
+            current_heading = "Introduction"
+            
+            
+            for para in doc.paragraphs:
+                text = para.text.strip()
+                
+                if not text:
+                    continue
+                
+                # Check if it's a heading
+                if para.style.name.startswith('Heading'):
+                    # Save previous section
+                    if current_content:
+                        section_text = "\n".join(current_content)
+                        
+                        if clean_text:
+                            section_text = TextCleaner.clean(section_text)
+                        
+                        sections[current_heading] = section_text
+                    
+                    # Start new section
+                    current_heading = text
+                    current_content = list()
+
+                else:
+                    current_content.append(text)
+            
+            # Save last section
+            if current_content:
+                section_text = "\n".join(current_content)
+                
+                if clean_text:
+                    section_text = TextCleaner.clean(section_text)
+                
+                sections[current_heading] = section_text
+            
+            return sections
+        
+        except Exception as e:
+            self.logger.error(f"Failed to extract heading sections: {repr(e)}")
+            raise DOCXParseError(str(file_path), original_error = e)

document_parser/ocr_engine.py

@@ -0,0 +1,781 @@
+# DEPENDENCIES
+import os
+import cv2
+import fitz
+import easyocr
+import numpy as np
+from PIL import Image
+from io import BytesIO
+from typing import List
+from typing import Dict
+from typing import Tuple
+from pathlib import Path
+from PIL import ImageFilter
+from typing import Optional
+from PIL import ImageEnhance
+from paddleocr import PaddleOCR
+from config.settings import get_settings
+from utils.error_handler import OCRException
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class OCREngine:
+    """
+    OCR engine with layout preservation - maintains document structure and formatting
+    """
+    def __init__(self, use_paddle: bool = True, lang: str = 'en', gpu: bool = False):
+        """
+        Initialize OCR engine
+        
+        Arguments:
+        ----------
+            use_paddle  { bool } : Use PaddleOCR as primary (better accuracy)
+
+            lang        { str }  : Language code ('en', 'es', 'fr', 'de', etc.)
+            
+            gpu         { bool } : Use GPU acceleration if available
+        """
+        self.logger       = logger
+        self.use_paddle   = use_paddle
+        self.lang         = lang
+        self.gpu          = gpu
+        self.paddle_ocr   = None
+        self.easy_ocr     = None
+        self._initialized = False
+        
+        self._initialize_engines()
+    
+
+    def _initialize_engines(self):
+        """
+        Initialize OCR engines with proper error handling
+        """
+        if self.use_paddle:
+            try:
+                self.paddle_ocr = PaddleOCR(use_angle_cls     = True,
+                                            lang              = self.lang,
+                                            use_gpu           = self.gpu,
+                                            show_log          = False,
+                                            det_db_thresh     = 0.3,
+                                            det_db_box_thresh = 0.5,
+                                           )
+
+                self.logger.info("PaddleOCR initialized successfully")
+
+            except Exception as e:
+                self.logger.warning(f"PaddleOCR not available: {repr(e)}. Falling back to EasyOCR.")
+                self.use_paddle = False
+        
+        if not self.use_paddle:
+            try: 
+                self.easy_ocr = easyocr.Reader([self.lang], gpu = self.gpu)
+                self.logger.info("EasyOCR initialized successfully")
+
+            except Exception as e:
+                self.logger.error(f"Failed to initialize EasyOCR: {repr(e)}")
+                raise OCRException(f"OCR engine initialization failed: {repr(e)}")
+        
+        self._initialized = True
+    
+
+    @handle_errors(error_type=OCRException, log_error=True, reraise=True)
+    def extract_text_from_pdf(self, pdf_path: Path, pages: Optional[List[int]] = None, preserve_layout: bool = True) -> str:
+        """
+        Extract text from PDF using OCR with layout preservation
+        
+        Arguments:
+        ----------
+            pdf_path        { Path } : Path to PDF file
+
+            pages           { list } : Specific pages to OCR (None = all pages)
+            
+            preserve_layout { bool } : Preserve document layout and structure
+            
+        Returns:
+        --------
+               { str }               : Extracted text with preserved formatting
+        """
+        pdf_path = Path(pdf_path)
+
+        self.logger.info(f"Starting OCR extraction from PDF: {pdf_path}")
+        
+        if not pdf_path.exists():
+            raise OCRException(f"PDF file not found: {pdf_path}")
+        
+        # Convert PDF pages to high-quality images
+        images = self._pdf_to_images(pdf_path = pdf_path, 
+                                     pages    = pages, 
+                                     dpi      = 300,
+                                    )
+
+        self.logger.info(f"Converted {len(images)} pages to images for OCR")
+        
+        # OCR each image with layout preservation
+        all_text = list()
+
+        for i, image in enumerate(images):
+            page_num = pages[i] if pages else i + 1
+
+            self.logger.info(f"Processing page {page_num}...")
+            
+            try:
+                if preserve_layout:
+                    # Extract text with layout information
+                    page_text = self._extract_text_with_layout(image    = image, 
+                                                               page_num = page_num,
+                                                              )
+                
+                else:
+                    # Simple extraction without layout
+                    img_array = np.array(image)
+                    page_text = self._ocr_image(img_array)
+
+                if page_text and page_text.strip():
+                    all_text.append(f"[PAGE {page_num}]\n{page_text}")
+                    self.logger.info(f"✓ Extracted {len(page_text)} characters from page {page_num}")
+                
+                else:
+                    self.logger.warning(f"No text extracted from page {page_num}")
+            
+            except Exception as e:
+                self.logger.error(f"OCR failed for page {page_num}: {repr(e)}")
+                all_text.append(f"[PAGE {page_num}]\n[OCR FAILED: {str(e)}]")
+        
+        combined_text = "\n\n".join(all_text)
+        self.logger.info(f"OCR completed: {len(combined_text)} total characters extracted")
+        
+        return combined_text
+    
+
+    def _extract_text_with_layout(self, image: Image.Image, page_num: int) -> str:
+        """
+        Extract text while preserving document layout and structure
+        
+        Arguments:
+        ----------
+            image    { Image.Image } : PIL Image
+            
+            page_num { int }         : Page number
+            
+        Returns:
+        --------
+                 { str }             : Formatted text with layout preserved
+        """
+        img_array = np.array(image)
+        
+        # Get OCR results with bounding boxes
+        if (self.use_paddle and self.paddle_ocr):
+            text_blocks = self._ocr_with_layout_paddle(image_array = img_array)
+        
+        elif self.easy_ocr:
+            text_blocks = self._ocr_with_layout_easyocr(image_array = img_array)
+        
+        else:
+            return ""
+        
+        if not text_blocks:
+            return ""
+        
+        # Organize text blocks into reading order with layout preservation
+        formatted_text = self._reconstruct_layout(text_blocks = text_blocks, 
+                                                  image_size  = image.size,
+                                                 )
+        
+        return formatted_text
+    
+
+    def _ocr_with_layout_paddle(self, image_array: np.ndarray) -> List[Dict]:
+        """
+        OCR using PaddleOCR and return structured text blocks with positions
+        
+        Returns:
+        --------
+            { list } : {'text': str, 'bbox': [...], 'confidence': float}
+        """
+        try:
+            result = self.paddle_ocr.ocr(image_array, cls=True)
+            
+            if not result or not result[0]:
+                return []
+            
+            text_blocks = list()
+            
+            for line in result[0]:
+                if (line and (len(line) >= 2)):
+                    bbox      = line[0]  # [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
+                    text_info = line[1]
+                    
+                    if (isinstance(text_info, (list, tuple)) and (len(text_info) >= 2)):
+                        text       = text_info[0]
+                        confidence = text_info[1]
+
+                    elif isinstance(text_info, str):
+                        text       = text_info
+                        confidence = 1.0
+
+                    else:
+                        continue
+                    
+                    if ((confidence > 0.5) and text and text.strip()):
+                        # Calculate bounding box coordinates
+                        x_coords = [point[0] for point in bbox]
+                        y_coords = [point[1] for point in bbox]
+                        
+                        text_blocks.append({'text'       : text.strip(),
+                                            'bbox'       : {'x1': min(x_coords),
+                                                            'y1': min(y_coords),
+                                                            'x2': max(x_coords),
+                                                            'y2': max(y_coords)
+                                                           },
+                                            'confidence' : confidence,
+                                            'center_y'   : (min(y_coords) + max(y_coords)) / 2,
+                                            'center_x'   : (min(x_coords) + max(x_coords)) / 2,
+                                          })
+            
+            return text_blocks
+            
+        except Exception as e:
+            self.logger.error(f"PaddleOCR layout extraction failed: {repr(e)}")
+            return []
+    
+
+    def _ocr_with_layout_easyocr(self, image_array: np.ndarray) -> List[Dict]:
+        """
+        OCR using EasyOCR and return structured text blocks with positions
+        """
+        try:
+            result = self.easy_ocr.readtext(image_array, paragraph=False)
+            
+            if not result:
+                return []
+            
+            text_blocks = list()
+            
+            for detection in result:
+                bbox       = detection[0]  # [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
+                text       = detection[1]
+                confidence = detection[2]
+                
+                if ((confidence > 0.5) and text and text.strip()):
+                    x_coords = [point[0] for point in bbox]
+                    y_coords = [point[1] for point in bbox]
+                    
+                    text_blocks.append({'text'       : text.strip(),
+                                        'bbox'       : {'x1' : min(x_coords),
+                                                        'y1' : min(y_coords),
+                                                        'x2' : max(x_coords),
+                                                        'y2' : max(y_coords),
+                                                       },
+                                        'confidence' : confidence,
+                                        'center_y'   : (min(y_coords) + max(y_coords)) / 2,
+                                        'center_x'   : (min(x_coords) + max(x_coords)) / 2,
+                                      })
+            
+            return text_blocks
+            
+        except Exception as e:
+            self.logger.error(f"EasyOCR layout extraction failed: {repr(e)}")
+            return []
+    
+
+    def _reconstruct_layout(self, text_blocks: List[Dict], image_size: Tuple[int, int]) -> str:
+        """
+        Reconstruct document layout from text blocks
+        
+        Strategy:
+        1. Group text blocks into lines (similar Y coordinates)
+        2. Detect columns, tables, lists
+        3. Sort lines top to bottom
+        4. Within each line, sort left to right
+        5. Detect paragraphs, headings, and lists
+        6. Add appropriate spacing and formatting
+        """
+        if not text_blocks:
+            return ""
+        
+        # Sort all blocks by Y position first
+        sorted_blocks         = sorted(text_blocks, key = lambda x: (x['center_y'], x['center_x']))
+        
+        # Detect multi-column layout
+        columns               = self._detect_columns(text_blocks = text_blocks, 
+                                                     image_size  = image_size,
+                                                    )
+        
+        # Group into lines (blocks with similar Y coordinates)
+        lines                 = list()
+
+        current_line          = [sorted_blocks[0]]
+
+        # pixels
+        line_height_threshold = 25  
+        
+        for block in sorted_blocks[1:]:
+            # Check if this block is on the same line as the previous one
+            y_diff = abs(block['center_y'] - current_line[-1]['center_y'])
+            
+            if (y_diff < line_height_threshold):
+                current_line.append(block)
+
+            else:
+                # Sort current line by X position and add to lines
+                current_line.sort(key = lambda x: x['center_x'])
+                lines.append(current_line)
+                
+                current_line = [block]
+        
+        # Don't forget the last line
+        if current_line:
+            current_line.sort(key = lambda x: x['center_x'])
+            lines.append(current_line)
+        
+        # Reconstruct text with formatting
+        formatted_lines = list()
+        prev_y          = 0
+        prev_indent     = 0
+        
+        for i, line_blocks in enumerate(lines):
+            # Calculate line metrics
+            current_y        = line_blocks[0]['center_y']
+            vertical_gap     = current_y - prev_y if (prev_y > 0) else 0
+            
+            # Detect indentation (left margin)
+            line_left_margin = line_blocks[0]['bbox']['x1']
+            
+            # Combine text blocks in this line with proper spacing
+            line_text        = self._combine_line_blocks(line_blocks = line_blocks)
+            
+            # Clean the text
+            line_text        = self._clean_ocr_text(text = line_text)
+            
+            # Skip if empty after cleaning
+            if not line_text.strip():
+                continue
+            
+            # Skip likely page numbers or artifacts (single numbers, very short text)
+            if self._is_page_artifact(line_text):
+                continue
+            
+            # Add extra newline for paragraph breaks (large vertical gaps)
+            # Threshold for paragraph break
+            if (vertical_gap > 35):  
+                formatted_lines.append("")
+            
+            # Detect and format different line types
+            if (self._is_heading(line_text, line_blocks)):
+                # Heading - add extra spacing
+                formatted_lines.append(f"\n{line_text}")
+            
+            elif (self._is_bullet_point(line_text)):
+                # Bullet point or list item
+                formatted_lines.append(f"  {line_text}")
+            
+            elif (self._is_table_row(line_blocks)):
+                # Table row - preserve spacing between columns
+                formatted_lines.append(self._format_table_row(line_blocks))
+            
+            else:
+                # Regular paragraph text
+                formatted_lines.append(line_text)
+            
+            prev_y      = current_y
+            prev_indent = line_left_margin
+        
+        return "\n".join(formatted_lines)
+    
+
+    def _combine_line_blocks(self, line_blocks: List[Dict]) -> str:
+        """
+        Combine text blocks in a line with intelligent spacing
+        """
+        if (len(line_blocks) == 1):
+            return line_blocks[0]['text']
+        
+        result = list()
+
+        for i, block in enumerate(line_blocks):
+            result.append(block['text'])
+            
+            # Add space between blocks if they're not touching
+            if (i < len(line_blocks) - 1):
+                next_block = line_blocks[i + 1]
+                gap        = next_block['bbox']['x1'] - block['bbox']['x2']
+                
+                # If gap is significant, add spacing
+                if (gap > 20):  # Threshold for adding extra space
+                    # Double space for columns/tables
+                    result.append("  ")  
+
+                elif (gap > 5):
+                    # Normal space
+                    result.append(" ")  
+        
+        return "".join(result)
+    
+
+    def _clean_ocr_text(self, text: str) -> str:
+        """
+        Clean OCR artifacts and normalize text
+        """
+        # Replace common OCR errors
+        replacements = {'''      : "'",  # Smart quote to regular quote
+                        '''      : "'",
+                        '"'      : '"',
+                        '"'      : '"',
+                        '—'      : '-',
+                        '–'      : '-',
+                        '…'      : '...',
+                        '\u00a0' : ' ',  # Non-breaking space
+                       }
+        
+        for old, new in replacements.items():
+            text = text.replace(old, new)
+        
+        # Fix common OCR mistakes
+        text = text.replace('l ', 'I ')    # lowercase L to I at start of sentence
+        text = text.replace(' l ', ' I ')  # lowercase L to I
+        
+        # Remove extra spaces
+        text = ' '.join(text.split())
+        
+        return text
+    
+
+    def _is_page_artifact(self, text: str) -> bool:
+        """
+        Detect page numbers, headers, footers, and other artifacts
+        """
+        text = text.strip()
+        
+        # Empty or very short
+        if (len(text) < 2):
+            return True
+        
+        # Just a number (likely page number)
+        if (text.isdigit() and (len(text) <= 3)):
+            return True
+        
+        # Common footer patterns
+        footer_patterns = ['page', 'of', 'for informational purposes', 'confidential', 'draft', 'version']
+        text_lower      = text.lower()
+
+        if ((len(text) < 50) and (any(pattern in text_lower for pattern in footer_patterns))):
+            # This is actually useful - don't skip
+            return False
+        
+        # Very short isolated text (likely artifact)
+        if ((len(text) <= 3) and not text.isalnum()):
+            return True
+        
+        return False
+    
+
+    def _is_bullet_point(self, text: str) -> bool:
+        """
+        Detect if text is a bullet point or list item
+        """
+        text           = text.strip()
+        
+        # Check for common bullet markers
+        bullet_markers = ['•', '·', '-', '○', '◦', '*', '►', '▪']
+       
+        if (text and (text[0] in bullet_markers)):
+            return True
+        
+        # Check for numbered lists
+        if (len(text) > 2):
+            
+            # Pattern: "1. ", "a) ", "i. "
+            if (text[0].isdigit() and text[1] in '.):'):
+                return True
+
+            if (text[0].isalpha() and len(text) > 1 and text[1] in '.):'):
+                return True
+        
+        return False
+    
+
+    def _is_table_row(self, line_blocks: List[Dict]) -> bool:
+        """
+        Detect if a line is part of a table (multiple separated columns)
+        """
+        if (len(line_blocks) < 2):
+            return False
+        
+        # Calculate gaps between blocks
+        gaps = list()
+
+        for i in range(len(line_blocks) - 1):
+            gap = line_blocks[i + 1]['bbox']['x1'] - line_blocks[i]['bbox']['x2']
+            gaps.append(gap)
+        
+        # If there are significant gaps, likely a table
+        significant_gaps = sum(1 for gap in gaps if gap > 30)
+        
+        return (significant_gaps >= 1) and (len(line_blocks) >= 2)
+    
+
+    def _format_table_row(self, line_blocks: List[Dict]) -> str:
+        """
+        Format a table row with proper column alignment
+        """
+        cells = list()
+
+        for block in line_blocks:
+            cells.append(block['text'].strip())
+        
+        # Join with tab or multiple spaces for better readability
+        return ("  |  ".join(cells))
+    
+
+    def _detect_columns(self, text_blocks: List[Dict], image_size: Tuple[int, int]) -> List[Dict]:
+        """
+        Detect multi-column layout
+        """
+        # Group blocks by X position to detect columns
+        if not text_blocks:
+            return []
+        
+        # Return single column
+        return [{'x_start': 0, 'x_end': image_size[0]}]
+    
+
+    def _is_heading(self, text: str, blocks: List[Dict]) -> bool:
+        """
+        Detect if a line is likely a heading
+        
+        Heuristics:
+        - All uppercase or Title Case
+        - Shorter than typical paragraph lines
+        - Often centered or left-aligned
+        - Larger font (if detectable from bbox height)
+        """
+        words = text.split()
+        if not words:
+            return False
+        
+        # Skip very short text (likely artifacts)
+        if len(text) < 3:
+            return False
+        
+        # Check for common heading keywords
+        heading_keywords    = ['summary', 'introduction', 'conclusion', 'analysis', 'report', 'overview', 'chapter', 'section', 'terms', 'points', 'protections', 'category', 'breakdown', 'recommendation', 'clause']
+        text_lower          = text.lower()
+        has_heading_keyword = any(keyword in text_lower for keyword in heading_keywords)
+        
+        # All caps or mostly caps
+        caps_ratio          = sum(1 for w in words if w.isupper() and len(w) > 1) / len(words)
+        
+        # Title case (each word starts with capital)
+        title_case_ratio    = sum(1 for w in words if w and w[0].isupper()) / len(words)
+        
+        # Short lines might be headings
+        is_short            = len(text) < 100
+        
+        # Check if text is likely a heading
+        is_likely_heading   = ((caps_ratio > 0.7 and is_short) or                                # Mostly uppercase and short
+                               (title_case_ratio > 0.8 and is_short and has_heading_keyword) or  # Title case with keywords
+                               (has_heading_keyword and is_short and title_case_ratio > 0.5)     # Keywords + some capitals
+                              )
+        
+        # Check font size (larger bounding box height indicates heading)
+        if blocks:
+            avg_height = sum(b['bbox']['y2'] - b['bbox']['y1'] for b in blocks) / len(blocks)
+            
+            # Headings often have larger font (taller bbox)
+            if (avg_height > 25):  # Threshold for heading font size
+                is_likely_heading = is_likely_heading or (is_short and title_case_ratio > 0.5)
+        
+        return is_likely_heading
+    
+
+    def _pdf_to_images(self, pdf_path: Path, pages: Optional[List[int]] = None, dpi: int = 300) -> List[Image.Image]:
+        """
+        Convert PDF pages to high-quality images
+        """
+        try:
+            doc    = fitz.open(str(pdf_path))
+            images = list()
+            
+            if pages is None:
+                pages_to_process = range(len(doc))
+
+            else:
+                pages_to_process = [p-1 for p in pages if (0 < p <= len(doc))]
+            
+            for page_num in pages_to_process:
+                page     = doc[page_num]
+                
+                # High-quality conversion
+                zoom     = dpi / 72.0
+                mat      = fitz.Matrix(zoom, zoom)
+                pix      = page.get_pixmap(matrix = mat, alpha = False)
+                
+                # Convert to PIL Image
+                img_data = pix.tobytes("png")
+                image    = Image.open(BytesIO(img_data))
+                
+                if (image.mode != 'RGB'):
+                    image = image.convert('RGB')
+                
+                images.append(image)
+            
+            doc.close()
+            return images
+            
+        except Exception as e:
+            raise OCRException(f"Failed to convert PDF to images: {repr(e)}")
+    
+
+    def _ocr_image(self, image_array: np.ndarray) -> str:
+        """
+        Simple OCR without layout preservation
+        """
+        if self.use_paddle and self.paddle_ocr:
+            try:
+                result = self._ocr_with_paddle_simple(image_array)
+                
+                if result:
+                    return result
+            
+            except Exception as e:
+                self.logger.debug(f"PaddleOCR failed: {repr(e)}")
+        
+        if self.easy_ocr:
+            try:
+                result = self._ocr_with_easyocr_simple(image_array)
+                
+                if result:
+                    return result
+            
+            except Exception as e:
+                self.logger.debug(f"EasyOCR failed: {repr(e)}")
+        
+        return ""
+    
+
+    def _ocr_with_paddle_simple(self, image_array: np.ndarray) -> str:
+        """
+        Simple PaddleOCR extraction
+        """
+        result = self.paddle_ocr.ocr(image_array, cls=True)
+        
+        if not result or not result[0]:
+            return ""
+        
+        texts = list()
+
+        for line in result[0]:
+            if (line and (len(line) >= 2)):
+                text_info = line[1]
+                if isinstance(text_info, (list, tuple)):
+                    text, conf = text_info[0], text_info[1]
+                
+                else:
+                    text, conf = text_info, 1.0
+                
+                if ((conf > 0.5) and text):
+                    texts.append(text.strip())
+        
+        return "\n".join(texts)
+    
+
+    def _ocr_with_easyocr_simple(self, image_array: np.ndarray) -> str:
+        """
+        Simple EasyOCR extraction
+        """
+        result = self.easy_ocr.readtext(image_array)
+        
+        if not result:
+            return ""
+        
+        texts  = list()
+
+        for detection in result:
+            text, conf = detection[1], detection[2]
+            if ((conf > 0.5) and text):
+                texts.append(text.strip())
+        
+        return "\n".join(texts)
+    
+
+    @handle_errors(error_type = OCRException, log_error = True, reraise = True)
+    def extract_text_from_image(self, image_path: Path, preserve_layout: bool = True) -> str:
+        """
+        Extract text from image file
+        """
+        image_path = Path(image_path)
+
+        self.logger.info(f"Extracting text from image: {image_path}")
+        
+        if not image_path.exists():
+            raise OCRException(f"Image file not found: {image_path}")
+        
+        image = Image.open(image_path)
+
+        if (image.mode != 'RGB'):
+            image = image.convert('RGB')
+        
+        if preserve_layout:
+            text = self._extract_text_with_layout(image, page_num=1)
+        
+        else:
+            img_array = np.array(image)
+            text      = self._ocr_image(img_array)
+        
+        self.logger.info(f"Image OCR completed: {len(text)} characters extracted")
+        return text
+    
+
+    def get_supported_languages(self) -> List[str]:
+        """
+        Get list of supported languages
+        """
+        return ['en', 'es', 'fr', 'de', 'it', 'pt', 'ru', 'zh', 'ja', 'ko', 'ar']
+    
+
+    def get_engine_info(self) -> dict:
+        """
+        Get information about OCR engine configuration
+        """
+        return {"primary_engine"      : "PaddleOCR" if self.use_paddle else "EasyOCR",
+                "language"            : self.lang,
+                "gpu_enabled"         : self.gpu,
+                "initialized"         : self._initialized,
+                "layout_preservation" : True,
+                "supported_languages" : self.get_supported_languages(),
+               }
+
+
+# Global OCR instance
+_global_ocr_engine = None
+
+
+def get_ocr_engine() -> OCREngine:
+    """
+    Get global OCR engine instance (singleton)
+    """
+    global _global_ocr_engine
+    
+    if _global_ocr_engine is None:
+        _global_ocr_engine = OCREngine()
+    
+    return _global_ocr_engine
+
+
+def extract_text_with_ocr(file_path: Path, preserve_layout: bool = True, **kwargs) -> str:
+    """
+    Convenience function for OCR text extraction with layout preservation
+    """
+    ocr_engine = get_ocr_engine()
+    
+    if (file_path.suffix.lower() == '.pdf'):
+        return ocr_engine.extract_text_from_pdf(file_path, preserve_layout=preserve_layout, **kwargs)
+
+    else:
+        return ocr_engine.extract_text_from_image(file_path, preserve_layout=preserve_layout, **kwargs)

document_parser/parser_factory.py

@@ -0,0 +1,534 @@
+# DEPENDENCIES
+from enum import Enum
+from typing import List
+from pathlib import Path
+from typing import Union
+from typing import Optional 
+from utils.helpers import IDGenerator
+from config.models import DocumentType
+from config.models import DocumentMetadata
+from utils.file_handler import FileHandler
+from utils.error_handler import RAGException
+from config.logging_config import get_logger
+from document_parser.pdf_parser import PDFParser
+from document_parser.txt_parser import TXTParser
+from document_parser.ocr_engine import OCREngine
+from document_parser.docx_parser import DOCXParser
+from utils.error_handler import InvalidFileTypeError
+from document_parser.zip_handler import ArchiveHandler
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class ParserFactory:
+    """
+    Factory class for creating appropriate document parsers: implements Factory pattern for extensible parser selection
+    """
+    def __init__(self):
+        self.logger             = logger
+        
+        # Initialize parsers (reusable instances)
+        self._parsers           = {DocumentType.PDF  : PDFParser(), 
+                                   DocumentType.DOCX : DOCXParser(), 
+                                   DocumentType.TXT  : TXTParser(),
+                                  }
+        
+        # Initialize helper components
+        self._ocr_engine        = None
+        self._archive_handler   = None
+        
+        # File extension to DocumentType mapping
+        self._extension_mapping = {'pdf'   : DocumentType.PDF,
+                                   'docx'  : DocumentType.DOCX,
+                                   'doc'   : DocumentType.DOCX,
+                                   'txt'   : DocumentType.TXT,
+                                   'text'  : DocumentType.TXT,
+                                   'md'    : DocumentType.TXT, 
+                                   'log'   : DocumentType.TXT,
+                                   'csv'   : DocumentType.TXT,
+                                   'json'  : DocumentType.TXT,
+                                   'xml'   : DocumentType.TXT,
+                                   'png'   : DocumentType.IMAGE, 
+                                   'jpg'   : DocumentType.IMAGE, 
+                                   'jpeg'  : DocumentType.IMAGE, 
+                                   'gif'   : DocumentType.IMAGE, 
+                                   'bmp'   : DocumentType.IMAGE, 
+                                   'tiff'  : DocumentType.IMAGE, 
+                                   'webp'  : DocumentType.IMAGE,
+                                   'zip'   : DocumentType.ARCHIVE, 
+                                   'tar'   : DocumentType.ARCHIVE, 
+                                   'gz'    : DocumentType.ARCHIVE, 
+                                   'tgz'   : DocumentType.ARCHIVE, 
+                                   'rar'   : DocumentType.ARCHIVE, 
+                                   '7z'    : DocumentType.ARCHIVE,
+                                  }
+
+    
+    def get_parser(self, file_path: Path):
+        """
+        Get appropriate parser for file
+        
+        Arguments:
+        ----------
+            file_path { Path }   : Path to document
+        
+        Returns:
+        --------
+             { object }          : Parser instance or handler
+        
+        Raises:
+        -------
+            InvalidFileTypeError : If file type not supported
+        """
+        doc_type = self.detect_document_type(file_path = file_path)
+        
+        # Handle special types (image, archive)
+        if (doc_type == DocumentType.IMAGE):
+            return self._get_ocr_engine()
+        
+        elif (doc_type == DocumentType.ARCHIVE): 
+            return self._get_archive_handler()
+        
+        # Handle standard document types
+        elif doc_type in self._parsers:
+            return self._parsers[doc_type]
+        
+        else:
+            raise InvalidFileTypeError(file_type     = str(doc_type),
+                                       allowed_types = [t.value for t in self._parsers.keys()] + [DocumentType.IMAGE.value, DocumentType.ARCHIVE.value],
+                                      )
+        
+    
+    def detect_document_type(self, file_path: Path) -> Union[DocumentType, str]:
+        """
+        Detect document type from file extension and content
+        
+        Arguments:
+        ----------
+            file_path { Path }   : Path to document
+        
+        Returns:
+        --------
+            { Union }            : DocumentType enum or string for special types
+        
+        Raises:
+        -------
+            InvalidFileTypeError : If type cannot be determined
+        """
+        file_path = Path(file_path)
+        
+        # Get extension
+        extension = file_path.suffix.lstrip('.').lower()
+        
+        # Check if extension is mapped
+        if extension in self._extension_mapping:
+            doc_type = self._extension_mapping[extension]
+            
+            self.logger.debug(f"Detected type {doc_type} from extension .{extension}")
+            
+            return doc_type
+        
+        # Try detecting from file content
+        detected_type = FileHandler.detect_file_type(file_path)
+        
+        if (detected_type and (detected_type in self._extension_mapping)):
+            doc_type = self._extension_mapping[detected_type]
+            
+            self.logger.debug(f"Detected type {doc_type} from content")
+            
+            return doc_type
+        
+        raise InvalidFileTypeError(file_type = extension, allowed_types = list(self._extension_mapping.keys()))
+    
+
+    def parse(self, file_path: Union[str, Path], extract_metadata: bool = True, clean_text: bool = True, **kwargs) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse document using appropriate parser
+        
+        Arguments:
+        ----------
+            file_path        { Path } : Path to document
+
+            extract_metadata { bool } : Extract document metadata
+            
+            clean_text       { bool } : Clean extracted text
+            
+            **kwargs                  : Additional parser-specific arguments
+        
+        Returns:
+        --------
+                { tuple }             : Tuple of (extracted_text, metadata)
+        
+        Raises:
+        -------
+            InvalidFileTypeError      : If file type not supported
+           
+            RAGException              : If parsing fails
+        """
+        file_path = Path(file_path)
+        
+        self.logger.info(f"Parsing document: {file_path}")
+        
+        # Get appropriate parser/handler
+        parser    = self.get_parser(file_path)
+        
+        # Handle different parser types
+        if isinstance(parser, (PDFParser, DOCXParser, TXTParser)):
+            # Standard document parser
+            text, metadata = parser.parse(file_path,
+                                          extract_metadata = extract_metadata,
+                                          clean_text       = clean_text,
+                                          **kwargs
+                                         )
+        
+        elif isinstance(parser, OCREngine):
+            # Image file - use OCR
+            text        = parser.extract_text_from_image(file_path)
+            metadata    = self._create_image_metadata(file_path) if extract_metadata else None
+        
+        elif isinstance(parser, ArchiveHandler):
+            # Archive file - extract and parse contents
+            return self._parse_archive(file_path        = file_path,
+                                       extract_metadata = extract_metadata,
+                                       clean_text       = clean_text,
+                                       **kwargs
+                                      )
+        
+        else:
+            raise InvalidFileTypeError(file_type = file_path.suffix, allowed_types = self.get_supported_extensions())
+        
+        self.logger.info(f"Successfully parsed {file_path.name}: {len(text)} chars, type={metadata.document_type if metadata else 'unknown'}")
+        
+        return text, metadata
+    
+
+    def _get_ocr_engine(self) -> OCREngine:
+        """
+        Get OCR engine instance (lazy initialization)
+        
+        Returns:
+        --------
+            { OCREngine } : OCR engine instance
+        """
+        if self._ocr_engine is None:
+            self._ocr_engine = OCREngine()
+        
+        return self._ocr_engine
+    
+
+    def _get_archive_handler(self) -> ArchiveHandler:
+        """
+        Get archive handler instance (lazy initialization)
+        
+        Returns:
+        --------
+            { ArchiveHandler } : Archive handler instance
+        """
+        if self._archive_handler is None:
+            self._archive_handler = ArchiveHandler()
+        
+        return self._archive_handler
+    
+
+    def _create_image_metadata(self, file_path: Path) -> DocumentMetadata:
+        """
+        Create metadata for image file
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to image file
+        
+        Returns:
+        --------
+            { DocumentMetadata } : DocumentMetadata object
+        """
+        stat = file_path.stat()
+        
+        return DocumentMetadata(document_id     = IDGenerator.generate_document_id(),
+                                filename        = file_path.name,
+                                file_path       = file_path,
+                                document_type   = DocumentType.IMAGE,
+                                file_size_bytes = stat.st_size,
+                                created_date    = stat.st_ctime,
+                                modified_date   = stat.st_mtime,
+                                extra           = {"file_type": "image"},
+                               )
+    
+
+    def _parse_archive(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, **kwargs) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse archive file: extract contents and parse all supported files
+        
+        Arguments:
+        ----------
+            file_path        { Path } : Path to archive file
+
+            extract_metadata { bool } : Extract document metadata
+            
+            clean_text       { bool } : Clean extracted text
+            
+            **kwargs                  : Additional arguments
+        
+        Returns:
+        --------
+                { tuple }             : Tuple of (combined_text, metadata)
+        """
+        archive_handler = self._get_archive_handler()
+        
+        # Extract archive contents
+        extracted_files = archive_handler.extract_archive(file_path)
+        
+        # Parse all extracted files
+        combined_text   = ""
+        all_metadata    = list()
+        
+        for extracted_file in extracted_files:
+            if self.is_supported(extracted_file):
+                try:
+                    file_text, file_metadata = self.parse(extracted_file,
+                                                          extract_metadata = extract_metadata,
+                                                          clean_text       = clean_text,
+                                                          **kwargs
+                                                         )
+                    
+                    combined_text           += f"\n\n[FILE: {extracted_file.name}]\n{file_text}"
+                    
+                    if file_metadata:
+                        all_metadata.append(file_metadata)
+                
+                except Exception as e:
+                    self.logger.warning(f"Failed to parse extracted file {extracted_file}: {repr(e)}")
+                    continue
+        
+        # Create combined metadata
+        combined_metadata = None
+        
+        if extract_metadata and all_metadata:
+            combined_metadata = DocumentMetadata(document_id     = IDGenerator.generate_document_id(),
+                                                 filename        = file_path.name,
+                                                 file_path       = file_path,
+                                                 document_type   = DocumentType.ARCHIVE,
+                                                 file_size_bytes = file_path.stat().st_size,
+                                                 extra           = {"archive_contents"    : len(extracted_files),
+                                                                    "parsed_files"        : len(all_metadata),
+                                                                    "contained_documents" : [meta.document_id for meta in all_metadata],
+                                                                   }
+                                                )
+        
+        return combined_text.strip(), combined_metadata
+    
+
+    def is_supported(self, file_path: Path) -> bool:
+        """
+        Check if file type is supported.
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to document
+        
+        Returns:
+        --------
+               { bool }        : True if supported
+        """
+        try:
+            self.detect_document_type(file_path = file_path)
+            return True
+
+        except InvalidFileTypeError:
+            return False
+    
+
+    def get_supported_extensions(self) -> list[str]:
+        """
+        Get list of supported file extensions.
+        
+        Returns:
+        --------
+            { list } : List of extensions (without dot)
+        """
+        return list(self._extension_mapping.keys())
+    
+
+    def register_parser(self, doc_type: DocumentType, parser_instance, extensions: Optional[list[str]] = None):
+        """
+        Register a new parser type (for extensibility)
+        
+        Arguments:
+        ----------
+            doc_type        { DocumentType } : Document type enum
+
+            parser_instance                  : Parser instance
+            
+            extensions         { list }      : File extensions to map to this parser
+        """
+        self._parsers[doc_type] = parser_instance
+        
+        if extensions:
+            for ext in extensions:
+                self._extension_mapping[ext.lstrip('.')] = doc_type
+        
+        self.logger.info(f"Registered parser for {doc_type}")
+    
+
+    def batch_parse(self, file_paths: list[Path], extract_metadata: bool = True, clean_text: bool = True, skip_errors: bool = True) -> list[tuple[Path, str, Optional[DocumentMetadata]]]:
+        """
+        Parse multiple documents.
+        
+        Arguments:
+        ----------
+            file_paths       { list } : List of file paths
+
+            extract_metadata { bool } : Extract metadata
+            
+            clean_text       { str }  : Clean text
+            
+            skip_errors      { bool } : Skip files that fail to parse
+        
+        Returns:
+        --------
+                    { list }          : List of (file_path, text, metadata) tuples
+        """
+        results = list()
+        
+        for file_path in file_paths:
+            try:
+                text, metadata = self.parse(file_path,
+                                            extract_metadata = extract_metadata,
+                                            clean_text       = clean_text,
+                                           )
+
+                results.append((file_path, text, metadata))
+            
+            except Exception as e:
+                self.logger.error(f"Failed to parse {file_path}: {repr(e)}")
+                
+                if not skip_errors:
+                    raise
+                # Add placeholder for failed file
+                results.append((file_path, "", None))
+        
+        self.logger.info(f"Batch parsed {len(results)}/{len(file_paths)} files successfully")
+        
+        return results
+    
+
+    def parse_directory(self, directory: Path, recursive: bool = False, pattern: str = "*", **kwargs) -> list[tuple[Path, str, Optional[DocumentMetadata]]]:
+        """
+        Parse all supported documents in a directory
+        
+        Arguments:
+        ----------
+            directory { Path } : Directory path
+
+            recursive { bool } : Search recursively
+            
+            pattern   { str }  : File pattern (glob)
+            
+            **kwargs           : Additional parse arguments
+        
+        Returns:
+        --------
+                { list }       : List of (file_path, text, metadata) tuples
+        """
+        directory       = Path(directory)
+        
+        # Get all files
+        all_files       = FileHandler.list_files(directory, pattern=pattern, recursive=recursive)
+        
+        # Filter to supported types
+        supported_files = [f for f in all_files if self.is_supported(f)]
+        
+        self.logger.info(f"Found {len(supported_files)} supported files in {directory} ({len(all_files) - len(supported_files)} unsupported)")
+        
+        # Parse all files
+        return self.batch_parse(supported_files, **kwargs)
+    
+
+    def get_parser_info(self) -> dict:
+        """
+        Get information about registered parsers
+        
+        Returns:
+        --------
+            { dict }    : Dictionary with parser information
+        """
+        info = {"supported_types"      : [t.value for t in self._parsers.keys()] + ['image', 'archive'],
+                "supported_extensions" : self.get_supported_extensions(),
+                "parser_classes"       : {t.value: type(p).__name__  for t, p in self._parsers.items()},
+                "special_handlers"     : {"image"   : "OCREngine", 
+                                          "archive" : "ArchiveHandler"},
+               }
+
+        return info
+
+
+# Global factory instance
+_factory = None
+
+
+def get_parser_factory() -> ParserFactory:
+    """
+    Get global parser factory instance (singleton)
+    
+    Returns:
+    --------
+        { ParserFactory }    : ParserFactory instance
+    """
+    global _factory
+
+    if _factory is None:
+        _factory = ParserFactory()
+    
+    return _factory
+
+
+# Convenience functions
+def parse_document(file_path: Union[str, Path], **kwargs) -> tuple[str, Optional[DocumentMetadata]]:
+    """
+    Convenience function to parse a document
+    
+    Arguments:
+    ----------
+        file_path { Path } : Path to document
+
+        **kwargs           : Additional arguments
+    
+    Returns:
+    --------
+           { tuple }       : Tuple of (text, metadata)
+    """
+    factory = get_parser_factory()
+
+    return factory.parse(file_path, **kwargs)
+
+
+def is_supported_file(file_path: Union[str, Path]) -> bool:
+    """
+    Check if file is supported
+    
+    Arguments:
+    ----------
+        file_path { Path } : Path to file
+    
+    Returns:
+    --------
+           { bool }        : True if supported
+    """
+    factory = get_parser_factory()
+
+    return factory.is_supported(Path(file_path))
+
+
+def get_supported_extensions() -> list[str]:
+    """
+    Get list of supported extensions.
+    
+    Returns:
+    --------
+        { list }    : List of extensions
+    """
+    factory = get_parser_factory()
+
+    return factory.get_supported_extensions()

document_parser/pdf_parser.py

@@ -0,0 +1,908 @@
+# DEPENDENCIES
+import hashlib
+from typing import Any
+from typing import List
+from typing import Dict
+from typing import Tuple
+from pathlib import Path
+from typing import Optional
+from datetime import datetime
+from config.models import DocumentType
+from utils.text_cleaner import TextCleaner
+from config.models import DocumentMetadata
+from config.logging_config import get_logger
+from utils.error_handler import PDFParseError
+from utils.error_handler import handle_errors
+from document_parser.ocr_engine import OCREngine
+
+try:
+    import fitz 
+    PYMUPDF_AVAILABLE = True
+    
+except ImportError:
+    PYMUPDF_AVAILABLE = False
+
+try:
+    import PyPDF2
+    from PyPDF2 import PdfReader
+    PYPdf2_AVAILABLE = True
+
+except ImportError:
+    PYPdf2_AVAILABLE = False
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class PDFParser:
+    """
+    Comprehensive PDF parsing with metadata extraction: Uses PyMuPDF (fitz) as primary parser with PyPDF2 fallback
+    
+    Handles various PDF formats including encrypted and scanned documents
+    """
+    def __init__(self, prefer_pymupdf: bool = True):
+        """
+        Initialize PDF parser.
+        
+        Arguments:
+        ----------
+            prefer_pymupdf { bool } : Use PyMuPDF as primary parser if available
+        """
+        self.logger         = logger
+        self.prefer_pymupdf = prefer_pymupdf and PYMUPDF_AVAILABLE
+        self.ocr_engine     = None  
+        
+        try:
+            from document_parser.ocr_engine import OCREngine
+            self.ocr_available = True
+
+        except ImportError:
+            self.ocr_available = False
+            self.logger.warning("OCR engine not available - scanned PDFs may not be processed")
+
+        if (not PYMUPDF_AVAILABLE and not PYPdf2_AVAILABLE):
+            raise ImportError("Neither PyMuPDF nor PyPDF2 are available. Please install at least one.")
+        
+        self.logger.info(f"PDF Parser initialized - Primary: {'PyMuPDF' if self.prefer_pymupdf else 'PyPDF2'}, PyMuPDF available: {PYMUPDF_AVAILABLE}, PyPDF2 available: {PYPdf2_AVAILABLE}")
+    
+
+    @handle_errors(error_type=PDFParseError, log_error = True, reraise = True)
+    def parse(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, password: Optional[str] = None) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse PDF and extract text and metadata : tries PyMuPDF first, falls back to PyPDF2 if needed
+        
+        Arguments:
+        ----------
+            file_path        { Path } : Path to PDF file
+            
+            extract_metadata { bool } : Extract document metadata
+            
+            clean_text       { bool } : Clean extracted text
+            
+            password         { str  } : Password for encrypted PDFs
+        
+        Returns:
+        --------
+                   { tuple }          : Tuple of (extracted_text, metadata)
+        
+        Raises:
+        -------
+            PDFParseError             : If parsing fails
+        """
+        file_path = Path(file_path)
+        
+        if not file_path.exists():
+            raise PDFParseError(str(file_path), original_error = FileNotFoundError(f"PDF file not found: {file_path}"))
+        
+        self.logger.info(f"Parsing PDF: {file_path}")
+        
+        # Try PyMuPDF first if preferred and available
+        if (self.prefer_pymupdf and PYMUPDF_AVAILABLE):
+            try:
+                parsed_text = self._parse_with_pymupdf(file_path        = file_path, 
+                                                       extract_metadata = extract_metadata, 
+                                                       clean_text       = clean_text, 
+                                                       password         = password,
+                                                      )
+
+                return parsed_text
+            
+            except Exception as e:
+                self.logger.warning(f"PyMuPDF parsing failed for {file_path}, falling back to PyPDF2: {repr(e)}")
+        
+        # Fall back to PyPDF2
+        if PYPdf2_AVAILABLE:
+            try:
+                parsed_text = self._parse_with_pypdf2(file_path        = file_path, 
+                                                      extract_metadata = extract_metadata, 
+                                                      clean_text       = clean_text,
+                                                      password         = password,
+                                                     )
+
+                return parsed_text
+            
+            except Exception as e:
+                self.logger.error(f"PyPDF2 parsing also failed for {file_path}: {repr(e)}")
+                raise PDFParseError(str(file_path), original_error = e)
+        
+        else:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("No PDF parsing libraries available"))
+
+    
+    def _parse_with_pymupdf(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, password: Optional[str] = None) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse PDF using PyMuPDF (fitz) with OCR fallback for scanned documents
+        """
+        self.logger.debug(f"Using PyMuPDF for parsing: {file_path}")
+        
+        doc = None
+
+        try:
+            # Open PDF with PyMuPDF
+            self.logger.debug(f"Opening document: {file_path}")
+            doc = fitz.open(str(file_path))
+            
+            self.logger.debug(f"Document opened successfully, {len(doc)} pages")
+            
+            # Handle encrypted PDFs
+            if (doc.needs_pass and password):
+                if not doc.authenticate(password):
+                    raise PDFParseError(str(file_path), original_error = ValueError("Invalid password for encrypted PDF"))
+            
+            elif (doc.needs_pass and not password):
+                raise PDFParseError(str(file_path), original_error = ValueError("PDF is encrypted but no password provided"))
+            
+            # Extract text with per-page OCR fallback
+            text_content = self._extract_text_with_pymupdf(doc       = doc, 
+                                                           file_path = file_path,
+                                                          )
+
+            # Extract metadata
+            metadata = None
+            if extract_metadata:
+                metadata = self._extract_metadata_with_pymupdf(doc       = doc, 
+                                                               file_path = file_path,
+                                                              )
+            
+            # Clean text
+            if clean_text:
+                text_content = TextCleaner.clean(text_content,
+                                                 remove_html          = True,
+                                                 normalize_whitespace = True,
+                                                 preserve_structure   = True,
+                                                )
+            
+            self.logger.info(f"Successfully parsed PDF with PyMuPDF: {len(text_content)} characters, {len(doc)} pages")
+            return text_content, metadata
+            
+        except Exception as e:
+            self.logger.error(f"PyMuPDF parsing failed for {file_path}: {repr(e)}")
+            raise
+        
+        finally:
+            # Always close the document in finally block
+            if doc:
+                self.logger.debug("Closing PyMuPDF document")
+                doc.close()
+
+    def _parse_with_pypdf2(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, password: Optional[str] = None) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse PDF using PyPDF2
+        
+        Arguments:
+        ----------
+            file_path         { Path } : Path to PDF file
+            
+            extract_metadata  { bool } : Extract document metadata
+            
+            clean_text        { bool } : Clean extracted text
+            
+            password           { str } : Password for encrypted PDFs
+        
+        Returns:
+        --------
+                  { tuple }            : Tuple of (extracted_text, metadata)
+        """
+        self.logger.debug(f"Using PyPDF2 for parsing: {file_path}")
+        
+        try:
+            # Open PDF with PyPDF2
+            with open(file_path, 'rb') as pdf_file:
+                reader = PdfReader(pdf_file)
+                
+                # Handle encrypted PDFs
+                if reader.is_encrypted:
+                    if password:
+                        reader.decrypt(password)
+                        self.logger.info("Successfully decrypted PDF with PyPDF2")
+                    
+                    else:
+                        raise PDFParseError(str(file_path), original_error = ValueError("PDF is encrypted but no password provided"))
+                
+                # Extract text from all pages
+                text_content = self._extract_text_with_pypdf2(reader = reader)
+                
+                # Extract metadata
+                metadata     = None
+
+                if extract_metadata:
+                    metadata = self._extract_metadata_with_pypdf2(reader    = reader, 
+                                                                  file_path = file_path,
+                                                                 )
+                
+                # Clean text
+                if clean_text:
+                    text_content = TextCleaner.clean(text_content,
+                                                     remove_html          = True,
+                                                     normalize_whitespace = True,
+                                                     preserve_structure   = True,
+                                                    )
+                
+                self.logger.info(f"Successfully parsed PDF with PyPDF2: {len(text_content)} characters, {len(reader.pages)} pages")
+                
+                return text_content, metadata
+                
+        except Exception as e:
+            self.logger.error(f"PyPDF2 parsing failed for {file_path}: {repr(e)}")
+            raise
+
+    
+    def _extract_text_with_pymupdf(self, doc: "fitz.Document", file_path: Path = None) -> str:
+        """
+        Extract text from all pages using PyMuPDF with per-page OCR fallback.
+        
+        Arguments:
+        ----------
+            doc        : PyMuPDF document object
+            
+            file_path  : Path to PDF file (for OCR fallback)
+        
+        Returns:
+        --------
+            { str }    : Combined text from all pages
+        """
+        text_parts = list()
+        
+        for page_num in range(len(doc)):
+            try:
+                page      = doc[page_num]
+                page_text = page.get_text()
+                
+                if page_text and page_text.strip():
+                    # Add page marker for citation purposes
+                    text_parts.append(f"\n[PAGE {page_num + 1}]\n{page_text}")
+                    self.logger.debug(f"Extracted {len(page_text)} chars from page {page_num + 1} with PyMuPDF")
+                
+                else:
+                    # No text extracted - this page might be scanned
+                    self.logger.warning(f"No text extracted from page {page_num + 1} with PyMuPDF (might be scanned)")
+                    
+                    # Try OCR for this specific page if available
+                    if self.ocr_available and file_path:
+                        try:
+                            self.logger.info(f"Attempting OCR for page {page_num + 1}")
+                            ocr_text = self._extract_page_text_with_ocr(file_path, page_num + 1)
+                            
+                            if ocr_text and ocr_text.strip():
+                                text_parts.append(f"\n[PAGE {page_num + 1} - OCR]\n{ocr_text}")
+                                self.logger.info(f"OCR extracted {len(ocr_text)} chars from page {page_num + 1}")
+                            
+                            else:
+                                text_parts.append(f"\n[PAGE {page_num + 1} - NO TEXT]\n")
+                                self.logger.warning(f"OCR also failed to extract text from page {page_num + 1}")
+                        
+                        except Exception as ocr_error:
+                            self.logger.warning(f"OCR failed for page {page_num + 1}: {repr(ocr_error)}")
+                            text_parts.append(f"\n[PAGE {page_num + 1} - OCR FAILED]\n")
+                    
+                    else:
+                        # No OCR available or no file_path provided
+                        text_parts.append(f"\n[PAGE {page_num + 1} - NO TEXT]\n")
+            
+            except Exception as e:
+                self.logger.warning(f"Error extracting text from page {page_num + 1} with PyMuPDF: {repr(e)}")
+                text_parts.append(f"\n[PAGE {page_num + 1} - ERROR: {str(e)}]\n")
+                continue
+        
+        return "\n".join(text_parts)
+    
+
+    def _extract_text_with_pypdf2(self, reader: PdfReader) -> str:
+        """
+        Extract text from all pages using PyPDF2
+        
+        Arguments:
+        ----------
+            reader { PdfReader } : PdfReader object
+        
+        Returns:
+        --------
+                 { str }         : Combined text from all pages
+        """
+        text_parts = list()
+        num_pages  = len(reader.pages)
+        
+        for page_num in range(num_pages):
+            try:
+                page      = reader.pages[page_num]
+                page_text = page.extract_text()
+                
+                if page_text and page_text.strip():
+                    # Add page marker for citation purposes
+                    text_parts.append(f"\n[PAGE {page_num + 1}]\n{page_text}")
+                    self.logger.debug(f"Extracted {len(page_text)} chars from page {page_num + 1} with PyPDF2")
+                
+                else:
+                    self.logger.warning(f"No text extracted from page {page_num + 1} with PyPDF2 (might be scanned)")
+            
+            except Exception as e:
+                self.logger.warning(f"Error extracting text from page {page_num + 1} with PyPDF2: {repr(e)}")
+                continue
+        
+        return "\n".join(text_parts)
+
+    
+    def _extract_metadata_with_pymupdf(self, doc: "fitz.Document", file_path: Path) -> DocumentMetadata:
+        """
+        Extract metadata using PyMuPDF
+        
+        Arguments:
+        -----------
+            doc       { fitz.Document } : PyMuPDF document object
+
+            file_path      { Path }     : Path to PDF file
+        
+        Returns:
+        --------
+              { DocumentMetadata }      : DocumentMetadata object
+        """
+        # Get PDF metadata
+        pdf_metadata  = doc.metadata
+        
+        # Extract common fields
+        title         = pdf_metadata.get('title', '').strip()
+        author        = pdf_metadata.get('author', '').strip()
+        
+        # Parse dates
+        created_date  = self._parse_pdf_date(pdf_metadata.get('creationDate'))
+        modified_date = self._parse_pdf_date(pdf_metadata.get('modDate'))
+        
+        # Get file size
+        file_size     = file_path.stat().st_size
+        
+        # Count pages
+        num_pages     = len(doc)
+        
+        # Generate document ID
+        doc_hash      = hashlib.md5(str(file_path).encode()).hexdigest()
+        doc_id        = f"doc_{int(datetime.now().timestamp())}_{doc_hash}"
+        
+        # Create metadata object
+        metadata      = DocumentMetadata(document_id     = doc_id,
+                                         filename        = file_path.name,
+                                         file_path       = file_path,
+                                         document_type   = DocumentType.PDF,
+                                         title           = title or file_path.stem,
+                                         author          = author,
+                                         created_date    = created_date,
+                                         modified_date   = modified_date,
+                                         file_size_bytes = file_size,
+                                         num_pages       = num_pages,
+                                         extra           = {"pdf_version"  : pdf_metadata.get('producer', ''),
+                                                            "pdf_metadata" : {k: str(v) for k, v in pdf_metadata.items() if v},
+                                                            "parser_used"  : "pymupdf"
+                                                           }
+                                        )
+        
+        return metadata
+
+    
+    def _extract_metadata_with_pypdf2(self, reader: PdfReader, file_path: Path) -> DocumentMetadata:
+        """
+        Extract metadata using PyPDF2
+        
+        Arguments:
+        ----------
+            reader    { PdfReader } : PdfReader object
+
+            file_path    { Path }   : Path to PDF file
+        
+        Returns:
+        --------
+            { DocumentMetadata }    : DocumentMetadata object
+        """
+        # Get PDF metadata
+        pdf_info      = reader.metadata if reader.metadata else {}
+        
+        # Extract common fields
+        title         = self._get_metadata_field(pdf_info, ['/Title', 'title'])
+        author        = self._get_metadata_field(pdf_info, ['/Author', 'author'])
+        
+        # Parse dates
+        created_date  = self._parse_pdf_date(self._get_metadata_field(pdf_info, ['/CreationDate', 'creation_date']))
+        modified_date = self._parse_pdf_date(self._get_metadata_field(pdf_info, ['/ModDate', 'mod_date']))
+        
+        # Get file size
+        file_size     = file_path.stat().st_size
+        
+        # Count pages
+        num_pages     = len(reader.pages)
+        
+        # Generate document ID
+        doc_hash      = hashlib.md5(str(file_path).encode()).hexdigest()
+        doc_id        = f"doc_{int(datetime.now().timestamp())}_{doc_hash}"
+        
+        # Create metadata object
+        metadata      = DocumentMetadata(document_id     = doc_id,
+                                         filename        = file_path.name,
+                                         file_path       = file_path,
+                                         document_type   = DocumentType.PDF,
+                                         title           = title or file_path.stem,
+                                         author          = author,
+                                         created_date    = created_date,
+                                         modified_date   = modified_date,
+                                         file_size_bytes = file_size,
+                                         num_pages       = num_pages,
+                                         extra           = {"pdf_version"  : self._get_metadata_field(pdf_info, ['/Producer', 'producer']),
+                                                            "pdf_metadata" : {k: str(v) for k, v in pdf_info.items() if v},
+                                                            "parser_used"  : "pypdf2",
+                                                           }
+                                        )
+        
+        return metadata
+
+
+    def _extract_text_with_ocr(self, file_path: Path) -> str:
+        """
+        Extract text from scanned PDF using OCR
+        """
+        if not self.ocr_available:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("OCR engine not available"))
+        
+        if self.ocr_engine is None:
+            self.ocr_engine = OCREngine()
+        
+        return self.ocr_engine.extract_text_from_pdf(file_path)
+
+    
+    def _extract_page_text_with_ocr(self, file_path: Path, page_number: int) -> str:
+        """
+        Extract text from a specific page using OCR
+        
+        Arguments:
+        ----------
+            file_path   { Path }  : Path to PDF file
+
+            page_number { int }   : Page number (1-indexed)
+        
+        Returns:
+        --------
+            { str }               : Extracted text from the page
+        """
+        if not self.ocr_available:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("OCR engine not available"))
+        
+        if self.ocr_engine is None:
+            self.ocr_engine = OCREngine()
+        
+        try:
+            # Use OCR engine to extract text from specific page
+            return self.ocr_engine.extract_text_from_pdf(pdf_path = file_path, 
+                                                         pages    = [page_number],
+                                                        )
+        
+        except Exception as e:
+            self.logger.error(f"OCR failed for page {page_number}: {repr(e)}")
+            return ""
+            
+
+    @staticmethod
+    def _get_metadata_field(metadata: Dict, field_names: List[str]) -> Optional[str]:
+        """
+        Get metadata field with fallback names
+        
+        Arguments:
+        ----------
+            metadata    { dict } : Metadata dictionary
+
+            field_names { list } : List of possible field names
+        
+        Returns:
+        --------
+                 { str }         : Field value or None
+        """
+        for field_name in field_names:
+            if field_name in metadata:
+                value = metadata[field_name]
+
+                if value:
+                    return str(value).strip()
+                    
+        return None
+    
+
+    @staticmethod
+    def _parse_pdf_date(date_str: Optional[str]) -> Optional[datetime]:
+        """
+        Parse PDF date format : PDF dates are in format: D:YYYYMMDDHHmmSSOHH'mm'
+        
+        Arguments:
+        ----------
+            date_str { str } : PDF date string
+        
+        Returns:
+        --------
+             { datetime }    : Datetime object or None
+        """
+        if not date_str:
+            return None
+        
+        try:
+            # Remove 'D:' prefix if present
+            if date_str.startswith('D:'):
+                date_str = date_str[2:]
+            
+            # Parse basic format: YYYYMMDDHHMMSS
+            date_str = date_str[:14]
+            
+            return datetime.strptime(date_str, '%Y%m%d%H%M%S')
+
+        except Exception:
+            return None
+    
+
+    def extract_page_text(self, file_path: Path, page_number: int, clean_text: bool = True) -> str:
+        """
+        Extract text from a specific page
+        
+        Arguments:
+        ----------
+            file_path   { Path } : Path to PDF file
+            
+            page_number { int }  : Page number (1-indexed)
+            
+            clean_text  { bool } : Clean extracted text
+        
+        Returns:
+        --------
+                  { str }        : Page text
+        """
+        # Try PyMuPDF first if preferred and available
+        if self.prefer_pymupdf and PYMUPDF_AVAILABLE:
+            try:
+                page_text = self._extract_page_text_pymupdf(file_path   = file_path, 
+                                                            page_number = page_number, 
+                                                            clean_text  = clean_text,
+                                                           )
+                return page_text
+            
+            except Exception as e:
+                self.logger.warning(f"PyMuPDF page extraction failed, falling back to PyPDF2: {repr(e)}")
+        
+        # Fall back to PyPDF2
+        if PYPdf2_AVAILABLE:
+            page_text = self._extract_page_text_pypdf2(file_path   = file_path, 
+                                                       pagse_number = page_number, 
+                                                       clean_text  = clean_text,
+                                                      )
+            
+            return page_text
+
+        else:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("No PDF parsing libraries available"))
+
+    
+    def _extract_page_text_pymupdf(self, file_path: Path, page_number: int, clean_text: bool = True) -> str:
+        """
+        Extract page text using PyMuPDF
+        """
+        doc = None
+        try:
+            doc       = fitz.open(str(file_path))
+            num_pages = len(doc)
+            
+            if ((page_number < 1) or (page_number > num_pages)):
+                raise ValueError(f"Page number {page_number} out of range (1-{num_pages})")
+            
+            page      = doc[page_number - 1]
+            page_text = page.get_text()
+            
+            if clean_text:
+                page_text = TextCleaner.clean(page_text)
+            
+            return page_text
+            
+        except Exception as e:
+            self.logger.error(f"Failed to extract page {page_number} with PyMuPDF: {repr(e)}")
+            raise PDFParseError(str(file_path), original_error = e)
+            
+        finally:
+            if doc:
+                doc.close()
+    
+    
+    def _extract_page_text_pypdf2(self, file_path: Path, page_number: int, clean_text: bool = True) -> str:
+        """
+        Extract page text using PyPDF2
+        """
+        try:
+            with open(file_path, 'rb') as pdf_file:
+                reader    = PdfReader(pdf_file)
+                num_pages = len(reader.pages)
+                
+                if ((page_number < 1) or (page_number > num_pages)):
+                    raise ValueError(f"Page number {page_number} out of range (1-{num_pages})")
+                
+                page      = reader.pages[page_number - 1]
+                page_text = page.extract_text()
+                
+                if clean_text:
+                    page_text = TextCleaner.clean(page_text)
+                
+                return page_text
+        
+        except Exception as e:
+            self.logger.error(f"Failed to extract page {page_number} with PyPDF2: {repr(e)}")
+            raise PDFParseError(str(file_path), original_error = e)
+    
+
+    def get_page_count(self, file_path: Path) -> int:
+        """
+        Get number of pages in PDF
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to PDF file
+        
+        Returns:
+        --------
+                { int }        : Number of pages
+        """
+        # Try PyMuPDF first if available
+        if PYMUPDF_AVAILABLE:
+            doc = None
+            try:
+                doc        = fitz.open(str(file_path))
+                page_count = len(doc)
+                
+                return page_count
+            
+            except Exception as e:
+                self.logger.warning(f"PyMuPDF page count failed, trying PyPDF2: {repr(e)}")
+            
+            finally:
+                if doc:
+                    doc.close()
+        
+        # Fall back to PyPDF2
+        if PYPdf2_AVAILABLE:
+            try:
+                with open(file_path, 'rb') as pdf_file:
+                    reader = PdfReader(pdf_file)
+                    
+                    return len(reader.pages)
+
+            except Exception as e:
+                self.logger.error(f"Failed to get page count: {repr(e)}")
+                raise PDFParseError(str(file_path), original_error = e)
+        else:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("No PDF parsing libraries available"))
+
+    
+    def extract_page_range(self, file_path: Path, start_page: int, end_page: int, clean_text: bool = True) -> str:
+        """
+        Extract text from a range of pages
+        
+        Arguments:
+        ----------
+            file_path  { Path } : Path to PDF file
+            
+            start_page { int }  : Starting page (1-indexed, inclusive)
+            
+            end_page   { int }  : Ending page (1-indexed, inclusive)
+            
+            clean_text { bool } : Clean extracted text
+        
+        Returns:
+        --------
+                  { str }       : Combined text from pages
+        """
+        # Try PyMuPDF first if preferred and available
+        if self.prefer_pymupdf and PYMUPDF_AVAILABLE:
+            try:
+                page_range = self._extract_page_range_pymupdf(file_path  = file_path, 
+                                                              start_page = start_page, 
+                                                              end_page   = end_page, 
+                                                              clean_text = clean_text,
+                                                             )
+                return page_range
+            
+            except Exception as e:
+                self.logger.warning(f"PyMuPDF page range extraction failed, falling back to PyPDF2: {repr(e)}")
+        
+        # Fall back to PyPDF2
+        if PYPdf2_AVAILABLE:
+            page_range = self._extract_page_range_pypdf2(file_path  = file_path, 
+                                                         start_page = start_page, 
+                                                         end_page   = end_page, 
+                                                         clean_text = clean_text,
+                                                        )
+            
+            return page_range
+
+        else:
+            raise PDFParseError(str(file_path), original_error = RuntimeError("No PDF parsing libraries available"))
+    
+
+    def _extract_page_range_pymupdf(self, file_path: Path, start_page: int, end_page: int, clean_text: bool = True) -> str:
+        """
+        Extract page range using PyMuPDF
+        """
+        doc = None
+        try:
+            doc       = fitz.open(str(file_path))
+            num_pages = len(doc)
+            
+            if ((start_page < 1) or (end_page > num_pages) or (start_page > end_page)):
+                raise ValueError(f"Invalid page range {start_page}-{end_page} for PDF with {num_pages} pages")
+            
+            text_parts = list()
+
+            for page_num in range(start_page - 1, end_page):
+                page      = doc[page_num]
+                page_text = page.get_text()
+                
+                if page_text:
+                    text_parts.append(f"\n[PAGE {page_num + 1}]\n{page_text}")
+            
+            combined_text = "\n".join(text_parts)
+            
+            if clean_text:
+                combined_text = TextCleaner.clean(combined_text)
+            
+            return combined_text
+            
+        except Exception as e:
+            self.logger.error(f"Failed to extract page range with PyMuPDF: {repr(e)}")
+            raise PDFParseError(str(file_path), original_error = e)
+            
+        finally:
+            if doc:
+                doc.close()
+    
+
+    def _extract_page_range_pypdf2(self, file_path: Path, start_page: int, end_page: int, clean_text: bool = True) -> str:
+        """
+        Extract page range using PyPDF2
+        """
+        try:
+            with open(file_path, 'rb') as pdf_file:
+                reader    = PdfReader(pdf_file)
+                num_pages = len(reader.pages)
+                
+                if ((start_page < 1) or (end_page > num_pages) or (start_page > end_page)):
+                    raise ValueError(f"Invalid page range {start_page}-{end_page} for PDF with {num_pages} pages")
+                
+                text_parts = list()
+
+                for page_num in range(start_page - 1, end_page):
+                    page      = reader.pages[page_num]
+                    page_text = page.extract_text()
+                    
+                    if page_text:
+                        text_parts.append(f"\n[PAGE {page_num + 1}]\n{page_text}")
+                
+                combined_text = "\n".join(text_parts)
+                
+                if clean_text:
+                    combined_text = TextCleaner.clean(combined_text)
+                
+                return combined_text
+        
+        except Exception as e:
+            self.logger.error(f"Failed to extract page range with PyPDF2: {repr(e)}")
+            raise PDFParseError(str(file_path), original_error = e)
+    
+
+    def is_scanned(self, file_path: Path) -> bool:
+        """
+        Check if PDF is scanned (image-based): Scanned PDFs have very little or no extractable text
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to PDF file
+        
+        Returns:
+        --------
+                { bool }       : True if appears to be scanned
+        """
+        # Try PyMuPDF first if available for better detection
+        if PYMUPDF_AVAILABLE:
+            try:
+                return self._is_scanned_pymupdf(file_path = file_path)
+
+            except Exception as e:
+                self.logger.warning(f"PyMuPDF scanned detection failed, trying PyPDF2: {repr(e)}")
+        
+        # Fall back to PyPDF2
+        if PYPdf2_AVAILABLE:
+            return self._is_scanned_pypdf2(file_path = file_path)
+        
+        else:
+            self.logger.warning("No PDF parsing libraries available for scanned detection")
+            return False
+    
+
+    def _is_scanned_pymupdf(self, file_path: Path) -> bool:
+        """
+        Check if PDF is scanned using PyMuPDF
+        """
+        doc = None
+        try:
+            doc               = fitz.open(str(file_path))
+            
+            # Sample first 3 pages
+            pages_to_check    = min(3, len(doc))
+            total_text_length = 0
+            
+            for i in range(pages_to_check):
+                page               = doc[i]
+                text               = page.get_text()
+                total_text_length += len(text.strip())
+            
+            # If average text per page is very low, likely scanned
+            avg_text_per_page = total_text_length / pages_to_check
+
+            # characters per page
+            threshold         = 100 
+            
+            is_scanned        = (avg_text_per_page < threshold)
+            
+            if is_scanned:
+                self.logger.info(f"PDF appears to be scanned (avg {avg_text_per_page:.0f} chars/page)")
+            
+            return is_scanned
+            
+        except Exception as e:
+            self.logger.warning(f"Could not determine if PDF is scanned with PyMuPDF: {repr(e)}")
+            return False
+            
+        finally:
+            if doc:
+                doc.close()
+    
+
+    def _is_scanned_pypdf2(self, file_path: Path) -> bool:
+        """
+        Check if PDF is scanned using PyPDF2
+        """
+        try:
+            with open(file_path, 'rb') as pdf_file:
+                reader            = PdfReader(pdf_file)
+                
+                # Sample first 3 pages
+                pages_to_check    = min(3, len(reader.pages))
+                total_text_length = 0
+                
+                for i in range(pages_to_check):
+                    page               = reader.pages[i]
+                    text               = page.extract_text()
+                    total_text_length += len(text.strip())
+                
+                # If average text per page is very low, likely scanned
+                avg_text_per_page = total_text_length / pages_to_check
+
+                # characters per page
+                threshold         = 100  
+                
+                is_scanned        = (avg_text_per_page < threshold)
+                
+                if is_scanned:
+                    self.logger.info(f"PDF appears to be scanned (avg {avg_text_per_page:.0f} chars/page)")
+                
+                return is_scanned
+        
+        except Exception as e:
+            self.logger.warning(f"Could not determine if PDF is scanned with PyPDF2: {repr(e)}")
+            return False

document_parser/txt_parser.py

@@ -0,0 +1,336 @@
+# DEPENDENCIES
+import chardet
+import hashlib
+from pathlib import Path
+from typing import Optional
+from datetime import datetime
+from config.models import DocumentType
+from utils.text_cleaner import TextCleaner
+from config.models import DocumentMetadata 
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import TextEncodingError
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class TXTParser:
+    """
+    Plain text file parser with automatic encoding detection : handles various text encodings and formats
+    """
+    # Common encodings to try
+    COMMON_ENCODINGS = ['utf-8', 
+                        'utf-16', 
+                        'ascii', 
+                        'latin-1', 
+                        'cp1252', 
+                        'iso-8859-1',
+                       ]
+    
+    def __init__(self):
+        self.logger = logger
+
+    
+    @handle_errors(error_type = TextEncodingError, log_error = True, reraise = True)
+    def parse(self, file_path: Path, extract_metadata: bool = True, clean_text: bool = True, encoding: Optional[str] = None) -> tuple[str, Optional[DocumentMetadata]]:
+        """
+        Parse text file and extract content
+        
+        Arguments:
+        -----------
+            file_path        { Path } : Path to text file
+            
+            extract_metadata { bool } : Extract document metadata
+            
+            clean_text       { bool } : Clean extracted text
+            
+            encoding         { str }  : Force specific encoding (None = auto-detect)
+        
+        Returns:
+        --------
+                  { tuple }           : Tuple of (extracted_text, metadata)
+        
+        Raises:
+        -------
+            TextEncodingError         : If file cannot be decoded
+        """
+        file_path = Path(file_path)
+        
+        if not file_path.exists():
+            raise TextEncodingError(str(file_path), encoding = "unknown", original_error = FileNotFoundError(f"Text file not found: {file_path}"))
+        
+        self.logger.info(f"Parsing TXT: {file_path}")
+        
+        # Detect encoding if not specified
+        if encoding is None:
+            encoding = self.detect_encoding(file_path)
+            self.logger.info(f"Detected encoding: {encoding}")
+        
+        try:
+            # Read file with detected/specified encoding
+            with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+                text_content = f.read()
+            
+            # Extract metadata
+            metadata = None
+
+            if extract_metadata:
+                metadata = self._extract_metadata(file_path   = file_path, 
+                                                  encoding    = encoding, 
+                                                  text_length = len(text_content),
+                                                 )
+            
+            # Clean text
+            if clean_text:
+                text_content = TextCleaner.clean(text_content,
+                                                 remove_html          = False,
+                                                 normalize_whitespace = True,
+                                                 preserve_structure   = True,
+                                                )
+            
+            self.logger.info(f"Successfully parsed TXT: {len(text_content)} characters")
+            
+            return text_content, metadata
+        
+        except Exception as e:
+            self.logger.error(f"Failed to parse TXT {file_path}: {repr(e)}")
+            raise TextEncodingError(str(file_path), encoding = encoding, original_error = e)
+    
+
+    def detect_encoding(self, file_path: Path) -> str:
+        """
+        Detect file encoding using chardet
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to text file
+        
+        Returns:
+        --------
+                { str }        : Detected encoding name
+        """
+        try:
+            # Read raw bytes
+            with open(file_path, 'rb') as f:
+                # Read first 10KB for detection
+                raw_data = f.read(10000)  
+            
+            # Detect encoding
+            result     = chardet.detect(raw_data)
+            encoding   = result['encoding']
+            confidence = result['confidence']
+            
+            self.logger.debug(f"Encoding detection: {encoding} (confidence: {confidence:.2%})")
+            
+            # If confidence is low, try common encodings
+            if (confidence < 0.7):
+                self.logger.warning(f"Low confidence ({confidence:.2%}) for detected encoding {encoding}")
+                encoding = self._try_common_encodings(file_path = file_path)
+            
+            # Fallback to UTF-8
+            return encoding or 'utf-8'  
+        
+        except Exception as e:
+            self.logger.warning(f"Encoding detection failed: {repr(e)}, using UTF-8")
+            return 'utf-8'
+    
+
+    def _try_common_encodings(self, file_path: Path) -> Optional[str]:
+        """
+        Try reading file with common encodings
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to text file
+        
+        Returns:
+        --------
+                { str }        : Working encoding or None
+        """
+        for encoding in self.COMMON_ENCODINGS:
+            try:
+                with open(file_path, 'r', encoding = encoding) as f:
+                    # Try reading first 1000 chars
+                    f.read(1000)  
+
+                self.logger.info(f"Successfully read with encoding: {encoding}")
+                return encoding
+            
+            except (UnicodeDecodeError, LookupError):
+                continue
+        
+        return None
+    
+
+    def _extract_metadata(self, file_path: Path, encoding: str, text_length: int) -> DocumentMetadata:
+        """
+        Extract metadata from text file
+        
+        Arguments:
+        ----------
+            file_path   { Path } : Path to text file
+
+            encoding    { str }  : File encoding
+            
+            text_length { int }  : Length of text content
+        
+        Returns:
+        --------
+            { DocumentMetadata } : DocumentMetadata object
+        """
+        # Get file stats
+        stat            = file_path.stat()
+        file_size       = stat.st_size
+        created_time    = datetime.fromtimestamp(stat.st_ctime)
+        modified_time   = datetime.fromtimestamp(stat.st_mtime)
+        
+        # Generate document ID
+        doc_hash        = hashlib.md5(str(file_path).encode()).hexdigest()
+        doc_id          = f"doc_{int(datetime.now().timestamp())}_{doc_hash}"
+        
+        # Estimate pages (rough: 3000 characters per page)
+        estimated_pages = max(1, text_length // 3000)
+        
+        # Count lines
+        with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+            num_lines = sum(1 for _ in f)
+        
+        # Create metadata object
+        metadata = DocumentMetadata(document_id     = doc_id,
+                                    filename        = file_path.name,
+                                    file_path       = file_path,
+                                    document_type   = DocumentType.TXT,
+                                    title           = file_path.stem,
+                                    created_date    = created_time,
+                                    modified_date   = modified_time,
+                                    file_size_bytes = file_size,
+                                    num_pages       = estimated_pages,
+                                    extra           = {"encoding"    : encoding,
+                                                       "num_lines"   : num_lines,
+                                                       "text_length" : text_length,
+                                                      }
+                                   )
+        
+        return metadata
+
+    
+    def read_lines(self, file_path: Path, start_line: int = 0, end_line: Optional[int] = None, encoding: Optional[str] = None) -> list[str]:
+        """
+        Read specific lines from file
+        
+        Arguments:
+        -----------
+            file_path  { Path } : Path to text file
+
+            start_line { int }  : Starting line (0-indexed)
+            
+            end_line   { int }  : Ending line (None = end of file)
+            
+            encoding   { str }  : File encoding (None = auto-detect)
+        
+        Returns:
+        --------
+                { list }        : List of lines
+        """
+        if encoding is None:
+            encoding = self.detect_encoding(file_path)
+        
+        try:
+            with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+                lines = f.readlines()
+            
+            if end_line is None:
+                return lines[start_line:]
+            
+            else:
+                return lines[start_line:end_line]
+        
+        except Exception as e:
+            self.logger.error(f"Failed to read lines: {repr(e)}")
+            raise TextEncodingError(str(file_path), encoding = encoding, original_error = e)
+    
+
+    def count_lines(self, file_path: Path, encoding: Optional[str] = None) -> int:
+        """
+        Count number of lines in file
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to text file
+            
+            encoding   { str } : File encoding (None = auto-detect)
+        
+        Returns:
+        --------
+                { int }        : Number of lines
+        """
+        if encoding is None:
+            encoding = self.detect_encoding(file_path)
+        
+        try:
+            with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+                return sum(1 for _ in f)
+        
+        except Exception as e:
+            self.logger.error(f"Failed to count lines: {repr(e)}")
+            raise TextEncodingError(str(file_path), encoding = encoding, original_error = e)
+    
+
+    def get_file_info(self, file_path: Path) -> dict:
+        """
+        Get comprehensive file information
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to text file
+        
+        Returns:
+        --------
+               { dict }        : Dictionary with file info
+        """
+        encoding = self.detect_encoding(file_path)
+        
+        with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+            content = f.read()
+        
+        lines = content.split('\n')
+        
+        return {"encoding"        : encoding,
+                "size_bytes"      : file_path.stat().st_size,
+                "num_lines"       : len(lines),
+                "num_characters"  : len(content),
+                "num_words"       : len(content.split()),
+                "avg_line_length" : sum(len(line) for line in lines) / len(lines) if lines else 0,
+               }
+
+    
+    def is_empty(self, file_path: Path) -> bool:
+        """
+        Check if file is empty or contains only whitespace
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to text file
+        
+        Returns:
+        --------
+               { bool }        : True if empty
+        """
+        try:
+            # Check file size first
+            if file_path.stat().st_size == 0:
+                return True
+            
+            # Read and check content
+            encoding = self.detect_encoding(file_path)
+            
+            with open(file_path, 'r', encoding = encoding, errors = 'replace') as f:
+                content = f.read().strip()
+            
+            return len(content) == 0
+
+        except Exception as e:
+            self.logger.warning(f"Error checking if file is empty: {repr(e)}")
+            return True

document_parser/zip_handler.py

@@ -0,0 +1,716 @@
+# DEPENDENCIES
+import py7zr
+import zipfile
+import tarfile
+import rarfile
+from typing import List
+from typing import Dict
+from pathlib import Path
+from typing import Optional
+from tempfile import TemporaryDirectory
+from config.settings import get_settings
+from utils.file_handler import FileHandler
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import ArchiveException
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class ArchiveHandler:
+    """
+    Comprehensive archive file handler supporting multiple formats
+    ZIP, TAR, RAR, 7Z with recursive extraction and validation
+    """
+    # Supported archive formats and their handlers
+    SUPPORTED_FORMATS = {'.zip' : 'zip',
+                         '.tar' : 'tar', 
+                         '.gz'  : 'tar',
+                         '.tgz' : 'tar',
+                         '.rar' : 'rar',
+                         '.7z'  : '7z',
+                        }
+
+    
+    def __init__(self, max_size_mb: int = 2048, max_files: int = 10000, allow_recursive: bool = True):
+        """
+        Initialize archive handler
+        
+        Arguments:
+        ----------
+            max_size_mb     { int }  : Maximum uncompressed size in MB
+
+            max_files       { int }  : Maximum number of files to extract
+            
+            allow_recursive { bool } : Allow nested archives
+        """
+        self.logger                = logger
+        self.max_size_bytes        = max_size_mb * 1024 * 1024
+        self.max_files             = max_files
+        self.allow_recursive       = allow_recursive
+        self.extracted_files_count = 0
+        self.total_extracted_size  = 0
+        self._temp_dirs            = list()
+    
+
+    @handle_errors(error_type = ArchiveException, log_error = True, reraise = True)
+    def extract_archive(self, archive_path: Path, extract_dir: Optional[Path] = None, flatten_structure: bool = False, filter_extensions: Optional[List[str]] = None) -> List[Path]:
+        """
+        Extract archive and return list of extracted file paths
+        
+        Arguments:
+        ----------
+            archive_path      { Path } : Path to archive file
+            
+            extract_dir       { Path } : Directory to extract to (None = temp directory)
+            
+            flatten_structure { bool } : Ignore directory structure, extract all files to root
+            
+            filter_extensions { list } : Only extract files with these extensions
+            
+        Returns:
+        --------
+                   { list }            : List of paths to extracted files
+        """
+        archive_path = Path(archive_path)
+        
+        if not archive_path.exists():
+            raise ArchiveException(f"Archive file not found: {archive_path}")
+        
+        # Validate archive size
+        self._validate_archive_size(archive_path = archive_path)
+        
+        # Determine extraction directory
+        if extract_dir is None:
+            temp_dir    = TemporaryDirectory()
+            extract_dir = Path(temp_dir.name)
+            # Keep reference to prevent cleanup
+            self._temp_dirs.append(temp_dir) 
+
+        else:
+            extract_dir = Path(extract_dir)
+            extract_dir.mkdir(parents = True, exist_ok = True)
+        
+        self.logger.info(f"Extracting archive: {archive_path} to {extract_dir}")
+        
+        # Reset counters
+        self.extracted_files_count = 0
+        self.total_extracted_size  = 0
+        
+        # Extract based on format
+        archive_format             = self._detect_archive_format(archive_path = archive_path)
+
+        extracted_files            = self._extract_by_format(archive_path      = archive_path, 
+                                                             extract_dir       = extract_dir, 
+                                                             format            = archive_format,  
+                                                             flatten_structure = flatten_structure, 
+                                                             filter_extensions = filter_extensions,
+                                                            )
+        
+        self.logger.info(f"Extracted {len(extracted_files)} files from {archive_path} ({self.total_extracted_size} bytes)")
+        
+        return extracted_files
+    
+
+    def _extract_by_format(self, archive_path: Path, extract_dir: Path, format: str, flatten_structure: bool, filter_extensions: Optional[List[str]]) -> List[Path]:
+        """
+        Extract archive based on format
+        """
+        try:
+            if (format == 'zip'):
+                return self._extract_zip(archive_path      = archive_path, 
+                                         extract_dir       = extract_dir, 
+                                         flatten_structure = flatten_structure, 
+                                         filter_extensions = filter_extensions,
+                                        )
+
+            elif (format == 'tar'):
+                return self._extract_tar(archive_path      = archive_path, 
+                                         extract_dir       = extract_dir, 
+                                         flatten_structure = flatten_structure, 
+                                         filter_extensions = filter_extensions,
+                                        )
+
+            elif (format == 'rar'):
+                return self._extract_rar(archive_path      = archive_path, 
+                                         extract_dir       = extract_dir, 
+                                         flatten_structure = flatten_structure, 
+                                         filter_extensions = filter_extensions,
+                                        )
+
+            elif (format == '7z'):
+                return self._extract_7z(archive_path      = archive_path, 
+                                        extract_dir       = extract_dir, 
+                                        flatten_structure = flatten_structure, 
+                                        filter_extensions = filter_extensions,
+                                       )
+
+            else:
+                raise ArchiveException(f"Unsupported archive format: {format}")
+
+        except Exception as e:
+            raise ArchiveException(f"Failed to extract {format} archive: {repr(e)}")
+    
+
+    def _extract_zip(self, archive_path: Path, extract_dir: Path, flatten_structure: bool, filter_extensions: Optional[List[str]]) -> List[Path]:
+        """
+        Extract ZIP archive
+        """
+        extracted_files = list()
+        
+        with zipfile.ZipFile(archive_path, 'r') as zip_ref:
+            # Validate files before extraction
+            file_list = zip_ref.namelist()
+
+            self._validate_file_count(file_count = len(file_list))
+            
+            for file_info in zip_ref.infolist():
+                # Use enhanced filtering
+                if self._should_extract_file(file_info.filename, filter_extensions):
+                    try:
+                        extracted_path = self._extract_zip_file(zip_ref           = zip_ref, 
+                                                                file_info         = file_info, 
+                                                                extract_dir       = extract_dir, 
+                                                                flatten_structure = flatten_structure,
+                                                               )
+                        
+                        if extracted_path:
+                            extracted_files.append(extracted_path)
+                    
+                    except Exception as e:
+                        self.logger.warning(f"Failed to extract {file_info.filename}: {repr(e)}")
+                        continue
+        
+        return extracted_files
+    
+
+    def _extract_zip_file(self, zip_ref, file_info, extract_dir: Path, flatten_structure: bool) -> Optional[Path]:
+        """
+        Extract single file from ZIP
+        """
+        # Skip directories
+        if file_info.filename.endswith('/'):
+            return None
+        
+        # Determine extraction path
+        if flatten_structure:
+            target_filename = Path(file_info.filename).name
+            extract_path    = extract_dir / self._safe_filename(filename = target_filename)
+        
+        else:
+            extract_path = extract_dir / self._safe_archive_path(archive_path = file_info.filename)
+        
+        # Ensure parent directory exists
+        extract_path.parent.mkdir(parents = True, exist_ok = True)
+        
+        # Check limits
+        self._check_extraction_limits(file_size = file_info.file_size)
+        
+        # Extract file
+        zip_ref.extract(file_info, extract_dir)
+        
+        # Rename if flattening structure
+        if flatten_structure:
+            original_path = extract_dir / file_info.filename
+            if (original_path != extract_path):
+                original_path.rename(extract_path)
+        
+        self.extracted_files_count += 1
+        self.total_extracted_size  += file_info.file_size
+        
+        return extract_path
+    
+
+    def _extract_tar(self, archive_path: Path, extract_dir: Path, flatten_structure: bool, filter_extensions: Optional[List[str]]) -> List[Path]:
+        """
+        Extract TAR archive
+        """
+        extracted_files = list()
+        
+        # Determine compression
+        mode = 'r'
+
+        if (archive_path.suffix.lower() in ['.gz', '.tgz']):
+            mode = 'r:gz'
+
+        elif (archive_path.suffix.lower() == '.bz2'):
+            mode = 'r:bz2'
+
+        elif (archive_path.suffix.lower() == '.xz'):
+            mode = 'r:xz'
+        
+        with tarfile.open(archive_path, mode) as tar_ref:
+            # Validate files before extraction
+            file_list = tar_ref.getnames()
+            self._validate_file_count(file_count = len(file_list))
+            
+            for member in tar_ref.getmembers():
+                if self._should_extract_file(member.name, filter_extensions) and member.isfile():
+                    try:
+                        extracted_path = self._extract_tar_file(tar_ref           = tar_ref, 
+                                                                member            = member, 
+                                                                extract_dir       = extract_dir, 
+                                                                flatten_structure = flatten_structure,
+                                                               )
+
+                        if extracted_path:
+                            extracted_files.append(extracted_path)
+                    
+                    except Exception as e:
+                        self.logger.warning(f"Failed to extract {member.name}: {repr(e)}")
+                        continue
+        
+        return extracted_files
+    
+
+    def _extract_tar_file(self, tar_ref, member, extract_dir: Path, flatten_structure: bool) -> Optional[Path]:
+        """
+        Extract single file from TAR
+        """
+        # Determine extraction path
+        if flatten_structure:
+            target_filename = Path(member.name).name
+            extract_path    = extract_dir / self._safe_filename(filename = target_filename)
+        
+        else:
+            extract_path = extract_dir / self._safe_archive_path(archive_path = member.name)
+        
+        # Ensure parent directory exists
+        extract_path.parent.mkdir(parents = True, exist_ok = True)
+        
+        # Check limits
+        self._check_extraction_limits(file_size = member.size)
+        
+        # Extract file
+        tar_ref.extract(member, extract_dir)
+        
+        # Rename if flattening structure
+        if flatten_structure:
+            original_path = extract_dir / member.name
+            
+            if (original_path != extract_path):
+                original_path.rename(extract_path)
+        
+        self.extracted_files_count += 1
+        self.total_extracted_size  += member.size
+        
+        return extract_path
+    
+
+    def _extract_rar(self, archive_path: Path, extract_dir: Path, flatten_structure: bool, filter_extensions: Optional[List[str]]) -> List[Path]:
+        """
+        Extract RAR archive
+        """
+        extracted_files = list()
+        
+        try:
+            with rarfile.RarFile(archive_path) as rar_ref:
+                # Validate files before extraction
+                file_list = rar_ref.namelist()
+
+                self._validate_file_count(file_count = len(file_list))
+                
+                for file_info in rar_ref.infolist():
+                    if (self._should_extract_file(filename = file_info.filename, filter_extensions = filter_extensions) and not file_info.isdir()):
+                        try:
+                            extracted_path = self._extract_rar_file(rar_ref           = rar_ref, 
+                                                                    file_info         = file_info, 
+                                                                    extract_dir       = extract_dir, 
+                                                                    flatten_structure = flatten_structure,
+                                                                   )
+                            if extracted_path:
+                                extracted_files.append(extracted_path)
+                        
+                        except Exception as e:
+                            self.logger.warning(f"Failed to extract {file_info.filename}: {repr(e)}")
+                            continue
+        
+        except rarfile.NotRarFile:
+            raise ArchiveException(f"Not a valid RAR file: {archive_path}")
+
+        except rarfile.BadRarFile:
+            raise ArchiveException(f"Corrupted RAR file: {archive_path}")
+        
+        return extracted_files
+    
+
+    def _extract_rar_file(self, rar_ref, file_info, extract_dir: Path, flatten_structure: bool) -> Optional[Path]:
+        """
+        Extract single file from RAR
+        """
+        # Determine extraction path
+        if flatten_structure:
+            target_filename = Path(file_info.filename).name
+            extract_path    = extract_dir / self._safe_filename(filename = target_filename)
+        
+        else:
+            extract_path = extract_dir / self._safe_archive_path(archive_path = file_info.filename)
+        
+        # Ensure parent directory exists
+        extract_path.parent.mkdir(parents = True, exist_ok = True)
+        
+        # Check limits
+        self._check_extraction_limits(file_size = file_info.file_size)
+        
+        # Extract file
+        rar_ref.extract(file_info.filename, extract_dir)
+        
+        # Rename if flattening structure
+        if flatten_structure:
+            original_path = extract_dir / file_info.filename
+            
+            if (original_path != extract_path):
+                original_path.rename(extract_path)
+        
+        self.extracted_files_count += 1
+        self.total_extracted_size  += file_info.file_size
+        
+        return extract_path
+    
+
+    def _extract_7z(self, archive_path: Path, extract_dir: Path, flatten_structure: bool, filter_extensions: Optional[List[str]]) -> List[Path]:
+        """
+        Extract 7Z archive
+        """
+        extracted_files = list()
+        
+        with py7zr.SevenZipFile(archive_path, 'r') as sevenz_ref:
+            # Get file list
+            file_list = sevenz_ref.getnames()
+            self._validate_file_count(file_count = len(file_list))
+            
+            # Extract all files
+            sevenz_ref.extractall(extract_dir)
+            
+            # Process extracted files
+            for filename in file_list:
+                if self._should_extract_file(filename = filename, filter_extensions = filter_extensions):
+                    original_path = extract_dir / filename
+
+                    if original_path.is_file():
+                        if flatten_structure:
+                            target_path = extract_dir / self._safe_filename(filename = Path(filename).name)
+                            
+                            if (original_path != target_path):
+                                original_path.rename(target_path)
+                                extracted_files.append(target_path)
+                            
+                            else:
+                                extracted_files.append(original_path)
+                        
+                        else:
+                            extracted_files.append(original_path)
+                        
+                        # Update counters
+                        self.extracted_files_count += 1
+                        self.total_extracted_size  += original_path.stat().st_size
+        
+        return extracted_files
+    
+    
+    def _is_system_file(self, filename: str) -> bool:
+        """
+        Check if file is a system/metadata file that should be skipped
+        """
+        system_patterns = ['__MACOSX',
+                           '.DS_Store', 
+                           'Thumbs.db',
+                           'desktop.ini',
+                           '~$',           # Temporary office files
+                           '._',           # macOS resource fork
+                           '#recycle',     # Recycle bin
+                           '@eaDir',       # Synology index
+                          ]
+                        
+        filename_str    = str(filename).lower()
+        path_parts      = Path(filename).parts
+        
+        # Check for system patterns in filename or path
+        for pattern in system_patterns:
+            if pattern.lower() in filename_str:
+                return True
+        
+        # Skip hidden files and directories (except current/parent dir references)
+        for part in path_parts:
+            if part.startswith(('.', '_')) and part not in ['.', '..']:
+                return True
+        
+        # Skip common backup and temporary files
+        temp_extensions = ['.tmp', '.temp', '.bak', '.backup']
+
+        if any(Path(filename).suffix.lower() == ext for ext in temp_extensions):
+            return True
+        
+        return False
+
+
+    def _should_extract_file(self, filename: str, filter_extensions: Optional[List[str]]) -> bool:
+        """
+        Check if file should be extracted based on filters and system files
+        """
+        # Skip system files and metadata
+        if self._is_system_file(filename):
+            self.logger.debug(f"Skipping system file: {filename}")
+            return False
+        
+        # Apply extension filters if provided
+        if filter_extensions is not None:
+            file_ext = Path(filename).suffix.lower()
+            return file_ext in [ext.lower() for ext in filter_extensions]
+        
+        return True
+
+
+    def _safe_archive_path(self, archive_path: str) -> Path:
+        """
+        Convert archive path to safe filesystem path
+        """
+        safe_parts = list()
+
+        for part in Path(archive_path).parts:
+            safe_parts.append(self._safe_filename(filename = part))
+        
+        return Path(*safe_parts)
+    
+
+    def _safe_filename(self, filename: str) -> str:
+        """
+        Ensure filename is safe for filesystem
+        """
+        return FileHandler.safe_filename(filename)
+    
+
+    def _detect_archive_format(self, archive_path: Path) -> str:
+        """
+        Detect archive format from file extension
+        """
+        suffix = archive_path.suffix.lower()
+        
+        for ext, format_type in self.SUPPORTED_FORMATS.items():
+            if ((suffix == ext) or (ext == '.tar' and suffix in ['.gz', '.tgz', '.bz2', '.xz'])):
+                return format_type
+        
+        raise ArchiveException(f"Unsupported archive format: {suffix}")
+    
+
+    def _validate_archive_size(self, archive_path: Path):
+        """
+        Validate archive size against limits
+        """
+        file_size = archive_path.stat().st_size
+        
+        if (file_size > self.max_size_bytes):
+            raise ArchiveException(f"Archive size {file_size} exceeds maximum {self.max_size_bytes}")
+    
+
+    def _validate_file_count(self, file_count: int):
+        """
+        Validate number of files in archive
+        """
+        if (file_count > self.max_files):
+            raise ArchiveException(f"Archive contains {file_count} files, exceeds maximum {self.max_files}")
+    
+
+    def _check_extraction_limits(self, file_size: int):
+        """
+        Check if extraction limits are exceeded
+        """
+        if (self.extracted_files_count >= self.max_files):
+            raise ArchiveException(f"Maximum file count ({self.max_files}) exceeded")
+        
+        if (self.total_extracted_size + file_size > self.max_size_bytes):
+            raise ArchiveException(f"Maximum extraction size ({self.max_size_bytes}) exceeded")
+    
+
+    def list_contents(self, archive_path: Path) -> List[Dict]:
+        """
+        List contents of archive without extraction
+        
+        Arguments:
+        ----------
+            archive_path { Path } : Path to archive file
+            
+        Returns:
+        --------
+                { list }          : List of file information dictionaries
+        """
+        archive_path = Path(archive_path)
+        format_type  = self._detect_archive_format(archive_path)
+        
+        try:
+            if (format_type == 'zip'):
+                return self._list_zip_contents(archive_path)
+
+            elif (format_type == 'tar'):
+                return self._list_tar_contents(archive_path)
+            
+            elif (format_type == 'rar'):
+                return self._list_rar_contents(archive_path)
+
+            elif (format_type == '7z'):
+                return self._list_7z_contents(archive_path)
+
+            else:
+                return []
+
+        except Exception as e:
+            self.logger.error(f"Failed to list archive contents: {repr(e)}")
+            return []
+    
+
+    def _list_zip_contents(self, archive_path: Path) -> List[Dict]:
+        """
+        List ZIP archive contents
+        """
+        contents = list()
+
+        with zipfile.ZipFile(archive_path, 'r') as zip_ref:
+            for file_info in zip_ref.infolist():
+                contents.append({'filename'      : file_info.filename,
+                                 'file_size'     : file_info.file_size,
+                                 'compress_size' : file_info.compress_size,
+                                 'is_dir'        : file_info.filename.endswith('/'),
+                               })
+        return contents
+    
+
+    def _list_tar_contents(self, archive_path: Path) -> List[Dict]:
+        """
+        List TAR archive contents
+        """
+        contents = list()
+        mode     = 'r'
+
+        if archive_path.suffix.lower() in ['.gz', '.tgz']:
+            mode = 'r:gz'
+        
+        with tarfile.open(archive_path, mode) as tar_ref:
+            for member in tar_ref.getmembers():
+                contents.append({'filename'  : member.name,
+                                 'file_size' : member.size,
+                                 'is_dir'    : member.isdir(),
+                                 'is_file'   : member.isfile(),
+                               })
+        return contents
+    
+
+    def _list_rar_contents(self, archive_path: Path) -> List[Dict]:
+        """
+        List RAR archive contents
+        """
+        contents = list()
+
+        with rarfile.RarFile(archive_path) as rar_ref:
+            for file_info in rar_ref.infolist():
+                contents.append({'filename'      : file_info.filename,
+                                 'file_size'     : file_info.file_size,
+                                 'compress_size' : file_info.compress_size,
+                                 'is_dir'        : file_info.isdir(),
+                               })
+        return contents
+    
+
+    def _list_7z_contents(self, archive_path: Path) -> List[Dict]:
+        """
+        List 7Z archive contents
+        """
+        contents = list()
+
+        with py7zr.SevenZipFile(archive_path, 'r') as sevenz_ref:
+            for filename in sevenz_ref.getnames():
+                # 7z doesn't provide detailed file info in listing
+                contents.append({'filename'  : filename,
+                                 'file_size' : 0,
+                                 'is_dir'    : filename.endswith('/'),
+                               })
+        return contents
+    
+
+    def is_supported_archive(self, file_path: Path) -> bool:
+        """
+        Check if file is a supported archive format
+        
+        Arguments:
+        ----------
+            file_path { Path } : Path to file
+            
+        Returns:
+        --------
+               { bool }        : True if supported archive format
+        """
+        try:
+            self._detect_archive_format(file_path)
+            return True
+
+        except ArchiveException:
+            return False
+    
+
+    def get_supported_formats(self) -> List[str]:
+        """
+        Get list of supported archive formats
+        
+        Returns:
+        --------
+            { list }    : List of supported file extensions
+        """
+        return list(self.SUPPORTED_FORMATS.keys())
+
+
+# Global archive handler instance
+_global_archive_handler = None
+
+
+def get_archive_handler() -> ArchiveHandler:
+    """
+    Get global archive handler instance (singleton)
+    
+    Returns:
+    --------
+        { ArchiveHandler }    : ArchiveHandler instance
+    """
+    global _global_archive_handler
+
+    if _global_archive_handler is None:
+        _global_archive_handler = ArchiveHandler()
+
+    return _global_archive_handler
+
+
+def extract_archive(archive_path: Path, **kwargs) -> List[Path]:
+    """
+    Convenience function for archive extraction
+    
+    Arguments:
+    ----------
+        archive_path { Path } : Path to archive file
+
+        **kwargs              : Additional arguments for ArchiveHandler
+        
+    Returns:
+    --------
+              { list }        : List of extracted file paths
+    """
+    handler = get_archive_handler()
+
+    return handler.extract_archive(archive_path, **kwargs)
+
+
+def is_archive_file(file_path: Path) -> bool:
+    """
+    Check if file is a supported archive
+    
+    Arguments:
+    ----------
+        file_path { Path } : Path to file
+        
+    Returns:
+    --------
+            { bool }       : True if supported archive
+    """
+    handler = get_archive_handler()
+
+    return handler.is_supported_archive(file_path)

embeddings/batch_processor.py

@@ -0,0 +1,365 @@
+# DEPENDENCIES
+import numpy as np
+from typing import List
+from typing import Optional
+from numpy.typing import NDArray
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import EmbeddingError
+from chunking.token_counter import get_token_counter
+from sentence_transformers import SentenceTransformer
+from utils.helpers import BatchProcessor as BaseBatchProcessor
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class BatchProcessor:
+    """
+    Efficient batch processing for embeddings: Handles large batches with memory optimization and progress tracking
+    """
+    def __init__(self):
+        self.logger           = logger
+        self.base_processor   = BaseBatchProcessor()
+        
+        # Batch processing statistics
+        self.total_batches    = 0
+        self.total_texts      = 0
+        self.failed_batches   = 0
+    
+
+    @handle_errors(error_type = EmbeddingError, log_error = True, reraise = True)
+    def process_embeddings_batch(self, model: SentenceTransformer, texts: List[str], batch_size: Optional[int] = None, normalize: bool = True, **kwargs) -> List[NDArray]:
+        """
+        Process embeddings in optimized batches
+        
+        Arguments:
+        ----------
+            model      { SentenceTransformer } : Embedding model
+
+            texts             { list }         : List of texts to embed
+            
+            batch_size        { int }          : Batch size (default from settings)
+            
+            normalize         { bool }         : Normalize embeddings
+            
+            **kwargs                           : Additional model.encode parameters
+        
+        Returns:
+        --------
+                      { list }                 : List of embedding vectors
+        """
+        if not texts:
+            return []
+        
+        batch_size = batch_size or settings.EMBEDDING_BATCH_SIZE
+        
+        self.logger.debug(f"Processing {len(texts)} texts in batches of {batch_size}")
+        
+        try:
+            # Use model's built-in batching with optimization
+            embeddings          = model.encode(texts,
+                                               batch_size           = batch_size,
+                                               normalize_embeddings = normalize,
+                                               show_progress_bar    = False,
+                                               convert_to_numpy     = True,
+                                               **kwargs
+                                              )
+            
+            # Update statistics
+            self.total_batches += ((len(texts) + batch_size - 1) // batch_size)
+            self.total_texts   += len(texts)
+            
+            self.logger.debug(f"Successfully generated {len(embeddings)} embeddings")
+            
+            # Convert to list of arrays
+            return list(embeddings)  
+            
+        except Exception as e:
+            self.failed_batches += 1
+            self.logger.error(f"Batch embedding failed: {repr(e)}")
+            raise EmbeddingError(f"Batch processing failed: {repr(e)}")
+    
+
+    def process_embeddings_with_fallback(self, model: SentenceTransformer, texts: List[str], batch_size: Optional[int] = None, normalize: bool = True) -> List[NDArray]:
+        """
+        Process embeddings with automatic batch size reduction on failure
+        
+        Arguments:
+        ----------
+            model      { SentenceTransformer } : Embedding model
+
+            texts      { list }                : List of texts
+            
+            batch_size { int }                 : Initial batch size
+            
+            normalize  { bool }                : Normalize embeddings
+        
+        Returns:
+        --------
+                 { list }                      : List of embeddings
+        """
+        batch_size = batch_size or settings.EMBEDDING_BATCH_SIZE
+        
+        try:
+            return self.process_embeddings_batch(model      = model,
+                                                 texts      = texts,
+                                                 batch_size = batch_size,
+                                                 normalize  = normalize,
+                                                )
+        
+        except (MemoryError, RuntimeError) as e:
+            self.logger.warning(f"Batch size {batch_size} failed, reducing to {batch_size // 2}")
+            
+            # Reduce batch size and retry
+            return self.process_embeddings_batch(model      = model,
+                                                 texts      = texts,
+                                                 batch_size = batch_size // 2,
+                                                 normalize  = normalize,
+                                                )
+    
+
+    def split_into_optimal_batches(self, texts: List[str], target_batch_size: int, max_batch_size: int = 1000) -> List[List[str]]:
+        """
+        Split texts into optimal batches considering token counts
+        
+        Arguments:
+        ----------
+            texts            { list } : List of texts
+
+            target_batch_size { int } : Target batch size in texts
+            
+            max_batch_size    { int } : Maximum batch size to allow
+        
+        Returns:
+        --------
+                       { list }       : List of text batches
+        """
+        if not texts:
+            return []
+        
+        token_counter  = get_token_counter()
+        batches        = list()
+        current_batch  = list()
+        current_tokens = 0
+        
+        # Estimate tokens per text (average of first 10 or all if less)
+        sample_size    = min(10, len(texts))
+        sample_tokens  = [token_counter.count_tokens(text) for text in texts[:sample_size]]
+        avg_tokens     = sum(sample_tokens) / len(sample_tokens) if sample_tokens else 100
+        
+        # Target tokens per batch (approximate)
+        target_tokens  = target_batch_size * avg_tokens
+        
+        for text in texts:
+            text_tokens = token_counter.count_tokens(text)
+            
+            # If single text is too large, put it in its own batch
+            if (text_tokens > (target_tokens * 0.8)):
+                if current_batch:
+                    batches.append(current_batch)
+                    current_batch  = list()
+                    current_tokens = 0
+                
+                batches.append([text])
+                continue
+            
+            # Check if adding this text would exceed limits
+            if (((current_tokens + text_tokens) > target_tokens) and current_batch) or (len(current_batch) >= max_batch_size):
+                batches.append(current_batch)
+                current_batch  = list()
+                current_tokens = 0
+            
+            current_batch.append(text)
+            current_tokens += text_tokens
+        
+        # Add final batch
+        if current_batch:
+            batches.append(current_batch)
+        
+        self.logger.debug(f"Split {len(texts)} texts into {len(batches)} optimal batches")
+        
+        return batches
+    
+
+    def process_batches_with_progress(self, model: SentenceTransformer, texts: List[str], batch_size: Optional[int] = None, progress_callback: Optional[callable] = None, **kwargs) -> List[NDArray]:
+        """
+        Process batches with progress reporting
+        
+        Arguments:
+        ----------
+            model            { SentenceTransformer } : Embedding model
+
+            texts            { list }                : List of texts
+            
+            batch_size       { int }                 : Batch size
+            
+            progress_callback { callable }           : Callback for progress updates
+            
+            **kwargs                                 : Additional parameters
+        
+        Returns:
+        --------
+                         { list }                    : List of embeddings
+        """
+        if not texts:
+            return []
+        
+        batch_size     = batch_size or settings.EMBEDDING_BATCH_SIZE
+        
+        # Split into batches
+        batches        = self.split_into_optimal_batches(texts             = texts, 
+                                                         target_batch_size = batch_size,
+                                                        )
+        
+        all_embeddings = list()
+        
+        for i, batch_texts in enumerate(batches):
+            if progress_callback:
+                progress = (i / len(batches)) * 100
+                progress_callback(progress, f"Processing batch {i + 1}/{len(batches)}")
+            
+            try:
+                batch_embeddings = self.process_embeddings_batch(model      = model,
+                                                                 texts      = batch_texts,
+                                                                 batch_size = len(batch_texts),
+                                                                 **kwargs
+                                                                )
+                
+                all_embeddings.extend(batch_embeddings)
+                
+                self.logger.debug(f"Processed batch {i + 1}/{len(batches)}: {len(batch_texts)} texts")
+            
+            except Exception as e:
+                self.logger.error(f"Failed to process batch {i + 1}: {repr(e)}")
+                
+                # Add None placeholders for failed batch
+                all_embeddings.extend([None] * len(batch_texts))
+        
+        if progress_callback:
+            progress_callback(100, "Embedding complete")
+        
+        return all_embeddings
+    
+
+    def validate_embeddings_batch(self, embeddings: List[NDArray], expected_count: int) -> bool:
+        """
+        Validate a batch of embeddings
+        
+        Arguments:
+        ----------
+            embeddings     { list } : List of embedding vectors
+
+            expected_count { int }  : Expected number of embeddings
+        
+        Returns:
+        --------
+                   { bool }         : True if valid
+        """
+        if (len(embeddings) != expected_count):
+            self.logger.error(f"Embedding count mismatch: expected {expected_count}, got {len(embeddings)}")
+            return False
+        
+        valid_count = 0
+        
+        for i, emb in enumerate(embeddings):
+            if emb is None:
+                self.logger.warning(f"None embedding at index {i}")
+                continue
+            
+            if not isinstance(emb, np.ndarray):
+                self.logger.warning(f"Invalid embedding type at index {i}: {type(emb)}")
+                continue
+            
+            if (emb.ndim != 1):
+                self.logger.warning(f"Invalid embedding dimension at index {i}: {emb.ndim}")
+                continue
+            
+            if np.any(np.isnan(emb)):
+                self.logger.warning(f"NaN values in embedding at index {i}")
+                continue
+            
+            valid_count += 1
+        
+        validity_ratio = valid_count / expected_count
+        
+        if (validity_ratio < 0.9):
+            self.logger.warning(f"Low embedding validity: {valid_count}/{expected_count} ({validity_ratio:.1%})")
+            return False
+        
+        return True
+    
+
+    def get_processing_stats(self) -> dict:
+        """
+        Get batch processing statistics
+        
+        Returns:
+        --------
+            { dict }    : Statistics dictionary
+        """
+        success_rate = ((self.total_batches - self.failed_batches) / self.total_batches * 100) if (self.total_batches > 0) else 100
+        
+        stats        = {"total_batches"    : self.total_batches,
+                        "total_texts"      : self.total_texts,
+                        "failed_batches"   : self.failed_batches,
+                        "success_rate"     : success_rate,
+                        "avg_batch_size"   : self.total_texts / self.total_batches if (self.total_batches > 0) else 0,
+                       }
+        
+        return stats
+    
+
+    def reset_stats(self):
+        """
+        Reset processing statistics
+        """
+        self.total_batches  = 0
+        self.total_texts    = 0
+        self.failed_batches = 0
+        
+        self.logger.debug("Reset batch processing statistics")
+
+
+# Global batch processor instance
+_batch_processor = None
+
+
+def get_batch_processor() -> BatchProcessor:
+    """
+    Get global batch processor instance
+    
+    Returns:
+    --------
+        { BatchProcessor } : BatchProcessor instance
+    """
+    global _batch_processor
+
+    if _batch_processor is None:
+        _batch_processor = BatchProcessor()
+    
+    return _batch_processor
+
+
+def process_embeddings_batch(model: SentenceTransformer, texts: List[str], **kwargs) -> List[NDArray]:
+    """
+    Convenience function for batch embedding
+    
+    Arguments:
+    ----------
+        model { SentenceTransformer } : Embedding model
+
+        texts { list }                : List of texts
+        
+        **kwargs                      : Additional arguments
+    
+    Returns:
+    --------
+             { list }                 : List of embeddings
+    """
+    processor = get_batch_processor()
+
+    return processor.process_embeddings_batch(model, texts, **kwargs)

embeddings/bge_embedder.py

@@ -0,0 +1,351 @@
+# DEPENDENCIES
+import time
+import numpy as np
+from typing import List
+from typing import Optional
+from numpy.typing import NDArray
+from config.models import DocumentChunk
+from config.settings import get_settings
+from config.models import EmbeddingRequest
+from config.models import EmbeddingResponse
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import EmbeddingError
+from embeddings.model_loader import get_model_loader
+from embeddings.batch_processor import BatchProcessor
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class BGEEmbedder:
+    """
+    BGE (BAAI General Embedding) model wrapper: Optimized for BAAI/bge models with proper normalization and batching
+    """
+    def __init__(self, model_name: Optional[str] = None, device: Optional[str] = None):
+        """
+        Initialize BGE embedder
+        
+        Arguments:
+        ----------
+            model_name { str } : BGE model name (default from settings)
+            
+            device     { str } : Device to run on
+        """
+        self.logger          = logger
+        self.model_name      = model_name or settings.EMBEDDING_MODEL
+        self.device          = device
+        
+        # Initialize components
+        self.model_loader    = get_model_loader()
+        self.batch_processor = BatchProcessor()
+        
+        # Load model
+        self.model           = self.model_loader.load_model(model_name = self.model_name, 
+                                                            device     = self.device,
+                                                           )
+        
+        # Get model info
+        self.embedding_dim   = self.model.get_sentence_embedding_dimension()
+        self.supports_batch  = True
+        
+        self.logger.info(f"Initialized BGEEmbedder: model={self.model_name}, dim={self.embedding_dim}, device={self.model.device}")
+    
+
+    @handle_errors(error_type = EmbeddingError, log_error = True, reraise = True)
+    def embed_text(self, text: str, normalize: bool = True) -> NDArray:
+        """
+        Embed single text string
+        
+        Arguments:
+        ----------
+            text      { str } : Input text
+
+            normalize { bool } : Normalize embeddings to unit length
+        
+        Returns:
+        --------
+               { NDArray }     : Embedding vector
+        """
+        if not text or not text.strip():
+            raise EmbeddingError("Cannot embed empty text")
+        
+        try:
+            # Encode single text
+            embedding = self.model.encode([text],
+                                          normalize_embeddings = normalize,
+                                          show_progress_bar    = False,
+                                         )
+            
+            # Return single vector
+            return embedding[0]  
+            
+        except Exception as e:
+            self.logger.error(f"Failed to embed text: {repr(e)}")
+            raise EmbeddingError(f"Text embedding failed: {repr(e)}")
+    
+
+    @handle_errors(error_type = EmbeddingError, log_error = True, reraise = True)
+    def embed_texts(self, texts: List[str], batch_size: Optional[int] = None, normalize: bool = True) -> List[NDArray]:
+        """
+        Embed multiple texts with batching
+        
+        Arguments:
+        ----------
+            texts      { list } : List of text strings
+
+            batch_size { int }  : Batch size (default from settings)
+            
+            normalize  { bool } : Normalize embeddings
+        
+        Returns:
+        --------
+                 { list }       : List of embedding vectors
+        """
+        if not texts:
+            return []
+        
+        # Filter empty texts
+        valid_texts = [t for t in texts if t and t.strip()]
+        
+        if (len(valid_texts) != len(texts)):
+            self.logger.warning(f"Filtered {len(texts) - len(valid_texts)} empty texts")
+        
+        if not valid_texts:
+            return []
+        
+        batch_size = batch_size or settings.EMBEDDING_BATCH_SIZE
+        
+        try:
+            # Use batch processing for efficiency
+            embeddings = self.batch_processor.process_embeddings_batch(model      = self.model,
+                                                                       texts      = valid_texts,
+                                                                       batch_size = batch_size,
+                                                                       normalize  = normalize,
+                                                                      )
+            
+            self.logger.debug(f"Generated {len(embeddings)} embeddings for {len(texts)} texts")
+            
+            return embeddings
+            
+        except Exception as e:
+            self.logger.error(f"Batch embedding failed: {repr(e)}")
+            raise EmbeddingError(f"Batch embedding failed: {repr(e)}")
+    
+
+    @handle_errors(error_type = EmbeddingError, log_error = True, reraise = True)
+    def embed_chunks(self, chunks: List[DocumentChunk], batch_size: Optional[int] = None, normalize: bool = True) -> List[DocumentChunk]:
+        """
+        Embed document chunks and update them with embeddings
+        
+        Arguments:
+        ----------
+            chunks     { list } : List of DocumentChunk objects
+
+            batch_size { int }  : Batch size
+            
+            normalize  { bool } : Normalize embeddings
+        
+        Returns:
+        --------
+                 { list }       : Chunks with embeddings added
+        """
+        if not chunks:
+            return []
+        
+        # Extract texts from chunks
+        texts      = [chunk.text for chunk in chunks]
+        
+        # Generate embeddings
+        embeddings = self.embed_texts(texts       = texts,
+                                      batch_size  = batch_size,
+                                      normalize   = normalize,
+                                     )
+        
+        # Update chunks with embeddings
+        for chunk, embedding in zip(chunks, embeddings):
+            # Convert numpy to list for serialization
+            chunk.embedding = embedding.tolist()  
+        
+        self.logger.info(f"Embedded {len(chunks)} document chunks")
+        
+        return chunks
+    
+
+    def process_embedding_request(self, request: EmbeddingRequest) -> EmbeddingResponse:
+        """
+        Process embedding request from API
+        
+        Arguments:
+        ----------
+            request { EmbeddingRequest } : Embedding request
+        
+        Returns:
+        --------
+            { EmbeddingResponse }        : Embedding response
+        """
+        start_time      = time.time()
+        
+        # Generate embeddings
+        embeddings      = self.embed_texts(texts      = request.texts,
+                                           batch_size = request.batch_size,
+                                           normalize  = request.normalize,
+                                          )
+        
+        # Convert to milliseconds
+        processing_time = (time.time() - start_time) * 1000  
+        
+        # Convert to list for serialization
+        embedding_list  = [emb.tolist() for emb in embeddings]
+        
+        response        = EmbeddingResponse(embeddings         = embedding_list,
+                                            dimension          = self.embedding_dim,
+                                            num_embeddings     = len(embeddings),
+                                            processing_time_ms = processing_time,
+                                           )
+         
+        return response
+    
+
+    def get_embedding_dimension(self) -> int:
+        """
+        Get embedding dimension
+        
+        Returns:
+        --------
+            { int }    : Embedding vector dimension
+        """
+        return self.embedding_dim
+    
+
+    def cosine_similarity(self, emb1: NDArray, emb2: NDArray) -> float:
+        """
+        Calculate cosine similarity between two embeddings
+        
+        Arguments:
+        ----------
+            emb1 { NDArray } : First embedding
+
+            emb2 { NDArray } : Second embedding
+        
+        Returns:
+        --------
+               { float }     : Cosine similarity (-1 to 1)
+        """
+        # Ensure embeddings are normalized
+        emb1_norm = emb1 / np.linalg.norm(emb1)
+        emb2_norm = emb2 / np.linalg.norm(emb2)
+        
+        return float(np.dot(emb1_norm, emb2_norm))
+    
+
+    def validate_embedding(self, embedding: NDArray) -> bool:
+        """
+        Validate embedding vector
+        
+        Arguments:
+        ----------
+            embedding { NDArray } : Embedding vector
+        
+        Returns:
+        --------
+                 { bool }         : True if valid
+        """
+        if (embedding is None):
+            return False
+        
+        if (not isinstance(embedding, np.ndarray)):
+            return False
+        
+        if (embedding.shape != (self.embedding_dim,)):
+            return False
+        
+        if (np.all(embedding == 0)):
+            return False
+        
+        if (np.any(np.isnan(embedding))):
+            return False
+        
+        return True
+    
+
+    def get_model_info(self) -> dict:
+        """
+        Get embedder information
+        
+        Returns:
+        --------
+            { dict }    : Embedder information
+        """
+        return {"model_name"        : self.model_name,
+                "embedding_dim"     : self.embedding_dim,
+                "device"            : str(self.model.device),
+                "supports_batch"    : self.supports_batch,
+                "normalize_default" : True,
+               }
+
+
+# Global embedder instance
+_embedder = None
+
+
+def get_embedder(model_name: Optional[str] = None, device: Optional[str] = None) -> BGEEmbedder:
+    """
+    Get global embedder instance
+    
+    Arguments:
+    ----------
+        model_name { str } : Model name
+        
+        device     { str } : Device
+    
+    Returns:
+    --------
+        { BGEEmbedder }    : BGEEmbedder instance
+    """
+    global _embedder
+
+    if _embedder is None:
+        _embedder = BGEEmbedder(model_name, device)
+    
+    return _embedder
+
+
+def embed_texts(texts: List[str], **kwargs) -> List[NDArray]:
+    """
+    Convenience function to embed texts
+    
+    Arguments:
+    ----------
+        texts { list } : List of texts
+        
+        **kwargs       : Additional arguments for BGEEmbedder
+    
+    Returns:
+    --------
+             { list }  : List of embeddings
+    """
+    embedder = get_embedder()
+
+    return embedder.embed_texts(texts, **kwargs)
+
+
+def embed_chunks(chunks: List[DocumentChunk], **kwargs) -> List[DocumentChunk]:
+    """
+    Convenience function to embed document chunks
+    
+    Arguments:
+    ----------
+        chunks { list } : List of DocumentChunk objects
+        
+        **kwargs        : Additional arguments
+    
+    Returns:
+    --------
+              { list }  : Chunks with embeddings
+    """
+    embedder = get_embedder()
+
+    return embedder.embed_chunks(chunks, **kwargs)

embeddings/embedding_cache.py

@@ -0,0 +1,331 @@
+# DEPENDENCIES
+import numpy as np
+from typing import List
+from typing import Optional
+from numpy.typing import NDArray
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.cache_manager import EmbeddingCache as BaseEmbeddingCache
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class EmbeddingCache:
+    """
+    Embedding cache with numpy array support and statistics: Wraps the base cache with embedding-specific features
+    """
+    def __init__(self, max_size: int = None, ttl: int = None):
+        """
+        Initialize embedding cache
+        
+        Arguments:
+        ----------
+            max_size { int } : Maximum cache size
+            
+            ttl      { int } : Time to live in seconds
+        """
+        self.logger               = logger
+        self.max_size             = max_size or settings.CACHE_MAX_SIZE
+        self.ttl                  = ttl or settings.CACHE_TTL
+        
+        # Initialize base cache
+        self.base_cache           = BaseEmbeddingCache(max_size = self.max_size,
+                                                       ttl      = self.ttl,
+                                                      )
+        
+        # Enhanced statistics
+        self.hits                 = 0
+        self.misses               = 0
+        self.embeddings_generated = 0
+        
+        self.logger.info(f"Initialized EmbeddingCache: max_size={self.max_size}, ttl={self.ttl}")
+    
+
+    def get_embedding(self, text: str) -> Optional[NDArray]:
+        """
+        Get embedding from cache
+        
+        Arguments:
+        ----------
+            text { str }   : Input text
+        
+        Returns:
+        --------
+               { NDArray } : Cached embedding or None
+        """
+        cached = self.base_cache.get_embedding(text)
+        
+        if cached is not None:
+            self.hits += 1
+            
+            # Convert list back to numpy array
+            return np.array(cached)
+        
+        else:
+            self.misses += 1
+            return None
+    
+
+    def set_embedding(self, text: str, embedding: NDArray):
+        """
+        Store embedding in cache
+        
+        Arguments:
+        ----------
+            text      { str }     : Input text
+
+            embedding { NDArray } : Embedding vector
+        """
+        # Convert numpy array to list for serialization
+        embedding_list             = embedding.tolist()
+        
+        self.base_cache.set_embedding(text, embedding_list)
+
+        self.embeddings_generated += 1
+    
+
+    def batch_get_embeddings(self, texts: List[str]) -> tuple[List[Optional[NDArray]], List[str]]:
+        """
+        Get multiple embeddings from cache
+        
+        Arguments:
+        ----------
+            texts { list } : List of texts
+        
+        Returns:
+        --------
+             { tuple }     : Tuple of (cached_embeddings, missing_texts)
+        """
+        cached_embeddings = list()
+        missing_texts     = list()
+        
+        for text in texts:
+            embedding = self.get_embedding(text)
+            
+            if embedding is not None:
+                cached_embeddings.append(embedding)
+            
+            else:
+                missing_texts.append(text)
+                cached_embeddings.append(None)
+        
+        return cached_embeddings, missing_texts
+    
+
+    def batch_set_embeddings(self, texts: List[str], embeddings: List[NDArray]):
+        """
+        Store multiple embeddings in cache
+        
+        Arguments:
+        ----------
+            texts      { list } : List of texts
+
+            embeddings { list } : List of embedding vectors
+        """
+        if (len(texts) != len(embeddings)):
+            raise ValueError("Texts and embeddings must have same length")
+        
+        for text, embedding in zip(texts, embeddings):
+            self.set_embedding(text, embedding)
+    
+
+    def get_cached_embeddings(self, texts: List[str], embed_function: callable, batch_size: Optional[int] = None) -> List[NDArray]:
+        """
+        Smart embedding getter: uses cache for existing, generates for missing
+        
+        Arguments:
+        ----------
+            texts          { list }     : List of texts
+
+            embed_function { callable } : Function to generate embeddings for missing texts
+            
+            batch_size     { int }      : Batch size for generation
+        
+        Returns:
+        --------
+                       { list }         : List of embeddings
+        """
+        # Get cached embeddings
+        cached_embeddings, missing_texts = self.batch_get_embeddings(texts = texts)
+        
+        if not missing_texts:
+            self.logger.debug(f"All {len(texts)} embeddings found in cache")
+            return cached_embeddings
+        
+        # Generate missing embeddings
+        self.logger.info(f"Generating {len(missing_texts)} embeddings ({(len(missing_texts)/len(texts))*100:.1f}% cache miss)")
+        
+        missing_embeddings               = embed_function(missing_texts, batch_size = batch_size)
+        
+        # Store new embeddings in cache
+        self.batch_set_embeddings(missing_texts, missing_embeddings)
+        
+        # Combine results
+        result_embeddings = list()
+        missing_idx       = 0
+        
+        for emb in cached_embeddings:
+            if emb is not None:
+                result_embeddings.append(emb)
+            
+            else:
+                result_embeddings.append(missing_embeddings[missing_idx])
+                missing_idx += 1
+        
+        return result_embeddings
+    
+
+    def clear(self):
+        """
+        Clear entire cache
+        """
+        self.base_cache.clear()
+        self.hits                  = 0
+        self.misses                = 0
+        self.embeddings_generated  = 0
+        
+        self.logger.info("Cleared embedding cache")
+    
+
+    def get_stats(self) -> dict:
+        """
+        Get cache statistics
+        
+        Returns:
+        --------
+            { dict }    : Statistics dictionary
+        """
+        base_stats     = self.base_cache.get_stats()
+        
+        total_requests = self.hits + self.misses
+        hit_rate       = (self.hits / total_requests * 100) if (total_requests > 0) else 0
+        
+        stats = {**base_stats,
+                 "hits"                  : self.hits,
+                 "misses"                : self.misses,
+                 "hit_rate_percentage"   : hit_rate,
+                 "embeddings_generated"  : self.embeddings_generated,
+                 "cache_size"            : self.base_cache.cache.size(),
+                 "max_size"              : self.max_size,
+                }
+        
+        return stats
+    
+
+    def save_to_file(self, file_path: str) -> bool:
+        """
+        Save cache to file
+        
+        Arguments:
+        ----------
+            file_path { str } : Path to save file
+        
+        Returns:
+        --------
+               { bool }       : True if successful
+        """
+        return self.base_cache.save_to_file(file_path)
+    
+
+    def load_from_file(self, file_path: str) -> bool:
+        """
+        Load cache from file
+        
+        Arguments:
+        ----------
+            file_path { str } : Path to load file
+        
+        Returns:
+        --------
+               { bool }       : True if successful
+        """
+        return self.base_cache.load_from_file(file_path)
+    
+
+    def warm_cache(self, texts: List[str], embed_function: callable, batch_size: Optional[int] = None):
+        """
+        Pre-populate cache with embeddings
+        
+        Arguments:
+        ----------
+            texts            { list }   : List of texts to warm cache with
+
+            embed_function { callable } : Embedding generation function
+            
+            batch_size       { int }    : Batch size
+        """
+        # Check which texts are not in cache
+        _, missing_texts = self.batch_get_embeddings(texts = texts)
+        
+        if not missing_texts:
+            self.logger.info("Cache already warm for all texts")
+            return
+        
+        self.logger.info(f"Warming cache with {len(missing_texts)} embeddings")
+        
+        # Generate and cache embeddings
+        embeddings = embed_function(missing_texts, batch_size = batch_size)
+
+        self.batch_set_embeddings(missing_texts, embeddings)
+        
+        self.logger.info(f"Cache warming complete: added {len(missing_texts)} embeddings")
+
+
+# Global embedding cache instance
+_embedding_cache = None
+
+
+def get_embedding_cache() -> EmbeddingCache:
+    """
+    Get global embedding cache instance
+    
+    Returns:
+    --------
+        { EmbeddingCache } : EmbeddingCache instance
+    """
+    global _embedding_cache
+
+    if _embedding_cache is None:
+        _embedding_cache = EmbeddingCache()
+    
+    return _embedding_cache
+
+
+def cache_embeddings(texts: List[str], embeddings: List[NDArray]):
+    """
+    Convenience function to cache embeddings
+    
+    Arguments:
+    ----------
+        texts      { list } : List of texts
+
+        embeddings { list } : List of embeddings
+    """
+    cache = get_embedding_cache()
+
+    cache.batch_set_embeddings(texts, embeddings)
+
+
+def get_cached_embeddings(texts: List[str], embed_function: callable, **kwargs) -> List[NDArray]:
+    """
+    Convenience function to get cached embeddings
+    
+    Arguments:
+    ----------
+        texts          { list } : List of texts
+
+        embed_function { callable } : Embedding function
+        
+        **kwargs                   : Additional arguments
+    
+    Returns:
+    --------
+                 { list }          : List of embeddings
+    """
+    cache = get_embedding_cache()
+
+    return cache.get_cached_embeddings(texts, embed_function, **kwargs)

embeddings/model_loader.py

@@ -0,0 +1,327 @@
+# DEPENDENCIES
+import os
+import gc
+import torch
+from typing import Optional
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import EmbeddingError
+from sentence_transformers import SentenceTransformer
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class EmbeddingModelLoader:
+    """
+    Manages loading and caching of embedding models: Supports multiple models with efficient resource management
+    """
+    def __init__(self):
+        self.logger        = logger
+        self._loaded_model = None
+        self._model_name   = None
+        self._device       = None
+        
+        # Model cache for multiple models
+        self._model_cache  = dict()
+    
+
+    @handle_errors(error_type = EmbeddingError, log_error = True, reraise = True)
+    def load_model(self, model_name: Optional[str] = None, device: Optional[str] = None, force_reload: bool = False) -> SentenceTransformer:
+        """
+        Load embedding model with caching and device optimization
+        
+        Arguments:
+        ----------
+            model_name   { str }    : Name of model to load (default from settings)
+            
+            device       { str }    : Device to load on ('cpu', 'cuda', 'mps', 'auto')
+            
+            force_reload { bool }   : Force reload even if model is cached
+        
+        Returns:
+        --------
+            { SentenceTransformer } : Loaded model instance
+        
+        Raises:
+        -------
+            EmbeddingError          : If model loading fails
+        """
+        model_name = model_name or settings.EMBEDDING_MODEL
+        device     = self._resolve_device(device)
+        
+        # Check cache first
+        cache_key  = f"{model_name}_{device}"
+        
+        if ((not force_reload) and (cache_key in self._model_cache)):
+            self.logger.debug(f"Using cached model: {cache_key}")
+            
+            self._loaded_model = self._model_cache[cache_key]
+            self._model_name   = model_name
+            self._device       = device
+            
+            return self._loaded_model
+        
+        try:
+            self.logger.info(f"Loading embedding model: {model_name} on device: {device}")
+            
+            # Load model with optimized settings
+            model                        = SentenceTransformer(model_name,
+                                                               device       = device,
+                                                               cache_folder = os.path.expanduser("~/.cache/sentence_transformers"),
+                                                              )
+            
+            # Model-specific optimizations
+            model                        = self._optimize_model(model  = model, 
+                                                                device = device,
+                                                               )
+            
+            # Cache the model
+            self._model_cache[cache_key] = model
+            self._loaded_model           = model
+            self._model_name             = model_name
+            self._device                 = device
+            
+            # Log model info
+            self._log_model_info(model  = model, 
+                                 device = device,
+                                )
+            
+            self.logger.info(f"Successfully loaded model: {model_name}")
+            
+            return model
+            
+        except Exception as e:
+            self.logger.error(f"Failed to load model {model_name}: {repr(e)}")
+            raise EmbeddingError(f"Model loading failed: {repr(e)}")
+    
+
+    def _resolve_device(self, device: Optional[str] = None) -> str:
+        """
+        Resolve the best available device
+        
+        Arguments:
+        ----------
+            device { str } : Requested device
+        
+        Returns:
+        --------
+               { str }     : Actual device to use
+        """
+        if (device and (device != "auto")):
+            return device
+        
+        # Auto device selection
+        if (settings.EMBEDDING_DEVICE != "auto"):
+            return settings.EMBEDDING_DEVICE
+        
+        # Automatic detection
+        if torch.cuda.is_available():
+            return "cuda"
+        
+        elif torch.backends.mps.is_available():
+            return "mps"
+        
+        else:
+            return "cpu"
+    
+
+    def _optimize_model(self, model: SentenceTransformer, device: str) -> SentenceTransformer:
+        """
+        Apply optimizations to the model
+        
+        Arguments:
+        ----------
+            model  { SentenceTransformer } : Model to optimize
+
+            device { str }                 : Device model is on
+        
+        Returns:
+        --------
+            { SentenceTransformer }        : Optimized model
+        """
+        # Enable eval mode for inference
+        model.eval()
+        
+        # GPU optimizations
+        if (device == "cuda"):
+            # Use half precision for GPU if supported
+            try:
+                model = model.half()
+                self.logger.debug("Enabled half precision for GPU")
+            
+            except Exception as e:
+                self.logger.warning(f"Could not enable half precision: {repr(e)}")
+        
+        # Disable gradient computation
+        for param in model.parameters():
+            param.requires_grad = False
+        
+        return model
+    
+
+    def _log_model_info(self, model: SentenceTransformer, device: str):
+        """
+        Log detailed model information
+        
+        Arguments:
+        ----------
+            model  { SentenceTransformer } : Model to log info for
+
+            device { str }                 : Device model is on
+        """
+        try:
+            # Get model architecture info
+            if hasattr(model, '_modules'):
+                modules = list(model._modules.keys())
+            
+            else:
+                modules = ["unknown"]
+            
+            # Get embedding dimension
+            if hasattr(model, 'get_sentence_embedding_dimension'):
+                dimension = model.get_sentence_embedding_dimension()
+            
+            else:
+                dimension = "unknown"
+            
+            # Count parameters
+            total_params = sum(p.numel() for p in model.parameters())
+            
+            self.logger.info(f"Model Info: {len(modules)} modules, dimension={dimension}, parameters={total_params:,}, device={device}")
+                           
+        except Exception as e:
+            self.logger.debug(f"Could not get detailed model info: {repr(e)}")
+    
+
+    def get_loaded_model(self) -> Optional[SentenceTransformer]:
+        """
+        Get currently loaded model
+        
+        Returns:
+        --------
+            { SentenceTransformer } : Currently loaded model or None
+        """
+        return self._loaded_model
+    
+
+    def get_model_info(self) -> dict:
+        """
+        Get information about loaded model
+        
+        Returns:
+        --------
+            { dict }    : Model information dictionary
+        """
+        if self._loaded_model is None:
+            return {"loaded": False}
+        
+        info = {"loaded"       : True,
+                "model_name"   : self._model_name,
+                "device"       : self._device,
+                "cache_size"   : len(self._model_cache),
+               }
+        
+        try:
+            if hasattr(self._loaded_model, 'get_sentence_embedding_dimension'):
+                info["embedding_dimension"] = self._loaded_model.get_sentence_embedding_dimension()
+            
+            info["model_class"] = type(self._loaded_model).__name__
+            
+        except Exception as e:
+            self.logger.warning(f"Could not get detailed model info: {e}")
+        
+        return info
+    
+
+    def clear_cache(self, model_name: Optional[str] = None):
+        """
+        Clear model cache
+        
+        Arguments:
+        ----------
+            model_name { str } : Specific model to clear (None = all)
+        """
+        if model_name:
+            # Clear specific model from all devices
+            keys_to_remove = [k for k in self._model_cache.keys() if k.startswith(model_name)]
+            
+            for key in keys_to_remove:
+                del self._model_cache[key]
+            
+            self.logger.info(f"Cleared cache for model: {model_name}")
+        
+        else:
+            # Clear all cache
+            cache_size = len(self._model_cache)
+            self._model_cache.clear()
+            
+            self.logger.info(f"Cleared all model cache ({cache_size} models)")
+    
+
+    def unload_model(self):
+        """
+        Unload current model and free memory
+        """
+        if self._loaded_model:
+            model_name = self._model_name
+            
+            # Clear from cache
+            if self._model_name and self._device:
+                cache_key = f"{self._model_name}_{self._device}"
+                self._model_cache.pop(cache_key, None)
+            
+            # Clear references
+            self._loaded_model = None
+            self._model_name   = None
+            self._device       = None
+            
+            # Force garbage collection            
+            gc.collect()
+            
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            
+            self.logger.info(f"Unloaded model: {model_name}")
+
+
+# Global model loader instance
+_model_loader = None
+
+
+def get_model_loader() -> EmbeddingModelLoader:
+    """
+    Get global model loader instance (singleton)
+    
+    Returns:
+    --------
+        { EmbeddingModelLoader } : Model loader instance
+    """
+    global _model_loader
+
+    if _model_loader is None:
+        _model_loader = EmbeddingModelLoader()
+    
+    return _model_loader
+
+
+def load_embedding_model(model_name: Optional[str] = None, device: Optional[str] = None) -> SentenceTransformer:
+    """
+    Convenience function to load embedding model
+    
+    Arguments:
+    ----------
+        model_name { str } : Model name
+        
+        device     { str } : Device
+    
+    Returns:
+    --------
+        { SentenceTransformer } : Loaded model
+    """
+    loader = get_model_loader()
+
+    return loader.load_model(model_name, device)

evaluation/ragas_evaluator.py

@@ -0,0 +1,667 @@
+# DEPENDENCIES
+import os
+import math
+import logging
+import statistics
+from typing import Any
+from typing import List
+from typing import Dict
+from ragas import evaluate
+from typing import Optional
+from datasets import Dataset
+from datetime import datetime
+from ragas.metrics import faithfulness
+from config.settings import get_settings
+from ragas.metrics import context_recall
+from config.models import RAGASStatistics
+from config.models import RAGASExportData
+from ragas.metrics import answer_relevancy
+from ragas.metrics import context_precision
+from ragas.metrics import context_relevancy
+from ragas.metrics import answer_similarity
+from ragas.metrics import answer_correctness
+from config.logging_config import get_logger
+from ragas.metrics import context_utilization
+from config.models import RAGASEvaluationResult
+
+
+# Setup Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+# Set OpenAI API key from settings
+if (hasattr(settings, 'OPENAI_API_KEY') and settings.OPENAI_API_KEY):
+    os.environ["OPENAI_API_KEY"] = settings.OPENAI_API_KEY
+    logger.info("OpenAI API key loaded from settings")
+
+else:
+    logger.warning("OPENAI_API_KEY not found in settings. Please add it to your .env file.")
+
+# Supressing Warning
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def sanitize_ragas_score(value: Any, metric_name: str = "unknown") -> float:
+    """
+    Sanitize a single RAGAS score to handle NaN, None, and invalid values
+    
+    Arguments:
+    ----------
+        value       { Any } : Raw score value
+        
+        metric_name { str } : Name of the metric (for logging)
+    
+    Returns:
+    --------
+          { float }         : Valid float between 0.0 and 1.0
+    """
+    # Handle None
+    if value is None:
+        return 0.0
+    
+    # Handle NaN and infinity
+    try:
+        float_val = float(value)
+        
+        if math.isnan(float_val) or math.isinf(float_val):
+            logger.warning(f"Invalid RAGAS score for {metric_name}: {value}, defaulting to 0.0")
+            return 0.0
+        
+        # Clamp between 0 and 1
+        return max(0.0, min(1.0, float_val))
+    
+    except (ValueError, TypeError):
+        logger.warning(f"Could not convert RAGAS score for {metric_name}: {value}, defaulting to 0.0")
+        return 0.0
+
+
+class RAGASEvaluator:
+    """
+    RAGAS evaluation module for RAG system quality assessment
+    """
+    def __init__(self, enable_ground_truth_metrics: bool = False):
+        """
+        Initialize RAGAS evaluator
+        
+        Arguments:
+        ----------
+            enable_ground_truth_metrics { bool } : Whether to compute metrics requiring ground truth
+        """
+        self.enable_ground_truth                              = enable_ground_truth_metrics
+        
+        # Metrics that don't require ground truth (UPDATED)
+        self.base_metrics                                     = [answer_relevancy, 
+                                                                 faithfulness,
+                                                                 context_utilization, 
+                                                                 context_relevancy,
+                                                                ]
+        
+        # Metrics requiring ground truth
+        self.ground_truth_metrics                             = [context_precision,
+                                                                 context_recall,
+                                                                 answer_similarity,
+                                                                 answer_correctness,
+                                                                ]
+        
+        # Store evaluation history
+        self.evaluation_history : List[RAGASEvaluationResult] = list()
+        self.session_start                                    = datetime.now()
+        
+        logger.info(f"RAGAS Evaluator initialized (ground_truth_metrics: {enable_ground_truth_metrics})")
+    
+    
+    def evaluate_single(self, query: str, answer: str, contexts: List[str], ground_truth: Optional[str] = None, retrieval_time_ms: int = 0,
+                        generation_time_ms: int = 0, total_time_ms: int = 0, chunks_retrieved: int = 0, query_type: str = "rag") -> RAGASEvaluationResult:
+        """
+        Evaluate a single query-answer pair using RAGAS metrics
+        
+        Arguments:
+        ----------
+            query              { str }  : User query
+            
+            answer             { str }  : Generated answer
+            
+            contexts           { list } : Retrieved context chunks
+            
+            ground_truth       { str }  : Reference answer (optional)
+            
+            retrieval_time_ms  { int }  : Retrieval time in milliseconds
+            
+            generation_time_ms { int }  : Generation time in milliseconds
+            
+            total_time_ms      { int }  : Total time in milliseconds
+            
+            chunks_retrieved   { int }  : Number of chunks retrieved
+
+            query_type         { str }  : Type of the query : RAG or non-RAG
+            
+        Returns:
+        --------
+            { RAGASEvaluationResult }   : RAGASEvaluationResult object
+        """
+        try:
+            logger.info(f"Evaluating {query_type.upper()}, query: {query[:100]}...")
+
+            if ((query_type == "general") or (query_type == "non-rag")):
+                logger.info(f"Skipping detailed RAGAS evaluation for {query_type} query")
+            
+                return RAGASEvaluationResult(query               = query,
+                                             answer              = answer,
+                                             contexts            = contexts,
+                                             ground_truth        = ground_truth,
+                                             timestamp           = datetime.now().isoformat(),
+                                             answer_relevancy    = 0.0,  # N/A for non-RAG
+                                             faithfulness        = 0.0,  # N/A for non-RAG
+                                             context_utilization = None,
+                                             context_precision   = None,
+                                             context_relevancy   = 0.0,  # N/A for non-RAG
+                                             context_recall      = None,
+                                             answer_similarity   = None,
+                                             answer_correctness  = None,
+                                             retrieval_time_ms   = retrieval_time_ms,
+                                             generation_time_ms  = generation_time_ms,
+                                             total_time_ms       = total_time_ms,
+                                             chunks_retrieved    = chunks_retrieved,
+                                             query_type          = query_type,
+                                            )
+            
+            # Only for RAG queries : Validate inputs
+            if not contexts or not any(c.strip() for c in contexts):
+                logger.warning("No valid contexts provided for RAGAS evaluation")
+                raise ValueError("No valid contexts for evaluation")
+            
+            # Prepare dataset for RAGAS
+            eval_data = {"question" : [query],
+                         "answer"   : [answer],
+                         "contexts" : [contexts],
+                        }
+            
+            # Add ground truth if available
+            if ground_truth and self.enable_ground_truth:
+                eval_data["ground_truth"] = [ground_truth]
+            
+            # Create dataset
+            dataset = Dataset.from_dict(eval_data)
+            
+            # Select metrics based on ground truth availability
+            if (ground_truth and self.enable_ground_truth):
+                metrics_to_use = self.base_metrics + self.ground_truth_metrics
+            
+            else:
+                metrics_to_use = self.base_metrics
+            
+            # Run evaluation
+            logger.info(f"Running RAGAS evaluation with {len(metrics_to_use)} metrics...")
+
+            results                 = evaluate(dataset, metrics = metrics_to_use)
+            
+            # Extract scores
+            scores                  = results.to_pandas().iloc[0].to_dict()
+            
+            # Sanitize all scores to handle NaN values
+            answer_relevancy        = sanitize_ragas_score(scores.get('answer_relevancy'), 'answer_relevancy')
+            
+            faithfulness            = sanitize_ragas_score(scores.get('faithfulness'), 'faithfulness')
+            
+            context_utilization_val = sanitize_ragas_score(scores.get('context_utilization'), 'context_utilization') if not ground_truth else None
+            
+            context_relevancy_val   = sanitize_ragas_score(scores.get('context_relevancy'), 'context_relevancy')
+            
+            # Ground truth metrics (sanitized)
+            context_precision_val   = None
+            context_recall_val      = None
+            answer_similarity_val   = None
+            answer_correctness_val  = None
+
+            if (ground_truth and ('context_precision' in scores)):
+                context_precision_val  = sanitize_ragas_score(scores.get('context_precision'), 'context_precision')
+            
+            
+
+            if (ground_truth and ('context_recall' in scores)):
+                context_recall_val     = sanitize_ragas_score(scores.get('context_recall'), 'context_recall')
+            
+            
+            if ground_truth and 'answer_similarity' in scores:
+                answer_similarity_val  = sanitize_ragas_score(scores.get('answer_similarity'), 'answer_similarity')
+            
+            
+            if ground_truth and 'answer_correctness' in scores:
+                answer_correctness_val = sanitize_ragas_score(scores.get('answer_correctness'), 'answer_correctness')
+            
+            # Create result object with sanitized values
+            result = RAGASEvaluationResult(query               = query,
+                                           answer              = answer,
+                                           contexts            = contexts,
+                                           ground_truth        = ground_truth,
+                                           timestamp           = datetime.now().isoformat(),
+                                           answer_relevancy    = answer_relevancy,
+                                           faithfulness        = faithfulness,
+                                           context_utilization = context_utilization_val,
+                                           context_precision   = context_precision_val,
+                                           context_relevancy   = context_relevancy_val,
+                                           context_recall      = context_recall_val,
+                                           answer_similarity   = answer_similarity_val,
+                                           answer_correctness  = answer_correctness_val,
+                                           retrieval_time_ms   = retrieval_time_ms,
+                                           generation_time_ms  = generation_time_ms,
+                                           total_time_ms       = total_time_ms,
+                                           chunks_retrieved    = chunks_retrieved,
+                                           query_type          = query_type,
+                                          )
+            
+            # Store in history
+            self.evaluation_history.append(result)
+            
+            # Log results
+            if ground_truth:
+                logger.info(f"Evaluation complete: relevancy={result.answer_relevancy:.3f}, faithfulness={result.faithfulness:.3f}, precision={result.context_precision:.3f}, overall={result.overall_score:.3f}")
+            
+            else:
+                logger.info(f"Evaluation complete: relevancy={result.answer_relevancy:.3f}, faithfulness={result.faithfulness:.3f}, utilization={result.context_utilization:.3f}, overall={result.overall_score:.3f}")
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"RAGAS evaluation failed for {query_type} query: {e}", exc_info = True)
+            
+            # Return zero metrics on failure (all sanitized)
+            return RAGASEvaluationResult(query               = query,
+                                         answer              = answer,
+                                         contexts            = contexts,
+                                         ground_truth        = ground_truth,
+                                         timestamp           = datetime.now().isoformat(),
+                                         answer_relevancy    = 0.0,
+                                         faithfulness        = 0.0,
+                                         context_utilization = 0.0 if not ground_truth else None,
+                                         context_precision   = None if not ground_truth else 0.0,
+                                         context_relevancy   = 0.0,
+                                         context_recall      = None,
+                                         answer_similarity   = None,
+                                         answer_correctness  = None,
+                                         retrieval_time_ms   = retrieval_time_ms,
+                                         generation_time_ms  = generation_time_ms,
+                                         total_time_ms       = total_time_ms,
+                                         chunks_retrieved    = chunks_retrieved,
+                                         query_type          = query_type
+                                        )
+    
+
+    def evaluate_query_response(self, query_response: Any) -> Dict:
+        """
+        Evaluate based on actual response characteristics, not predictions
+        
+        Arguments:
+        ----------
+            query_response { Any } : QueryResponse object with metadata
+        
+        Returns:
+        --------
+                { dict }           : RAGAS evaluation results
+        """
+        try:
+            # Extract necessary data from response object: Check if it has the attributes we need
+            if (hasattr(query_response, 'sources')):
+                sources = query_response.sources
+
+            elif hasattr(query_response, 'contexts'):
+                sources = query_response.contexts
+
+            else:
+                sources = []
+            
+            # Extract context from sources
+            contexts = list()
+
+            if (sources and len(sources) > 0):
+                if (hasattr(sources[0], 'content')):
+                    contexts = [s.content for s in sources]
+
+                elif ((isinstance(sources[0], dict)) and ('content' in sources[0])):
+                    contexts = [s['content'] for s in sources]
+
+                elif (isinstance(sources[0], str)):
+                    contexts = sources
+            
+            # Check if this is actually a RAG response
+            is_actual_rag = ((sources and len(sources) > 0) or (contexts and len(contexts) > 0) or (hasattr(query_response, 'metrics') and  query_response.metrics and query_response.metrics.get("execution_path") == "rag_pipeline"))
+            
+            if not is_actual_rag:
+                logger.info(f"Non-RAG response, skipping RAGAS evaluation")
+                return {"evaluated" : False,
+                        "reason"    : "Not a RAG response",
+                        "is_rag"    : False,
+                       }
+            
+            # Get query and answer
+            query  = getattr(query_response, 'query', '')
+            answer = getattr(query_response, 'answer', '')
+            
+            if not query or not answer:
+                logger.warning("Missing query or answer for evaluation")
+                return {"evaluated" : False,
+                        "reason"    : "Missing query or answer",
+                        "is_rag"    : True,
+                       }
+            
+            # Check if context exists in metrics
+            if (hasattr(query_response, 'metrics') and query_response.metrics):
+                if (query_response.metrics.get("context_for_evaluation")):
+                    contexts = [query_response.metrics["context_for_evaluation"]]
+            
+            if ((not contexts) or (not any(c.strip() for c in contexts))):
+                logger.warning("No context available for RAGAS evaluation")
+                return {"evaluated" : False,
+                        "reason"    : "No context available",
+                        "is_rag"    : True,
+                       }
+            
+            # Try to get query_type from query_response
+            if (hasattr(query_response, 'query_type')):
+                detected_query_type = query_response.query_type
+            
+            elif (hasattr(query_response, 'metrics') and query_response.metrics):
+                detected_query_type = query_response.metrics.get("query_type", "rag")
+            
+            else:
+                # Determine based on contexts
+                detected_query_type = "rag" if (contexts and (len(contexts) > 0)) else "general"
+
+            # Now use the existing evaluate_single method
+            result                       = self.evaluate_single(query              = query,
+                                                                answer             = answer,
+                                                                contexts           = contexts,
+                                                                ground_truth       = None,
+                                                                retrieval_time_ms  = getattr(query_response, 'retrieval_time_ms', 0),
+                                                                generation_time_ms = getattr(query_response, 'generation_time_ms', 0),
+                                                                total_time_ms      = getattr(query_response, 'total_time_ms', 0),
+                                                                chunks_retrieved   = len(sources) if sources else len(contexts),
+                                                                query_type         = detected_query_type,
+                                                               )
+            
+            # Convert to dict and add metadata
+            result_dict                  = result.to_dict() if hasattr(result, 'to_dict') else vars(result)
+            
+            # Add evaluation metadata
+            result_dict["evaluated"]     = True
+            result_dict["is_rag"]        = True
+            result_dict["context_count"] = len(contexts)
+            
+            # Add prediction vs reality info if available
+            if ((hasattr(query_response, 'metrics')) and query_response.metrics):
+                result_dict["predicted_type"]      = query_response.metrics.get("predicted_type", "unknown")
+                result_dict["actual_type"]         = query_response.metrics.get("actual_type", "unknown")
+                result_dict["confidence_mismatch"] = (query_response.metrics.get("predicted_type") != query_response.metrics.get("actual_type"))
+            
+            logger.info(f"RAGAS evaluation completed for RAG response")
+            return result_dict
+            
+        except Exception as e:
+            logger.error(f"Query response evaluation failed: {repr(e)}", exc_info = True)
+            
+            return {"evaluated" : False,
+                    "error"     : str(e),
+                    "is_rag"    : True,
+                   }
+
+    
+    def evaluate_batch(self, queries: List[str], answers: List[str], contexts_list: List[List[str]], ground_truths: Optional[List[str]] = None,
+                       query_types: Optional[List[str]] = None) -> List[RAGASEvaluationResult]:
+        """
+        Evaluate multiple query-answer pairs in batch
+        
+        Arguments:
+        ----------
+            queries       { list } : List of user queries
+
+            answers       { list } : List of generated answers
+            
+            contexts_list { list } : List of context lists
+            
+            ground_truths { list } : List of reference answers (optional)
+
+            query_types   { list } : List of query types RAG / non-RAG
+            
+        Returns:
+        --------
+                  { list }         : List of RAGASEvaluationResult objects
+        """
+        try:
+            logger.info(f"Batch evaluating {len(queries)} queries...")
+            
+            # Prepare dataset
+            eval_data = {"question" : queries,
+                         "answer"   : answers,
+                         "contexts" : contexts_list,
+                        }
+            
+            if ground_truths and self.enable_ground_truth:
+                eval_data["ground_truth"] = ground_truths
+            
+            # Create dataset
+            dataset = Dataset.from_dict(eval_data)
+            
+            # Select metrics
+            if (ground_truths and self.enable_ground_truth):
+                metrics_to_use = self.base_metrics + self.ground_truth_metrics
+
+            else:
+                metrics_to_use = self.base_metrics
+            
+            # Run evaluation
+            results            = evaluate(dataset, metrics = metrics_to_use)
+            results_df         = results.to_pandas()
+            
+            # Create result objects
+            evaluation_results = list()
+
+            for idx, row in results_df.iterrows():
+                # Determine query_type for this item
+                if query_types and idx < len(query_types):
+                    current_query_type = query_types[idx]
+
+                else:
+                    # Default based on whether contexts are available
+                    current_query_type = "rag" if contexts_list[idx] and len(contexts_list[idx]) > 0 else "general"
+            
+                # Sanitize all scores
+                answer_relevancy_val    = sanitize_ragas_score(row.get('answer_relevancy', 0.0), f'answer_relevancy_{idx}')
+                
+                faithfulness_val        = sanitize_ragas_score(row.get('faithfulness', 0.0), f'faithfulness_{idx}')
+                
+                context_relevancy_val   = sanitize_ragas_score(row.get('context_relevancy', 0.0), f'context_relevancy_{idx}')
+                
+                # Handle context_utilization vs context_precision
+                context_utilization_val = sanitize_ragas_score(row.get('context_utilization'), f'context_utilization_{idx}') if not ground_truths else None
+                
+                context_precision_val   = sanitize_ragas_score(row.get('context_precision'), f'context_precision_{idx}') if (ground_truths and 'context_precision' in row) else None
+            
+                # Ground truth metrics
+                context_recall_val      = sanitize_ragas_score(row.get('context_recall'), f'context_recall_{idx}') if (ground_truths and 'context_recall' in row) else None
+                
+                answer_similarity_val   = sanitize_ragas_score(row.get('answer_similarity'), f'answer_similarity_{idx}') if (ground_truths and 'answer_similarity' in row) else None
+                
+                answer_correctness_val  = sanitize_ragas_score(row.get('answer_correctness'), f'answer_correctness_{idx}') if (ground_truths and 'answer_correctness' in row) else None
+                
+                # For non-RAG queries, set appropriate scores
+                if ((current_query_type == "general") or (current_query_type == "non-rag")):
+                    # Non-RAG queries shouldn't have RAGAS metrics
+                    answer_relevancy_val    = 0.0
+                    faithfulness_val        = 0.0
+                    context_relevancy_val   = 0.0
+                    context_utilization_val = None
+                    context_precision_val   = None
+            
+                result                  = RAGASEvaluationResult(query               = queries[idx],
+                                                                answer              = answers[idx],
+                                                                contexts            = contexts_list[idx],
+                                                                ground_truth        = ground_truths[idx] if ground_truths else None,
+                                                                timestamp           = datetime.now().isoformat(),
+                                                                answer_relevancy    = answer_relevancy_val,
+                                                                faithfulness        = faithfulness_val,
+                                                                context_precision   = context_precision_val,
+                                                                context_utilization = context_utilization_val,
+                                                                context_relevancy   = context_relevancy_val,
+                                                                context_recall      = context_recall_val,
+                                                                answer_similarity   = answer_similarity_val,
+                                                                answer_correctness  = answer_correctness_val,
+                                                                retrieval_time_ms   = 0,
+                                                                generation_time_ms  = 0,
+                                                                total_time_ms       = 0,
+                                                                chunks_retrieved    = len(contexts_list[idx]),
+                                                                query_type          = current_query_type,
+                                                               )
+
+                evaluation_results.append(result)
+
+                self.evaluation_history.append(result)
+            
+            logger.info(f"Batch evaluation complete for {len(evaluation_results)} queries")
+            return evaluation_results
+            
+        except Exception as e:
+            logger.error(f"Batch evaluation failed: {e}", exc_info = True)
+            return []
+    
+    
+    def get_session_statistics(self) -> RAGASStatistics:
+        """
+        Get aggregate statistics for the current evaluation session
+        
+        Returns:
+        ---------
+            { RAGASStatistics } : RAGASStatistics object with aggregate metrics
+        """
+        if not self.evaluation_history:
+            # Return empty statistics
+            return RAGASStatistics(total_evaluations       = 0,
+                                   avg_answer_relevancy    = 0.0,
+                                   avg_faithfulness        = 0.0,
+                                   avg_context_precision   = 0.0,
+                                   avg_context_utilization = 0.0,
+                                   avg_context_relevancy   = 0.0,
+                                   avg_overall_score       = 0.0,
+                                   avg_retrieval_time_ms   = 0.0,
+                                   avg_generation_time_ms  = 0.0,
+                                   avg_total_time_ms       = 0.0,
+                                   min_score               = 0.0,
+                                   max_score               = 0.0,
+                                   std_dev                 = 0.0,
+                                   session_start           = self.session_start,
+                                   last_updated            = datetime.now(),
+                                  )
+        
+        n                  = len(self.evaluation_history)
+        
+        # Calculate averages
+        avg_relevancy      = sum(r.answer_relevancy for r in self.evaluation_history) / n
+        avg_faithfulness   = sum(r.faithfulness for r in self.evaluation_history) / n
+        
+        # Calculate context_precision and context_utilization separately
+        precision_values   = [r.context_precision for r in self.evaluation_history if r.context_precision is not None]
+        utilization_values = [r.context_utilization for r in self.evaluation_history if r.context_utilization is not None]
+        
+        avg_precision      = sum(precision_values) / len(precision_values) if precision_values else 0.0
+        avg_utilization    = sum(utilization_values) / len(utilization_values) if utilization_values else 0.0
+        
+        avg_relevancy_ctx  = sum(r.context_relevancy for r in self.evaluation_history) / n
+        
+        # Overall scores
+        overall_scores     = [r.overall_score for r in self.evaluation_history]
+        avg_overall        = sum(overall_scores) / n
+        min_score          = min(overall_scores)
+        max_score          = max(overall_scores)
+        std_dev            = statistics.stdev(overall_scores) if n > 1 else 0.0
+        
+        # Performance averages
+        avg_retrieval      = sum(r.retrieval_time_ms for r in self.evaluation_history) / n
+        avg_generation     = sum(r.generation_time_ms for r in self.evaluation_history) / n
+        avg_total          = sum(r.total_time_ms for r in self.evaluation_history) / n
+        
+        # Ground truth metrics (if available)
+        recall_values      = [r.context_recall for r in self.evaluation_history if r.context_recall is not None]
+        similarity_values  = [r.answer_similarity for r in self.evaluation_history if r.answer_similarity is not None]
+        correctness_values = [r.answer_correctness for r in self.evaluation_history if r.answer_correctness is not None]
+        
+        return RAGASStatistics(total_evaluations       = n,
+                               avg_answer_relevancy    = round(avg_relevancy, 3),
+                               avg_faithfulness        = round(avg_faithfulness, 3),
+                               avg_context_precision   = round(avg_precision, 3) if precision_values else None,
+                               avg_context_utilization = round(avg_utilization, 3) if utilization_values else None,
+                               avg_context_relevancy   = round(avg_relevancy_ctx, 3),
+                               avg_overall_score       = round(avg_overall, 3),
+                               avg_context_recall      = round(sum(recall_values) / len(recall_values), 3) if recall_values else None,
+                               avg_answer_similarity   = round(sum(similarity_values) / len(similarity_values), 3) if similarity_values else None,
+                               avg_answer_correctness  = round(sum(correctness_values) / len(correctness_values), 3) if correctness_values else None,
+                               avg_retrieval_time_ms   = round(avg_retrieval, 2),
+                               avg_generation_time_ms  = round(avg_generation, 2),
+                               avg_total_time_ms       = round(avg_total, 2),
+                               min_score               = round(min_score, 3),
+                               max_score               = round(max_score, 3),
+                               std_dev                 = round(std_dev, 3),
+                               session_start           = self.session_start,
+                               last_updated            = datetime.now(),
+                              )
+
+
+    def get_evaluation_history(self) -> List[Dict]:
+        """
+        Get full evaluation history as list of dictionaries
+        
+        Returns:
+        --------
+            { list }    : List of evaluation results as dictionaries
+        """
+        return [result.to_dict() for result in self.evaluation_history]
+    
+    
+    def clear_history(self):
+        """
+        Clear evaluation history and reset session
+        """
+        self.evaluation_history.clear()
+        self.session_start = datetime.now()
+
+        logger.info("Evaluation history cleared, new session started")
+    
+
+    def export_to_dict(self) -> RAGASExportData:
+        """
+        Export all evaluations to structured format
+        
+        Returns:
+        --------
+            { RAGASExportData }    : RAGASExportData object with complete evaluation data
+        """
+        return RAGASExportData(export_timestamp     = datetime.now().isoformat(),
+                               total_evaluations    = len(self.evaluation_history),
+                               statistics           = self.get_session_statistics(),
+                               evaluations          = self.evaluation_history,
+                               ground_truth_enabled = self.enable_ground_truth,
+                              )
+
+
+
+# Global evaluator instance
+_ragas_evaluator : Optional[RAGASEvaluator] = None
+
+
+def get_ragas_evaluator(enable_ground_truth_metrics: bool = False) -> RAGASEvaluator:
+    """
+    Get or create global RAGAS evaluator instance
+    
+    Arguments:
+    ----------
+        enable_ground_truth_metrics { bool } : Whether to enable ground truth metrics
+        
+    Returns:
+    --------
+               { RAGASEvaluator }            : RAGASEvaluator instance
+    """
+    global _ragas_evaluator
+    
+    if _ragas_evaluator is None:
+        _ragas_evaluator = RAGASEvaluator(enable_ground_truth_metrics)
+    
+    return _ragas_evaluator

generation/citation_formatter.py

@@ -0,0 +1,378 @@
+# DEPENDENCIES
+import re
+from enum import Enum
+from typing import List
+from typing import Dict
+from typing import Optional
+from collections import defaultdict
+from config.models import CitationStyle
+from config.models import ChunkWithScore
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import CitationFormattingError
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class CitationFormatter:
+    """
+    Citation formatting and management: Formats citations in generated text according to different styles and ensures citation consistency and validity
+    """
+    def __init__(self, style: CitationStyle = CitationStyle.NUMERIC):
+        """
+        Initialize citation formatter
+        
+        Arguments:
+        ----------
+            style { CitationStyle } : Citation style to use
+        """
+        self.logger           = logger
+        self.style            = style
+        self.citation_pattern = re.compile(r'\[(\d+)\]')
+        
+        # Style configurations
+        self.style_configs    = {CitationStyle.NUMERIC  : {"inline_format" : "[{number}]", "reference_format" : "[{number}] {source_info}", "separator" : " ",},
+                                 CitationStyle.VERBOSE  : {"inline_format" : "[{number}]", "reference_format" : "Citation {number}: {source_info}", "separator" : "\n",},
+                                 CitationStyle.MINIMAL  : {"inline_format" : "[{number}]", "reference_format" : "[{number}]", "separator" : " ",},
+                                 CitationStyle.ACADEMIC : {"inline_format" : "({number})", "reference_format" : "{number}. {source_info}", "separator" : "\n",},
+                                 CitationStyle.LEGAL    : {"inline_format" : "[{number}]", "reference_format" : "[{number}] {source_info}", "separator" : "\n",}
+                                }
+    
+
+    def format_citations_in_text(self, text: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Format citations in generated text
+        
+        Arguments:
+        ----------
+            text    { str }  : Text containing citation markers
+            
+            sources { list } : List of sources for citation mapping
+        
+        Returns:
+        --------
+              { str }        : Text with formatted citations
+        """
+        if not text or not sources:
+            return text
+        
+        try:
+            # Extract citation numbers from text
+            citation_numbers = self._extract_citation_numbers(text = text)
+            
+            if not citation_numbers:
+                return text
+            
+            # Create citation mapping
+            citation_map     = self._create_citation_map(sources = sources)
+            
+            # Replace citation markers with formatted citations
+            formatted_text   = self._replace_citation_markers(text         = text, 
+                                                              citation_map = citation_map,
+                                                             )
+            
+            self.logger.debug(f"Formatted {len(citation_numbers)} citations in text")
+            
+            return formatted_text
+        
+        except Exception as e:
+            self.logger.error(f"Citation formatting failed: {repr(e)}")
+            raise CitationFormattingError(f"Citation formatting failed: {repr(e)}")
+    
+
+    def generate_reference_section(self, sources: List[ChunkWithScore], cited_numbers: List[int]) -> str:
+        """
+        Generate reference section for cited sources
+        
+        Arguments:
+        ----------
+            sources       { list } : All available sources
+            
+            cited_numbers { list } : Numbers of actually cited sources
+        
+        Returns:
+        --------
+                  { str }          : Formatted reference section
+        """
+        if not sources or not cited_numbers:
+            return ""
+        
+        try:
+            style_config  = self.style_configs[self.style]
+            references    = list()
+            
+            # Get only cited sources
+            cited_sources = [sources[num-1] for num in cited_numbers if (1 <= num <= len(sources))]
+            
+            for i, source in enumerate(cited_sources, 1):
+                source_info = self._format_source_info(source, i)
+                reference   = style_config["reference_format"].format(number = i, source_info = source_info)
+                
+                references.append(reference)
+            
+            separator         = style_config["separator"]
+            reference_section = separator.join(references)
+            
+            # Add section header if appropriate
+            if (self.style in [CitationStyle.VERBOSE, CitationStyle.ACADEMIC]):
+                reference_section = "References:\n" + reference_section
+            
+            self.logger.debug(f"Generated reference section with {len(references)} entries")
+            
+            return reference_section
+        
+        except Exception as e:
+            self.logger.error(f"Reference section generation failed: {repr(e)}")
+            return ""
+    
+
+    def _extract_citation_numbers(self, text: str) -> List[int]:
+        """
+        Extract citation numbers from text
+        """
+        matches          = self.citation_pattern.findall(text)
+        citation_numbers = [int(match) for match in matches]
+        
+        # Unique and sorted
+        return sorted(set(citation_numbers))  
+    
+
+    def _create_citation_map(self, sources: List[ChunkWithScore]) -> Dict[int, str]:
+        """
+        Create mapping from citation numbers to formatted citations
+        """
+        citation_map = dict()
+        style_config = self.style_configs[self.style]
+        
+        for i, source in enumerate(sources, 1):
+            formatted_citation = style_config["inline_format"].format(number=i)
+            citation_map[i]    = formatted_citation
+        
+        return citation_map
+    
+
+    def _replace_citation_markers(self, text: str, citation_map: Dict[int, str]) -> str:
+        """
+        Replace citation markers in text
+        """
+        def replacement(match):
+            citation_num = int(match.group(1))
+
+            return citation_map.get(citation_num, match.group(0))
+        
+        return self.citation_pattern.sub(replacement, text)
+    
+
+    def _format_source_info(self, source: ChunkWithScore, citation_number: int) -> str:
+        """
+        Format source information based on style
+        """
+        chunk = source.chunk
+        
+        if (self.style == CitationStyle.MINIMAL):
+            return f"Source {citation_number}"
+        
+        # Build source components
+        components = list()
+        
+        # Document information
+        if hasattr(chunk, 'metadata'):
+            meta = chunk.metadata
+            
+            if ('filename' in meta):
+                components.append(f"Document: {meta['filename']}")
+            
+            if (('title' in meta) and meta['title']):
+                components.append(f"\"{meta['title']}\"")
+            
+            if (('author' in meta) and meta['author']):
+                components.append(f"by {meta['author']}")
+        
+        # Location information
+        location_parts = list()
+
+        if chunk.page_number:
+            location_parts.append(f"p. {chunk.page_number}")
+        
+        if chunk.section_title:
+            location_parts.append(f"Section: {chunk.section_title}")
+        
+        if location_parts:
+            components.append("(" + ", ".join(location_parts) + ")")
+        
+        # Relevance score (for verbose styles)
+        if ((self.style in [CitationStyle.VERBOSE, CitationStyle.ACADEMIC]) and (source.score > 0)):
+            components.append(f"[relevance: {source.score:.3f}]")
+        
+        return " ".join(components)
+    
+
+    def validate_citations(self, text: str, sources: List[ChunkWithScore]) -> tuple[bool, List[int]]:
+        """
+        Validate citations in text
+        
+        Arguments:
+        ----------
+            text    { str }  : Text to validate
+            
+            sources { list } : Available sources
+        
+        Returns:
+        --------
+              { tuple }      : (is_valid, invalid_citations)
+        """
+        citation_numbers  = self._extract_citation_numbers(text = text)
+        invalid_citations = list()
+        
+        for number in citation_numbers:
+            if ((number < 1) or (number > len(sources))):
+                invalid_citations.append(number)
+        
+        is_valid = (len(invalid_citations) == 0)
+        
+        if not is_valid:
+            self.logger.warning(f"Invalid citations found: {invalid_citations}")
+        
+        return is_valid, invalid_citations
+    
+
+    def normalize_citations(self, text: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Normalize citations to ensure sequential numbering
+        
+        Arguments:
+        ----------
+            text    { str }  : Text with citations
+            
+            sources { list } : Available sources
+        
+        Returns:
+        --------
+              { str }        : Text with normalized citations
+        """
+        citation_numbers = self._extract_citation_numbers(text = text)
+        
+        if not citation_numbers:
+            return text
+        
+        # Create mapping from old to new numbers
+        citation_mapping = dict()
+
+        for i, old_num in enumerate(sorted(set(citation_numbers)), 1):
+            if (1 <= old_num <= len(sources)):
+                citation_mapping[old_num] = i
+        
+        # Replace citations
+        def normalize_replacement(match):
+            old_num      = int(match.group(1))
+            new_num      = citation_mapping.get(old_num, old_num)
+            style_config = self.style_configs[self.style]
+            
+            return style_config["inline_format"].format(number = new_num)
+        
+        normalized_text = self.citation_pattern.sub(normalize_replacement, text)
+        
+        if citation_mapping:
+            self.logger.info(f"Normalized citations: {citation_numbers} -> {list(citation_mapping.values())}")
+        
+        return normalized_text
+    
+
+    def get_citation_statistics(self, text: str, sources: List[ChunkWithScore]) -> Dict:
+        """
+        Get citation statistics
+        
+        Arguments:
+        ----------
+            text    { str }  : Text with citations
+            
+            sources { list } : Available sources
+        
+        Returns:
+        --------
+              { dict }       : Citation statistics
+        """
+        citation_numbers = self._extract_citation_numbers(text = text)
+        
+        if not citation_numbers:
+            return {"total_citations": 0}
+        
+        # Calculate distribution
+        source_usage = defaultdict(int)
+
+        for number in citation_numbers:
+            if (1 <= number <= len(sources)):
+                source                = sources[number-1]
+                doc_id                = source.chunk.document_id
+                source_usage[doc_id] += 1
+        
+        return {"total_citations"      : len(citation_numbers),
+                "unique_citations"     : len(set(citation_numbers)),
+                "sources_used"         : len(source_usage),
+                "citations_per_source" : dict(source_usage),
+                "citation_density"     : len(citation_numbers) / max(1, len(text.split())),  
+               }
+    
+
+    def set_style(self, style: CitationStyle):
+        """
+        Set citation style
+        
+        Arguments:
+        ----------
+            style { CitationStyle } : New citation style
+        """
+        if (style not in self.style_configs):
+            raise CitationFormattingError(f"Unsupported citation style: {style}")
+        
+        old_style  = self.style
+        self.style = style
+        
+        self.logger.info(f"Citation style changed: {old_style} -> {style}")
+
+
+# Global citation formatter instance
+_citation_formatter = None
+
+
+def get_citation_formatter() -> CitationFormatter:
+    """
+    Get global citation formatter instance (singleton)
+    
+    Returns:
+    --------
+        { CitationFormatter }    : CitationFormatter instance
+    """
+    global _citation_formatter
+    
+    if _citation_formatter is None:
+        _citation_formatter = CitationFormatter()
+    
+    return _citation_formatter
+
+
+@handle_errors(error_type = CitationFormattingError, log_error = True, reraise = False)
+def format_citations(text: str, sources: List[ChunkWithScore], style: CitationStyle = None) -> str:
+    """
+    Convenience function for citation formatting
+    
+    Arguments:
+    ----------
+        text          { str }     : Text containing citations
+        
+        sources      { list }     : List of sources
+        
+        style   { CitationStyle } : Citation style to use
+    
+    Returns:
+    --------
+                 { str }          : Formatted text
+    """
+    formatter = get_citation_formatter()
+    
+    if style is not None:
+        formatter.set_style(style)
+    
+    return formatter.format_citations_in_text(text, sources)

generation/general_responder.py

@@ -0,0 +1,155 @@
+# DEPENDENCIES
+import random
+import datetime
+from typing import List
+from typing import Dict
+from config.models import LLMProvider
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from generation.llm_client import get_llm_client
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class GeneralResponder:
+    """
+    Handles general/conversational queries using LLM without RAG
+    """
+    def __init__(self, provider: LLMProvider = None, model_name: str = None):
+        self.logger             = logger
+        self.provider           = provider or LLMProvider.OLLAMA
+        self.model_name         = model_name or settings.OLLAMA_MODEL
+        
+        # Initialize LLM client for general responses
+        self.llm_client         = get_llm_client(provider   = self.provider, 
+                                                 model_name = self.model_name,
+                                                )
+        
+        # System prompt for general conversation
+        self.system_prompt      = """
+                                     You are a helpful, friendly AI assistant. You're part of a larger system called the "AI Universal Knowledge Ingestion System" which specializes in document analysis and retrieval.
+
+                                     When users ask general questions, answer helpfully and conversationally. When they ask about your capabilities, explain that you can:
+                                     1. Answer general knowledge questions
+                                     2. Help with document analysis (when they upload documents)
+                                     3. Provide explanations on various topics
+                                     4. Engage in friendly conversation
+
+                                     If a question is better answered by searching through documents, politely suggest uploading documents first.
+
+                                     Be concise but thorough. Use a friendly, professional tone.
+                                  """
+
+        # Fallback responses (if LLM fails)
+        self.fallback_responses = {"greeting" : ["Hello! 👋 I'm your AI assistant. How can I help you today?", "Hi there! I'm here to help with questions or document analysis. What's on your mind?", "Greetings! I can answer questions or help analyze documents. What would you like to know?"],
+                                   "farewell" : ["Goodbye! Feel free to come back if you have more questions.", "See you later! Don't hesitate to ask if you need help with documents.", "Take care! Remember I'm here for document analysis too."],
+                                   "thanks"   : ["You're welcome! Happy to help.", "My pleasure! Let me know if you need anything else.", "Glad I could help! Don't hesitate to ask more questions."],
+                                   "default"  : ["I'm here to help! You can ask me general questions or upload documents for analysis.", "That's an interesting question! I'd be happy to discuss it with you.", "I can help with that. What specific aspect would you like to know about?"],
+                                  }
+    
+
+    async def respond(self, query: str, conversation_history: List[Dict] = None) -> Dict:
+        """
+        Generate a response to a general query
+        
+        Arguments:
+        ----------
+            query                { str } : User query
+
+            conversation_history { list } : Previous messages in conversation
+        
+        Returns:
+        --------
+                     { dict }             : Response dictionary
+        """
+        try:
+            # Prepare messages for LLM
+            messages = list()
+            
+            # Add system prompt
+            messages.append({"role"    : "system", 
+                             "content" : self.system_prompt,
+                           })
+            
+            # Add conversation history if available
+            if conversation_history:
+                # Last 5 messages for context
+                messages.extend(conversation_history[-5:])  
+            
+            # Add current query
+            messages.append({"role"    : "user", 
+                             "content" : query,
+                           })
+            
+            # Generate response: Slightly higher temp for conversational
+            llm_response  = await self.llm_client.generate(messages    = messages,
+                                                           temperature = 0.7, 
+                                                           max_tokens  = 500,
+                                                          )
+            
+            response_text = llm_response.get("content", "").strip()
+            
+            if not response_text:
+                response_text = self._get_fallback_response(query)
+            
+            return {"answer"       : response_text,
+                    "is_general"   : True,
+                    "requires_rag" : False,
+                    "tokens_used"  : llm_response.get("usage", {}),
+                    "model"        : self.model_name,
+                   }
+            
+        except Exception as e:
+            logger.error(f"General response generation failed: {e}")
+            return {"answer"       : self._get_fallback_response(query),
+                    "is_general"   : True,
+                    "requires_rag" : False,
+                    "error"        : str(e),
+                    "model"        : self.model_name,
+                   }
+    
+
+    def _get_fallback_response(self, query: str) -> str:
+        """
+        Get a fallback response if LLM fails
+        """
+        query_lower = query.lower()
+        
+        if (any(word in query_lower for word in ["hello", "hi", "hey", "greetings"])):
+            return random.choice(self.fallback_responses["greeting"])
+        
+        elif (any(word in query_lower for word in ["thank", "thanks", "appreciate"])):
+            return random.choice(self.fallback_responses["thanks"])
+        
+        elif (any(word in query_lower for word in ["bye", "goodbye", "see you"])):
+            return random.choice(self.fallback_responses["farewell"])
+        
+        else:
+            return random.choice(self.fallback_responses["default"])
+
+
+
+# Global responder instance
+_general_responder = None
+
+
+def get_general_responder(provider: LLMProvider = None, model_name: str = None) -> GeneralResponder:
+    """
+    Get global general responder instance
+    
+    Returns:
+    --------
+        { GeneralResponder } : GeneralResponder instance
+    """
+    global _general_responder
+    
+    if _general_responder is None:
+        _general_responder = GeneralResponder(provider   = provider, 
+                                              model_name = model_name,
+                                             )
+    
+    return _general_responder

generation/llm_client.py

@@ -0,0 +1,396 @@
+# DEPENDENCIES
+import os
+import json
+import time
+import aiohttp
+import requests
+from typing import List
+from typing import Dict
+from typing import Optional
+from typing import AsyncGenerator
+from config.models import LLMProvider
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import LLMClientError
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class LLMClient:
+    """
+    Unified LLM client supporting multiple providers (Ollama, OpenAI): Provides consistent interface for text generation across different LLM services
+    """
+    def __init__(self, provider: LLMProvider = None, model_name: str = None, api_key: str = None, base_url: str = None):
+        """
+        Initialize LLM client
+        
+        Arguments:
+        ----------
+            provider   { LLMProvider } : LLM provider to use
+            
+            model_name { str }         : Model name to use
+            
+            api_key    { str }         : API key (for OpenAI)
+            
+            base_url   { str }         : Base URL for API (for Ollama)
+        """
+        self.logger     = logger
+        self.settings   = get_settings()
+        self.provider   = provider or LLMProvider.OLLAMA
+        self.model_name = model_name or self.settings.OLLAMA_MODEL
+        self.api_key    = api_key
+        self.base_url   = base_url or self.settings.OLLAMA_BASE_URL
+        self.timeout    = self.settings.OLLAMA_TIMEOUT
+        
+        # Initialize provider-specific configurations
+        self._initialize_provider()
+    
+
+    def _initialize_provider(self):
+        """
+        Initialize provider-specific configurations
+        """
+        # Auto-detect provider if not explicitly set
+        if (self.settings.IS_HF_SPACE and not self.settings.OLLAMA_ENABLED):
+            if (self.settings.USE_OPENAI and self.settings.OPENAI_API_KEY):
+                self.provider = LLMProvider.OPENAI
+                logger.info("HF Space detected: Using OpenAI API")
+            
+            else:
+                raise LLMClientError("Running in HF Space without Ollama. Set OPENAI_API_KEY in Space secrets.")
+        
+        # Provider initialization
+        if (self.provider == LLMProvider.OLLAMA):
+            if not self.base_url:
+                raise LLMClientError("Ollama base URL is required")
+            
+            self.logger.info(f"Initialized Ollama client: {self.base_url}, model: {self.model_name}")
+        
+        elif (self.provider == LLMProvider.OPENAI):
+            if not self.api_key:
+                # Try to get from environment
+                self.api_key = os.getenv('OPENAI_API_KEY')
+                if not self.api_key:
+                    raise LLMClientError("OpenAI API key is required")
+            
+            self.base_url = "https://api.openai.com/v1"
+
+            self.logger.info(f"Initialized OpenAI client, model: {self.model_name}")
+        
+        else:
+            raise LLMClientError(f"Unsupported provider: {self.provider}")
+    
+
+    async def generate(self, messages: List[Dict], **generation_params) -> Dict:
+        """
+        Generate text completion (async)
+        
+        Arguments:
+        ----------
+            messages          { list } : List of message dictionaries
+            
+            **generation_params        : Generation parameters (temperature, max_tokens, etc.)
+        
+        Returns:
+        --------
+                   { dict }            : Generation response
+        """
+        try:
+            if (self.provider == LLMProvider.OLLAMA):
+                return await self._generate_ollama(messages, **generation_params)
+            
+            elif (self.provider == LLMProvider.OPENAI):
+                return await self._generate_openai(messages, **generation_params)
+            
+            else:
+                raise LLMClientError(f"Unsupported provider: {self.provider}")
+        
+        except Exception as e:
+            self.logger.error(f"Generation failed: {repr(e)}")
+            raise LLMClientError(f"Generation failed: {repr(e)}")
+    
+
+    async def generate_stream(self, messages: List[Dict], **generation_params) -> AsyncGenerator[str, None]:
+        """
+        Generate text completion with streaming (async)
+        
+        Arguments:
+        ----------
+            messages          { list } : List of message dictionaries
+            
+            **generation_params        : Generation parameters
+        
+        Returns:
+        --------
+                   { AsyncGenerator }  : Async generator yielding response chunks
+        """
+        try:
+            if (self.provider == LLMProvider.OLLAMA):
+                async for chunk in self._generate_ollama_stream(messages, **generation_params):
+                    yield chunk
+            
+            elif (self.provider == LLMProvider.OPENAI):
+                async for chunk in self._generate_openai_stream(messages, **generation_params):
+                    yield chunk
+            
+            else:
+                raise LLMClientError(f"Unsupported provider: {self.provider}")
+        
+        except Exception as e:
+            self.logger.error(f"Stream generation failed: {repr(e)}")
+            raise LLMClientError(f"Stream generation failed: {repr(e)}")
+    
+
+    async def _generate_ollama(self, messages: List[Dict], **generation_params) -> Dict:
+        """
+        Generate using Ollama API
+        """
+        url     = f"{self.base_url}/api/chat"
+        
+        # Prepare request payload
+        payload = {"model"    : self.model_name,
+                   "messages" : messages,
+                   "stream"   : False,
+                   "options"  : {"temperature" : generation_params.get("temperature", 0.1),
+                                 "top_p"       : generation_params.get("top_p", 0.9),
+                                 "num_predict" : generation_params.get("max_tokens", 1000),
+                                }
+                  }
+        
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, json = payload, timeout = self.timeout) as response:
+                if (response.status != 200):
+                    error_text = await response.text()
+                    raise LLMClientError(f"Ollama API error: {response.status} - {error_text}")
+                
+                result = await response.json()
+                
+                return {"content"       : result["message"]["content"],
+                        "usage"         : {"prompt_tokens"     : result.get("prompt_eval_count", 0),
+                                           "completion_tokens" : result.get("eval_count", 0),
+                                           "total_tokens"      : result.get("prompt_eval_count", 0) + result.get("eval_count", 0),
+                                          },
+                        "finish_reason" : result.get("done_reason", "stop"),
+                       }
+    
+
+    async def _generate_ollama_stream(self, messages: List[Dict], **generation_params) -> AsyncGenerator[str, None]:
+        """
+        Generate stream using Ollama API - FIXED VERSION
+        """
+        url     = f"{self.base_url}/api/chat"
+        
+        payload = {"model"    : self.model_name,
+                   "messages" : messages,
+                   "stream"   : True,
+                   "options"  : {"temperature" : generation_params.get("temperature", 0.1),
+                                 "top_p"       : generation_params.get("top_p", 0.9),
+                                 "num_predict" : generation_params.get("max_tokens", 1000),
+                                }
+                  }
+        
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, json = payload, timeout = self.timeout) as response:
+                if (response.status != 200):
+                    error_text = await response.text()
+                    raise LLMClientError(f"Ollama API error: {response.status} - {error_text}")
+                
+                async for line in response.content:
+                    line_str = line.decode('utf-8').strip()
+                    
+                    # Skip empty lines
+                    if not line_str:
+                        continue
+                    
+                    try:
+                        chunk_data = json.loads(line_str)
+                    
+                        # Check if this is the final chunk
+                        if (chunk_data.get("done", False)):
+                            break
+                        
+                        # Extract content regardless of whether it's empty: Ollama sends incremental content in each chunk
+                        if ("message" in chunk_data):
+                            content = chunk_data["message"].get("content", "")
+                            
+                            # Only yield non-empty content
+                            if content:  
+                                yield content
+                    
+                    except json.JSONDecodeError as e:
+                        self.logger.warning(f"Failed to parse streaming chunk: {line_str[:100]}")
+                        continue
+    
+
+    async def _generate_openai(self, messages: List[Dict], **generation_params) -> Dict:
+        """
+        Generate using OpenAI API
+        """
+        url     = f"{self.base_url}/chat/completions"
+        
+        headers = {"Authorization" : f"Bearer {self.api_key}",
+                   "Content-Type"  : "application/json",
+                  }
+        
+        payload = {"model"       : self.model_name,
+                   "messages"    : messages,
+                   "temperature" : generation_params.get("temperature", 0.1),
+                   "top_p"       : generation_params.get("top_p", 0.9),
+                   "max_tokens"  : generation_params.get("max_tokens", 1000),
+                   "stream"      : False,
+                 }
+        
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, headers = headers, json = payload, timeout = self.timeout) as response:
+                if (response.status != 200):
+                    error_text = await response.text()
+                    raise LLMClientError(f"OpenAI API error: {response.status} - {error_text}")
+                
+                result = await response.json()
+                
+                return {"content"       : result["choices"][0]["message"]["content"],
+                        "usage"         : result["usage"],
+                        "finish_reason" : result["choices"][0]["finish_reason"],
+                       }
+    
+
+    async def _generate_openai_stream(self, messages: List[Dict], **generation_params) -> AsyncGenerator[str, None]:
+        """
+        Generate stream using OpenAI API
+        """
+        url     = f"{self.base_url}/chat/completions"
+        
+        headers = {"Authorization" : f"Bearer {self.api_key}",
+                   "Content-Type"  : "application/json",
+                  }
+        
+        payload = {"model"       : self.model_name,
+                   "messages"    : messages,
+                   "temperature" : generation_params.get("temperature", 0.1),
+                   "top_p"       : generation_params.get("top_p", 0.9),
+                   "max_tokens"  : generation_params.get("max_tokens", 1000),
+                   "stream"      : True,
+                  }
+        
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, headers = headers, json = payload, timeout = self.timeout) as response:
+                if (response.status != 200):
+                    error_text = await response.text()
+                    
+                    raise LLMClientError(f"OpenAI API error: {response.status} - {error_text}")
+                
+                async for line in response.content:
+                    line = line.decode('utf-8').strip()
+                    
+                    if (line.startswith('data: ')):
+                        # Remove 'data: ' prefix
+                        data = line[6:]  
+                        
+                        if (data == '[DONE]'):
+                            break
+                        
+                        try:
+                            chunk_data = json.loads(data)
+                            if ("choices" in chunk_data) and (chunk_data["choices"]):
+                                delta = chunk_data["choices"][0].get("delta", {})
+                                
+                                if ("content" in delta):
+                                    yield delta["content"]
+                        
+                        except json.JSONDecodeError:
+                            continue
+    
+
+    def check_health(self) -> bool:
+        """
+        Check if LLM provider is healthy and accessible
+        
+        Returns:
+        --------
+            { bool }    : True if healthy
+        """
+        try:
+            if (self.provider == LLMProvider.OLLAMA):
+                response = requests.get(f"{self.base_url}/api/tags", timeout = 30)
+                return (response.status_code == 200)
+            
+            elif (self.provider == LLMProvider.OPENAI):
+                # Simple models list check
+                headers  = {"Authorization": f"Bearer {self.api_key}"}
+                response = requests.get(f"{self.base_url}/models", headers=headers, timeout=10)
+                return (response.status_code == 200)
+            
+            return False
+        
+        except Exception as e:
+            self.logger.warning(f"Health check failed: {repr(e)}")
+            return False
+    
+
+    def get_provider_info(self) -> Dict:
+        """
+        Get provider information
+        
+        Returns:
+        --------
+            { dict }    : Provider information
+        """
+        return {"provider" : self.provider.value,
+                "model"    : self.model_name,
+                "base_url" : self.base_url,
+                "healthy"  : self.check_health(),
+                "timeout"  : self.timeout,
+               }
+
+
+# Global LLM client instance
+_llm_client = None
+
+
+def get_llm_client(provider: LLMProvider = None, **kwargs) -> LLMClient:
+    """
+    Get global LLM client instance (singleton)
+    
+    Arguments:
+    ----------
+        provider { LLMProvider } : LLM provider to use
+        
+        **kwargs                 : Additional client configuration
+    
+    Returns:
+    --------
+        { LLMClient }           : LLMClient instance
+    """
+    global _llm_client
+    
+    if _llm_client is None or (provider and _llm_client.provider != provider):
+        _llm_client = LLMClient(provider, **kwargs)
+    
+    return _llm_client
+
+
+@handle_errors(error_type = LLMClientError, log_error = True, reraise = False)
+async def generate_text(messages: List[Dict], provider: LLMProvider = LLMProvider.OLLAMA, **kwargs) -> str:
+    """
+    Convenience function for text generation
+    
+    Arguments:
+    ----------
+        messages     { list }    : List of message dictionaries
+        
+        provider { LLMProvider } : LLM provider to use
+        
+        **kwargs                 : Generation parameters
+    
+    Returns:
+    --------
+             { str }             : Generated text
+    """
+    client   = get_llm_client(provider, **kwargs)
+    response = await client.generate(messages, **kwargs)
+    
+    return response["content"]

generation/prompt_builder.py

@@ -0,0 +1,542 @@
+# DEPENDENCIES
+from typing import List
+from typing import Dict
+from typing import Optional
+from config.models import PromptType
+from config.settings import get_settings
+from config.models import ChunkWithScore
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from generation.token_manager import TokenManager
+from utils.error_handler import PromptBuildingError
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class PromptBuilder:
+    """
+    Intelligent prompt building for LLM generation: Constructs optimized prompts for different task types with proper context management and citation handling
+    """
+    def __init__(self, model_name: str = None):
+        """
+        Initialize prompt builder
+        
+        Arguments:
+        ----------
+            model_name { str } : Model name for token management
+        """
+        self.logger           = logger
+        self.settings         = get_settings()
+        self.model_name       = model_name or self.settings.OLLAMA_MODEL
+        self.token_manager    = TokenManager(model_name)
+        
+        # Prompt templates for different tasks
+        self.prompt_templates = {PromptType.QA             : {"system": self._get_qa_system_prompt(), "user": self._get_qa_user_template(), "max_context_ratio": 0.6},
+                                 PromptType.SUMMARY        : {"system": self._get_summary_system_prompt(), "user": self._get_summary_user_template(), "max_context_ratio": 0.8,},
+                                 PromptType.ANALYTICAL     : {"system": self._get_analytical_system_prompt(), "user": self._get_analytical_user_template(), "max_context_ratio": 0.7},
+                                 PromptType.COMPARISON     : {"system": self._get_comparison_system_prompt(), "user": self._get_comparison_user_template(), "max_context_ratio": 0.5},
+                                 PromptType.EXTRACTION     : {"system": self._get_extraction_system_prompt(), "user": self._get_extraction_user_template(), "max_context_ratio": 0.6},
+                                 PromptType.CREATIVE       : {"system": self._get_creative_system_prompt(), "user": self._get_creative_user_template(), "max_context_ratio": 0.4},
+                                 PromptType.CONVERSATIONAL : {"system": self._get_conversational_system_prompt(), "user": self._get_conversational_user_template(), "max_context_ratio": 0.5}
+                                }
+    
+
+    def build_prompt(self, query: str, context: str, sources: List[ChunkWithScore], prompt_type: PromptType = PromptType.QA,  include_citations: bool = True,
+                     max_completion_tokens: int = 1000) -> Dict[str, str]:
+        """
+        Build complete prompt for LLM generation
+        
+        Arguments:
+        ----------
+            query                      { str }   : User query
+            
+            context                    { str }   : Retrieved context
+            
+            sources                   { list }   : Source chunks
+            
+            prompt_type           { PromptType } : Type of prompt to build
+            
+            include_citations         { bool }   : Whether to include citation instructions
+            
+            max_completion_tokens     { int }    : Tokens to reserve for completion
+        
+        Returns:
+        --------
+                         { dict }                : Dictionary with 'system' and 'user' prompts
+        """
+        if not query or not context:
+            raise PromptBuildingError("Query and context cannot be empty")
+        
+        try:
+            # Get template for prompt type
+            template          = self.prompt_templates.get(prompt_type, self.prompt_templates[PromptType.QA])
+            
+            # Optimize context to fit within token limits
+            optimized_context = self._optimize_context(context               = context, 
+                                                       template              = template, 
+                                                       max_completion_tokens = max_completion_tokens,
+                                                      )
+            
+            # Build system prompt
+            system_prompt     = self._build_system_prompt(template          = template, 
+                                                          include_citations = include_citations, 
+                                                          prompt_type       = prompt_type,
+                                                         )
+            
+            # Build user prompt
+            user_prompt       = self._build_user_prompt(template    = template, 
+                                                        query       = query, 
+                                                        context     = optimized_context, 
+                                                        sources     = sources, 
+                                                        prompt_type = prompt_type,
+                                                       )
+            
+            # Validate token usage
+            self._validate_prompt_length(system_prompt         = system_prompt, 
+                                         user_prompt           = user_prompt, 
+                                         max_completion_tokens = max_completion_tokens,
+                                        )
+            
+            self.logger.debug(f"Built {prompt_type.value} prompt: {self.token_manager.count_tokens(system_prompt + user_prompt)} tokens")
+            
+            return {"system"   : system_prompt,
+                    "user"     : user_prompt,
+                    "metadata" : {"prompt_type"    : prompt_type.value,
+                                  "context_tokens" : self.token_manager.count_tokens(optimized_context),
+                                  "total_tokens"   : self.token_manager.count_tokens(system_prompt + user_prompt),
+                                  "sources_count"  : len(sources),
+                                 }
+                  }
+        
+        except Exception as e:
+            self.logger.error(f"Prompt building failed: {repr(e)}")
+            raise PromptBuildingError(f"Prompt building failed: {repr(e)}")
+    
+
+    def _optimize_context(self, context: str, template: Dict, max_completion_tokens: int) -> str:
+        """
+        Optimize context to fit within token limits
+        
+        Arguments:
+        ----------
+            context               { str }  : Context text to optimize
+
+            template              { Dict } : Prompt template
+            
+            max_completion_tokens { int }  : Tokens to reserve for completion
+        
+        Returns:
+        --------
+                        { str }            : Optimized context
+        """
+        # Calculate tokens for system prompt
+        system_tokens          = self.token_manager.count_tokens(template["system"])
+        
+        # Estimate user template tokens by removing placeholders
+        user_template_clean    = template["user"]
+        
+        # Remove all known placeholders
+        placeholders_to_remove = ["{query}", "{context}", "{sources_info}", "{focus}"]
+        
+        for placeholder in placeholders_to_remove:
+            user_template_clean = user_template_clean.replace(placeholder, "")
+        
+        user_template_tokens = self.token_manager.count_tokens(user_template_clean)
+        
+        # Calculate available tokens for context - Reserve: system + user template + completion + buffer
+        reserved_tokens      = system_tokens + user_template_tokens + max_completion_tokens + 100
+        max_context_tokens   = self.token_manager.context_window - reserved_tokens
+        
+        # Ensure we don't exceed zero
+        if (max_context_tokens <= 0):
+            self.logger.warning(f"No tokens available for context. Reserved: {reserved_tokens}, Window: {self.token_manager.context_window}")
+            return ""
+        
+        # Apply max context ratio from template config
+        ratio_limit        = int(self.token_manager.context_window * template["max_context_ratio"])
+        max_context_tokens = min(max_context_tokens, ratio_limit)
+        
+        # Log optimization details
+        self.logger.debug(f"Context optimization: max_tokens={max_context_tokens}, ratio={template['max_context_ratio']}")
+        
+        # Truncate context to fit
+        optimized_context  = self.token_manager.truncate_to_fit(context, max_context_tokens, strategy="end")
+        
+        # Calculate reduction percentage
+        original_tokens    = self.token_manager.count_tokens(context)
+        optimized_tokens   = self.token_manager.count_tokens(optimized_context)
+        
+        if (original_tokens > optimized_tokens):
+            reduction_pct = ((original_tokens - optimized_tokens) / original_tokens) * 100
+            
+            self.logger.info(f"Context reduced by {reduction_pct:.1f}% ({original_tokens} → {optimized_tokens} tokens)")
+        
+        return optimized_context
+    
+
+    def _build_system_prompt(self, template: Dict, include_citations: bool, prompt_type: PromptType) -> str:
+        """
+        Build system prompt
+        """
+        system_prompt = template["system"]
+        
+        # Add citation instructions if needed
+        if (include_citations and (prompt_type != PromptType.CREATIVE)):
+            citation_instructions = self._get_citation_instructions()
+            system_prompt        += "\n\n" + citation_instructions
+        
+        return system_prompt
+    
+
+    def _build_user_prompt(self, template: Dict, query: str, context: str, sources: List[ChunkWithScore], prompt_type: PromptType) -> str:
+        """
+        Build user prompt
+        """
+        # Format sources information
+        sources_info = self._format_sources_info(sources) if sources else ""
+        
+        # Build user prompt using template
+        user_prompt  = template["user"].format(query        = query,
+                                               context      = context,
+                                               sources_info = sources_info,
+                                             )
+        
+        # Add task-specific formatting
+        if (prompt_type == PromptType.COMPARISON):
+            user_prompt = self._enhance_comparison_prompt(user_prompt = user_prompt, 
+                                                          sources     = sources,
+                                                         )
+
+        elif (prompt_type == PromptType.ANALYTICAL):
+            user_prompt = self._enhance_analytical_prompt(user_prompt = user_prompt, 
+                                                          sources     = sources,
+                                                         )
+        
+        return user_prompt
+    
+
+    def _format_sources_info(self, sources: List[ChunkWithScore]) -> str:
+        """
+        Format sources information for the prompt
+        """
+        if not sources:
+            return ""
+        
+        sources_list = list()
+
+        for i, source in enumerate(sources, 1):
+            chunk       = source.chunk
+            source_info = f"Source [{i}]:"
+            
+            if hasattr(chunk, 'metadata') and 'filename' in chunk.metadata:
+                source_info += f" {chunk.metadata['filename']}"
+            
+            if chunk.page_number:
+                source_info += f" (page {chunk.page_number})"
+            
+            if chunk.section_title:
+                source_info += f" - {chunk.section_title}"
+            
+            sources_list.append(source_info)
+        
+        return "Available sources:\n" + "\n".join(sources_list)
+    
+
+    def _enhance_comparison_prompt(self, user_prompt: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Enhance prompt for comparison tasks
+        
+        Arguments:
+        ----------
+            user_prompt { str }  : Base user prompt
+
+            sources     { list } : Source chunks
+        
+        Returns:
+        --------
+                { str }          : Enhanced prompt
+        """
+        if (len(sources) < 2):
+            return user_prompt
+        
+        enhancement  = "\n\nPlease compare and contrast the information from different sources. "
+        enhancement += "Highlight agreements, disagreements, and complementary information. "
+        enhancement += "If sources conflict, present both perspectives clearly."
+        
+        return user_prompt + enhancement
+    
+
+    def _enhance_analytical_prompt(self, user_prompt: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Enhance prompt for analytical tasks - FIXED SIGNATURE
+        
+        Arguments:
+        ----------
+            user_prompt { str }  : Base user prompt
+
+            sources     { list } : Source chunks
+        
+        Returns:
+        --------
+                { str }          : Enhanced prompt
+        """
+        enhancement  = "\n\nProvide analytical insights by:"
+        enhancement += "\n1. Identifying patterns and relationships in the information"
+        enhancement += "\n2. Analyzing implications and consequences"
+        enhancement += "\n3. Evaluating the strength of evidence from different sources"
+        enhancement += "\n4. Drawing well-reasoned conclusions"
+        
+        return user_prompt + enhancement
+    
+
+    def _validate_prompt_length(self, system_prompt: str, user_prompt: str, max_completion_tokens: int):
+        """
+        Validate that prompt fits within context window
+        
+        Arguments:
+        ----------
+            system_prompt         { str } : System prompt
+
+            user_prompt           { str } : User prompt
+            
+            max_completion_tokens { int } : Tokens needed for completion
+        
+        Raises:
+        -------
+            PromptBuildingError           : If prompt exceeds context window
+        """
+        system_tokens  = self.token_manager.count_tokens(system_prompt)
+        user_tokens    = self.token_manager.count_tokens(user_prompt)
+        total_tokens   = system_tokens + user_tokens
+        total_required = total_tokens + max_completion_tokens
+        
+        if (total_required > self.token_manager.context_window):
+            error_msg = (f"Prompt exceeds context window:\n"
+                         f"- System prompt: {system_tokens} tokens\n"
+                         f"- User prompt: {user_tokens} tokens\n"
+                         f"- Completion reserve: {max_completion_tokens} tokens\n"
+                         f"- Total required: {total_required} tokens\n"
+                         f"- Context window: {self.token_manager.context_window} tokens\n"
+                         f"- Overflow: {total_required - self.token_manager.context_window} tokens"
+                        )
+            self.logger.error(error_msg)
+
+            raise PromptBuildingError(error_msg)
+        
+        # Log successful validation
+        utilization = (total_required / self.token_manager.context_window) * 100
+
+        self.logger.debug(f"Prompt validation passed: {total_required}/{self.token_manager.context_window} tokens ({utilization:.1f}% utilization)")
+
+
+    def _get_citation_instructions(self) -> str:
+        """
+        Get citation instructions for system prompt
+        """
+        return ("CITATION INSTRUCTIONS:\n"
+                "1. Always cite your sources using [number] notation\n"
+                "2. Use the citation number that corresponds to the source in the context\n"
+                "3. Cite sources for all factual claims and specific information\n"
+                "4. If information appears in multiple sources, cite the most relevant one\n"
+                "5. Do not make up information not present in the provided context\n"
+                "6. If the context doesn't contain the answer, explicitly state this"
+               )
+    
+
+    # Template methods for different prompt types
+    def _get_qa_system_prompt(self) -> str:
+        return ("You are a precise and helpful AI assistant that answers questions based solely on the provided context.\n\n"
+                "Core Principles:\n"
+                "1. ONLY use information from the provided context\n"
+                "2. Be concise but complete - don't omit important details\n"
+                "3. Structure complex answers clearly\n"
+                "4. If information is ambiguous or conflicting, acknowledge this\n"
+                "5. Never make up or infer information not present in the context"
+               )
+    
+
+    def _get_qa_user_template(self) -> str:
+        return ("Context Information:\n{context}\n\n"
+                "{sources_info}\n\n"
+                "Question: {query}\n\n"
+                "Instructions: Answer the question using ONLY the information provided in the context above. "
+                "Cite your sources using [number] notation. If the context doesn't contain enough information "
+                "to answer fully, state this clearly.\n\n"
+                "Answer:"
+               )
+    
+
+    def _get_summary_system_prompt(self) -> str:
+        return ("You are a thorough AI assistant that provides comprehensive summaries based on the provided context.\n\n"
+                "Summary Guidelines:\n"
+                "1. Capture all key points and main ideas\n"
+                "2. Maintain the original meaning and intent\n"
+                "3. Organize information logically\n"
+                "4. Be comprehensive but concise\n"
+                "5. Highlight important findings and conclusions"
+               )
+    
+
+    def _get_summary_user_template(self) -> str:
+        return ("Content to summarize:\n{context}\n\n"
+                "Please provide a comprehensive summary that captures all key points and main ideas. "
+                "Organize the summary logically and ensure it reflects the original content accurately.\n\n"
+                "Summary:"
+               )
+    
+
+    def _get_analytical_system_prompt(self) -> str:
+        return ("You are an analytical AI assistant that provides insights based on the provided context.\n\n"
+                "Analytical Guidelines:\n"
+                "1. Analyze patterns and connections in the information\n"
+                "2. Compare different perspectives if multiple sources exist\n"
+                "3. Highlight key findings and implications\n"
+                "4. Identify gaps or limitations in the available information\n"
+                "5. Provide well-reasoned analysis and conclusions"
+               )
+
+    
+    def _get_analytical_user_template(self) -> str:
+        return ("Context for analysis:\n{context}\n\n"
+                "{sources_info}\n\n"
+                "Analytical task: {query}\n\n"
+                "Please provide a detailed analysis based on the context above. "
+                "Identify patterns, relationships, and implications. "
+                "Cite sources for all analytical claims.\n\n"
+                "Analysis:"
+               )
+
+    
+    def _get_comparison_system_prompt(self) -> str:
+        return ("You are an AI assistant that compares information across multiple sources.\n\n"
+                "Comparison Guidelines:\n"
+                "1. Identify similarities and differences between sources\n"
+                "2. Note if sources agree or disagree on specific points\n"
+                "3. Highlight complementary information\n"
+                "4. Present conflicting perspectives fairly\n"
+                "5. Draw conclusions about the overall consensus or disagreement"
+               )
+
+    
+    def _get_comparison_user_template(self) -> str:
+        return ("Context from multiple sources:\n{context}\n\n"
+                "{sources_info}\n\n"
+                "Comparison task: {query}\n\n"
+                "Please compare how different sources address this topic. "
+                "Identify agreements, disagreements, and complementary information. "
+                "Cite specific sources for each point of comparison.\n\n"
+                "Comparison:"
+               )
+
+    
+    def _get_extraction_system_prompt(self) -> str:
+        return ("You are a precise AI assistant that extracts specific information from context.\n\n"
+                "Extraction Guidelines:\n"
+                "1. Extract only the requested information\n"
+                "2. Be thorough and complete\n"
+                "3. Maintain accuracy and precision\n"
+                "4. Organize extracted information clearly\n"
+                "5. Cite sources for all extracted information"
+               )
+
+    
+    def _get_extraction_user_template(self) -> str:
+        return ("Context:\n{context}\n\n"
+                "{sources_info}\n\n"
+                "Extraction task: {query}\n\n"
+                "Please extract the requested information from the context above. "
+                "Be thorough and precise. Cite sources for all extracted information.\n\n"
+                "Extracted Information:"
+               )
+
+    
+    def _get_creative_system_prompt(self) -> str:
+        return ("You are a creative AI assistant that generates content based on provided context.\n\n"
+                "Creative Guidelines:\n"
+                "1. Use the context as inspiration and foundation\n"
+                "2. Be creative and engaging\n"
+                "3. Maintain coherence with the source material\n"
+                "4. You may extrapolate and build upon the provided information\n"
+                "5. Clearly distinguish between source-based content and creative additions"
+               )
+
+    
+    def _get_creative_user_template(self) -> str:
+        return ("Context and inspiration:\n{context}\n\n"
+                "Creative task: {query}\n\n"
+                "Please create content based on the context above. "
+                "You may use the context as inspiration and build upon it creatively. "
+                "If you add information beyond what's in the context, make this clear.\n\n"
+                "Creative Response:"
+               )
+
+    
+    def _get_conversational_system_prompt(self) -> str:
+        return ("You are a helpful and engaging conversational AI assistant.\n\n"
+                "Conversational Guidelines:\n"
+                "1. Be natural and engaging in conversation\n"
+                "2. Use the provided context to inform your responses\n"
+                "3. Maintain a friendly and helpful tone\n"
+                "4. Ask clarifying questions when needed\n"
+                "5. Cite sources when providing specific information"
+               )
+
+    
+    def _get_conversational_user_template(self) -> str:
+        return ("Context for our conversation:\n{context}\n\n"
+                "Current message: {query}\n\n"
+                "Please respond naturally and helpfully based on the context above. "
+                "If providing specific information, cite your sources.\n\n"
+                "Response:"
+               )
+
+
+# Global prompt builder instance
+_prompt_builder = None
+
+
+def get_prompt_builder(model_name: str = None) -> PromptBuilder:
+    """
+    Get global prompt builder instance (singleton)
+    
+    Arguments:
+    ----------
+        model_name { str } : Model name for token management
+    
+    Returns:
+    --------
+        { PromptBuilder }  : PromptBuilder instance
+    """
+    global _prompt_builder
+    
+    if _prompt_builder is None or (model_name and _prompt_builder.model_name != model_name):
+        _prompt_builder = PromptBuilder(model_name)
+    
+    return _prompt_builder
+
+
+@handle_errors(error_type = PromptBuildingError, log_error = True, reraise = False)
+def build_qa_prompt(query: str, context: str, sources: List[ChunkWithScore], **kwargs) -> Dict[str, str]:
+    """
+    Convenience function for building QA prompts
+    
+    Arguments:
+    ----------
+        query   { str }  : User query
+        
+        context { str }  : Retrieved context
+        
+        sources { list } : Source chunks
+        
+        **kwargs         : Additional prompt building arguments
+    
+    Returns:
+    --------
+             { dict }    : Dictionary with system and user prompts
+    """
+    builder = get_prompt_builder()
+    
+    return builder.build_prompt(query, context, sources, PromptType.QA, **kwargs)

generation/query_classifier.py

@@ -0,0 +1,247 @@
+# DEPENDENCIES
+import json
+from typing import Dict
+from config.models import LLMProvider
+from config.settings import get_settings
+from config.logging_config import get_logger
+from generation.llm_client import get_llm_client
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class QueryClassifier:
+    """
+    LLM-based query classifier that intelligently routes queries to:
+    1. General/Conversational (no document context needed)
+    2. RAG/Document-based (needs retrieval from documents)
+    
+    Uses the LLM itself for classification instead of hardcoded patterns.
+    """
+    def __init__(self, provider: LLMProvider = None, model_name: str = None):
+        self.logger        = logger
+        self.provider      = provider or LLMProvider.OLLAMA
+        self.model_name    = model_name or settings.OLLAMA_MODEL
+        
+        # Initialize LLM client for classification
+        self.llm_client    = get_llm_client(provider   = self.provider,
+                                            model_name = self.model_name,
+                                           )
+        
+        # Classification prompt
+        self.system_prompt = """
+                                You are a query classification system for a RAG (Retrieval-Augmented Generation) system. 
+                                Your job is to determine if a user query should be answered using the user's uploaded documents.
+
+                                **IMPORTANT CONTEXT**: The user has uploaded documents to the system. All queries related to the content of those uploaded documents should use RAG.
+
+                                Classify queries into TWO categories:
+
+                                **RAG (Document-based)** - Use when ANY of these are true:
+                                1. Query asks about ANY content that could be in the uploaded documents
+                                2. Query asks factual questions that could be answered from document content
+                                3. Query asks for lists, summaries, or analysis of information
+                                4. Query mentions specific details, data, statistics, names, dates, or facts
+                                5. Query asks "what", "how", "list", "explain", "summarize", "compare", "analyze" about any topic
+                                6. Query could reasonably be answered by searching through documents
+                                7. **CRITICAL**: When documents are uploaded, DEFAULT TO RAG for most factual/content queries
+
+                                **GENERAL (Conversational)** - Use ONLY when MOST of these are true:
+                                1. Query is purely conversational (greetings, thanks, casual chat)
+                                2. Query asks about the RAG system itself or its functionality
+                                3. Query asks for general knowledge that is NOT specific to uploaded documents
+                                4. Query is a meta-question about how to use the system
+                                5. Query contains NO request for factual information from documents
+
+                                **EXAMPLES FOR ANY DOCUMENT TYPE**:
+                                - For business documents: "What sales channels does the company use?" → RAG
+                                - For research papers: "What were the study's findings?" → RAG
+                                - For legal documents: "What are the key clauses?" → RAG
+                                - For technical manuals: "How do I configure the system?" → RAG
+                                - For personal documents: "What dates are mentioned?" → RAG
+                                - "Hi, how are you?" → GENERAL
+                                - "How do I upload a document?" → GENERAL
+                                - "What is the capital of France?" → GENERAL (unless geography documents were uploaded)
+
+                                **KEY RULES**:
+                                1. When documents exist, assume queries are about them unless clearly not
+                                2. When in doubt, classify as RAG (safer to search than hallucinate)
+                                3. If query could be answered from document content, use RAG
+                                4. Only use GENERAL for purely conversational or system-related queries
+
+                                Respond with ONLY a JSON object (no markdown, no extra text):
+                                {
+                                "type": "rag" or "general",
+                                "confidence": 0.0 to 1.0,
+                                "reason": "brief explanation"
+                                }
+                             """
+    
+    
+    async def classify(self, query: str, has_documents: bool = True) -> Dict:
+        """
+        Classify a query using LLM
+        
+        Arguments:
+        ----------
+            query         { str }  : User query
+
+            has_documents { bool } : Whether documents are available in the system
+        
+        Returns:
+        --------
+                 { dict }          : Classification result
+        """
+        try:
+            # If no documents are available, everything should be general
+            if not has_documents:
+                return {"type"              : "general",
+                        "confidence"        : 1.0,
+                        "reason"            : "No documents available in system",
+                        "suggested_action"  : "respond_with_general_llm",
+                        "is_llm_classified" : False,
+                       }
+            
+            # Build classification prompt
+            user_prompt    = f"""
+                                 Query: "{query}"
+
+                                 System status: {"Documents are available" if has_documents else "No documents uploaded"}
+
+                                 Classify this query. Remember: if uncertain, prefer RAG.
+                              """
+            
+            messages       = [{"role"    : "system", 
+                               "content" : self.system_prompt,
+                              },
+                              {"role"    : "user", 
+                               "content" : user_prompt,
+                              }
+                             ]
+            
+            # Get LLM classification (use low temperature for consistency)
+            llm_response   = await self.llm_client.generate(messages    = messages,
+                                                            temperature = 0.1,  # Low temperature for consistent classification
+                                                            max_tokens  = 150,
+                                                           )
+            
+            response_text  = llm_response.get("content", "").strip()
+            
+            # Parse JSON response
+            classification = self._parse_llm_response(response_text)
+            
+            # Add suggested action based on classification
+            if (classification["type"] == "rag"):
+                classification["suggested_action"] = "respond_with_rag"
+
+            elif (classification["type"] == "general"):
+                classification["suggested_action"] = "respond_with_general_llm"
+
+            else:
+                # Default to RAG if uncertain
+                classification["suggested_action"] = "respond_with_rag"
+            
+            classification["is_llm_classified"] = True
+            
+            logger.info(f"LLM classified query as: {classification['type']} (confidence: {classification['confidence']:.2f})")
+            logger.debug(f"Classification reason: {classification['reason']}")
+            
+            return classification
+            
+        except Exception as e:
+            logger.error(f"LLM classification failed: {e}, defaulting to RAG")
+            # On error, default to RAG (safer to try document search)
+            return {"type"              : "rag",
+                    "confidence"        : 0.5,
+                    "reason"            : f"Classification failed: {str(e)}, defaulting to RAG",
+                    "suggested_action"  : "respond_with_rag",
+                    "is_llm_classified" : False,
+                    "error"             : str(e)
+                   }
+    
+    
+    def _parse_llm_response(self, response_text: str) -> Dict:
+        """
+        Parse LLM JSON response
+        
+        Arguments:
+        ----------
+            response_text { str } : LLM response text
+        
+        Returns:
+        --------
+                 { dict }         : Parsed classification
+        """
+        try:
+            # Remove markdown code blocks if present
+            if ("```json" in response_text):
+                response_text = response_text.split("```json")[1].split("```")[0].strip()
+            
+            elif ("```" in response_text):
+                response_text = response_text.split("```")[1].split("```")[0].strip()
+            
+            # Parse JSON
+            result = json.loads(response_text)
+            
+            # Validate required fields
+            if ("type" not in result) or (result["type"] not in ["rag", "general"]):
+                raise ValueError(f"Invalid type in response: {result.get('type')}")
+            
+            # Set defaults for missing fields
+            result.setdefault("confidence", 0.8)
+            result.setdefault("reason", "LLM classification")
+            
+            # Clamp confidence to valid range
+            result["confidence"] = max(0.0, min(1.0, float(result["confidence"])))
+            
+            return result
+            
+        except (json.JSONDecodeError, ValueError, KeyError) as e:
+            logger.warning(f"Failed to parse LLM response: {e}")
+            logger.debug(f"Raw response: {response_text}")
+            
+            # Try to extract type from text if JSON parsing fails
+            response_lower = response_text.lower()
+            
+            if (("general" in response_lower) and ("rag" not in response_lower)):
+                return {"type"       : "general",
+                        "confidence" : 0.6,
+                        "reason"     : "Parsed from non-JSON response",
+                       }
+
+            else:
+                # Default to RAG if parsing fails
+                return {"type"       : "rag",
+                        "confidence" : 0.6,
+                        "reason"     : "Failed to parse response, defaulting to RAG",
+                       }
+
+
+# Global classifier instance
+_query_classifier = None
+
+
+def get_query_classifier(provider: LLMProvider = None, model_name: str = None) -> QueryClassifier:
+    """
+    Get global query classifier instance
+    
+    Arguments:
+    ----------
+        provider   { LLMProvider } : LLM provider
+
+        model_name    { str }      : Model name
+    
+    Returns:
+    --------
+        { QueryClassifier }        : QueryClassifier instance
+    """
+    global _query_classifier
+    
+    if _query_classifier is None:
+        _query_classifier = QueryClassifier(provider   = provider, 
+                                            model_name = model_name,
+                                           )
+    
+    return _query_classifier

generation/response_generator.py

@@ -0,0 +1,880 @@
+# DEPENDENCIES
+import time
+import asyncio
+from typing import Dict
+from typing import List
+from typing import Optional
+from datetime import datetime
+from typing import AsyncGenerator
+from config.models import PromptType
+from config.models import LLMProvider
+from config.models import QueryRequest
+from config.models import QueryResponse
+from config.models import ChunkWithScore
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from generation.llm_client import get_llm_client
+from utils.error_handler import ResponseGenerationError
+from generation.prompt_builder import get_prompt_builder
+from retrieval.hybrid_retriever import get_hybrid_retriever
+from generation.query_classifier import get_query_classifier
+from generation.general_responder import get_general_responder
+from generation.citation_formatter import get_citation_formatter
+from generation.temperature_controller import get_temperature_controller
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class ResponseGenerator:
+    """
+    Main orchestrator for RAG response generation with LLM-based intelligent query routing
+    
+    Handles both:
+    1. Generic/conversational queries (greetings, system info, general knowledge)
+    2. Document-based RAG queries (retrieval + generation)
+    
+    Pipeline: Query → LLM Classifier → Route to (General LLM | RAG Pipeline) → Response
+    """
+    def __init__(self, provider: LLMProvider = None, model_name: str = None):
+        """
+        Initialize response generator with LLM-based query routing capabilities
+        
+        Arguments:
+        ----------
+            provider   { LLMProvider } : LLM provider (Ollama/OpenAI)
+
+            model_name { str }         : Model name to use
+        """
+        self.logger                 = logger
+        self.settings               = get_settings()
+
+        # Auto-detect provider for HF Spaces
+        if provider is None:
+            if (self.settings.IS_HF_SPACE and not self.settings.OLLAMA_ENABLED):
+                if (self.settings.USE_OPENAI and self.settings.OPENAI_API_KEY):
+                    provider   = LLMProvider.OPENAI
+                    model_name = model_name or self.settings.OPENAI_MODEL
+                    logger.info("Auto-detected: Using OpenAI")
+                
+                else:
+                    raise ValueError("No LLM provider configured for HF Space. Set OPENAI_API_KEY in Space secrets.")
+            
+            else:
+                # Local development - use Ollama
+                provider = LLMProvider.OLLAMA
+
+        self.provider               = provider 
+        self.model_name             = model_name
+        
+        # Initialize components
+        self.llm_client             = get_llm_client(provider   = self.provider, 
+                                                     model_name = self.model_name,
+                                                    )
+        
+        # Query routing components (NOW USES LLM FOR CLASSIFICATION)
+        self.query_classifier       = get_query_classifier(provider   = self.provider,
+                                                           model_name = self.model_name,
+                                                          )
+        
+        self.general_responder      = get_general_responder(provider   = self.provider,
+                                                            model_name = self.model_name,
+                                                           )
+        
+        # RAG components
+        self.hybrid_retriever       = get_hybrid_retriever()
+        self.prompt_builder         = get_prompt_builder(model_name = self.model_name)
+        self.citation_formatter     = get_citation_formatter()
+        self.temperature_controller = get_temperature_controller()
+        
+        # Statistics
+        self.generation_count       = 0
+        self.total_generation_time  = 0.0
+        self.general_query_count    = 0
+        self.rag_query_count        = 0
+        
+        self.logger.info(f"Initialized ResponseGenerator with LLM-Based Query Routing: provider={self.provider.value}, model={self.model_name}")
+    
+
+    @handle_errors(error_type = ResponseGenerationError, log_error = True, reraise = True)
+    async def generate_response(self, request: QueryRequest, conversation_history: List[Dict] = None, has_documents: bool = True) -> QueryResponse:
+        """
+        Generate response with LLM-based intelligent query routing
+        
+        Arguments:
+        ----------
+            request              { QueryRequest } : Query request object
+
+            conversation_history { list }        : Previous conversation messages
+            
+            has_documents        { bool }        : Whether documents are available in the system
+        
+        Returns:
+        --------
+            { QueryResponse }                    : Complete query response
+        """
+        start_time = time.time()
+        
+        self.logger.info(f"Processing query: '{request.query[:100]}...'")
+        
+        try:
+            # Classify query using LLM
+            classification = await self.query_classifier.classify(query         = request.query,
+                                                                  has_documents = has_documents,
+                                                                 )
+            
+            self.logger.info(f"Query classified as: {classification['type']} (confidence: {classification['confidence']:.2f}, LLM-based: {classification.get('is_llm_classified', False)})")
+            self.logger.debug(f"Classification reason: {classification['reason']}")
+            
+            # Route based on classification
+            if (classification['suggested_action'] == 'respond_with_general_llm'):
+                # Handle as general query
+                response                  = await self._handle_general_query(request        = request,
+                                                                             classification = classification,
+                                                                             start_time     = start_time,
+                                                                             history        = conversation_history,
+                                                                            )
+                                    
+                self.general_query_count += 1
+                
+                return response
+            
+            elif (classification['suggested_action'] == 'respond_with_rag'):
+                # Handle as RAG query
+                response              = await self._handle_rag_query(request        = request,
+                                                                     classification = classification,
+                                                                     start_time     = start_time,
+                                                                    )
+                
+                self.rag_query_count += 1
+                
+                return response
+            
+            else:
+                # Default to RAG if unclear
+                self.logger.info("Unclear classification - defaulting to RAG...")
+                
+                try:
+                    response = await self._handle_rag_query(request        = request,
+                                                            classification = classification,
+                                                            start_time     = start_time,
+                                                            allow_fallback = True,
+                                                           )
+                    
+                    # If no results from RAG, fall back to general
+                    if ((not response.sources) or (len(response.sources) == 0)):
+                        self.logger.info("No RAG results - falling back to general response")
+
+                        response                  = await self._handle_general_query(request        = request,
+                                                                                     classification = classification,
+                                                                                     start_time     = start_time,
+                                                                                     history        = conversation_history,
+                                                                                    )
+                        self.general_query_count += 1
+                    
+                    else:
+                        self.rag_query_count     += 1
+                    
+                    return response
+                    
+                except Exception as e:
+                    self.logger.warning(f"RAG attempt failed, falling back to general: {e}")
+                    response                  = await self._handle_general_query(request        = request,
+                                                                                 classification = classification,
+                                                                                 start_time     = start_time,
+                                                                                 history        = conversation_history,
+                                                                                )
+                    self.general_query_count += 1
+                    
+                    return response
+        
+        except Exception as e:
+            self.logger.error(f"Response generation failed: {repr(e)}", exc_info = True)
+            raise ResponseGenerationError(f"Response generation failed: {repr(e)}")
+    
+
+    async def _handle_general_query(self, request: QueryRequest, classification: Dict, start_time: float, history: List[Dict] = None) -> QueryResponse:
+        """
+        Handle general/conversational queries without RAG
+        
+        Arguments:
+        ----------
+            request        { QueryRequest } : Original request
+
+            classification { dict }         : Classification result
+            
+            start_time     { float }        : Start timestamp
+        
+            history        { list }         : Conversation history
+        
+        Returns:
+        --------
+            { QueryResponse }               : Response without RAG
+        """
+        self.logger.debug("Handling as general query...")
+        
+        # Use general responder
+        general_response = await self.general_responder.respond(query                = request.query,
+                                                                conversation_history = history,
+                                                               )
+        
+        answer           = general_response.get("answer", "I'm here to help! Please let me know how I can assist you.")
+        total_time       = (time.time() - start_time) * 1000
+        
+        # Create QueryResponse object
+        response         = QueryResponse(query              = request.query,
+                                         answer             = answer,
+                                         sources            = [],  # No sources for general queries
+                                         retrieval_time_ms  = 0.0,
+                                         generation_time_ms = total_time,
+                                         total_time_ms      = total_time,
+                                         tokens_used        = general_response.get("tokens_used", {"input": 0, "output": 0, "total": 0}),
+                                         model_used         = self.model_name,
+                                         timestamp          = datetime.now(),
+                                        )
+        
+        # Add metadata about query type
+        if request.include_metrics:
+            response.metrics = {"query_type"        : "general",
+                                "classification"    : classification['type'],
+                                "confidence"        : classification['confidence'],
+                                "requires_rag"      : False,
+                                "conversation_mode" : True,
+                                "llm_classified"    : classification.get('is_llm_classified', False),
+                               }
+        
+        self.logger.info(f"General response generated in {total_time:.0f}ms")
+        
+        return response
+    
+
+    async def _handle_rag_query(self, request: QueryRequest, classification: Dict, start_time: float, allow_fallback: bool = False) -> QueryResponse:
+        """
+        Handle RAG-based queries with document retrieval
+        
+        Arguments:
+        ----------
+            request        { QueryRequest } : Original request
+
+            classification { dict }         : Classification result
+            
+            start_time     { float }        : Start timestamp
+            
+            allow_fallback { bool }         : Whether to allow fallback to general
+        
+        Returns:
+        --------
+            { QueryResponse }               : RAG response
+        """
+        self.logger.debug("Handling as RAG query...")
+        
+        try:
+            # Retrieve relevant context
+            self.logger.debug("Retrieving context...")
+            retrieval_start  = time.time()
+            
+            retrieval_result = self.hybrid_retriever.retrieve_with_context(query             = request.query,
+                                                                           top_k             = request.top_k or self.settings.TOP_K_RETRIEVE,
+                                                                           enable_reranking  = request.enable_reranking,
+                                                                           include_citations = request.include_sources,
+                                                                          )
+            
+            retrieval_time   = (time.time() - retrieval_start) * 1000
+            
+            chunks           = retrieval_result["chunks"]
+            context          = retrieval_result["context"]
+            
+            if not chunks:
+                self.logger.warning("No relevant context found for query")
+                
+                if allow_fallback:
+                    # Return empty response to trigger fallback
+                    return QueryResponse(query              = request.query,
+                                         answer             = "",
+                                         sources            = [],
+                                         retrieval_time_ms  = retrieval_time,
+                                         generation_time_ms = 0.0,
+                                         total_time_ms      = retrieval_time,
+                                         tokens_used        = {"input": 0, "output": 0, "total": 0},
+                                         model_used         = self.model_name,
+                                         timestamp          = datetime.now(),
+                                        )
+
+                else:
+                    return self._create_no_results_response(request           = request, 
+                                                            retrieval_time_ms = retrieval_time,
+                                                           )
+            
+            self.logger.info(f"Retrieved {len(chunks)} chunks in {retrieval_time:.0f}ms")
+            
+            # Determine prompt type and temperature
+            self.logger.debug("Determining prompt strategy...")
+
+            prompt_type = self._infer_prompt_type(query = request.query)
+            
+            temperature = self._get_adaptive_temperature(request          = request,
+                                                         query            = request.query,
+                                                         context          = context,
+                                                         retrieval_scores = [chunk.score for chunk in chunks],
+                                                         prompt_type      = prompt_type,
+                                                        )
+            
+            self.logger.debug(f"Prompt type: {prompt_type.value}, Temperature: {temperature}")
+            
+            # Build optimized prompt
+            self.logger.debug("Building prompt...")
+
+            prompt      = self.prompt_builder.build_prompt(query                 = request.query,
+                                                           context               = context,
+                                                           sources               = chunks,
+                                                           prompt_type           = prompt_type,
+                                                           include_citations     = request.include_sources,
+                                                           max_completion_tokens = request.max_tokens or self.settings.MAX_TOKENS,
+                                                          )
+            
+            # Generate LLM response
+            self.logger.debug("Generating LLM response...")
+            generation_start = time.time()
+            
+            messages         = [{"role"    : "system", 
+                                 "content" : prompt["system"]
+                                },
+                                {"role"    : "user", 
+                                 "content" : prompt["user"],
+                                }
+                               ]
+            
+            llm_response     = await self.llm_client.generate(messages    = messages,
+                                                              temperature = temperature,
+                                                              top_p       = request.top_p or self.settings.TOP_P,
+                                                              max_tokens  = request.max_tokens or self.settings.MAX_TOKENS,
+                                                             )
+            
+            generation_time  = (time.time() - generation_start) * 1000
+            
+            answer           = llm_response["content"]
+            
+            self.logger.info(f"Generated response in {generation_time:.0f}ms ({llm_response['usage']['completion_tokens']} tokens)")
+            
+            # Format citations (if enabled)
+            if request.include_sources:
+                self.logger.debug("Formatting citations...")
+                answer = self._post_process_citations(answer  = answer, 
+                                                      sources = chunks,
+                                                     )
+            
+            # Create response object
+            total_time = (time.time() - start_time) * 1000
+            
+            response   = QueryResponse(query              = request.query,
+                                       answer             = answer,
+                                       sources            = chunks if request.include_sources else [],
+                                       retrieval_time_ms  = retrieval_time,
+                                       generation_time_ms = generation_time,
+                                       total_time_ms      = total_time,
+                                       tokens_used        = {"input"  : llm_response["usage"]["prompt_tokens"],
+                                                             "output" : llm_response["usage"]["completion_tokens"],
+                                                             "total"  : llm_response["usage"]["total_tokens"],
+                                                            },
+                                       model_used         = self.model_name,
+                                       timestamp          = datetime.now(),
+                                      )
+            
+            # Add quality metrics if requested
+            if request.include_metrics:
+                response.metrics                           = self._calculate_quality_metrics(query    = request.query,
+                                                                                             answer   = answer,
+                                                                                             context  = context,
+                                                                                             sources  = chunks,
+                                                                                            )
+                # Track both: prediction & reality 
+                response.metrics["predicted_type"]         = classification.get('type', 'unknown')
+                response.metrics["predicted_confidence"]   = classification.get('confidence', 0.0)
+                response.metrics["actual_type"]            = "rag"  # Always rag if we're here
+                response.metrics["execution_path"]         = "rag_pipeline"
+                response.metrics["has_context"]            = len(chunks) > 0
+                response.metrics["context_chunks"]         = len(chunks)
+                response.metrics["rag_confidence"]         = min(1.0, sum(c.score for c in chunks) / len(chunks) if chunks else 0.0)
+                response.metrics["is_forced_rag"]          = classification.get('is_forced_rag', False)
+                response.metrics["llm_classified"]         = classification.get('is_llm_classified', False)
+
+                # Add context for evaluation
+                response.metrics["context_for_evaluation"] = context
+            
+            # Update statistics
+            self.generation_count      += 1
+            self.total_generation_time += total_time
+            
+            self.logger.info(f"RAG response generated successfully in {total_time:.0f}ms")
+            
+            return response
+            
+        except Exception as e:
+            self.logger.error(f"RAG query handling failed: {repr(e)}", exc_info = True)
+            
+            if allow_fallback:
+                # Return empty to trigger fallback
+                return QueryResponse(query              = request.query,
+                                     answer             = "",
+                                     sources            = [],
+                                     retrieval_time_ms  = 0.0,
+                                     generation_time_ms = 0.0,
+                                     total_time_ms      = 0.0,
+                                     tokens_used        = {"input": 0, "output": 0, "total": 0},
+                                     model_used         = self.model_name,
+                                     timestamp          = datetime.now(),
+                                    )
+            else:
+                raise
+    
+
+    @handle_errors(error_type = ResponseGenerationError, log_error = True, reraise = True)
+    async def generate_response_stream(self, request: QueryRequest, has_documents: bool = True) -> AsyncGenerator[str, None]:
+        """
+        Generate streaming RAG response
+        
+        Arguments:
+        ----------
+            request       { QueryRequest } : Query request object
+            
+            has_documents { bool }         : Whether documents are available
+        
+        Yields:
+        -------
+            { str }                        : Response chunks (tokens)
+        """
+        self.logger.info(f"Generating streaming response for query: '{request.query[:100]}...'")
+        
+        try:
+            # Classify query first
+            classification = await self.query_classifier.classify(query         = request.query,
+                                                                  has_documents = has_documents,
+                                                                 )
+            
+            if (classification['suggested_action'] == 'respond_with_general_llm'):
+                # Stream general response
+                general_response = await self.general_responder.respond(query = request.query)
+                yield general_response.get("answer", "")
+                
+                return
+            
+            # Otherwise proceed with RAG streaming - Procced with Retrieving context
+            retrieval_result = self.hybrid_retriever.retrieve_with_context(query             = request.query,
+                                                                           top_k             = request.top_k or self.settings.TOP_K_RETRIEVE,
+                                                                           enable_reranking  = request.enable_reranking,
+                                                                           include_citations = request.include_sources,
+                                                                          )
+            
+            chunks           = retrieval_result["chunks"]
+            context          = retrieval_result["context"]
+            
+            if not chunks:
+                yield "I couldn't find relevant information to answer your question."
+                return
+            
+            # Determine strategy
+            prompt_type = self._infer_prompt_type(query = request.query)
+            temperature = self._get_adaptive_temperature(request          = request,
+                                                         query            = request.query,
+                                                         context          = context,
+                                                         retrieval_scores = [chunk.score for chunk in chunks],
+                                                         prompt_type      = prompt_type,
+                                                        )
+            
+            # Build prompt
+            prompt      = self.prompt_builder.build_prompt(query                 = request.query,
+                                                           context               = context,
+                                                           sources               = chunks,
+                                                           prompt_type           = prompt_type,
+                                                           include_citations     = request.include_sources,
+                                                           max_completion_tokens = request.max_tokens or self.settings.MAX_TOKENS,
+                                                          )
+            
+            # Stream LLM response
+            messages    = [{"role"    : "system", 
+                            "content" : prompt["system"],
+                           },
+                           {"role"    : "user", 
+                            "content" : prompt["user"],
+                           },
+                          ]
+            
+            async for chunk_text in self.llm_client.generate_stream(messages    = messages,
+                                                                    temperature = temperature,
+                                                                    top_p       = request.top_p or self.settings.TOP_P,
+                                                                    max_tokens  = request.max_tokens or self.settings.MAX_TOKENS,
+                                                                   ):
+                yield chunk_text
+            
+            self.logger.info("Streaming response completed")
+            
+        except Exception as e:
+            self.logger.error(f"Streaming generation failed: {repr(e)}", exc_info = True)
+            
+            yield f"\n\n[Error: {str(e)}]"
+    
+
+    def _infer_prompt_type(self, query: str) -> PromptType:
+        """
+        Infer appropriate prompt type from query
+        
+        Arguments:
+        ----------
+            query { str }  : User query
+        
+        Returns:
+        --------
+            { PromptType } : Inferred prompt type
+        """
+        query_lower = query.lower()
+        
+        # Summary indicators
+        if (any(word in query_lower for word in ['summarize', 'summary', 'overview', 'tldr', 'brief'])):
+            return PromptType.SUMMARY
+        
+        # Comparison indicators
+        if (any(word in query_lower for word in ['compare', 'contrast', 'difference', 'versus', 'vs'])):
+            return PromptType.COMPARISON
+        
+        # Analytical indicators
+        if (any(word in query_lower for word in ['analyze', 'analysis', 'evaluate', 'assess', 'examine'])):
+            return PromptType.ANALYTICAL
+        
+        # Extraction indicators
+        if (any(word in query_lower for word in ['extract', 'list', 'find all', 'identify', 'enumerate'])):
+            return PromptType.EXTRACTION
+        
+        # Creative indicators
+        if (any(word in query_lower for word in ['create', 'write', 'compose', 'generate', 'imagine'])):
+            return PromptType.CREATIVE
+        
+        # Conversational indicators
+        if (any(word in query_lower for word in ['tell me about', 'explain', 'discuss', 'talk about'])):
+            return PromptType.CONVERSATIONAL
+        
+        # Default to QA
+        return PromptType.QA
+    
+
+    def _get_adaptive_temperature(self, request: QueryRequest, query: str, context: str, retrieval_scores: List[float], prompt_type: PromptType) -> float:
+        """
+        Get adaptive temperature based on query characteristics
+        
+        Arguments:
+        ----------
+            request          { QueryRequest } : Original request
+
+            query            { str }          : User query
+            
+            context          { str }          : Retrieved context
+            
+            retrieval_scores { list }         : Retrieval scores
+            
+            prompt_type      { PromptType }   : Inferred prompt type
+        
+        Returns:
+        --------
+                  { float }                   : Temperature value
+        """
+        # Use request temperature if explicitly provided
+        if (request.temperature is not None):
+            self.logger.debug(f"Using request temperature: {request.temperature}")
+            
+            return request.temperature
+        
+        # Otherwise, use adaptive temperature controller
+        temperature = self.temperature_controller.get_temperature(query            = query,
+                                                                  context          = context,
+                                                                  retrieval_scores = retrieval_scores,
+                                                                  query_type       = prompt_type.value,
+                                                                 )
+        
+        return temperature
+    
+
+    def _post_process_citations(self, answer: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Post-process answer to format citations
+        
+        Arguments:
+        ----------
+            answer  { str }  : Generated answer with citation markers
+
+            sources { list } : Source chunks
+        
+        Returns:
+        --------
+              { str }        : Answer with formatted citations
+        """
+        try:
+            # Validate citations
+            is_valid, invalid = self.citation_formatter.validate_citations(answer, sources)
+            
+            if not is_valid:
+                self.logger.warning(f"Invalid citations found: {invalid}")
+                # Normalize to fix issues
+                answer = self.citation_formatter.normalize_citations(answer, sources)
+            
+            # Format citations according to style
+            formatted_answer = self.citation_formatter.format_citations_in_text(answer, sources)
+            
+            return formatted_answer
+            
+        except Exception as e:
+            self.logger.error(f"Citation post-processing failed: {repr(e)}")
+            # Return original answer if formatting fails
+            return answer
+    
+
+    def _create_no_results_response(self, request: QueryRequest, retrieval_time_ms: float) -> QueryResponse:
+        """
+        Create response when no results are found
+        
+        Arguments:
+        ----------
+            request           { QueryRequest } : Original request
+
+            retrieval_time_ms { float }        : Time spent on retrieval
+        
+        Returns:
+        --------
+            { QueryResponse }                  : Response indicating no results
+        """
+        no_results_answer = ("I couldn't find relevant information in the available documents to answer your question. "
+                             "This could mean:\n"
+                             "1. The information is not present in the indexed documents\n"
+                             "2. The question may need to be rephrased for better matching\n"
+                             "3. The relevant documents haven't been uploaded yet\n\n"
+                             "Please try:\n"
+                             "- Rephrasing your question with different keywords\n"
+                             "- Asking a more specific or general question\n"
+                             "- Ensuring the relevant documents are uploaded\n"
+                            )
+        
+        return QueryResponse(query              = request.query,
+                             answer             = no_results_answer,
+                             sources            = [],
+                             retrieval_time_ms  = retrieval_time_ms,
+                             generation_time_ms = 0.0,
+                             total_time_ms      = retrieval_time_ms,
+                             tokens_used        = {"input": 0, "output": 0, "total": 0},
+                             model_used         = self.model_name,
+                             timestamp          = datetime.now(),
+                            )
+    
+
+    def _calculate_quality_metrics(self, query: str, answer: str, context: str, sources: List[ChunkWithScore]) -> Dict[str, float]:
+        """
+        Calculate quality metrics for the response
+        
+        Arguments:
+        ----------
+            query   { str }  : User query
+
+            answer  { str }  : Generated answer
+            
+            context { str }  : Retrieved context
+            
+            sources { list } : Source chunks
+        
+        Returns:
+        --------
+             { dict }        : Quality metrics
+        """
+        metrics = dict()
+        
+        try:
+            # Answer length metrics
+            metrics["answer_length"]       = len(answer.split())
+            metrics["answer_char_length"]  = len(answer)
+            
+            # Citation metrics
+            citation_stats                 = self.citation_formatter.get_citation_statistics(answer, sources)
+            metrics["citations_used"]      = citation_stats.get("total_citations", 0)
+            metrics["unique_citations"]    = citation_stats.get("unique_citations", 0)
+            metrics["citation_density"]    = citation_stats.get("citation_density", 0.0)
+            
+            # Context utilization
+            context_length                 = len(context.split())
+            metrics["context_utilization"] = min(1.0, metrics["answer_length"] / max(1, context_length))
+            
+            # Retrieval quality
+            if sources:
+                avg_score                      = sum(s.score for s in sources) / len(sources)
+                metrics["avg_retrieval_score"] = avg_score
+                metrics["top_retrieval_score"] = sources[0].score if sources else 0.0
+            
+            # Query-answer alignment (simple keyword overlap)
+            query_words                     = set(query.lower().split())
+            answer_words                    = set(answer.lower().split())
+            overlap                         = len(query_words & answer_words)
+            metrics["query_answer_overlap"] = overlap / max(1, len(query_words))
+            
+        except Exception as e:
+            self.logger.warning(f"Failed to calculate some quality metrics: {repr(e)}")
+        
+        return metrics
+    
+
+    async def generate_batch_responses(self, requests: List[QueryRequest], has_documents: bool = True) -> List[QueryResponse]:
+        """
+        Generate responses for multiple queries in batch
+        
+        Arguments:
+        ----------
+            requests      { list } : List of query requests
+            
+            has_documents { bool } : Whether documents are available
+        
+        Returns:
+        --------
+                  { list }         : List of query responses
+        """
+        self.logger.info(f"Generating batch responses for {len(requests)} queries")
+        
+        tasks     = [self.generate_response(request       = request,
+                                            has_documents = has_documents) for request in requests]
+        
+        responses = await asyncio.gather(*tasks, return_exceptions = True)
+        
+        # Handle exceptions
+        results   = list()
+
+        for i, response in enumerate(responses):
+            if isinstance(response, Exception):
+                self.logger.error(f"Batch query {i} failed: {repr(response)}")
+                # Create error response
+                error_response = self._create_error_response(requests[i], str(response))
+                results.append(error_response)
+            
+            else:
+                results.append(response)
+        
+        self.logger.info(f"Completed batch generation: {len(results)} responses")
+        
+        return results
+    
+
+    def _create_error_response(self, request: QueryRequest, error_message: str) -> QueryResponse:
+        """
+        Create error response for failed generation
+        
+        Arguments:
+        ----------
+            request       { QueryRequest } : Original request
+
+            error_message { str }          : Error message
+        
+        Returns:
+        --------
+            { QueryResponse }              : Error response
+        """
+        return QueryResponse(query              = request.query,
+                             answer             = f"An error occurred while generating the response: {error_message}",
+                             sources            = [],
+                             retrieval_time_ms  = 0.0,
+                             generation_time_ms = 0.0,
+                             total_time_ms      = 0.0,
+                             tokens_used        = {"input": 0, "output": 0, "total": 0},
+                             model_used         = self.model_name,
+                             timestamp          = datetime.now(),
+                            )
+    
+
+    def get_generation_stats(self) -> Dict:
+        """
+        Get generation statistics including query type breakdown
+        
+        Returns:
+        --------
+            { dict }    : Generation statistics
+        """
+        avg_time = (self.total_generation_time / self.generation_count) if self.generation_count > 0 else 0
+        
+        return {"total_generations"      : self.generation_count,
+                "general_queries"        : self.general_query_count,
+                "rag_queries"            : self.rag_query_count,
+                "total_generation_time"  : self.total_generation_time,
+                "avg_generation_time_ms" : avg_time,
+                "provider"               : self.provider.value,
+                "model"                  : self.model_name,
+                "llm_health"             : self.llm_client.check_health(),
+                "query_routing_enabled"  : True,
+                "llm_based_routing"      : True,
+               }
+    
+
+    def reset_stats(self):
+        """
+        Reset generation statistics
+        """
+        self.generation_count      = 0
+        self.general_query_count   = 0
+        self.rag_query_count       = 0
+        self.total_generation_time = 0.0
+        
+        self.logger.info("Generation statistics reset")
+
+
+# Global response generator instance
+_response_generator = None
+
+
+def get_response_generator(provider: LLMProvider = None, model_name: str = None) -> ResponseGenerator:
+    """
+    Get global response generator instance (singleton)
+    
+    Arguments:
+    ----------
+        provider   { LLMProvider } : LLM provider
+
+        model_name { str }         : Model name
+    
+    Returns:
+    --------
+        { ResponseGenerator }      : ResponseGenerator instance
+    """
+    global _response_generator
+    
+    if _response_generator is None or (provider and _response_generator.provider != provider):
+        _response_generator = ResponseGenerator(provider, model_name)
+    
+    return _response_generator
+
+
+@handle_errors(error_type = ResponseGenerationError, log_error = True, reraise = False)
+async def generate_answer(query: str, top_k: int = 5, temperature: float = None, has_documents: bool = True, **kwargs) -> str:
+    """
+    Convenience function for quick answer generation
+    
+    Arguments:
+    ----------
+        query         { str }   : User query
+
+        top_k         { int }   : Number of chunks to retrieve
+        
+        temperature   { float } : Temperature for generation
+        
+        has_documents { bool }  : Whether documents are available
+        
+        **kwargs                : Additional parameters
+    
+    Returns:
+    --------
+             { str }            : Generated answer
+    """
+    request   = QueryRequest(query       = query,
+                             top_k       = top_k,
+                             temperature = temperature,
+                             **kwargs
+                            )
+    
+    generator = get_response_generator()
+    response  = await generator.generate_response(request       = request,
+                                                  has_documents = has_documents,
+                                                 )
+    
+    return response.answer

generation/temperature_controller.py

@@ -0,0 +1,430 @@
+# DEPENDENCIES
+import math
+from typing import Any
+from typing import Dict
+from typing import Optional
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from config.models import TemperatureStrategy
+from utils.error_handler import TemperatureControlError
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class TemperatureController:
+    """
+    Intelligent temperature control for LLM generation: Implements adaptive temperature strategies based on query type, complexity, and desired output characteristics
+    """
+    def __init__(self, base_temperature: float = None, strategy: TemperatureStrategy = None):
+        """
+        Initialize temperature controller
+        
+        Arguments:
+        ----------
+            base_temperature { float } : Base temperature value (default from settings)
+            
+            strategy          { str }  : Temperature control strategy
+        """
+        self.logger           = logger
+        self.settings         = get_settings()
+        self.base_temperature = base_temperature or self.settings.DEFAULT_TEMPERATURE
+        self.strategy         = strategy or TemperatureStrategy.ADAPTIVE
+        
+        # Validate base temperature
+        if not (0.0 <= self.base_temperature <= 1.0):
+            raise TemperatureControlError(f"Temperature must be between 0 and 1: {self.base_temperature}")
+        
+        # Strategy configurations
+        self.strategy_configs = {TemperatureStrategy.FIXED       : {"description" : "Fixed temperature for all queries", "range" : (0.0, 1.0)},
+                                 TemperatureStrategy.ADAPTIVE    : {"description" : "Adapt temperature based on query complexity", "range" : (0.1, 0.8), "complexity_threshold" : 0.6},
+                                 TemperatureStrategy.CONFIDENCE  : {"description" : "Adjust temperature based on retrieval confidence", "range" : (0.1, 0.9), "high_confidence_temp" : 0.1, "low_confidence_temp" : 0.7},
+                                 TemperatureStrategy.PROGRESSIVE : {"description" : "Progressively increase temperature for creative tasks", "range" : (0.1, 0.9), "creative_threshold" : 0.7}
+                                }
+        
+        self.logger.info(f"Initialized TemperatureController: base={self.base_temperature}, strategy={self.strategy}")
+    
+
+    def get_temperature(self, query: str = "", context: str = "", retrieval_scores: Optional[list] = None, query_type: str = "qa") -> float:
+        """
+        Get appropriate temperature for generation
+        
+        Arguments:
+        ----------
+            query            { str }  : User query
+            
+            context          { str }  : Retrieved context
+            
+            retrieval_scores { list } : Scores of retrieved chunks
+            
+            query_type       { str }  : Type of query ('qa', 'creative', 'analytical', 'summary')
+        
+        Returns:
+        --------
+                   { float }          : Temperature value (0.0 - 1.0)
+        """
+        if (self.strategy == TemperatureStrategy.FIXED):
+            return self._fixed_temperature()
+        
+        elif (self.strategy == TemperatureStrategy.ADAPTIVE):
+            return self._adaptive_temperature(query      = query, 
+                                              context    = context, 
+                                              query_type = query_type,
+                                             )
+        
+        elif (self.strategy == TemperatureStrategy.CONFIDENCE):
+            return self._confidence_based_temperature(retrieval_scores = retrieval_scores, 
+                                                      query_type       = query_type,
+                                                     )
+        
+        elif (self.strategy == TemperatureStrategy.PROGRESSIVE):
+            return self._progressive_temperature(query_type = query_type, 
+                                                 query      = query,
+                                                )
+        
+        else:
+            self.logger.warning(f"Unknown strategy: {self.strategy}, using fixed")
+            return self.base_temperature
+    
+
+    def _fixed_temperature(self) -> float:
+        """
+        Fixed temperature strategy
+        """
+        return self.base_temperature
+    
+
+    def _adaptive_temperature(self, query: str, context: str, query_type: str) -> float:
+        """
+        Adaptive temperature based on query complexity and type
+        """
+        base_temp        = self.base_temperature
+        
+        # Adjust based on query type
+        type_adjustments = {"qa"         : -0.2,      # More deterministic for Q&A
+                            "creative"   : 0.3,       # More creative for creative tasks
+                            "analytical" : -0.1,      # Slightly deterministic for analysis
+                            "summary"    : -0.15,     # Deterministic for summarization
+                            "comparison" : 0.1,       # Slightly creative for comparisons
+                           }
+        
+        adjustment       = type_adjustments.get(query_type, 0.0)
+        temp             = base_temp + adjustment
+        
+        # Adjust based on query complexity
+        complexity       = self._calculate_query_complexity(query = query)
+        
+        if (complexity > 0.7): 
+            # High complexity
+            temp += 0.1
+
+        elif (complexity < 0.3):  
+            # Low complexity
+            temp -= 0.1
+        
+        # Adjust based on context quality
+        if context:
+            context_quality = self._calculate_context_quality(context = context)
+
+            # Poor context
+            if (context_quality < 0.5):  
+                # More creative when context is poor
+                temp += 0.15  
+        
+        return self._clamp_temperature(temperature = temp)
+    
+
+    def _confidence_based_temperature(self, retrieval_scores: Optional[list], query_type: str) -> float:
+        """
+        Temperature based on retrieval confidence
+        """
+        if not retrieval_scores:
+            self.logger.debug("No retrieval scores, using base temperature")
+            return self.base_temperature
+        
+        # Calculate average confidence
+        avg_confidence = sum(retrieval_scores) / len(retrieval_scores)
+        
+        config         = self.strategy_configs[TemperatureStrategy.CONFIDENCE]
+        high_temp      = config["high_confidence_temp"]
+        low_temp       = config["low_confidence_temp"]
+        
+        # High confidence -> low temperature (deterministic) & Low confidence -> high temperature (creative)
+        if (avg_confidence > 0.8):
+            temperature = high_temp
+
+        elif (avg_confidence < 0.3):
+            temperature = low_temp
+
+        else:
+            # Linear interpolation between high and low temps
+            normalized_confidence = (avg_confidence - 0.3) / (0.8 - 0.3)
+            temperature           = high_temp + (low_temp - high_temp) * (1 - normalized_confidence)
+        
+        # Adjust for query type
+        if (query_type == "creative"):
+            temperature = min(0.9, temperature + 0.2)
+
+        elif (query_type == "qa"):
+            temperature = max(0.1, temperature - 0.1)
+        
+        return self._clamp_temperature(temperature = temperature)
+    
+
+    def _progressive_temperature(self, query_type: str, query: str) -> float:
+        """
+        Progressive temperature based on task requirements
+        """
+        base_temp = self.base_temperature
+        
+        # Task-based progression
+        if (query_type == "creative"):
+            # High creativity
+            return self._clamp_temperature(temperature = 0.8)  
+        
+        elif (query_type == "analytical"):
+            # Balanced
+            return self._clamp_temperature(temperature = 0.3)  
+        
+        elif (query_type == "qa"):
+            # For factual Q&A, use lower temperature
+            if self._is_factual_query(query):
+                return self._clamp_temperature(temperature = 0.1)
+            
+            else:
+                return self._clamp_temperature(temperature = 0.4)
+        
+        elif (query_type == "summary"):
+            # Deterministic summaries
+            return self._clamp_temperature(temperature = 0.2)  
+        
+        else:
+            return self._clamp_temperature(temperature = base_temp)
+    
+
+    def _calculate_query_complexity(self, query: str) -> float:
+        """
+        Simple, predictable complexity score
+        """
+        if not query:
+            return 0.5
+        
+        # Count words and questions
+        words       = len(query.split())
+        has_why_how = any(word in query.lower() for word in ['why', 'how', 'explain'])
+        has_compare = any(word in query.lower() for word in ['compare', 'contrast', 'difference'])
+        
+        # Simple rules
+        if has_compare:
+            # Complex
+            return 0.8  
+        
+        elif (has_why_how and( words > 15)):
+            return 0.7
+        
+        elif words > 20:
+            return 0.6
+        
+        else:
+            # Simple
+            return 0.3  
+    
+
+    def _calculate_context_quality(self, context: str) -> float:
+        """
+        Calculate context quality (0.0 - 1.0)
+        """
+        if not context:
+            return 0.0
+        
+        factors          = list()
+        
+        # Length factor (adequate context)
+        words            = len(context.split())
+
+        # Normalize
+        length_factor    = min(words / 500, 1.0)  
+
+        factors.append(length_factor)
+        
+        # Diversity factor (multiple sources/citations)
+        citation_count   = context.count('[')
+        diversity_factor = min(citation_count / 5, 1.0)
+        
+        factors.append(diversity_factor)
+        
+        # Coherence factor (simple measure)
+        sentence_count   = context.count('.')
+        
+        if (sentence_count > 0):
+            avg_sentence_length = words / sentence_count
+            # Ideal ~20 words/sentence
+            coherence_factor    = 1.0 - min(abs(avg_sentence_length - 20) / 50, 1.0)  
+
+            factors.append(coherence_factor)
+        
+        return sum(factors) / len(factors)
+    
+
+    def _is_factual_query(self, query: str) -> bool:
+        """
+        Check if query is factual (requires precise answers)
+        """
+        factual_indicators = ['what is', 'who is', 'when did', 'where is', 'how many', 'how much', 'definition of', 'meaning of', 'calculate', 'number of']
+        
+        query_lower        = query.lower()
+
+        return any(indicator in query_lower for indicator in factual_indicators)
+    
+
+    def _clamp_temperature(self, temperature: float) -> float:
+        """
+        Clamp temperature to valid range
+        """
+        strategy_config = self.strategy_configs.get(self.strategy, {})
+        temp_range      = strategy_config.get("range", (0.0, 1.0))
+        
+        clamped         = max(temp_range[0], min(temperature, temp_range[1]))
+
+        # Round to 2 decimal places
+        clamped         = round(clamped, 2)  
+        
+        return clamped
+    
+
+    def get_temperature_parameters(self, temperature: float) -> Dict[str, Any]:
+        """
+        Get additional parameters based on temperature
+        
+        Arguments:
+        ----------
+            temperature { float } : Temperature value
+        
+        Returns:
+        --------
+                   { dict }       : Additional generation parameters
+        """
+        params = {"temperature" : temperature,
+                  "top_p"       : 0.9,
+                 }
+        
+        # Adjust top_p based on temperature
+        if (temperature < 0.3):
+            # Broader distribution for low temp
+            params["top_p"] = 0.95  
+
+        elif (temperature > 0.7):
+            # Narrower distribution for high temp
+            params["top_p"] = 0.7  
+        
+        # Adjust presence_penalty based on temperature
+        if (temperature > 0.5):
+            # Encourage novelty for creative tasks
+            params["presence_penalty"] = 0.1  
+
+        else:
+            params["presence_penalty"] = 0.0
+        
+        return params
+    
+
+    def explain_temperature_choice(self, query: str, context: str, retrieval_scores: list, query_type: str, final_temperature: float) -> Dict[str, Any]:
+        """
+        Explain why a particular temperature was chosen
+        
+        Arguments:
+        ----------
+            query             { str }   : User query
+            
+            context           { str }   : Retrieved context
+            
+            retrieval_scores  { list }  : Retrieval scores
+            
+            query_type        { str }   : Query type
+            
+            final_temperature { float } : Chosen temperature
+        
+        Returns:
+        --------
+                      { dict }          : Explanation dictionary
+        """
+        explanation = {"strategy"          : self.strategy.value,
+                       "final_temperature" : final_temperature,
+                       "base_temperature"  : self.base_temperature,
+                       "factors"           : {},
+                      }
+        
+        if (self.strategy == TemperatureStrategy.ADAPTIVE):
+            complexity             = self._calculate_query_complexity(query = query)
+            context_quality        = self._calculate_context_quality(context = context)
+            
+            explanation["factors"] = {"query_complexity" : round(complexity, 3),
+                                      "context_quality"  : round(context_quality, 3),
+                                      "query_type"       : query_type,
+                                     }
+        
+        elif (self.strategy == TemperatureStrategy.CONFIDENCE):
+            if retrieval_scores:
+                avg_confidence         = sum(retrieval_scores) / len(retrieval_scores)
+                explanation["factors"] = {"average_retrieval_confidence" : round(avg_confidence, 3),
+                                          "query_type"                   : query_type,
+                                         }
+        
+        elif (self.strategy == TemperatureStrategy.PROGRESSIVE):
+            is_factual             = self._is_factual_query(query)
+            explanation["factors"] = {"query_type"       : query_type,
+                                      "is_factual_query" : is_factual,
+                                     }
+        
+        return explanation
+
+
+# Global temperature controller instance
+_temperature_controller = None
+
+
+def get_temperature_controller() -> TemperatureController:
+    """
+    Get global temperature controller instance (singleton)
+    
+    Returns:
+    --------
+        { TemperatureController }    : TemperatureController instance
+    """
+    global _temperature_controller
+    
+    if _temperature_controller is None:
+        _temperature_controller = TemperatureController()
+    
+    return _temperature_controller
+
+
+@handle_errors(error_type=TemperatureControlError, log_error=True, reraise=False)
+def get_adaptive_temperature(query: str = "", context: str = "", retrieval_scores: list = None, query_type: str = "qa") -> float:
+    """
+    Convenience function for getting adaptive temperature
+    
+    Arguments:
+    ----------
+        query            { str }   : User query
+        
+        context          { str }   : Retrieved context
+         
+        retrieval_scores { list }  : Retrieval scores
+        
+        query_type       { str }   : Query type
+    
+    Returns:
+    --------
+               { float }           : Temperature value
+    """
+    controller = get_temperature_controller()
+    
+    return controller.get_temperature(query            = query, 
+                                      context          = context, 
+                                      retrieval_scores = retrieval_scores, 
+                                      query_type       = query_type,
+                                     )

generation/token_manager.py

@@ -0,0 +1,431 @@
+# DEPENDENCIES
+import tiktoken
+from typing import List
+from typing import Dict
+from typing import Optional
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import TokenManagementError
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class TokenManager:
+    """
+    Token management for LLM context windows: Handles token counting, context window management, and optimization for different LLM providers (Ollama, OpenAI)
+    """
+    def __init__(self, model_name: str = None):
+        """
+        Initialize token manager
+        
+        Arguments:
+        ----------
+            model_name { str } : Model name for tokenizer selection
+        """
+        self.logger         = logger
+        self.settings       = get_settings()
+        self.model_name     = model_name or self.settings.OLLAMA_MODEL
+        self.encoding       = None
+        self.context_window = self._get_context_window()
+        
+        self._initialize_tokenizer()
+    
+
+    def _initialize_tokenizer(self):
+        """
+        Initialize appropriate tokenizer based on model
+        """
+        try:
+            # Determine tokenizer based on model
+            if self.model_name.startswith(('gpt-3.5', 'gpt-4')):
+                # OpenAI models
+                self.encoding = tiktoken.encoding_for_model(self.model_name)
+                self.logger.debug(f"Initialized tiktoken for {self.model_name}")
+            
+            else:
+                # Default for Ollama/local models
+                self.encoding = tiktoken.get_encoding("cl100k_base")
+                self.logger.debug(f"Using cl100k_base tokenizer for local model {self.model_name}")
+        
+        except Exception as e:
+            self.logger.warning(f"Failed to initialize specific tokenizer: {repr(e)}, using approximation")
+            self.encoding = None
+    
+
+    def _get_context_window(self) -> int:
+        """
+        Get context window size based on model
+        
+        Returns:
+        --------
+            { int }    : Context window size in tokens
+        """
+        model_contexts = {"gpt-3.5-turbo" : 4096,
+                          "mistral:7b"    : 8192,
+                         }
+        
+        # Find matching model
+        for model_pattern, context_size in model_contexts.items():
+            if model_pattern in self.model_name.lower():
+                return context_size
+        
+        # Default context window
+        default_context = self.settings.CONTEXT_WINDOW
+        self.logger.info(f"Using default context window {default_context} for model {self.model_name}")
+        
+        return default_context
+    
+
+    def count_tokens(self, text: str) -> int:
+        """
+        Count tokens in text
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+             { int }     : Number of tokens
+        """
+        if not text:
+            return 0
+        
+        if self.encoding is not None:
+            try:
+                return len(self.encoding.encode(text))
+            
+            except Exception as e:
+                self.logger.warning(f"Tokenizer failed, using approximation: {repr(e)}")
+        
+        # Fallback approximation
+        return self._approximate_token_count(text = text)
+    
+
+    def _approximate_token_count(self, text: str) -> int:
+        """
+        Approximate token count when tokenizer is unavailable
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+             { int }     : Approximate token count
+        """
+        if not text:
+            return 0
+        
+        # Use word-based approximation (more reliable than char-based)
+        words            = text.split()
+        
+        # English text averages ~1.3 tokens per word : (accounting for punctuation and subword tokenization)
+        estimated_tokens = int(len(words) * 1.3)
+        
+        # Add 5% buffer for punctuation and special tokens
+        estimated_tokens = int(estimated_tokens * 1.05)
+        
+        return estimated_tokens
+        
+
+    def count_message_tokens(self, messages: List[Dict]) -> int:
+        """
+        Count tokens in chat messages
+        
+        Arguments:
+        ----------
+            messages { list } : List of message dictionaries
+        
+        Returns:
+        --------
+                 { int }      : Total tokens in messages
+        """
+        if not messages:
+            return 0
+        
+        total_tokens = 0
+        
+        for message in messages:
+            # Count content tokens
+            content       = message.get('content', '')
+            total_tokens += self.count_tokens(text = content)
+            
+            # Count role tokens (approximate)
+            role          = message.get('role', '')
+            total_tokens += self.count_tokens(text = role)
+            
+            # Add overhead for message structure: Approximate overhead per message
+            total_tokens += 5  
+        
+        return total_tokens
+    
+
+    def fits_in_context(self, prompt: str, max_completion_tokens: int = 1000) -> bool:
+        """
+        Check if prompt fits in context window with room for completion
+        
+        Arguments:
+        ----------
+            prompt                { str } : Prompt text
+            
+            max_completion_tokens { int } : Tokens to reserve for completion
+        
+        Returns:
+        --------
+                      { bool }            : True if prompt fits
+        """
+        prompt_tokens  = self.count_tokens(text = prompt)
+        total_required = prompt_tokens + max_completion_tokens
+        
+        return (total_required <= self.context_window)
+    
+
+    def truncate_to_fit(self, text: str, max_tokens: int, strategy: str = "end") -> str:
+        """
+        Truncate text to fit within token limit
+        
+        Arguments:
+        ----------
+            text       { str } : Text to truncate
+            
+            max_tokens { int } : Maximum tokens allowed
+            
+            strategy   { str } : Truncation strategy ('end', 'start', 'middle')
+        
+        Returns:
+        --------
+                { str }        : Truncated text
+        """
+        current_tokens = self.count_tokens(text = text)
+        
+        if (current_tokens <= max_tokens):
+            return text
+        
+        if (strategy == "end"):
+            return self._truncate_from_end(text       = text, 
+                                           max_tokens = max_tokens,
+                                          )
+        
+        elif (strategy == "start"):
+            return self._truncate_from_start(text       = text, 
+                                             max_tokens = max_tokens,
+                                            )
+        
+        elif (strategy == "middle"):
+            return self._truncate_from_middle(text       = text, 
+                                              max_tokens = max_tokens,
+                                             )
+        
+        else:
+            self.logger.warning(f"Unknown truncation strategy: {strategy}, using 'end'")
+            return self._truncate_from_end(text       = text, 
+                                           max_tokens = max_tokens,
+                                          )
+    
+
+    def _truncate_from_end(self, text: str, max_tokens: int) -> str:
+        """
+        Truncate from the end of the text
+        """
+        if self.encoding is not None:
+            tokens           = self.encoding.encode(text)
+            truncated_tokens = tokens[:max_tokens]
+            return self.encoding.decode(truncated_tokens)
+        
+        # Approximate truncation
+        words           = text.split()
+        # Conservative estimate
+        target_words    = int(max_tokens * 0.75)  
+        truncated_words = words[:target_words]
+        
+        return " ".join(truncated_words)
+    
+
+    def _truncate_from_start(self, text: str, max_tokens: int) -> str:
+        """
+        Truncate from the start of the text
+        """
+        if self.encoding is not None:
+            tokens           = self.encoding.encode(text)
+            # Take from end
+            truncated_tokens = tokens[-max_tokens:]  
+            return self.encoding.decode(truncated_tokens)
+        
+        # Approximate truncation
+        words           = text.split()
+        target_words    = int(max_tokens * 0.75)
+
+        # Take from end
+        truncated_words = words[-target_words:]  
+        
+        return " ".join(truncated_words)
+    
+
+    def _truncate_from_middle(self, text: str, max_tokens: int) -> str:
+        """
+        Truncate from the middle of the text
+        """
+        if self.encoding is not None:
+            tokens = self.encoding.encode(text)
+
+            if (len(tokens) <= max_tokens):
+                return text
+            
+            # Keep beginning and end, remove middle
+            keep_start   = max_tokens // 3
+            keep_end     = max_tokens - keep_start
+            
+            start_tokens = tokens[:keep_start]
+            end_tokens   = tokens[-keep_end:]
+            
+            return self.encoding.decode(start_tokens) + " [...] " + self.encoding.decode(end_tokens)
+        
+        # Approximate truncation
+        words       = text.split()
+
+        if (len(words) <= max_tokens):
+            return text
+        
+        keep_start  = max_tokens // 3
+        keep_end    = max_tokens - keep_start
+        
+        start_words = words[:keep_start]
+        end_words   = words[-keep_end:]
+        
+        return " ".join(start_words) + " [...] " + " ".join(end_words)
+    
+
+    def calculate_max_completion_tokens(self, prompt: str, reserve_tokens: int = 100) -> int:
+        """
+        Calculate maximum completion tokens given prompt length
+        
+        Arguments:
+        ----------
+            prompt         { str } : Prompt text
+            
+            reserve_tokens { int } : Tokens to reserve for safety
+        
+        Returns:
+        --------
+                   { int }         : Maximum completion tokens
+        """
+        prompt_tokens    = self.count_tokens(text = prompt)
+        available_tokens = self.context_window - prompt_tokens - reserve_tokens
+        
+        return max(0, available_tokens)
+    
+
+    def optimize_context_usage(self, context: str, prompt: str, max_completion_tokens: int = 1000) -> str:
+        """
+        Optimize context to fit within context window
+        
+        Arguments:
+        ----------
+            context              { str } : Context text
+            
+            prompt               { str } : Prompt template
+            
+            max_completion_tokens { int } : Tokens needed for completion
+        
+        Returns:
+        --------
+                      { str }            : Optimized context
+        """
+        total_prompt_tokens   = self.count_tokens(text = prompt.format(context=""))
+        available_for_context = self.context_window - total_prompt_tokens - max_completion_tokens
+        
+        if (available_for_context <= 0):
+            self.logger.warning("Prompt too large for context window")
+            return ""
+        
+        context_tokens = self.count_tokens(text = context)
+        
+        if (context_tokens <= available_for_context):
+            return context
+        
+        # Truncate context to fit
+        optimized_context = self.truncate_to_fit(context, available_for_context, strategy="end")
+        
+        reduction_pct     = ((context_tokens - self.count_tokens(text = optimized_context)) / context_tokens) * 100
+
+        self.logger.info(f"Context reduced by {reduction_pct:.1f}% to fit context window")
+        
+        return optimized_context
+    
+
+    def get_token_stats(self, text: str) -> Dict:
+        """
+        Get detailed token statistics
+        
+        Arguments:
+        ----------
+            text { str } : Input text
+        
+        Returns:
+        --------
+            { dict }     : Token statistics
+        """
+        tokens = self.count_tokens(text = text)
+        chars  = len(text)
+        words  = len(text.split())
+        
+        return {"tokens"          : tokens,
+                "characters"      : chars,
+                "words"           : words,
+                "chars_per_token" : chars / tokens if tokens > 0 else 0,
+                "tokens_per_word" : tokens / words if words > 0 else 0,
+                "context_window"  : self.context_window,
+                "model"           : self.model_name,
+               }
+
+
+# Global token manager instance
+_token_manager = None
+
+
+def get_token_manager(model_name: str = None) -> TokenManager:
+    """
+    Get global token manager instance
+    
+    Arguments:
+    ----------
+        model_name { str } : Model name for tokenizer selection
+    
+    Returns:
+    --------
+        { TokenManager }   : TokenManager instance
+    """
+    global _token_manager
+    
+    if _token_manager is None or (model_name and _token_manager.model_name != model_name):
+        _token_manager = TokenManager(model_name)
+    
+    return _token_manager
+
+
+@handle_errors(error_type = TokenManagementError, log_error = True, reraise = False)
+def count_tokens_safe(text: str, model_name: str = None) -> int:
+    """
+    Safe token counting with error handling
+    
+    Arguments:
+    ----------
+        text       { str } : Text to count tokens for
+        
+        model_name { str } : Model name for tokenizer
+    
+    Returns:
+    --------
+             { int }       : Token count (0 on error)
+    """
+    try:
+        manager = get_token_manager(model_name = model_name)
+        return manager.count_tokens(text = text)
+    
+    except Exception:
+        return 0

ingestion/async_coordinator.py

@@ -0,0 +1,414 @@
+# DEPENDENCIES
+import asyncio
+from typing import Any
+from typing import List
+from typing import Dict
+from pathlib import Path
+import concurrent.futures
+from typing import Optional
+from datetime import datetime
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.error_handler import ProcessingException
+from ingestion.progress_tracker import get_progress_tracker
+from document_parser.parser_factory import get_parser_factory
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class AsyncCoordinator:
+    """
+    Asynchronous document processing coordinator: Manages parallel processing of multiple documents with resource optimization
+    """
+    def __init__(self, max_workers: Optional[int] = None):
+        """
+        Initialize async coordinator
+        
+        Arguments:
+        ----------
+            max_workers { int } : Maximum parallel workers (default from settings)
+        """
+        self.logger               = logger
+        self.max_workers          = max_workers or settings.MAX_WORKERS
+        self.parser_factory       = get_parser_factory()
+        self.progress_tracker     = get_progress_tracker()
+        
+        # Processing statistics
+        self.total_processed      = 0
+        self.total_failed         = 0
+        self.avg_processing_time  = 0.0
+        
+        self.logger.info(f"Initialized AsyncCoordinator: max_workers={self.max_workers}")
+    
+
+    @handle_errors(error_type = ProcessingException, log_error = True, reraise = True)
+    async def process_documents_async(self, file_paths: List[Path], progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Process multiple documents asynchronously with progress tracking
+        
+        Arguments:
+        ----------
+            file_paths        { list }     : List of file paths to process
+
+            progress_callback { callable } : Callback for progress updates
+        
+        Returns:
+        --------
+                       { dict }            : Processing results
+        """
+        if not file_paths:
+            return {"processed" : 0, 
+                    "failed"    : 0, 
+                    "results"   : [],
+                   }
+        
+        self.logger.info(f"Starting async processing of {len(file_paths)} documents")
+        
+        # Initialize progress tracking
+        task_id = self.progress_tracker.start_task(total_items = len(file_paths),
+                                                   description = "Document processing",
+                                                  )
+        
+        try:
+            # Process files in parallel with semaphore for resource control
+            semaphore     = asyncio.Semaphore(self.max_workers)
+            tasks         = [self._process_single_file_async(file_path        = file_path, 
+                                                             semaphore        = semaphore, 
+                                                             task_id          = task_id,
+                                                             progress_callback = progress_callback,
+                                                            ) for file_path in file_paths]
+            
+            results       = await asyncio.gather(*tasks, return_exceptions = True)
+            
+            # Process results
+            processed     = list()
+            failed        = list()
+            
+            for file_path, result in zip(file_paths, results):
+                if isinstance(result, Exception):
+                    self.logger.error(f"Failed to process {file_path}: {repr(result)}")
+                    failed.append({"file_path": file_path, "error": str(result)})
+                    self.total_failed += 1
+                
+                else:
+                    processed.append(result)
+                    self.total_processed += 1
+            
+            # Update statistics
+            self._update_statistics(processed_count = len(processed))
+            
+            final_result = {"processed"      : len(processed),
+                            "failed"         : len(failed),
+                            "success_rate"   : (len(processed) / len(file_paths)) * 100,
+                            "results"        : processed,
+                            "failures"       : failed,
+                            "task_id"        : task_id,
+                           }
+            
+            self.progress_tracker.complete_task(task_id)
+            
+            self.logger.info(f"Async processing completed: {len(processed)} successful, {len(failed)} failed")
+            
+            return final_result
+            
+        except Exception as e:
+            self.progress_tracker.fail_task(task_id, str(e))
+            raise ProcessingException(f"Async processing failed: {repr(e)}")
+    
+
+    async def _process_single_file_async(self, file_path: Path, semaphore: asyncio.Semaphore, task_id: str, progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Process single file asynchronously
+        
+        Arguments:
+        ----------
+            file_path            { Path }   : File to process
+
+            semaphore         { Semaphore } : Resource semaphore
+            
+            task_id              { str }    : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing result
+        """
+        async with semaphore:
+            try:
+                self.logger.debug(f"Processing file: {file_path}")
+                
+                # Update progress
+                self.progress_tracker.update_task(task_id        = task_id, 
+                                                  current_item   = file_path.name,
+                                                  current_status = "parsing",
+                                                 )
+
+                processed_so_far = len([t for t in self.progress_tracker.active_tasks.get(task_id, {}) if t])
+                self.progress_tracker.update_task(task_id = task_id, processed_items = processed_so_far)
+
+                if progress_callback:
+                    progress_callback(self.progress_tracker.get_task_progress(task_id))
+                
+                # Parse document
+                start_time      = datetime.now()
+                text, metadata  = await asyncio.to_thread(self.parser_factory.parse, 
+                                                          file_path        = file_path,
+                                                          extract_metadata = True,
+                                                          clean_text       = True,
+                                                         )
+                
+                processing_time = (datetime.now() - start_time).total_seconds()
+                
+                # Update progress
+                self.progress_tracker.update_task(task_id        = task_id,
+                                                  current_item   = file_path.name, 
+                                                  current_status = "completed",
+                                                 )
+                
+                # Handle metadata (could be Pydantic model or dict)
+                metadata_dict    = metadata.dict() if hasattr(metadata, 'dict') else (metadata if isinstance(metadata, dict) else {})
+                
+                result           = {"file_path"       : str(file_path),
+                                    "file_name"       : file_path.name,
+                                    "text_length"     : len(text),
+                                    "processing_time" : processing_time,
+                                    "metadata"        : metadata_dict,
+                                    "success"         : True,
+                                   }
+                
+                # Increment processed items count
+                current_progress = self.progress_tracker.get_task_progress(task_id)
+
+                if current_progress:
+                    self.progress_tracker.update_task(task_id         = task_id, 
+                                                      processed_items = current_progress.processed_items + 1,
+                                                     )
+                
+                if progress_callback:
+                    progress_callback(self.progress_tracker.get_task_progress(task_id))
+                
+                return result
+                
+            except Exception as e:
+                self.logger.error(f"Failed to process {file_path}: {repr(e)}")
+                
+                self.progress_tracker.update_task(task_id        = task_id,
+                                                  current_item   = file_path.name,
+                                                  current_status = f"failed: {str(e)}",
+                                                 )
+                
+                raise ProcessingException(f"File processing failed: {repr(e)}")
+    
+
+    def process_documents_threaded(self, file_paths: List[Path], progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Process documents using thread pool (alternative to async)
+        
+        Arguments:
+        ----------
+            file_paths          { list }   : List of file paths
+
+            progress_callback { callable } : Progress callback
+        
+        Returns:
+        --------
+                       { dict }            : Processing results
+        """
+        self.logger.info(f"Starting threaded processing of {len(file_paths)} documents")
+        
+        task_id = self.progress_tracker.start_task(total_items = len(file_paths),
+                                                   description = "Threaded document processing",
+                                                  )
+        
+        try:
+            with concurrent.futures.ThreadPoolExecutor(max_workers = self.max_workers) as executor:
+                # Submit all tasks
+                future_to_file = {executor.submit(self._process_single_file_sync, file_path, task_id, progress_callback): file_path for file_path in file_paths}
+                
+                results        = list()
+                failed         = list()
+                
+                for future in concurrent.futures.as_completed(future_to_file):
+                    file_path = future_to_file[future]
+                    
+                    try:
+                        result = future.result()
+                        results.append(result)
+                        self.total_processed += 1
+                    
+                    except Exception as e:
+                        self.logger.error(f"Failed to process {file_path}: {repr(e)}")
+                        failed.append({"file_path": file_path, "error": str(e)})
+                        self.total_failed += 1
+                
+                final_result = {"processed"    : len(results),
+                                "failed"       : len(failed),
+                                "success_rate" : (len(results) / len(file_paths)) * 100,
+                                "results"      : results,
+                                "failures"     : failed,
+                                "task_id"      : task_id,
+                               }
+                
+                self.progress_tracker.complete_task(task_id)
+                
+                self.logger.info(f"Threaded processing completed: {len(results)} successful, {len(failed)} failed")
+                
+                return final_result
+                
+        except Exception as e:
+            self.progress_tracker.fail_task(task_id, str(e))
+            raise ProcessingException(f"Threaded processing failed: {repr(e)}")
+    
+
+    def _process_single_file_sync(self, file_path: Path, task_id: str, progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Process single file synchronously (for thread pool)
+        
+        Arguments:
+        ----------
+            file_path            { Path }   : File to process
+
+            task_id              { str }    : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing result
+        """
+        self.logger.debug(f"Processing file (sync): {file_path}")
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = file_path.name,
+                                          current_status = "parsing",
+                                         )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        # Parse document
+        start_time      = datetime.now()
+        text, metadata  = self.parser_factory.parse(file_path        = file_path,
+                                                    extract_metadata = True,
+                                                    clean_text       = True,
+                                                   )
+        
+        processing_time = (datetime.now() - start_time).total_seconds()
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = file_path.name,
+                                          current_status = "completed",
+                                         )
+        
+        # Handle metadata (could be Pydantic model or dict)
+        metadata_dict = metadata.dict() if hasattr(metadata, 'dict') else (metadata if isinstance(metadata, dict) else {})
+        
+        result        = {"file_path"       : str(file_path),
+                         "file_name"       : file_path.name,
+                         "text_length"     : len(text),
+                         "processing_time" : processing_time,
+                         "metadata"        : metadata_dict,
+                         "success"         : True,
+                        }
+        
+        # Increment processed items count
+        current_progress = self.progress_tracker.get_task_progress(task_id)
+        
+        if current_progress:
+            self.progress_tracker.update_task(task_id         = task_id,
+                                              processed_items = current_progress.processed_items + 1,
+                                             )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        return result
+    
+
+    def _update_statistics(self, processed_count: int):
+        """
+        Update processing statistics
+        
+        Arguments:
+        ----------
+            processed_count { int } : Number of documents processed in current batch
+        """
+        # Update average processing time (simplified)
+        if (processed_count > 0):
+            self.avg_processing_time = (self.avg_processing_time + (processed_count * 1.0)) / 2
+    
+
+    def get_coordinator_stats(self) -> Dict[str, Any]:
+        """
+        Get coordinator statistics
+        
+        Returns:
+        --------
+            { dict }    : Statistics dictionary
+        """
+        return {"total_processed"     : self.total_processed,
+                "total_failed"        : self.total_failed,
+                "success_rate"        : (self.total_processed / (self.total_processed + self.total_failed)) * 100 if (self.total_processed + self.total_failed) > 0 else 0,
+                "avg_processing_time" : self.avg_processing_time,
+                "max_workers"         : self.max_workers,
+                "active_tasks"        : self.progress_tracker.get_active_task_count(),
+               }
+    
+
+    def cleanup(self):
+        """
+        Cleanup resources
+        """
+        self.progress_tracker.cleanup_completed_tasks()
+        
+        self.logger.debug("AsyncCoordinator cleanup completed")
+
+
+# Global async coordinator instance
+_async_coordinator = None
+
+
+def get_async_coordinator(max_workers: Optional[int] = None) -> AsyncCoordinator:
+    """
+    Get global async coordinator instance
+    
+    Arguments:
+    ----------
+        max_workers { int }  : Maximum workers
+    
+    Returns:
+    --------
+        { AsyncCoordinator } : AsyncCoordinator instance
+    """
+    global _async_coordinator
+
+    if _async_coordinator is None:
+        _async_coordinator = AsyncCoordinator(max_workers)
+    
+    return _async_coordinator
+
+
+async def process_documents_async(file_paths: List[Path], **kwargs) -> Dict[str, Any]:
+    """
+    Convenience function for async document processing
+    
+    Arguments:
+    ----------
+        file_paths { list } : List of file paths
+        
+        **kwargs            : Additional arguments
+    
+    Returns:
+    --------
+             { dict }       : Processing results
+    """
+    coordinator = get_async_coordinator()
+
+    return await coordinator.process_documents_async(file_paths, **kwargs)

ingestion/progress_tracker.py

@@ -0,0 +1,518 @@
+# DEPENDENCIES
+import time
+import uuid
+from enum import Enum
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from datetime import datetime
+from dataclasses import dataclass
+from config.settings import get_settings
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class TaskStatus(str, Enum):
+    """
+    Task status enumeration
+    """
+    PENDING    = "pending"
+    RUNNING    = "running"
+    COMPLETED  = "completed"
+    FAILED     = "failed"
+    CANCELLED  = "cancelled"
+
+
+@dataclass
+class TaskProgress:
+    """
+    Progress tracking data structure
+    """
+    task_id                     : str
+    description                 : str
+    status                      : TaskStatus
+    total_items                 : int
+    processed_items             : int
+    current_item                : Optional[str]
+    current_status              : str
+    start_time                  : datetime
+    end_time                    : Optional[datetime]
+    progress_percent            : float
+    estimated_seconds_remaining : Optional[float]
+    metadata                    : Dict[str, Any]
+
+
+class ProgressTracker:
+    """
+    Comprehensive progress tracking for long-running operations: Provides real-time progress monitoring and status updates
+    """
+    def __init__(self, max_completed_tasks: int = 100):
+        """
+        Initialize progress tracker
+        
+        Arguments:
+        ----------
+            max_completed_tasks { int } : Maximum number of completed tasks to keep in history
+        """
+        self.logger                                         = logger
+        self.max_completed_tasks                            = max_completed_tasks
+        
+        # Task storage
+        self.active_tasks         : Dict[str, TaskProgress] = dict()
+        self.completed_tasks      : List[TaskProgress]      = list()
+        self.failed_tasks         : List[TaskProgress]      = list()
+        
+        self.logger.info("Initialized ProgressTracker")
+    
+
+    def start_task(self, total_items: int, description: str, metadata: Optional[Dict[str, Any]] = None) -> str:
+        """
+        Start tracking a new task
+        
+        Arguments:
+        ----------
+            total_items { int }  : Total number of items to process
+
+            description { str }  : Task description
+            
+            metadata    { dict } : Additional task metadata
+        
+        Returns:
+        --------
+               { str }           : Generated task ID
+        """
+        task_id                    = self._generate_task_id()
+        
+        task_progress              = TaskProgress(task_id                     = task_id,
+                                                  description                 = description,
+                                                  status                      = TaskStatus.RUNNING,
+                                                  total_items                 = total_items,
+                                                  processed_items             = 0,
+                                                  current_item                = None,
+                                                  current_status              = "Starting...",
+                                                  start_time                  = datetime.now(),
+                                                  end_time                    = None,
+                                                  progress_percent            = 0.0,
+                                                  estimated_seconds_remaining = None,
+                                                  metadata                    = metadata or {},
+                                                 )
+        
+        self.active_tasks[task_id] = task_progress
+        
+        self.logger.info(f"Started task {task_id}: {description} ({total_items} items)")
+        
+        return task_id
+    
+
+    def update_task(self, task_id: str, processed_items: Optional[int] = None, current_item: Optional[str] = None, 
+                    current_status: Optional[str] = None, metadata_update: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Update task progress
+        
+        Arguments:
+        ----------
+            task_id         { str }  : Task ID to update
+
+            processed_items { int }  : Number of items processed
+            
+            current_item    { str }  : Current item being processed
+            
+            current_status  { str }  : Current status message
+            
+            metadata_update { dict } : Metadata updates
+        
+        Returns:
+        --------
+                 { bool }            : True if update successful
+        """
+        if task_id not in self.active_tasks:
+            self.logger.warning(f"Task {task_id} not found in active tasks")
+            return False
+        
+        task = self.active_tasks[task_id]
+        
+        # Update fields
+        if processed_items is not None:
+            task.processed_items = processed_items
+        
+        if current_item is not None:
+            task.current_item = current_item
+        
+        if current_status is not None:
+            task.current_status = current_status
+        
+        if metadata_update:
+            task.metadata.update(metadata_update)
+        
+        # Calculate progress
+        if (task.total_items > 0):
+            task.progress_percent = (task.processed_items / task.total_items) * 100.0
+            
+            # Estimate remaining time
+            if (task.processed_items > 0):
+                elapsed_seconds  = (datetime.now() - task.start_time).total_seconds()
+                items_per_second = task.processed_items / elapsed_seconds
+                
+                if (items_per_second > 0):
+                    remaining_items                  = task.total_items - task.processed_items
+                    task.estimated_seconds_remaining = remaining_items / items_per_second
+        
+        # Ensure progress doesn't exceed 100%
+        if (task.progress_percent > 100.0):
+            task.progress_percent = 100.0
+        
+        self.logger.debug(f"Updated task {task_id}: {task.progress_percent:.1f}% complete")
+        
+        return True
+    
+
+    def complete_task(self, task_id: str, final_metadata: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Mark task as completed
+        
+        Arguments:
+        ----------
+            task_id         { str }  : Task ID to complete
+
+            final_metadata  { dict } : Final metadata updates
+        
+        Returns:
+        --------
+                 { bool }            : True if completion successful
+        """
+        if task_id not in self.active_tasks:
+            self.logger.warning(f"Task {task_id} not found for completion")
+            return False
+        
+        task                             = self.active_tasks[task_id]
+        
+        # Update task
+        task.status                      = TaskStatus.COMPLETED
+        task.end_time                    = datetime.now()
+        task.processed_items             = task.total_items  # Ensure 100% completion
+        task.progress_percent            = 100.0
+        task.estimated_seconds_remaining = 0.0
+        task.current_status              = "Completed"
+        
+        if final_metadata:
+            task.metadata.update(final_metadata)
+        
+        # Move to completed tasks
+        self.completed_tasks.append(task)
+        del self.active_tasks[task_id]
+        
+        # Maintain history size
+        if (len(self.completed_tasks) > self.max_completed_tasks):
+            self.completed_tasks = self.completed_tasks[-self.max_completed_tasks:]
+        
+        total_time = (task.end_time - task.start_time).total_seconds()
+        
+        self.logger.info(f"Completed task {task_id}: {task.description} in {total_time:.2f}s")
+        
+        return True
+    
+
+    def fail_task(self, task_id: str, error_message: str, error_details: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Mark task as failed
+        
+        Arguments:
+        ----------
+            task_id        { str }  : Task ID to mark as failed
+
+            error_message  { str }  : Error message
+            
+            error_details  { dict } : Additional error details
+        
+        Returns:
+        --------
+                 { bool }           : True if failure marking successful
+        """
+        if (task_id not in self.active_tasks):
+            self.logger.warning(f"Task {task_id} not found for failure marking")
+            return False
+        
+        task                           = self.active_tasks[task_id]
+        
+        # Update task
+        task.status                    = TaskStatus.FAILED
+        task.end_time                  = datetime.now()
+        task.current_status            = f"Failed: {error_message}"
+        
+        if error_details:
+            task.metadata["error_details"] = error_details
+        
+        task.metadata["error_message"] = error_message
+        
+        # Move to failed tasks
+        self.failed_tasks.append(task)
+        del self.active_tasks[task_id]
+        
+        total_time                     = (task.end_time - task.start_time).total_seconds()
+        
+        self.logger.error(f"Task {task_id} failed after {total_time:.2f}s: {error_message}")
+        
+        return True
+    
+
+    def cancel_task(self, task_id: str, reason: str = "User cancelled") -> bool:
+        """
+        Cancel a running task
+        
+        Arguments:
+        ----------
+            task_id { str } : Task ID to cancel
+
+            reason  { str } : Cancellation reason
+        
+        Returns:
+        --------
+               { bool }     : True if cancellation successful
+        """
+        if task_id not in self.active_tasks:
+            self.logger.warning(f"Task {task_id} not found for cancellation")
+            return False
+        
+        task                                 = self.active_tasks[task_id]
+        
+        # Update task
+        task.status                          = TaskStatus.CANCELLED
+        task.end_time                        = datetime.now()
+        task.current_status                  = f"Cancelled: {reason}"
+        task.metadata["cancellation_reason"] = reason
+        
+        # Move to completed tasks (as cancelled)
+        self.completed_tasks.append(task)
+        del self.active_tasks[task_id]
+        
+        total_time                           = (task.end_time - task.start_time).total_seconds()
+        
+        self.logger.info(f"Cancelled task {task_id} after {total_time:.2f}s: {reason}")
+        
+        return True
+    
+
+    def get_task_progress(self, task_id: str) -> Optional[TaskProgress]:
+        """
+        Get current progress for a task
+        
+        Arguments:
+        ----------
+            task_id { str }     : Task ID
+        
+        Returns:
+        --------
+               { TaskProgress } : Task progress or None if not found
+        """
+        # Check active tasks first
+        if task_id in self.active_tasks:
+            return self.active_tasks[task_id]
+        
+        # Check completed tasks
+        for task in self.completed_tasks:
+            if (task.task_id == task_id):
+                return task
+        
+        # Check failed tasks
+        for task in self.failed_tasks:
+            if (task.task_id == task_id):
+                return task
+        
+        return None
+    
+
+    def get_all_active_tasks(self) -> List[TaskProgress]:
+        """
+        Get all active tasks
+        
+        Returns:
+        --------
+            { list }    : List of active task progresses
+        """
+        return list(self.active_tasks.values())
+    
+
+    def get_active_task_count(self) -> int:
+        """
+        Get number of active tasks
+        
+        Returns:
+        --------
+            { int }    : Number of active tasks
+        """
+        return len(self.active_tasks)
+    
+
+    def get_recent_completed_tasks(self, limit: int = 10) -> List[TaskProgress]:
+        """
+        Get recently completed tasks
+        
+        Arguments:
+        ----------
+            limit { int } : Maximum number of tasks to return
+        
+        Returns:
+        --------
+               { list }   : List of completed tasks (newest first)
+        """
+        # Return newest first
+        return self.completed_tasks[-limit:][::-1]  
+    
+
+    def get_task_statistics(self, task_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Get detailed statistics for a task
+        
+        Arguments:
+        ----------
+            task_id { str } : Task ID
+        
+        Returns:
+        --------
+               { dict }     : Task statistics or None
+        """
+        task = self.get_task_progress(task_id)
+        
+        if not task:
+            return None
+        
+        stats = {"task_id"          : task.task_id,
+                 "description"      : task.description,
+                 "status"           : task.status.value,
+                 "progress_percent" : task.progress_percent,
+                 "processed_items"  : task.processed_items,
+                 "total_items"      : task.total_items,
+                 "current_item"     : task.current_item,
+                 "current_status"   : task.current_status,
+                 "start_time"       : task.start_time.isoformat(),
+                 "metadata"         : task.metadata,
+                }
+        
+        if task.end_time:
+            total_seconds               = (task.end_time - task.start_time).total_seconds()
+            stats["total_time_seconds"] = total_seconds
+            stats["end_time"]           = task.end_time.isoformat()
+        
+        if task.estimated_seconds_remaining:
+            stats["estimated_seconds_remaining"] = task.estimated_seconds_remaining
+        
+        return stats
+    
+
+    def get_global_statistics(self) -> Dict[str, Any]:
+        """
+        Get global progress statistics
+        
+        Returns:
+        --------
+            { dict }    : Global statistics
+        """
+        total_completed     = len(self.completed_tasks)
+        total_failed        = len(self.failed_tasks)
+        total_tasks         = total_completed + total_failed + len(self.active_tasks)
+        
+        # Calculate average completion time
+        avg_completion_time = 0.0
+        
+        if (total_completed > 0):
+            total_time          = sum((task.end_time - task.start_time).total_seconds() for task in self.completed_tasks if task.end_time)
+            avg_completion_time = total_time / total_completed
+        
+        return {"active_tasks"                : len(self.active_tasks),
+                "completed_tasks"             : total_completed,
+                "failed_tasks"                : total_failed,
+                "total_tasks"                 : total_tasks,
+                "success_rate"                : (total_completed / total_tasks * 100) if total_tasks > 0 else 0,
+                "avg_completion_time_seconds" : avg_completion_time,
+                "max_completed_tasks"         : self.max_completed_tasks,
+               }
+    
+
+    def cleanup_completed_tasks(self, older_than_hours: int = 24):
+        """
+        Clean up old completed tasks
+        
+        Arguments:
+        ----------
+            older_than_hours { int } : Remove tasks older than this many hours
+        """
+        cutoff_time          = datetime.now().timestamp() - (older_than_hours * 3600)
+        initial_count        = len(self.completed_tasks)
+        
+        self.completed_tasks = [task for task in self.completed_tasks if task.end_time and task.end_time.timestamp() > cutoff_time]
+        
+        removed_count        = initial_count - len(self.completed_tasks)
+        
+        if (removed_count > 0):
+            self.logger.info(f"Cleaned up {removed_count} completed tasks older than {older_than_hours} hours")
+    
+
+    @staticmethod
+    def _generate_task_id() -> str:
+        """
+        Generate unique task ID
+        
+        Returns:
+        --------
+            { str }    : Generated task ID
+        """
+        return f"task_{uuid.uuid4().hex[:8]}_{int(time.time())}"
+    
+
+    def __del__(self):
+        """
+        Cleanup on destruction
+        """
+        try:
+            self.cleanup_completed_tasks()
+
+        except Exception:
+            # Ignore cleanup errors during destruction
+            pass  
+
+
+# Global progress tracker instance
+_progress_tracker = None
+
+
+def get_progress_tracker() -> ProgressTracker:
+    """
+    Get global progress tracker instance
+    
+    Returns:
+    --------
+        { ProgressTracker } : ProgressTracker instance
+    """
+    global _progress_tracker
+
+    if _progress_tracker is None:
+        _progress_tracker = ProgressTracker()
+    
+    return _progress_tracker
+
+
+def start_progress_task(total_items: int, description: str, **kwargs) -> str:
+    """
+    Convenience function to start a progress task
+    
+    Arguments:
+    ----------
+        total_items { int } : Total items
+        
+        description { str } : Task description
+        
+        **kwargs            : Additional arguments
+    
+    Returns:
+    --------
+               { str }      : Task ID
+    """
+    tracker = get_progress_tracker()
+
+    return tracker.start_task(total_items, description, **kwargs)

ingestion/router.py

@@ -0,0 +1,516 @@
+# DEPENDENCIES
+import os
+import tempfile
+from enum import Enum
+from typing import Any
+from typing import List
+from typing import Dict
+from pathlib import Path        
+from typing import Optional
+from config.settings import get_settings
+from utils.file_handler import FileHandler
+from config.models import IngestionInputType
+from config.logging_config import get_logger
+from utils.error_handler import handle_errors
+from utils.validators import validate_upload_file
+from utils.error_handler import ProcessingException
+from ingestion.progress_tracker import get_progress_tracker
+from document_parser.zip_handler import get_archive_handler
+from ingestion.async_coordinator import get_async_coordinator
+
+
+# Setup Settings and Logging
+settings = get_settings()
+logger   = get_logger(__name__)
+
+
+class IngestionRouter:
+    """
+    Intelligent ingestion router: Determines optimal processing strategy based on input type and characteristics
+    """
+    def __init__(self):
+        """
+        Initialize ingestion router
+        """
+        self.logger                = logger
+        self.async_coordinator     = get_async_coordinator()
+        self.progress_tracker      = get_progress_tracker()
+        self.file_handler          = FileHandler()
+        
+        # Processing strategies
+        self.processing_strategies = {IngestionInputType.FILE    : self._process_single_file,
+                                      IngestionInputType.ARCHIVE : self._process_archive,
+                                      IngestionInputType.URL     : self._process_url,
+                                      IngestionInputType.TEXT    : self._process_text,
+                                     }
+        
+        self.logger.info("Initialized IngestionRouter")
+    
+
+    @handle_errors(error_type = ProcessingException, log_error = True, reraise = True)
+    def route_and_process(self, input_data: Any, input_type: IngestionInputType, metadata: Optional[Dict[str, Any]] = None, progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Route input to appropriate processing strategy
+        
+        Arguments:
+        ----------
+            input_data         { Any }          : Input data (file path, URL, text, etc.)
+
+            input_type   { IngestionInputType } : Type of input
+            
+            metadata            { dict }        : Additional metadata
+            
+            progress_callback { callable }      : Progress callback
+        
+        Returns:
+        --------
+                        { dict }                : Processing results
+        """
+        self.logger.info(f"Routing {input_type.value} input for processing")
+        
+        # Validate input
+        self._validate_input(input_data, input_type)
+        
+        # Get processing strategy
+        processor = self.processing_strategies.get(input_type)
+        
+        if not processor:
+            raise ProcessingException(f"No processor available for input type: {input_type}")
+        
+        # Process with progress tracking
+        task_id   = self.progress_tracker.start_task(total_items = 1,
+                                                     description = f"Processing {input_type.value}",
+                                                     metadata    = metadata or {},
+                                                    )
+        
+        try:
+            result = processor(input_data, metadata, task_id, progress_callback)
+            
+            self.progress_tracker.complete_task(task_id, {"processed_items": 1})
+            
+            self.logger.info(f"Successfully processed {input_type.value} input")
+            
+            return result
+            
+        except Exception as e:
+            self.progress_tracker.fail_task(task_id, str(e))
+            raise ProcessingException(f"Failed to process {input_type.value} input: {repr(e)}")
+    
+
+    def _process_single_file(self, file_path: Path, metadata: Optional[Dict[str, Any]], task_id: str, progress_callback: Optional[callable]) -> Dict[str, Any]:
+        """
+        Process single file
+        
+        Arguments:
+        ----------
+            file_path         { Path }      : File path to process
+
+            metadata          { dict }      : File metadata
+            
+            task_id           { str }       : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing results
+        """
+        self.logger.info(f"Processing single file: {file_path}")
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = file_path.name,
+                                          current_status = "validating_file",
+                                         )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        # Validate file
+        validate_upload_file(file_path)
+        
+        # Process file
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = file_path.name,
+                                          current_status = "processing_file",
+                                         )
+        
+        result = self.async_coordinator.process_documents_threaded([file_path], progress_callback)
+        
+        # Extract single result (since we processed one file)
+        if result["results"]:
+            file_result = result["results"][0]
+
+        else:
+            raise ProcessingException(f"File processing failed: {result.get('failures', [])}")
+        
+        return {"success"      : True,
+                "file_path"    : str(file_path),
+                "file_name"    : file_path.name,
+                "text_length"  : file_result.get("text_length", 0),
+                "content_type" : self._detect_content_type(file_path),
+                "metadata"     : file_result.get("metadata", {}),
+               }
+    
+
+    def _process_archive(self, archive_path: Path, metadata: Optional[Dict[str, Any]], task_id: str, progress_callback: Optional[callable]) -> Dict[str, Any]:
+        """
+        Process archive file (ZIP, RAR, etc.)
+        
+        Arguments:
+        ----------
+            archive_path        { Path }    : Archive file path
+
+            metadata            { dict }    : Archive metadata
+            
+            task_id             { str }     : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing results
+        """
+        self.logger.info(f"Processing archive: {archive_path}")
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = archive_path.name,
+                                          current_status = "extracting_archive",
+                                         )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        # Extract archive
+        archive_handler = get_archive_handler()
+        extracted_files = archive_handler.extract_archive(archive_path)
+        
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = archive_path.name,
+                                          current_status = f"processing_{len(extracted_files)}_files",
+                                          processed_items = 0,
+                                          total_items    = len(extracted_files),
+                                         )
+        
+        # Process extracted files
+        result = self.async_coordinator.process_documents_threaded(extracted_files, progress_callback)
+        
+        return {"success"          : True,
+                "archive_path"     : str(archive_path),
+                "archive_name"     : archive_path.name,
+                "extracted_files"  : len(extracted_files),
+                "processed_files"  : result["processed"],
+                "failed_files"     : result["failed"],
+                "success_rate"     : result["success_rate"],
+                "results"          : result["results"],
+                "failures"         : result["failures"],
+               }
+    
+
+    def _process_url(self, url: str, metadata: Optional[Dict[str, Any]], task_id: str, progress_callback: Optional[callable]) -> Dict[str, Any]:
+        """
+        Process URL (web page scraping)
+        
+        WARNING: URL processing not implemented yet. Requires Playwright/BeautifulSoup integration
+        TODO: Implement web scraping functionality
+
+        Arguments:
+        ----------
+            url                 { str }     : URL to process
+
+            metadata           { dict }     : URL metadata
+            
+            task_id             { str }     : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing results
+        """
+        self.logger.info(f"Processing URL: {url}")
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = url,
+                                          current_status = "scraping_url",
+                                         )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        # Note: Web scraping would be implemented here: For now, return placeholder
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = url,
+                                          current_status = "processing_content",
+                                         )
+        
+        # Placeholder implementation: In production, this would use playwright/beautifulsoup scrapers
+        return {"success"      : True,
+                "url"          : url,
+                "content_type" : "web_page",
+                "text_length"  : 0,  # Would be actual content length
+                "message"      : "URL processing placeholder - implement web scraping",
+                "metadata"     : metadata or {},
+               }
+    
+
+    def _process_text(self, text: str, metadata: Optional[Dict[str, Any]], task_id: str, progress_callback: Optional[callable]) -> Dict[str, Any]:
+        """
+        Process raw text input
+        
+        Arguments:
+        ----------
+            text                { str }     : Text content to process
+
+            metadata            { dict }    : Text metadata
+            
+            task_id             { str }     : Progress task ID
+            
+            progress_callback { callable }  : Progress callback
+        
+        Returns:
+        --------
+                       { dict }             : Processing results
+        """
+        self.logger.info(f"Processing text input ({len(text)} characters)")
+        
+        # Update progress
+        self.progress_tracker.update_task(task_id        = task_id,
+                                          current_item   = "text_input",
+                                          current_status = "processing_text",
+                                         )
+        
+        if progress_callback:
+            progress_callback(self.progress_tracker.get_task_progress(task_id))
+        
+        # For text input, create a temporary file and process it
+        with tempfile.NamedTemporaryFile(mode = 'w', suffix = '.txt', delete = False, encoding = 'utf-8') as temp_file:
+            temp_file.write(text)
+            temp_path = Path(temp_file.name)
+        
+        try:
+            # Process as a file
+            file_result                         = self._process_single_file(file_path         = temp_path, 
+                                                                            metadata          = metadata, 
+                                                                            task_id           = task_id, 
+                                                                            progress_callback = progress_callback,
+                                                                           )
+            
+            # Add text-specific metadata
+            file_result["input_type"]           = "direct_text"
+            file_result["original_text_length"] = len(text)
+            
+            return file_result
+            
+        finally:
+            # Cleanup temporary file
+            try:
+                os.unlink(temp_path)
+
+            except Exception as e:
+                self.logger.warning(f"Failed to delete temporary file: {repr(e)}")
+    
+
+    def _validate_input(self, input_data: Any, input_type: IngestionInputType):
+        """
+        Validate input based on type
+        
+        Arguments:
+        ----------
+            input_data         { Any }        : Input data to validate
+
+            input_type { IngestionInputType } : Type of input
+        
+        Raises:
+        -------
+            ProcessingException               : If validation fails
+        """
+        if (input_type == IngestionInputType.FILE):
+            if not isinstance(input_data, (str, Path)):
+                raise ProcessingException("File input must be a path string or Path object")
+            
+            file_path = Path(input_data)
+            if not file_path.exists():
+                raise ProcessingException(f"File not found: {file_path}")
+        
+        elif (input_type == IngestionInputType.URL):
+            if not isinstance(input_data, str):
+                raise ProcessingException("URL input must be a string")
+            
+            if not input_data.startswith(('http://', 'https://')):
+                raise ProcessingException("URL must start with http:// or https://")
+        
+        elif (input_type == IngestionInputType.TEXT):
+            if not isinstance(input_data, str):
+                raise ProcessingException("Text input must be a string")
+            
+            if len(input_data.strip()) == 0:
+                raise ProcessingException("Text input cannot be empty")
+        
+        elif (input_type == IngestionInputType.ARCHIVE):
+            if not isinstance(input_data, (str, Path)):
+                raise ProcessingException("Archive input must be a path string or Path object")
+            
+            file_path = Path(input_data)
+            if not file_path.exists():
+                raise ProcessingException(f"Archive file not found: {file_path}")
+    
+
+    def _detect_content_type(self, file_path: Path) -> str:
+        """
+        Detect content type from file extension
+        
+        Arguments:
+        ----------
+            file_path { Path } : File path
+        
+        Returns:
+        --------
+               { str }         : Content type
+        """
+        extension     = file_path.suffix.lower()
+        
+        content_types = {'.pdf'  : 'application/pdf',
+                         '.docx' : 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                         '.doc'  : 'application/msword',
+                         '.txt'  : 'text/plain',
+                         '.zip'  : 'application/zip',
+                         '.rar'  : 'application/vnd.rar',
+                         '.7z'   : 'application/x-7z-compressed',
+                        }
+        
+        return content_types.get(extension, 'application/octet-stream')
+    
+
+    def batch_process(self, inputs: List[Dict[str, Any]], progress_callback: Optional[callable] = None) -> Dict[str, Any]:
+        """
+        Process multiple inputs in batch
+        
+        Arguments:
+        ----------
+            inputs            { list }     : List of input dictionaries
+
+            progress_callback { callable } : Progress callback
+        
+        Returns:
+        --------
+                       { dict }            : Batch processing results
+        """
+        self.logger.info(f"Starting batch processing of {len(inputs)} inputs")
+        
+        task_id     = self.progress_tracker.start_task(total_items = len(inputs),
+                                                       description = "Batch processing",
+                                                      )
+        
+        results     = list()
+        failed      = list()
+        
+        for i, input_config in enumerate(inputs):
+            try:
+                self.progress_tracker.update_task(task_id         = task_id,
+                                                  processed_items = i,
+                                                  current_item    = f"Input {i + 1}",
+                                                  current_status  = "processing",
+                                                 )
+                
+                input_type = IngestionInputType(input_config.get("type"))
+                input_data = input_config.get("data")
+                metadata   = input_config.get("metadata", {})
+                
+                result     = self.route_and_process(input_data        = input_data,
+                                                    input_type        = input_type,
+                                                    metadata          = metadata,
+                                                    progress_callback = None,
+                                                   )
+                
+                results.append(result)
+                
+                if progress_callback:
+                    progress_callback(self.progress_tracker.get_task_progress(task_id))
+            
+            except Exception as e:
+                self.logger.error(f"Failed to process input {i + 1}: {repr(e)}")
+                failed.append({"input_index": i, "input_config": input_config, "error": str(e)})
+        
+        self.progress_tracker.complete_task(task_id, {"processed_items": len(results)})
+        
+        return {"processed"    : len(results),
+                "failed"       : len(failed),
+                "success_rate" : (len(results) / len(inputs)) * 100,
+                "results"      : results,
+                "failures"     : failed,
+               }
+    
+
+    def cleanup(self):
+        """
+        Cleanup router resources
+        """
+        if hasattr(self, 'async_coordinator'):
+            self.async_coordinator.cleanup()
+        
+        self.logger.debug("IngestionRouter cleanup completed")
+
+
+    def get_processing_capabilities(self) -> Dict[str, Any]:
+        """
+        Get supported processing capabilities
+        
+        Returns:
+        --------
+            { dict }    : Capabilities information
+        """
+        # URL type is not fully implemented yet
+        supported_types = [t.value for t in IngestionInputType if (t != IngestionInputType.URL)]
+        
+        return {"supported_input_types" : supported_types,
+                "max_file_size_mb"      : settings.MAX_FILE_SIZE_MB,
+                "max_batch_files"       : settings.MAX_BATCH_FILES,
+                "allowed_extensions"    : settings.ALLOWED_EXTENSIONS,
+                "max_workers"           : settings.MAX_WORKERS,
+                "async_supported"       : True,
+                "batch_supported"       : True,
+               }
+
+
+# Global ingestion router instance
+_ingestion_router = None
+
+
+def get_ingestion_router() -> IngestionRouter:
+    """
+    Get global ingestion router instance
+    
+    Returns:
+    --------
+        { IngestionRouter } : IngestionRouter instance
+    """
+    global _ingestion_router
+
+    if _ingestion_router is None:
+        _ingestion_router = IngestionRouter()
+    
+    return _ingestion_router
+
+
+def process_input(input_data: Any, input_type: IngestionInputType, **kwargs) -> Dict[str, Any]:
+    """
+    Convenience function for input processing
+    
+    Arguments:
+    ----------
+        input_data         { Any }        : Input data
+        
+        input_type { IngestionInputType } : Input type
+        
+        **kwargs                          : Additional arguments
+    
+    Returns:
+    --------
+                  { dict }                : Processing results
+    """
+    router = get_ingestion_router()
+
+    return router.route_and_process(input_data, input_type, **kwargs)

requirements.txt

@@ -0,0 +1,135 @@
+# Core Dependencies
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+pydantic==2.12.5
+pydantic-settings==2.12.0
+python-multipart==0.0.6
+
+# AI/ML & NLP
+#ollama==0.1.7
+sentence-transformers==2.2.2
+transformers==4.37.2
+torch==2.9.1
+tokenizers==0.15.2
+accelerate==0.24.1
+
+# Vector Database & Search
+faiss-cpu==1.7.4
+rank-bm25==0.2.2
+
+# Document Processing
+PyMuPDF==1.23.8
+PyPDF2==3.0.1
+python-docx==1.1.0
+python-pptx==0.6.23
+
+# OCR & Image Processing
+paddleocr==2.7.3
+easyocr==1.7.2
+paddlepaddle==3.2.2
+Pillow==10.0.1
+opencv-python==4.6.0.66
+
+# Archive Handling
+py7zr==0.20.6
+rarfile==4.1
+python-magic==0.4.27
+
+# Text Processing & Utilities
+chardet==5.2.0
+numpy==1.24.3
+nltk==3.8.1
+tqdm==4.66.1
+filetype==1.2.0
+sentencepiece==0.2.1
+
+# Caching & Performance
+redis==5.0.1
+psutil==5.9.6
+
+# Ragas Evaluation Framework
+ragas==0.1.2
+datasets==2.14.6
+evaluate==0.4.1
+
+# Evaluation Metrics & Utilities
+scikit-learn==1.3.2
+scipy==1.11.4
+pandas==2.0.3
+seaborn==0.13.0
+matplotlib==3.10.7
+
+# Development
+black==25.11.0
+flake8==7.3.0
+mypy==1.18.2
+
+# Async & Concurrency
+aiofiles==23.2.1
+
+# System & Utilities
+python-dateutil==2.8.2
+typing-extensions==4.15.0
+protobuf==4.25.8
+
+# Additional dependencies from your environment
+aiohttp>=3.9.3
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==25.4.0
+click==8.1.7
+colorlog==6.10.1
+cryptography==42.0.2
+filelock==3.20.0
+h11==0.16.0
+huggingface-hub==0.20.0
+httpx==0.25.2
+idna==3.11
+importlib-metadata==6.11.0
+joblib==1.5.2
+jsonpatch==1.33
+jsonschema==4.25.0
+langchain==0.1.7
+langchain-community==0.0.20
+langchain-core==0.1.23
+langsmith==0.0.87
+loguru==0.7.2
+lxml==5.1.0
+MarkupSafe==3.0.2
+msgpack==1.0.7
+networkx==3.4.2
+openai>=1.54.0
+orjson==3.11.4
+packaging==23.2
+pandas==2.0.3
+pip==25.2
+platformdirs==4.5.0
+pluggy==1.6.0
+pydantic_core==2.41.5
+pygments==2.19.2
+pypdf==4.3.1
+pytest==9.0.1
+python-dotenv==1.0.1
+PyYAML==6.0.2
+requests==2.32.5
+rich==13.7.0
+scikit-image==0.21.0
+setuptools==80.9.0
+six==1.17.0
+sniffio==1.3.0
+SQLAlchemy==2.0.27
+starlette==0.27.0
+sympy==1.14.0
+tenacity==8.5.0
+threadpoolctl==3.6.0
+tiktoken==0.5.2
+tomli==2.2.1
+torchvision==0.24.1
+typer==0.9.4
+urllib3==2.5.0
+wasabi==1.1.3
+Werkzeug==3.1.4
+wheel==0.45.1
+xxhash==3.6.0
+yarl==1.22.0
+zipp==3.23.0

retrieval/citation_tracker.py

@@ -0,0 +1,452 @@
+# DEPENDENCIES
+import re
+from typing import List
+from typing import Dict
+from typing import Tuple 
+from typing import Optional
+from collections import defaultdict
+from config.models import DocumentChunk
+from config.models import ChunkWithScore
+from config.logging_config import get_logger
+from utils.error_handler import CitationError
+from utils.error_handler import handle_errors
+
+
+# Setup Logging
+logger = get_logger(__name__)
+
+
+class CitationTracker:
+    """
+    Citation tracking and management: Tracks source citations in generated text and provides citation formatting and validation
+    """
+    def __init__(self):
+        """
+        Initialize citation tracker
+        """
+        self.logger           = logger
+        self.citation_pattern = re.compile(r'\[(\d+)\]')
+    
+
+    def extract_citations(self, text: str) -> List[int]:
+        """
+        Extract citation numbers from text
+        
+        Arguments:
+        ----------
+            text { str } : Text containing citations
+        
+        Returns:
+        --------
+             { list }    : List of citation numbers found in text
+        """
+        if not text:
+            return []
+        
+        try:
+            matches          = self.citation_pattern.findall(text)
+            citation_numbers = [int(match) for match in matches]
+            
+            # Remove duplicates and sort
+            unique_citations = sorted(set(citation_numbers))
+            
+            self.logger.debug(f"Extracted {len(unique_citations)} citations from text")
+            
+            return unique_citations
+        
+        except Exception as e:
+            self.logger.error(f"Citation extraction failed: {repr(e)}")
+            return []
+    
+
+    def validate_citations(self, text: str, sources: List[ChunkWithScore]) -> Tuple[bool, List[int]]:
+        """
+        Validate that all citations in text reference existing sources
+        
+        Arguments:
+        ----------
+            text    { str }  : Text containing citations
+            
+            sources { list } : List of available sources
+        
+        Returns:
+        --------
+              { Tuple[bool, List[int]] } : (is_valid, invalid_citations)
+        """
+        citation_numbers = self.extract_citations(text = text)
+        
+        if not citation_numbers:
+            return True, []
+        
+        # Check if all citation numbers are within valid range
+        max_valid         = len(sources)
+        invalid_citations = [num for num in citation_numbers if (num < 1) or (num > max_valid)]
+        
+        if invalid_citations:
+            self.logger.warning(f"Invalid citations found: {invalid_citations}. Valid range: 1-{max_valid}")
+            return False, invalid_citations
+        
+        return True, []
+    
+
+    def format_citations(self, sources: List[ChunkWithScore], style: str = "numeric") -> str:
+        """
+        Format citations as reference list
+        
+        Arguments:
+        ----------
+            sources { list } : List of sources to format
+            
+            style   { str }  : Citation style ('numeric', 'verbose')
+        
+        Returns:
+        --------
+               { str }       : Formatted citation text
+        """
+        if not sources:
+            return ""
+        
+        try:
+            citations = list()
+
+            for i, source in enumerate(sources, 1):
+                if (style == "verbose"):
+                    citation = self._format_verbose_citation(source = source, 
+                                                             number = i,
+                                                            )
+                
+                else:
+                    citation = self._format_numeric_citation(source = source, 
+                                                             number = i,
+                                                            )
+                
+                citations.append(citation)
+            
+            citation_text = "\n".join(citations)
+            
+            self.logger.debug(f"Formatted {len(citations)} citations in {style} style")
+            
+            return citation_text
+        
+        except Exception as e:
+            self.logger.error(f"Citation formatting failed: {repr(e)}")
+            return ""
+    
+
+    def _format_numeric_citation(self, source: ChunkWithScore, number: int) -> str:
+        """
+        Format citation in numeric style with sanitization
+        
+        Arguments:
+        ----------
+            source { ChunkWithScore } : Source to format
+
+            number      { int }       : Citation number
+        
+        Returns:
+        --------
+                { str }              : Formatted citation
+        """
+        chunk = source.chunk
+        
+        parts = [f"[{number}]"]
+        
+        # Add source information with proper sanitization
+        if (hasattr(chunk, 'metadata') and chunk.metadata):
+            if ('filename' in chunk.metadata):
+                # Sanitize filename more thoroughly
+                filename = str(chunk.metadata['filename'])
+                
+                # Remove problematic characters that could break citation parsing: Keep only alphanumeric, spaces, dots, hyphens, underscores
+                filename = re.sub(r'[^\w\s\.\-]', '_', filename)
+                
+                # Limit length to prevent overflow
+                if (len(filename) > 50):
+                    filename = filename[:47] + "..."
+                
+                parts.append(f"Source: {filename}")
+        
+        if chunk.page_number:
+            parts.append(f"Page {chunk.page_number}")
+        
+        if chunk.section_title:
+            # Sanitize section title similarly
+            section = str(chunk.section_title)
+            section = re.sub(r'[^\w\s\.\-]', '_', section)
+            
+            if (len(section) > 40):
+                section = section[:37] + "..."
+            
+            parts.append(f"Section: {section}")
+        
+        # Add relevance score if available
+        if (source.score > 0):
+            parts.append(f"(Relevance: {source.score:.2f})")
+        
+        return " ".join(parts)
+    
+
+    def _format_verbose_citation(self, source: ChunkWithScore, number: int) -> str:
+        """
+        Format citation in verbose style - SAFER VERSION
+        
+        Arguments:
+        ----------
+            source { ChunkWithScore } : Source to format
+
+            number     { int }        : Citation number
+        
+        Returns:
+        --------
+                { str }              : Formatted citation
+        """
+        chunk = source.chunk
+        
+        parts = [f"Citation {number}:"]
+        
+        # Document information with sanitization
+        if (hasattr(chunk, 'metadata')):
+            meta = chunk.metadata
+            
+            if ('filename' in meta):
+                filename = str(meta['filename'])
+                filename = re.sub(r'[^\w\s\.\-]', '_', filename)
+                
+                if (len(filename) > 50):
+                    filename = filename[:47] + "..."
+
+                parts.append(f"Document: {filename}")
+            
+            if ('title' in meta):
+                title = str(meta['title'])
+                title = re.sub(r'[^\w\s\.\-]', '_', title)
+                
+                if (len(title) > 60):
+                    title = title[:57] + "..."
+                
+                parts.append(f"Title: {title}")
+            
+            if ('author' in meta):
+                author = str(meta['author'])
+                author = re.sub(r'[^\w\s\.\-]', '_', author)
+                
+                if (len(author) > 40):
+                    author = author[:37] + "..."
+                    
+                parts.append(f"Author: {author}")
+        
+        # Location information
+        location_parts = list()
+        
+        if chunk.page_number:
+            location_parts.append(f"page {chunk.page_number}")
+        
+        if chunk.section_title:
+            section = str(chunk.section_title)
+            section = re.sub(r'[^\w\s\.\-]', '_', section)
+            
+            if (len(section) > 40):
+                section = section[:37] + "..."
+            
+            location_parts.append(f"section '{section}'")
+        
+        if location_parts:
+            parts.append("(" + ", ".join(location_parts) + ")")
+        
+        # Relevance information
+        if (source.score > 0):
+            parts.append(f"[Relevance score: {source.score:.3f}]")
+        
+        return " ".join(parts)
+    
+
+    def generate_citation_map(self, sources: List[ChunkWithScore]) -> Dict[int, Dict]:
+        """
+        Generate mapping from citation numbers to source details
+        
+        Arguments:
+        ----------
+            sources { list } : List of sources
+        
+        Returns:
+        --------
+               { dict }      : Dictionary mapping citation numbers to source details
+        """
+        citation_map = dict()
+
+        for i, source in enumerate(sources, 1):
+            chunk           = source.chunk
+            
+            citation_map[i] = {'chunk_id'      : chunk.chunk_id,
+                               'document_id'   : chunk.document_id,
+                               'score'         : source.score,
+                               'text_preview'  : chunk.text[:200] + "..." if (len(chunk.text) > 200) else chunk.text,
+                               'metadata'      : getattr(chunk, 'metadata', {}),
+                               'page_number'   : chunk.page_number,
+                               'section_title' : chunk.section_title,
+                              }
+        
+        return citation_map
+    
+
+    def replace_citation_markers(self, text: str, citation_map: Dict[int, str]) -> str:
+        """
+        Replace citation markers with formatted citations - FIXED
+        
+        Arguments:
+        ----------
+            text         { str }  : Text containing citation markers
+            
+            citation_map { dict } : Mapping of citation numbers to formatted strings
+        
+        Returns:
+        --------
+                  { str }         : Text with replaced citations
+        """
+        def replacement(match):
+            try:
+                citation_num     = int(match.group(1))
+                
+                # Get replacement text and sanitize it
+                replacement_text = citation_map.get(citation_num, match.group(0))
+                
+                return str(replacement_text)
+            
+            except (ValueError, IndexError):
+                # Return original match if parsing fails
+                return match.group(0)
+        
+        try:
+            return self.citation_pattern.sub(replacement, text)
+        
+        except Exception as e:
+            self.logger.error(f"Citation replacement failed: {repr(e)}")
+            # Return original text on error
+            return text  
+    
+
+    def get_citation_statistics(self, text: str, sources: List[ChunkWithScore]) -> Dict:
+        """
+        Get statistics about citations in text
+        
+        Arguments:
+        ----------
+            text    { str }  : Text containing citations
+            
+            sources { list } : List of sources
+        
+        Returns:
+        --------
+              { dict }       : Citation statistics
+        """
+        citation_numbers = self.extract_citations(text = text)
+        
+        if not citation_numbers:
+            return {"total_citations": 0}
+        
+        # Calculate citation distribution
+        citation_counts = defaultdict(int)
+
+        for num in citation_numbers:
+            if 1 <= num <= len(sources):
+                source = sources[num - 1]
+                doc_id = source.chunk.document_id
+                citation_counts[doc_id] += 1
+        
+        return {"total_citations"      : len(citation_numbers),
+                "unique_citations"     : len(set(citation_numbers)),
+                "citation_distribution": dict(citation_counts),
+                "citations_per_source" : {i: citation_numbers.count(i) for i in set(citation_numbers)},
+               }
+    
+
+    def ensure_citation_consistency(self, text: str, sources: List[ChunkWithScore]) -> str:
+        """
+        Ensure citation numbers are consistent and sequential
+        
+        Arguments:
+        ----------
+            text    { str }  : Text containing citations
+            
+            sources { list } : List of sources
+        
+        Returns:
+        --------
+              { str }        : Text with consistent citations
+        """
+        is_valid, invalid_citations = self.validate_citations(text, sources)
+        
+        if not is_valid:
+            self.logger.warning("Invalid citations found, attempting to fix consistency")
+            
+            # Extract current citations and create mapping
+            current_citations = self.extract_citations(text = text)
+            
+            if not current_citations:
+                return text
+            
+            # Create mapping from old to new citation numbers
+            citation_mapping = dict()
+            
+            for i, old_num in enumerate(sorted(set(current_citations)), 1):
+                if (old_num <= len(sources)):
+                    citation_mapping[old_num] = i
+            
+            # Replace citations in text
+            def consistent_replacement(match):
+                old_num = int(match.group(1))
+                new_num = citation_mapping.get(old_num, old_num)
+                
+                return f"[{new_num}]"
+            
+            fixed_text = self.citation_pattern.sub(consistent_replacement, text)
+            
+            self.logger.info(f"Fixed citation consistency: {current_citations} -> {list(citation_mapping.values())}")
+            
+            return fixed_text
+        
+        return text
+
+
+# Global citation tracker instance
+_citation_tracker = None
+
+
+def get_citation_tracker() -> CitationTracker:
+    """
+    Get global citation tracker instance (singleton)
+    
+    Returns:
+    --------
+        { CitationTracker }    : CitationTracker instance
+    """
+    global _citation_tracker
+    
+    if _citation_tracker is None:
+        _citation_tracker = CitationTracker()
+    
+    return _citation_tracker
+
+
+@handle_errors(error_type = CitationError, log_error = True, reraise = False)
+def extract_and_validate_citations(text: str, sources: List[ChunkWithScore]) -> Tuple[List[int], bool]:
+    """
+    Convenience function for citation extraction and validation
+    
+    Arguments:
+    ----------
+        text    { str }  : Text containing citations
+        
+        sources { list } : List of sources
+    
+    Returns:
+    --------
+             { Tuple[List[int], bool] } : (citation_numbers, is_valid)
+    """
+    tracker   = get_citation_tracker()
+    citations = tracker.extract_citations(text = text)
+    is_valid, _ = tracker.validate_citations(text    = text, 
+                                             sources = sources,
+                                            )
+    
+    return citations, is_valid