# Import the core PyTorch library for tensor operations
import torch

# Import LangChain's wrappers for Hugging Face models
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline

# Import core components from the transformers library for model loading and configuration
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig

# Import LangChain's tools for prompt engineering and output handling
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser

# --- Configure Embedding Model ---
embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2"

# Use GPU if available, otherwise fallback to CPU
model_kwargs = {"device": "cuda"}

# Initialize embeddings with LangChain's wrapper
embeddings = HuggingFaceEmbeddings(
    model_name=embedding_model_name,
    model_kwargs=model_kwargs
)

# --- Configure LLM for Summarization and Generation ---
llm_id = "mistralai/Mistral-7B-Instruct-v0.2"

# Quantization: reduces memory footprint while preserving performance
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_quant_type="nf4"
)

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(llm_id)

# Load LLM with quantization
model = AutoModelForCausalLM.from_pretrained(
    llm_id,
    torch_dtype=torch.float16,
    device_map="auto",
    quantization_config=quantization_config
)

# Create a text-generation pipeline using the loaded model and tokenizer.
pipe = pipeline(
    "text-generation", 
    model=model, 
    tokenizer=tokenizer, 
    max_new_tokens=512 # Controls the max length of the generated summaries and answers
)

# Wrap pipeline for LangChain compatibility
llm = HuggingFacePipeline(pipeline=pipe)

# Define the documentation sections to scrape, with varying crawl depths.
urls_to_load = [
    {"url": "https://huggingface.co/docs/transformers/index", "max_depth": 3},
    {"url": "https://huggingface.co/docs/datasets/index", "max_depth": 2},
    {"url": "https://huggingface.co/docs/tokenizers/index", "max_depth": 2},
    {"url": "https://huggingface.co/docs/peft/index", "max_depth": 1},
    {"url": "https://huggingface.co/docs/accelerate/index", "max_depth": 1}
]

from langchain_community.document_loaders import RecursiveUrlLoader
from bs4 import BeautifulSoup as Soup

# Empty list to append components
docs = []

# Iterate through the list and crawl each documentation section.
for item in urls_to_load:
    # Initialize the loader with the specific URL and parameters.
    loader = RecursiveUrlLoader(
        url=item["url"],
        max_depth=item["max_depth"],
        extractor=lambda x: Soup(x, "html.parser").text, # Use BeautifulSoup to extract text
        prevent_outside=True, # Ensure we stay within the documentation pages
        use_async=True, # Use asynchronous requests for faster crawling
        timeout=600, # Set a generous timeout for slow pages
    )
    # Load the documents and add them to our master list.
    loaded_docs = loader.load()
    docs.extend(loaded_docs)
    print(f"Loaded {len(loaded_docs)} documents from {item['url']}")

import numpy as np

# We need a consistent way to count tokens, using the LLM's tokenizer is the most accurate method.
def count_tokens(text: str) -> int:
    """Counts the number of tokens in a text using the configured tokenizer."""
    # Ensure text is not None and is a string
    ifnot isinstance(text, str):
        return0
    return len(tokenizer.encode(text))

# Extract the text content from the loaded LangChain Document objects
docs_texts = [d.page_content for d in docs]

# Calculate token counts for each document
token_counts = [count_tokens(text) for text in docs_texts]

# Print statistics to understand the document size distribution
print(f"Total documents: {len(docs_texts)}")
print(f"Total tokens in corpus: {np.sum(token_counts)}")
print(f"Average tokens per document: {np.mean(token_counts):.2f}")
print(f"Min tokens in a document: {np.min(token_counts)}")
print(f"Max tokens in a document: {np.max(token_counts)}")

# Output
# Total documents: 145
# Total tokens in corpus: 312566
# Average tokens per document: 2155.59
# Min tokens in a document: 312
# Max tokens in a document: 12453

from langchain.text_splitter import RecursiveCharacterTextSplitter

# We join all the documents into a single string for more efficient processing.
# The '---' separator helps maintain document boundaries if needed later.
concatenated_content = "\n\n --- \n\n".join(docs_texts)

# Create the text splitter using our LLM's tokenizer for accuracy.
text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
    tokenizer=tokenizer,
    chunk_size=1000, # The max number of tokens in a chunk
    chunk_overlap=100# The number of tokens to overlap between chunks
)

# Split the text into chunks, which will be our leaf nodes.
leaf_texts = text_splitter.split_text(concatenated_content)

print(f"Created {len(leaf_texts)} leaf nodes (chunks) for the RAPTOR tree.")
# Output
# Created 412 leaf nodes (chunks) for the RAPTOR tree.

from langchain_community.vectorstores import FAISS
from langchain_core.runnables import RunnablePassthrough

# In a simple RAG, the vector store is built only on the leaf-level chunks.
vectorstore_normal = FAISS.from_texts(
    texts=leaf_texts, 
    embedding=embeddings
)

# Create a retriever from this vector store that fetches the top 5 results.
retriever_normal = vectorstore_normal.as_retriever(
    search_kwargs={'k': 5}
)

print(f"Built Simple RAG vector store with {len(leaf_texts)} documents.")
# Output
# Built Simple RAG vector store with 412 documents.

# This prompt template instructs the LLM to answer based ONLY on the provided context.
final_prompt_text = """You are an expert assistant for the Hugging Face ecosystem. 
Answer the user's question based ONLY on the following context. If the context does not contain the answer, state that you don't know.
CONTEXT:
{context}
QUESTION:
{question}
ANSWER:"""
final_prompt = ChatPromptTemplate.from_template(final_prompt_text)

# A helper function to format the retrieved documents.
def format_docs(docs):
    return"\n\n".join(doc.page_content for doc in docs)

# Construct the RAG chain for the simple approach.
rag_chain_normal = (
    {"context": retriever_normal | format_docs, "question": RunnablePassthrough()}
    | final_prompt
    | llm
    | StrOutputParser()
)

# Let's ask a broad, conceptual question.
question = "What is the core philosophy of the Hugging Face ecosystem?"
answer = rag_chain_normal.invoke(question)

print(f"Question: {question}\n")
print(f"Answer: {answer}")

Question: What is the core philosophy of the Hugging Face ecosystem?

Answer: The Hugging Face ecosystem is built around the `transformers` 
library, which provides APIs to easily download and use pretrained models.
The core idea is to make these models accessible. For example, the `pipeline`
function is a key part of this, offering a simple way to use models for 
inference. It also includes libraries like `datasets` for data loading and
`accelerate` for training.

from typing import Dict, List, Optional, Tuple
import numpy as np
import pandas as pd
import umap
from sklearn.mixture import GaussianMixture

RANDOM_SEED = 42

def global_cluster_embeddings(embeddings: np.ndarray, dim: int, n_neighbors: Optional[int] = None, metric: str = "cosine") -> np.ndarray:
    """Perform global dimensionality reduction on the embeddings using UMAP."""
    # Heuristically set n_neighbors if not provided
    if n_neighbors isNone:
        n_neighbors = int((len(embeddings) - 1) ** 0.5)
    # Return the UMAP-transformed embeddings
    return umap.UMAP(
        n_neighbors=n_neighbors, 
        n_compnotallow=dim, 
        metric=metric, 
        random_state=RANDOM_SEED
    ).fit_transform(embeddings)

def get_optimal_clusters(embeddings: np.ndarray, max_clusters: int = 50) -> int:
    """Determine the optimal number of clusters using the Bayesian Information Criterion (BIC)."""
    # Limit the max number of clusters to be less than the number of data points
    max_clusters = min(max_clusters, len(embeddings))
    # If there's only one point, there can only be one cluster
    if max_clusters <= 1: 
        return1
    
    # Test different numbers of clusters from 1 to max_clusters
    n_clusters_range = np.arange(1, max_clusters)
    bics = []
    for n in n_clusters_range:
        # Initialize and fit the GMM for the current number of clusters
        gmm = GaussianMixture(n_compnotallow=n, random_state=RANDOM_SEED)
        gmm.fit(embeddings)
        # Calculate and store the BIC for the current model
        bics.append(gmm.bic(embeddings))
        
    # Return the number of clusters that resulted in the lowest BIC score
    return n_clusters_range[np.argmin(bics)]

def GMM_cluster(embeddings: np.ndarray, threshold: float) -> Tuple[List[np.ndarray], int]:
    """Cluster embeddings using a GMM and a probability threshold."""
    # Find the optimal number of clusters for this set of embeddings
    n_clusters = get_optimal_clusters(embeddings)
    
    # Fit the GMM with the optimal number of clusters
    gmm = GaussianMixture(n_compnotallow=n_clusters, random_state=RANDOM_SEED)
    gmm.fit(embeddings)
    
    # Get the probability of each point belonging to each cluster
    probs = gmm.predict_proba(embeddings)
    
    # Assign a point to a cluster if its probability is above the threshold
    # A single point can be assigned to multiple clusters.
    labels = [np.where(prob > threshold)[0] for prob in probs]
    
    return labels, n_clusters

def perform_clustering(embeddings: np.ndarray, dim: int = 10, threshold: float = 0.1) -> List[np.ndarray]:
    """Perform hierarchical clustering (global and local) on the embeddings."""
    # Handle cases with very few documents to avoid errors during dimensionality reduction.
    if len(embeddings) <= dim + 1:
        return [np.array([0]) for _ in range(len(embeddings))]

    # --- Global Clustering Stage ---
    # First, reduce the dimensionality of all embeddings globally.
    reduced_embeddings_global = global_cluster_embeddings(embeddings, dim)
    # Then, perform GMM clustering on the reduced-dimensional data.
    global_clusters, n_global_clusters = GMM_cluster(reduced_embeddings_global, threshold)

    # --- Local Clustering Stage ---
    # Initialize a list to hold all final local cluster assignments for each document.
    all_local_clusters = [np.array([]) for _ in range(len(embeddings))]
    # Keep track of the total number of clusters found so far to ensure unique IDs.
    total_clusters = 0

    # Iterate through each global cluster to find sub-clusters.
    for i in range(n_global_clusters):
        # Get all original indices for embeddings that are part of the current global cluster.
        global_cluster_indices = [idx for idx, gc in enumerate(global_clusters) if i in gc]
        ifnot global_cluster_indices:
            continue
        
        # Get the actual embeddings for this global cluster.
        global_cluster_embeddings_ = embeddings[global_cluster_indices]

        # Perform local clustering on this subset of embeddings.
        if len(global_cluster_embeddings_) <= dim + 1:
            local_clusters, n_local_clusters = ([np.array([0])] * len(global_cluster_embeddings_)), 1
        else:
            # We don't need a separate 'local_cluster_embeddings' function.
            # The global one works, as it adapts n_neighbors to the input
            reduced_embeddings_local = global_cluster_embeddings(global_cluster_embeddings_, dim)
            local_clusters, n_local_clusters = GMM_cluster(reduced_embeddings_local, threshold)

        # Map the local cluster IDs back to the original document indices.
        for original_idx, local_cluster_ids in zip(global_cluster_indices, local_clusters):
            # We add 'total_clusters' to ensure each cluster ID is globally unique.
            all_local_clusters[original_idx] = local_cluster_ids + total_clusters
            
        total_clusters += n_local_clusters
    
    # Return the final, globally unique cluster assignments for each document.
    return all_local_clusters