Summarizer in Machine Learning

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
import networkx as nx
import re

def textrank_summarize(text: str, num_sentences: int = 3) -> str:
    """Extractive summarization using TextRank with sentence embeddings."""
    # Split into sentences
    sentences = re.split(r'(?<=[.!?])\s+', text.strip())
    if len(sentences) <= num_sentences:
        return text

    # Encode sentences using a lightweight model
    model = SentenceTransformer('all-MiniLM-L6-v2')
    embeddings = model.encode(sentences)

    # Build similarity matrix
    sim_matrix = cosine_similarity(embeddings)
    np.fill_diagonal(sim_matrix, 0)  # No self-loops

    # Apply PageRank
    graph = nx.from_numpy_array(sim_matrix)
    scores = nx.pagerank(graph, alpha=0.85, max_iter=100)

    # Select top sentences (preserve original order)
    ranked_indices = sorted(scores, key=scores.get, reverse=True)
    selected = sorted(ranked_indices[:num_sentences])

    return ' '.join(sentences[i] for i in selected)

# Usage
text = """India's digital payments ecosystem has seen remarkable growth.
UPI processed 12.02 billion transactions in October 2024 alone.
PhonePe and Google Pay dominate with a combined 85% market share.
The Reserve Bank of India continues to expand the infrastructure.
Rural adoption remains a challenge despite government incentives.
New features like UPI Lite and credit-on-UPI are driving further adoption."""

print(textrank_summarize(text, num_sentences=2))

from transformers import pipeline, BartForConditionalGeneration, BartTokenizer
import torch

# Option 1: Quick setup with pipeline API
summarizer = pipeline(
    "summarization",
    model="facebook/bart-large-cnn",
    device=0 if torch.cuda.is_available() else -1
)

text = """The Indian Space Research Organisation (ISRO) successfully launched 
the Chandrayaan-3 mission, which achieved a historic soft landing on the 
lunar south pole on August 23, 2023. India became the fourth country to 
land on the Moon and the first to land near the south pole. The Pragyan 
rover deployed from the Vikram lander conducted multiple experiments 
including thermal measurements and seismic activity detection. The mission 
cost approximately Rs 615 crore (about $75 million), making it one of the 
most cost-effective lunar missions in history. The success was celebrated 
across India and positioned ISRO as a leader in cost-effective space 
exploration. The data collected continues to provide insights about the 
lunar surface composition and potential water ice deposits."""

result = summarizer(
    text,
    max_length=80,
    min_length=30,
    do_sample=False,
    num_beams=4,
    length_penalty=2.0,
    no_repeat_ngram_size=3
)
print(result[0]['summary_text'])

# Option 2: Fine-grained control with model + tokenizer
model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn")
tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn")

inputs = tokenizer(
    text,
    return_tensors="pt",
    max_length=1024,
    truncation=True
)

summary_ids = model.generate(
    inputs["input_ids"],
    max_length=80,
    min_length=30,
    num_beams=4,
    length_penalty=2.0,
    no_repeat_ngram_size=3,
    early_stopping=True
)

print(tokenizer.decode(summary_ids[0], skip_special_tokens=True))

from openai import OpenAI
import json

client = OpenAI()  # Uses OPENAI_API_KEY env variable

def chain_of_density_summarize(
    article: str,
    iterations: int = 5,
    target_words: int = 80,
    model: str = "gpt-4o-mini"
) -> dict:
    """Chain of Density summarization: iteratively increases entity density."""

    prompt = f"""Article: {article}

You will generate {iterations} increasingly concise, entity-dense summaries
of the above article. Each summary should be approximately {target_words} words.

Guidelines:
- Summary 1: Write a sparse summary covering only 1-2 main entities.
- Each subsequent summary: Identify 1-3 informative entities from the article
  that are MISSING from the previous summary. Add them by:
  (a) Replacing less specific phrases with more specific ones
  (b) Fusing multiple sentences to make room
  (c) Compressing existing information
- NEVER increase the summary length. Every word must earn its place.
- NEVER drop entities mentioned in earlier summaries.
- An entity is a real-world object: named events, people, organizations,
  specific numbers, dates, technical terms.

Return a JSON array of {iterations} objects, each with keys:
  "summary": the summary text,
  "missing_entities": list of entities added in this iteration,
  "density_score": number of entities per 100 words
"""

    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.3,
        response_format={"type": "json_object"}
    )

    result = json.loads(response.choices[0].message.content)
    return result

# Usage
article = """India's Unified Payments Interface (UPI) has revolutionized
digital payments in the country. In 2024, UPI processed over 130 billion
transactions worth Rs 200 lakh crore ($2.4 trillion). PhonePe leads with
48% market share, followed by Google Pay at 37%. The National Payments
Corporation of India (NPCI) has expanded UPI internationally, with
singapore, UAE, France, and Sri Lanka now accepting UPI payments.
The RBI introduced UPI Lite for small transactions under Rs 500 without
PIN verification, processing 300 million transactions monthly.
Credit-on-UPI, launched in partnership with RuPay, has seen 15 million
users adopt the feature within six months."""

result = chain_of_density_summarize(article)
for i, step in enumerate(result.get("summaries", [result])):
    print(f"\n--- Iteration {i+1} ---")
    print(step.get("summary", step))

from transformers import pipeline
import torch
from typing import List
import re

class MapReduceSummarizer:
    """Summarize documents longer than the model's context window."""

    def __init__(
        self,
        model_name: str = "facebook/bart-large-cnn",
        chunk_size: int = 900,
        chunk_overlap: int = 100,
        map_max_length: int = 150,
        reduce_max_length: int = 200
    ):
        self.summarizer = pipeline(
            "summarization",
            model=model_name,
            device=0 if torch.cuda.is_available() else -1
        )
        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap
        self.map_max_length = map_max_length
        self.reduce_max_length = reduce_max_length

    def _split_into_chunks(self, text: str) -> List[str]:
        """Split text into overlapping chunks at sentence boundaries."""
        sentences = re.split(r'(?<=[.!?])\s+', text.strip())
        chunks = []
        current_chunk = []
        current_length = 0

        for sentence in sentences:
            words = len(sentence.split())
            if current_length + words > self.chunk_size and current_chunk:
                chunks.append(' '.join(current_chunk))
                # Keep last few sentences for overlap
                overlap_sentences = []
                overlap_len = 0
                for s in reversed(current_chunk):
                    if overlap_len + len(s.split()) > self.chunk_overlap:
                        break
                    overlap_sentences.insert(0, s)
                    overlap_len += len(s.split())
                current_chunk = overlap_sentences
                current_length = overlap_len
            current_chunk.append(sentence)
            current_length += words

        if current_chunk:
            chunks.append(' '.join(current_chunk))

        return chunks

    def summarize(self, text: str) -> str:
        """Map-reduce summarization pipeline."""
        chunks = self._split_into_chunks(text)

        if len(chunks) == 1:
            result = self.summarizer(
                chunks[0],
                max_length=self.reduce_max_length,
                min_length=50,
                do_sample=False
            )
            return result[0]['summary_text']

        # Map phase: summarize each chunk
        print(f"Map phase: summarizing {len(chunks)} chunks...")
        chunk_summaries = []
        for i, chunk in enumerate(chunks):
            result = self.summarizer(
                chunk,
                max_length=self.map_max_length,
                min_length=30,
                do_sample=False
            )
            chunk_summaries.append(result[0]['summary_text'])

        # Reduce phase: combine chunk summaries
        combined = ' '.join(chunk_summaries)
        print(f"Reduce phase: combining {len(chunk_summaries)} summaries...")

        # If combined summaries are still too long, recurse
        if len(combined.split()) > self.chunk_size:
            return self.summarize(combined)

        result = self.summarizer(
            combined,
            max_length=self.reduce_max_length,
            min_length=50,
            do_sample=False
        )
        return result[0]['summary_text']

# Usage
summarizer = MapReduceSummarizer()
long_document = "..."  # Your long document here
print(summarizer.summarize(long_document))

from rouge_score import rouge_scorer
from bert_score import score as bert_score
import numpy as np

def evaluate_summary(
    candidate: str,
    reference: str,
    verbose: bool = True
) -> dict:
    """Evaluate a summary using ROUGE and BERTScore metrics."""

    # ROUGE scores
    scorer = rouge_scorer.RougeScorer(
        ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'],
        use_stemmer=True
    )
    rouge_results = scorer.score(reference, candidate)

    # BERTScore
    P, R, F1 = bert_score(
        [candidate],
        [reference],
        model_type="microsoft/deberta-xlarge-mnli",
        lang="en",
        verbose=False
    )

    results = {
        "rouge1_f1": rouge_results['rouge1'].fmeasure,
        "rouge2_f1": rouge_results['rouge2'].fmeasure,
        "rougeL_f1": rouge_results['rougeL'].fmeasure,
        "bertscore_precision": P.item(),
        "bertscore_recall": R.item(),
        "bertscore_f1": F1.item()
    }

    if verbose:
        print("=== ROUGE Scores ===")
        for key in ['rouge1', 'rouge2', 'rougeL']:
            r = rouge_results[key]
            print(f"  {key}: P={r.precision:.4f} R={r.recall:.4f} F1={r.fmeasure:.4f}")
        print(f"\n=== BERTScore ===")
        print(f"  P={P.item():.4f} R={R.item():.4f} F1={F1.item():.4f}")

    return results

# Example
reference = "India's ISRO achieved a historic lunar south pole landing with Chandrayaan-3 at a cost of $75 million."
candidate = "ISRO's Chandrayaan-3 mission successfully landed near the Moon's south pole, making India the first country to achieve this feat, at a cost of approximately Rs 615 crore."

results = evaluate_summary(candidate, reference)

# Summarization pipeline config (YAML)
pipeline:
  name: document-summarizer
  strategy: map-reduce  # options: single-pass, map-reduce, refine

preprocessor:
  max_input_tokens: 100000
  sentence_splitter: spacy  # options: spacy, nltk, regex
  clean_html: true
  remove_boilerplate: true

chunker:
  chunk_size_tokens: 900
  overlap_tokens: 100
  respect_sentence_boundaries: true

model:
  name: facebook/bart-large-cnn
  device: cuda:0
  quantization: int8  # options: none, int8, int4
  generation:
    max_length: 150
    min_length: 40
    num_beams: 4
    length_penalty: 2.0
    no_repeat_ngram_size: 3
    early_stopping: true

quality:
  faithfulness_checker: summac  # options: summac, alignscore, llm-judge
  min_rouge_l: 0.30
  max_hallucination_score: 0.15
  reject_on_failure: false  # log warning instead of rejecting

monitoring:
  log_rouge_scores: true
  log_latency_p99: true
  alert_on_quality_drop: true
  evaluation_sample_rate: 0.05  # evaluate 5% of outputs