Step-by-Step Guide to Creating Custom LangChain Agents in 2025

# .env file
OPENAI_API_KEY=your_openai_api_key_here
ANTHROPIC_API_KEY=your_anthropic_api_key_here
LANGCHAIN_TRACING_V2=true
LANGCHAIN_API_KEY=your_langchain_api_key_here
LANGCHAIN_PROJECT=my_custom_agent_project

# Load environment variables
from dotenv import load_dotenv
import os

load_dotenv()

# Verify keys are loaded
openai_key = os.getenv("OPENAI_API_KEY")
anthropic_key = os.getenv("ANTHROPIC_API_KEY")

if not openai_key or not anthropic_key:
    raise ValueError("API keys not found in environment variables")

research-assistant/
├── .env                  # Environment variables
├── main.py               # Main application
├── agent/
│   ├── __init__.py
│   ├── config.py         # Agent configuration
│   ├── prompts.py        # System prompts
│   ├── memory.py         # Memory implementation
│   └── tools/
│       ├── __init__.py
│       ├── research.py   # Research tools
│       └── analysis.py   # Analysis tools
├── data/
│   └── cache/            # Cache for research results
└── deployment/
    ├── Dockerfile
    └── requirements.txt

mkdir -p research-assistant/agent/tools research-assistant/data/cache research-assistant/deployment
touch research-assistant/.env research-assistant/main.py
touch research-assistant/agent/__init__.py research-assistant/agent/config.py research-assistant/agent/prompts.py research-assistant/agent/memory.py
touch research-assistant/agent/tools/__init__.py research-assistant/agent/tools/research.py research-assistant/agent/tools/analysis.py
touch research-assistant/deployment/Dockerfile research-assistant/deployment/requirements.txt

# agent/config.py
from langchain_openai import ChatOpenAI
from langchain_anthropic import ChatAnthropic
import os

def get_openai_llm(model_name="gpt-5-0314", temperature=0.2):
    """Configure and return an OpenAI LLM instance."""
    return ChatOpenAI(
        model=model_name,
        temperature=temperature,
        max_tokens=4000,
        top_p=0.95,
        frequency_penalty=0,
        presence_penalty=0,
        openai_api_key=os.getenv("OPENAI_API_KEY")
    )

def get_anthropic_llm(model_name="claude-3-7-sonnet-20250219", temperature=0.3):
    """Configure and return an Anthropic LLM instance."""
    return ChatAnthropic(
        model=model_name,
        temperature=temperature,
        max_tokens=4000,
        top_k=12,
        top_p=0.9,
        anthropic_api_key=os.getenv("ANTHROPIC_API_KEY")
    )

# Configuration options for different agent types
AGENT_CONFIGS = {
    "research": {
        "llm": get_anthropic_llm,
        "description": "Research assistant specialized in finding and summarizing academic papers",
        "default_tools": ["research_paper_search", "paper_summarizer", "citation_formatter"]
    },
    "analysis": {
        "llm": get_openai_llm,
        "description": "Data analysis assistant specialized in extracting insights from research papers",
        "default_tools": ["data_extractor", "statistical_analyzer", "trend_identifier"]
    }
}

# agent/tools/research.py
from langchain.tools.base import BaseTool
from pydantic import BaseModel, Field
import requests
import json
import os
from typing import Optional, Type, List, Dict, Any
import time

class ResearchPaperSearchInput(BaseModel):
    """Input schema for the research paper search tool."""
    query: str = Field(..., description="The search query for finding research papers")
    max_results: int = Field(5, description="Maximum number of papers to return")
    sort_by: str = Field("relevance", description="Sort results by: relevance, date, citation_count")
    
class ResearchPaperSearch(BaseTool):
    """Tool for searching academic papers using the ArXiv API."""
    name = "research_paper_search"
    description = "Searches for academic papers on a given topic using the ArXiv API."
    args_schema: Type[BaseModel] = ResearchPaperSearchInput
    
    def _run(self, query: str, max_results: int = 5, sort_by: str = "relevance") -> str:
        """Execute the tool functionality."""
        # Cache directory for search results
        cache_dir = os.path.join(os.getcwd(), "data", "cache")
        os.makedirs(cache_dir, exist_ok=True)
        
        # Generate cache key based on query parameters
        cache_key = f"{query}_{max_results}_{sort_by}".replace(" ", "_").lower()
        cache_file = os.path.join(cache_dir, f"{cache_key}.json")
        
        # Check for cached results
        if os.path.exists(cache_file):
            with open(cache_file, 'r') as f:
                return json.load(f)['result']
        
        # If not cached, make API call
        base_url = "http://export.arxiv.org/api/query"
        params = {
            "search_query": f"all:{query}",
            "start": 0,
            "max_results": max_results,
            "sortBy": "submittedDate" if sort_by == "date" else "relevance",
            "sortOrder": "descending"
        }
        
        try:
            response = requests.get(base_url, params=params)
            response.raise_for_status()
            
            # Parse XML response (simplified implementation)
            # In production, use a proper XML parser like ElementTree
            import xml.etree.ElementTree as ET
            
            root = ET.fromstring(response.text)
            
            # Define namespace
            namespace = {'atom': 'http://www.w3.org/2005/Atom'}
            
            papers = []
            for entry in root.findall('.//atom:entry', namespace):
                paper = {}
                paper['title'] = entry.find('./atom:title', namespace).text.strip()
                paper['authors'] = [author.find('./atom:name', namespace).text for author 
                                   in entry.findall('./atom:author', namespace)]
                paper['summary'] = entry.find('./atom:summary', namespace).text.strip()
                paper['published'] = entry.find('./atom:published', namespace).text
                paper['id'] = entry.find('./atom:id', namespace).text
                paper['pdf_url'] = next((link.get('href') for link in entry.findall('./atom:link', namespace) 
                                        if link.get('title') == 'pdf'), None)
                papers.append(paper)
            
            # Format results as a readable string
            if papers:
                result = f"Found {len(papers)} papers about '{query}':\n\n"
                for i, paper in enumerate(papers, 1):
                    result += f"{i}. Title: {paper['title']}\n"
                    result += f"   Authors: {', '.join(paper['authors'])}\n"
                    result += f"   Published: {paper['published'][:10]}\n"
                    result += f"   ID: {paper['id'].split('/')[-1]}\n"
                    result += f"   PDF: {paper['pdf_url']}\n"
                    result += f"   Summary: {paper['summary'][:200]}...\n\n"
            else:
                result = f"No papers found for query: {query}"
            
            # Cache results
            with open(cache_file, 'w') as f:
                json.dump({
                    'timestamp': time.time(),
                    'parameters': {'query': query, 'max_results': max_results, 'sort_by': sort_by},
                    'result': result,
                    'raw_data': papers
                }, f)
            
            return result
            
        except Exception as e:
            return f"Error searching for papers: {str(e)}"

class PaperSummarizerInput(BaseModel):
    """Input schema for the paper summarizer tool."""
    paper_id: str = Field(..., description="The ID of the paper to summarize (arxiv ID)")
    summary_length: int = Field(500, description="Target length of the summary in words")
    focus_areas: List[str] = Field(["methodology", "results", "implications"], 
                                 description="Areas to focus on in the summary")

class PaperSummarizer(BaseTool):
    """Tool for summarizing academic papers."""
    name = "paper_summarizer"
    description = "Retrieves and summarizes a scientific paper given its ID."
    args_schema: Type[BaseModel] = PaperSummarizerInput
    
    def _run(self, paper_id: str, summary_length: int = 500, 
            focus_areas: List[str] = ["methodology", "results", "implications"]) -> str:
        """Execute the tool functionality."""
        # Implementation would download and summarize the paper
        # For this tutorial, we'll simulate the process
        
        # Simplified implementation - in production, you would:
        # 1. Download the PDF using a library like PyPDF2
        # 2. Extract text from the PDF
        # 3. Use the foundation model to generate a summary
        
        # Simulated implementation
        paper_details = {
            "title": f"Example Paper about Advanced AI Models (ID: {paper_id})",
            "authors": ["Jane Smith", "John Doe"],
            "published": "2025-01-15",
            "abstract": "This paper introduces a novel approach to neural architectures that improves performance on benchmark NLP tasks by 15%."
        }
        
        # Simulate a focused summary based on request parameters
        summary_sections = {
            "methodology": "The paper presents a multi-stage training process that combines supervised learning with reinforcement learning from human feedback. The architecture incorporates a mixture-of-experts layer with 128 specialists that can be dynamically activated based on input characteristics.",
            
            "results": "Experiments on the SuperGLUE benchmark show a 15% improvement over previous state-of-the-art models. The approach is particularly effective on reasoning tasks, with a 23% improvement on the ReasoningBench-2024 dataset.",
            
            "implications": "The findings suggest that specialized neural pathways activated selectively based on task requirements can significantly improve efficiency and performance. This approach may enable more powerful models with fewer parameters.",
            
            "limitations": "The authors note that their approach requires more computational resources during training, though inference is more efficient than comparable models. Additionally, the specialization mechanism sometimes leads to over-indexing on specific features."
        }
        
        # Build summary based on requested focus areas
        summary = f"# Summary of {paper_details['title']}\n"
        summary += f"Authors: {', '.join(paper_details['authors'])}\n"
        summary += f"Published: {paper_details['published']}\n\n"
        summary += f"## Abstract\n{paper_details['abstract']}\n\n"
        
        for area in focus_areas:
            if area in summary_sections:
                summary += f"## {area.title()}\n{summary_sections[area]}\n\n"
        
        # Adjust length (simplified)
        current_word_count = len(summary.split())
        if current_word_count > summary_length:
            summary += f"\nNote: Summary truncated to approximately {summary_length} words from original {current_word_count}."
        
        return summary

# Register tools for easy access
RESEARCH_TOOLS = {
    "research_paper_search": ResearchPaperSearch,
    "paper_summarizer": PaperSummarizer
}

# agent/memory.py
from langchain.memory import HierarchicalMemory, ConversationBufferMemory
from langchain.memory.models import BaseMemoryStore
from langchain_community.memory.stores import RedisMemoryStore
from langchain_openai import OpenAIEmbeddings
import redis
import os
import json
import time
from typing import Dict, List, Any, Optional

class CustomHierarchicalMemory:
    """Enhanced hierarchical memory implementation for research assistant."""
    
    def __init__(self, llm, use_redis=True, redis_url=None, namespace="research_agent"):
        """Initialize the memory system."""
        self.llm = llm
        self.namespace = namespace
        self.embeddings = OpenAIEmbeddings()
        
        # Set up memory stores
        if use_redis and (redis_url or os.getenv("REDIS_URL")):
            redis_url = redis_url or os.getenv("REDIS_URL")
            try:
                self.persistent_store = RedisMemoryStore(
                    redis_url=redis_url,
                    namespace=namespace,
                    ttl=60*60*24*30,  # 30-day retention
                    embeddings=self.embeddings
                )
                self.redis_available = True
            except redis.ConnectionError:
                print("Warning: Redis connection failed. Falling back to in-memory storage.")
                self.redis_available = False
                self.persistent_store = {}
        else:
            self.redis_available = False
            self.persistent_store = {}
        
        # Initialize different memory types
        self.working_memory = ConversationBufferMemory(
            llm=self.llm,
            memory_key="chat_history",
            return_messages=True,
            input_key="input",
            output_key="output"
        )
        
        # Initialize concept store for semantic memory
        self.semantic_memory = {}
        
        # Initialize episodic memory
        self.episodic_memory = {}
    
    def add_to_working_memory(self, input_text: str, output_text: str) -> None:
        """Add an interaction to working memory."""
        self.working_memory.save_context(
            {"input": input_text},
            {"output": output_text}
        )
    
    def add_to_semantic_memory(self, concept: str, information: str) -> None:
        """Add or update a concept in semantic memory."""
        timestamp = time.time()
        
        if self.redis_available:
            key = f"semantic:{concept}"
            self.persistent_store.redis_client.hset(
                key,
                mapping={
                    "information": information,
                    "last_updated": timestamp,
                    "access_count": self.persistent_store.redis_client.hget(key, "access_count") or 0
                }
            )
        else:
            # In-memory fallback
            self.semantic_memory[concept] = {
                "information": information,
                "last_updated": timestamp,
                "access_count": self.semantic_memory.get(concept, {}).get("access_count", 0)
            }
    
    def add_to_episodic_memory(self, episode_name: str, events: List[Dict[str, str]]) -> None:
        """Add an episode to episodic memory."""
        timestamp = time.time()
        
        if self.redis_available:
            key = f"episodic:{episode_name}"
            self.persistent_store.redis_client.hset(
                key,
                mapping={
                    "events": json.dumps(events),
                    "created": timestamp,
                    "access_count": self.persistent_store.redis_client.hget(key, "access_count") or 0
                }
            )
        else:
            # In-memory fallback
            self.episodic_memory[episode_name] = {
                "events": events,
                "created": timestamp,
                "access_count": self.episodic_memory.get(episode_name, {}).get("access_count", 0)
            }
    
    def retrieve_from_semantic_memory(self, concept: str) -> Optional[str]:
        """Retrieve information about a concept from semantic memory."""
        if self.redis_available:
            key = f"semantic:{concept}"
            if self.persistent_store.redis_client.exists(key):
                # Update access count
                self.persistent_store.redis_client.hincrby(key, "access_count", 1)
                return self.persistent_store.redis_client.hget(key, "information")
        else:
            if concept in self.semantic_memory:
                self.semantic_memory[concept]["access_count"] += 1
                return self.semantic_memory[concept]["information"]
        
        return None
    
    def retrieve_relevant_context(self, query: str, max_results: int = 3) -> str:
        """Retrieve relevant information based on the query."""
        # This would use vector search with embeddings in a real implementation
        # For simplicity, we'll simulate relevant retrievals
        
        # In production:
        # 1. Convert query to embedding
        # 2. Search for similar concepts in semantic memory
        # 3. Search for similar episodes in episodic memory
        # 4. Combine and return the most relevant information
        
        # Simplified implementation
        context = [
            "The user previously asked about searching for papers on neural networks.",
            "You provided summaries of 3 papers related to transformers architecture.",
            "The user mentioned they're working on a research project about self-attention mechanisms."
        ]
        
        return "\n".join(context)
    
    def get_conversation_history(self, k: int = 10) -> str:
        """Get the last k conversation turns."""
        memory_variables = self.working_memory.load_memory_variables({})
        
        if "chat_history" in memory_variables and memory_variables["chat_history"]:
            chat_history = memory_variables["chat_history"]
            # Return last k turns
            limited_history = chat_history[-2*k:] if len(chat_history) > 2*k else chat_history
            
            # Format as string
            history_str = ""
            for i in range(0, len(limited_history), 2):
                if i+1 < len(limited_history):
                    history_str += f"User: {limited_history[i].content}\n"
                    history_str += f"Assistant: {limited_history[i+1].content}\n\n"
            
            return history_str
        
        return "No conversation history available."

# agent/prompts.py
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate
from langchain.schema.messages import SystemMessage

def get_research_agent_prompt():
    """Return the prompt template for the research agent."""
    return ChatPromptTemplate.from_messages([
        SystemMessage(content="""
        You are an advanced Research Assistant agent specialized in finding, analyzing, and summarizing scientific papers.
        
        ## Your Capabilities
        - Searching for academic papers using keywords and phrases
        - Summarizing papers with focus on specific aspects (methodology, results, implications)
        - Extracting key insights and contributions from papers
        - Comparing multiple papers to identify trends and contradictions
        - Formulating research questions based on literature gaps
        
        ## Guidelines for Interaction
        1. When a user requests information about a topic, first clarify the specific research question
        2. Use the research_paper_search tool to find relevant papers
        3. When presenting papers, prioritize:
           - Recency (papers from 2023-2025 unless specifically asked for historical context)
           - Relevance to the specific question
           - Citation count when available
        4. Provide focused summaries that highlight the most important aspects for the user's needs
        5. Always cite papers properly with authors, year, and title
        6. When uncertain about details, acknowledge limitations rather than speculating
        
        ## Output Format
        Present information in a structured, scannable format:
        - Use headers to organize information
        - Use bullet points for key findings
        - Bold important conclusions or methodological innovations
        - Include direct quotes sparingly and only when highly relevant
        
        Remember to maintain scientific accuracy and rigor in all your responses.
        """),
        MessagesPlaceholder(variable_name="chat_history"),
        HumanMessagePromptTemplate.from_template("{input}"),
        MessagesPlaceholder(variable_name="agent_scratchpad"),
    ])

def get_analysis_agent_prompt():
    """Return the prompt template for the analysis agent."""
    return ChatPromptTemplate.from_messages([
        SystemMessage(content="""
        You are an advanced Research Analysis agent specialized in extracting insights and patterns from scientific papers.
        
        ## Your Capabilities
        - Analyzing methodologies used in research papers
        - Identifying statistical techniques and experimental designs
        - Extracting quantitative results and performance metrics
        - Comparing results across multiple studies
        - Identifying methodological limitations and potential biases
        
        ## Guidelines for Interaction
        1. When analyzing papers, focus on:
           - Methodological rigor and appropriateness
           - Statistical validity of conclusions
           - Comparative performance against baselines and state-of-the-art
           - Limitations acknowledged by authors and those you identify
        2. Provide balanced assessment of strengths and weaknesses
        3. Contextualize findings within the broader field
        4. Highlight methodological innovations
        
        ## Output Format
        Present analysis in a structured format:
        - Methodology assessment (strengths, weaknesses)
        - Results validity and significance
        - Comparative analysis (if multiple papers)
        - Identified limitations and biases
        - Suggestions for improvement or future work
        
        Remember that critical analysis should be constructive and evidence-based.
        """),
        MessagesPlaceholder(variable_name="chat_history"),
        HumanMessagePromptTemplate.from_template("{input}"),
        MessagesPlaceholder(variable_name="agent_scratchpad"),
    ])

# main.py
import os
from dotenv import load_dotenv
from typing import Dict, List, Optional

# Load environment variables
load_dotenv()

# Import langchain components
from langchain.agents import AgentExecutor, create_react_agent
from langchain.tools import Tool
from langchain.chains import LLMChain

# Import our custom components
from agent.config import get_anthropic_llm, get_openai_llm
from agent.memory import CustomHierarchicalMemory
from agent.prompts import get_research_agent_prompt
from agent.tools.research import ResearchPaperSearch, PaperSummarizer

class ResearchAssistantAgent:
    """Complete implementation of a research assistant agent."""
    
    def __init__(
        self, 
        model_provider: str = "anthropic",
        use_redis: bool = False,
        verbose: bool = True
    ):
        """Initialize the research assistant agent."""
        # Set up foundation model
        if model_provider.lower() == "anthropic":
            self.llm = get_anthropic_llm()
        else:
            self.llm = get_openai_llm()
        
        # Set up memory system
        self.memory = CustomHierarchicalMemory(
            llm=self.llm,
            use_redis=use_redis,
            namespace="research_assistant"
        )
        
        # Set up tools
        self.tools = [
            ResearchPaperSearch(),
            PaperSummarizer()
        ]
        
        # Set up agent prompt
        self.prompt = get_research_agent_prompt()
        
        # Create the agent
        self.agent = create_react_agent(
            llm=self.llm,
            tools=self.tools,
            prompt=self.prompt
        )
        
        # Create the agent executor
        self.agent_executor = AgentExecutor(
            agent=self.agent,
            tools=self.tools,
            verbose=verbose,
            max_iterations=10,
            handle_parsing_errors=True,
            early_stopping_method="force",
            memory=self.memory.working_memory
        )
    
    def add_tool(self, tool):
        """Add a new tool to the agent."""
        self.tools.append(tool)
        
        # Recreate agent with updated tools
        self.agent = create_react_agent(
            llm=self.llm,
            tools=self.tools,
            prompt=self.prompt
        )
        
        # Update agent executor
        self.agent_executor = AgentExecutor(
            agent=self.agent,
            tools=self.tools,
            verbose=self.agent_executor.verbose,
            max_iterations=self.agent_executor.max_iterations,
            handle_parsing_errors=True,
            early_stopping_method="force",
            memory=self.memory.working_memory
        )
    
    def process_query(self, query: str) -> Dict:
        """Process a user query and return the response."""
        # Add relevant context from long-term memory
        context = self.memory.retrieve_relevant_context(query)
        
        # Execute agent
        response = self.agent_executor.invoke({
            "input": query,
            "context": context
        })
        
        # Update memory with this interaction
        self.memory.add_to_working_memory(query, response["output"])
        
        # Extract and store any concepts mentioned
        # This would typically use a Named Entity Recognition system
        # Simplified implementation for tutorial
        potential_concepts = ["neural networks", "transformers", "attention mechanism"]
        for concept in potential_concepts:
            if concept in query.lower() or concept in response["output"].lower():
                self.memory.add_to_semantic_memory(
                    concept=concept,
                    information=f"The user asked about {concept} on {time.strftime('%Y-%m-%d')}. " +
                               f"You provided information about its relevance in recent research."
                )
        
        return response

# Example usage of the agent
if __name__ == "__main__":
    # Create the agent
    agent = ResearchAssistantAgent(verbose=True)
    
    # Example queries
    queries = [
        "What are the latest developments in transformer architectures?",
        "Find papers about self-attention mechanisms in language models from 2024.",
        "Summarize the methodologies used in the latest research on multimodal LLMs."
    ]
    
    # Process each query
    for query in queries:
        print(f"\n\n==== QUERY: {query} ====\n")
        response = agent.process_query(query)
        print(f"\n==== RESPONSE ====\n{response['output']}")

# deployment/app.py
from fastapi import FastAPI, HTTPException, Depends, Request, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import List, Dict, Any, Optional
import os
import sys
import time
import json
import asyncio

# Add parent directory to path to import agent module
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Import our agent
from main import ResearchAssistantAgent

# Create FastAPI app
app = FastAPI(
    title="Research Assistant API",
    description="API for a custom LangChain Research Assistant Agent",
    version="1.0.0"
)

# Add CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Initialize agent
research_agent = ResearchAssistantAgent(verbose=False)

# Define request/response models
class QueryRequest(BaseModel):
    query: str
    conversation_id: Optional[str] = None
    stream: bool = False

class QueryResponse(BaseModel):
    response: str
    conversation_id: str
    thinking_process: Optional[List[Dict[str, Any]]] = None
    execution_time: float

# Define API endpoints
@app.post("/query", response_model=QueryResponse)
async def process_query(request: QueryRequest):
    """Process a research query."""
    start_time = time.time()
    
    try:
        # Process query
        result = research_agent.process_query(request.query)
        
        # Prepare response
        response = QueryResponse(
            response=result["output"],
            conversation_id=request.conversation_id or f"conv_{int(time.time())}",
            thinking_process=result.get("intermediate_steps"),
            execution_time=time.time() - start_time
        )
        
        return response
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error processing query: {str(e)}")

@app.get("/healthcheck")
async def healthcheck():
    """Check if the API is running."""
    return {"status": "healthy", "timestamp": time.time()}

# Run the app
if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

# deployment/Dockerfile
FROM python:3.11-slim

WORKDIR /app

# Install system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
    gcc \
    python3-dev \
    && rm -rf /var/lib/apt/lists/*

# Copy requirements first to leverage Docker cache
COPY deployment/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy application code
COPY . .

# Set environment variables
ENV PYTHONUNBUFFERED=1
ENV PORT=8000

# Expose port
EXPOSE ${PORT}

# Run the application
CMD uvicorn deployment.app:app --host 0.0.0.0 --port ${PORT}

# test_agent.py
from main import ResearchAssistantAgent
import time
import json
import os

class AgentTester:
    """Test suite for the research assistant agent."""
    
    def __init__(self, agent):
        self.agent = agent
        
        # Create results directory
        self.results_dir = os.path.join(os.getcwd(), "test_results")
        os.makedirs(self.results_dir, exist_ok=True)
        
        # Define test cases
        self.test_cases = [
            {
                "id": "basic_search",
                "query": "Find recent papers about transformer architectures",
                "criteria": {
                    "contains_paper_mentions": True,
                    "contains_recent_years": True,
                    "response_length_min": 200
                }
            },
            {
                "id": "specific_methodology",
                "query": "What methodologies are used for evaluating transformer models in 2024?",
                "criteria": {
                    "mentions_evaluation_metrics": True,
                    "mentions_methodologies": True,
                    "contains_recent_years": True
                }
            },
            {
                "id": "compare_papers",
                "query": "Compare the approaches in the most recent papers about self-attention mechanisms",
                "criteria": {
                    "contains_comparison": True,
                    "mentions_multiple_papers": True,
                    "mentions_self_attention": True
                }
            }
        ]
    
    def evaluate_response(self, response, criteria):
        """Evaluate response against test criteria."""
        results = {}
        
        # Check for paper mentions
        if "contains_paper_mentions" in criteria:
            paper_indicators = ["et al.", "published", "paper", "research", "study"]
            results["contains_paper_mentions"] = any(indicator in response.lower() for indicator in paper_indicators)
        
        # Check for recent years
        if "contains_recent_years" in criteria:
            years = ["2023", "2024", "2025"]
            results["contains_recent_years"] = any(year in response for year in years)
        
        # Check response length
        if "response_length_min" in criteria:
            results["response_length_min"] = len(response) >= criteria["response_length_min"]
        
        # Check for evaluation metrics
        if "mentions_evaluation_metrics" in criteria:
            metrics = ["accuracy", "precision", "recall", "f1", "perplexity", "BLEU", "ROUGE"]
            results["mentions_evaluation_metrics"] = any(metric.lower() in response.lower() for metric in metrics)
        
        # Check for methodologies
        if "mentions_methodologies" in criteria:
            methods = ["benchmark", "evaluation", "testing", "validation", "comparison"]
            results["mentions_methodologies"] = any(method.lower() in response.lower() for method in methods)
        
        # Check for comparisons
        if "contains_comparison" in criteria:
            comparison_terms = ["compared to", "versus", "unlike", "similar to", "difference", "contrast"]
            results["contains_comparison"] = any(term in response.lower() for term in comparison_terms)
        
        # Check for multiple papers
        if "mentions_multiple_papers" in criteria:
            # Check if response mentions multiple papers (heuristic)
            paper_mentions = sum(1 for term in ["et al.", "paper", "study", "research"] if term in response.lower())
            results["mentions_multiple_papers"] = paper_mentions >= 3
        
        # Check for self-attention mentions
        if "mentions_self_attention" in criteria:
            attention_terms = ["self-attention", "attention mechanism", "attention head", "attention layer"]
            results["mentions_self_attention"] = any(term in response.lower() for term in attention_terms)
        
        # Calculate overall success
        criteria_met = sum(1 for result in results.values() if result)
        total_criteria = len(results)
        results["success_rate"] = criteria_met / total_criteria if total_criteria > 0 else 0
        results["overall_success"] = results["success_rate"] >= 0.7
        
        return results
    
    def run_tests(self):
        """Run all test cases and generate report."""
        test_results = []
        
        for case in self.test_cases:
            print(f"Running test case: {case['id']}")
            start_time = time.time()
            
            # Process query
            result = self.agent.process_query(case["query"])
            
            # Evaluate response
            evaluation = self.evaluate_response(result["output"], case["criteria"])
            
            # Compile test result
            test_result = {
                "test_id": case["id"],
                "query": case["query"],
                "response": result["output"],
                "execution_time": time.time() - start_time,
                "evaluation": evaluation
            }
            
            test_results.append(test_result)
            
            # Print summary
            print(f"Test case: {case['id']}")
            print(f"Success: {evaluation['overall_success']}")
            print(f"Execution time: {test_result['execution_time']:.2f} seconds")
            print("---")
        
        # Generate report
        report = {
            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
            "total_tests": len(test_results),
            "successful_tests": sum(1 for r in test_results if r["evaluation"]["overall_success"]),
            "average_execution_time": sum(r["execution_time"] for r in test_results) / len(test_results),
            "detailed_results": test_results
        }
        
        # Save report
        report_path = os.path.join(self.results_dir, f"test_report_{int(time.time())}.json")
        with open(report_path, 'w') as f:
            json.dump(report, f, indent=2)
        
        print(f"Test report saved to {report_path}")
        return report

# Run tests if script is executed directly
if __name__ == "__main__":
    agent = ResearchAssistantAgent(verbose=False)
    tester = AgentTester(agent)
    test_report = tester.run_tests()
    
    # Print summary
    print("\n=== TEST SUMMARY ===")
    print(f"Total tests: {test_report['total_tests']}")
    print(f"Successful tests: {test_report['successful_tests']}")
    print(f"Success rate: {test_report['successful_tests'] / test_report['total_tests'] * 100:.1f}%")
    print(f"Average execution time: {test_report['average_execution_time']:.2f} seconds")