import { Tool } from 'langchain/tools'
import { Zapserp, SearchEngine, SearchResponse } from 'zapserp'
import { OpenAI } from 'langchain/llms/openai'
import { initializeAgentExecutorWithOptions } from 'langchain/agents'

interface ZapserpToolConfig {
  zapserp: Zapserp
  maxResults?: number
  searchStrategy?: 'news' | 'web' | 'academic'
  includeContent?: boolean
}

class ZapserpSearchTool extends Tool {
  name = 'zapserp_search'
  description = `Useful for searching the web for current information, news, and facts. 
  Input should be a search query. 
  Returns relevant web content with sources and URLs.
  Use this when you need up-to-date information that might not be in your training data.`
  
  private zapserp: Zapserp
  private config: ZapserpToolConfig
  
  constructor(config: ZapserpToolConfig) {
    super()
    this.zapserp = config.zapserp
    this.config = config
  }
  
  async _call(query: string): Promise<string> {
    try {
      // Perform search
      const searchResponse: SearchResponse = await this.zapserp.search({
        query: query.trim(),
        engines: this.getSearchEngines(),
        limit: this.config.maxResults || 5,
        language: 'en',
        country: 'us'
      })
      
      if (!searchResponse.results || searchResponse.results.length === 0) {
        return `No results found for query: "${query}"`
      }
      
      // Extract content if requested
      let results = searchResponse.results
      
      if (this.config.includeContent) {
        const urls = results.slice(0, 3).map(r => r.url) // Top 3 for content extraction
        
        try {
          const contentResponse = await this.zapserp.readerBatch({ urls })
          
          // Combine search results with extracted content
          results = results.map((result, index) => {
            const content = contentResponse.results[index]
            return {
              ...result,
              extractedContent: content ? this.summarizeContent(content.content) : null
            }
          })
        } catch (error) {
          console.error('Content extraction failed:', error)
        }
      }
      
      // Format results for LLM consumption
      return this.formatSearchResults(results, query)
      
    } catch (error) {
      console.error('Zapserp search failed:', error)
      return `Search failed for query: "${query}". Error: ${error.message}`
    }
  }
  
  private getSearchEngines(): SearchEngine[] {
    switch (this.config.searchStrategy) {
      case 'news':
        return [SearchEngine.GOOGLE, SearchEngine.BING]
      case 'academic':
        return [SearchEngine.GOOGLE]
      default:
        return [SearchEngine.GOOGLE, SearchEngine.BING]
    }
  }
  
  private summarizeContent(content: string): string {
    // Summarize content to fit within context limits
    const words = content.split(' ')
    if (words.length <= 100) return content
    
    // Take first 100 words
    return words.slice(0, 100).join(' ') + '...'
  }
  
  private formatSearchResults(results: any[], query: string): string {
    const formattedResults = results.map((result, index) => {
      let formatted = `Result ${index + 1}:
Title: ${result.title}
URL: ${result.url}
Snippet: ${result.snippet || 'No snippet available'}`

      if (result.extractedContent) {
        formatted += `\nExtracted Content: ${result.extractedContent}`
      }
      
      return formatted
    }).join('\n\n')
    
    return `Search results for "${query}":\n\n${formattedResults}\n\nTotal results: ${results.length}`
  }
}

// Advanced Zapserp Tool with Context Awareness
class ContextAwareZapserpTool extends ZapserpSearchTool {
  private conversationContext: string[] = []
  
  async _call(query: string): Promise<string> {
    // Enhance query with conversation context
    const enhancedQuery = this.enhanceQueryWithContext(query)
    
    // Perform search with enhanced query
    const result = await super._call(enhancedQuery)
    
    // Update conversation context
    this.updateContext(query, result)
    
    return result
  }
  
  private enhanceQueryWithContext(query: string): string {
    if (this.conversationContext.length === 0) return query
    
    // Get recent context (last 2 interactions)
    const recentContext = this.conversationContext.slice(-2).join(' ')
    
    // Combine context with current query
    return `${recentContext} ${query}`.trim()
  }
  
  private updateContext(query: string, result: string) {
    // Extract key information from result for context
    const keyInfo = this.extractKeyInformation(result)
    this.conversationContext.push(`Query: ${query} Key findings: ${keyInfo}`)
    
    // Keep context manageable (last 5 interactions)
    if (this.conversationContext.length > 5) {
      this.conversationContext = this.conversationContext.slice(-5)
    }
  }
  
  private extractKeyInformation(result: string): string {
    // Simple extraction of key information
    const lines = result.split('\n')
    const titles = lines.filter(line => line.startsWith('Title:'))
    return titles.slice(0, 2).map(title => title.replace('Title: ', '')).join(', ')
  }
}

// Usage with LangChain Agent
async function createZapserpAgent() {
  const zapserp = new Zapserp({ apiKey: 'YOUR_ZAPSERP_API_KEY' })
  
  const tools = [
    new ContextAwareZapserpTool({
      zapserp,
      maxResults: 5,
      searchStrategy: 'web',
      includeContent: true
    })
  ]
  
  const llm = new OpenAI({
    openAIApiKey: 'YOUR_OPENAI_API_KEY',
    temperature: 0.1,
    modelName: 'gpt-4-turbo-preview'
  })
  
  const agent = await initializeAgentExecutorWithOptions(tools, llm, {
    agentType: 'zero-shot-react-description',
    verbose: true,
    maxIterations: 3,
    returnIntermediateSteps: true
  })
  
  return agent
}

// Example usage
const agent = await createZapserpAgent()

const response = await agent.call({
  input: "What are the latest developments in AI safety research and what are the main concerns being discussed?"
})

console.log('Agent Response:', response.output)
console.log('Intermediate Steps:', response.intermediateSteps)

import { BaseDocumentLoader } from 'langchain/document_loaders/base'
import { Document } from 'langchain/document'
import { Zapserp } from 'zapserp'

interface WebDocumentLoaderConfig {
  zapserp: Zapserp
  searchQuery: string
  maxDocuments?: number
  chunkSize?: number
  chunkOverlap?: number
}

class ZapserpDocumentLoader extends BaseDocumentLoader {
  private config: WebDocumentLoaderConfig
  
  constructor(config: WebDocumentLoaderConfig) {
    super()
    this.config = config
  }
  
  async load(): Promise<Document[]> {
    try {
      // Perform search
      const searchResponse = await this.config.zapserp.search({
        query: this.config.searchQuery,
        limit: this.config.maxDocuments || 10,
        engines: ['google', 'bing']
      })
      
      if (!searchResponse.results.length) return []
      
      // Extract content from URLs
      const urls = searchResponse.results.map(r => r.url)
      const contentResponse = await this.config.zapserp.readerBatch({ urls })
      
      // Convert to LangChain documents
      const documents: Document[] = []
      
      for (let i = 0; i < contentResponse.results.length; i++) {
        const content = contentResponse.results[i]
        const searchResult = searchResponse.results[i]
        
        if (content && content.content) {
          // Split content into chunks if needed
          const chunks = this.splitContent(content.content)
          
          chunks.forEach((chunk, chunkIndex) => {
            documents.push(new Document({
              pageContent: chunk,
              metadata: {
                source: content.url,
                title: content.title,
                author: content.metadata?.author,
                publishedTime: content.metadata?.publishedTime,
                searchQuery: this.config.searchQuery,
                searchRank: i + 1,
                chunkIndex,
                totalChunks: chunks.length
              }
            }))
          })
        }
      }
      
      return documents
      
    } catch (error) {
      console.error('Failed to load documents:', error)
      return []
    }
  }
  
  private splitContent(content: string): string[] {
    const chunkSize = this.config.chunkSize || 1000
    const chunkOverlap = this.config.chunkOverlap || 200
    
    if (content.length <= chunkSize) return [content]
    
    const chunks: string[] = []
    let start = 0
    
    while (start < content.length) {
      const end = Math.min(start + chunkSize, content.length)
      const chunk = content.slice(start, end)
      
      // Try to end at a sentence boundary
      if (end < content.length) {
        const lastSentence = chunk.lastIndexOf('.')
        if (lastSentence > start + chunkSize * 0.5) {
          chunks.push(chunk.slice(0, lastSentence + 1))
          start = lastSentence + 1 - chunkOverlap
        } else {
          chunks.push(chunk)
          start = end - chunkOverlap
        }
      } else {
        chunks.push(chunk)
        break
      }
    }
    
    return chunks
  }
}

// Usage with LangChain Vector Store
import { OpenAIEmbeddings } from 'langchain/embeddings/openai'
import { MemoryVectorStore } from 'langchain/vectorstores/memory'

async function buildKnowledgeBase(searchQueries: string[]) {
  const zapserp = new Zapserp({ apiKey: 'YOUR_ZAPSERP_API_KEY' })
  const embeddings = new OpenAIEmbeddings()
  
  // Load documents from multiple search queries
  const allDocuments: Document[] = []
  
  for (const query of searchQueries) {
    const loader = new ZapserpDocumentLoader({
      zapserp,
      searchQuery: query,
      maxDocuments: 5,
      chunkSize: 1000,
      chunkOverlap: 200
    })
    
    const docs = await loader.load()
    allDocuments.push(...docs)
  }
  
  // Create vector store
  const vectorStore = await MemoryVectorStore.fromDocuments(allDocuments, embeddings)
  
  return vectorStore
}

// Example: Building a knowledge base about AI research
const knowledgeBase = await buildKnowledgeBase([
  'artificial intelligence research 2024',
  'machine learning breakthroughs',
  'AI safety recent developments',
  'large language models advances'
])

// Query the knowledge base
const relevantDocs = await knowledgeBase.similaritySearch(
  'What are the latest developments in AI safety?',
  5
)

console.log('Relevant documents:', relevantDocs)

import { BaseReader, Document } from 'llamaindex'
import { Zapserp, SearchEngine } from 'zapserp'

interface ZapserpReaderConfig {
  zapserp: Zapserp
  searchQueries: string[]
  maxDocumentsPerQuery?: number
  contentExtraction?: boolean
  filterDomains?: string[]
}

class ZapserpReader extends BaseReader {
  private config: ZapserpReaderConfig
  
  constructor(config: ZapserpReaderConfig) {
    super()
    this.config = config
  }
  
  async loadData(): Promise<Document[]> {
    const documents: Document[] = []
    
    for (const query of this.config.searchQueries) {
      try {
        const queryDocs = await this.processSearchQuery(query)
        documents.push(...queryDocs)
      } catch (error) {
        console.error(`Failed to process query "${query}":`, error)
      }
    }
    
    return documents
  }
  
  private async processSearchQuery(query: string): Promise<Document[]> {
    // Perform search
    const searchResponse = await this.config.zapserp.search({
      query,
      engines: [SearchEngine.GOOGLE, SearchEngine.BING],
      limit: this.config.maxDocumentsPerQuery || 10,
      language: 'en'
    })
    
    if (!searchResponse.results.length) return []
    
    // Filter URLs if domain filter is provided
    let urls = searchResponse.results.map(r => r.url)
    if (this.config.filterDomains) {
      urls = urls.filter(url => 
        this.config.filterDomains!.some(domain => url.includes(domain))
      )
    }
    
    if (!urls.length) return []
    
    // Extract content if enabled
    if (this.config.contentExtraction) {
      const contentResponse = await this.config.zapserp.readerBatch({ urls })
      
      return contentResponse.results
        .filter(content => content && content.content)
        .map(content => new Document({
          text: content.content,
          metadata: {
            source: content.url,
            title: content.title,
            author: content.metadata?.author,
            publishedTime: content.metadata?.publishedTime,
            searchQuery: query,
            contentLength: content.contentLength
          }
        }))
    } else {
      // Use search snippets
      return searchResponse.results.map(result => new Document({
        text: result.snippet || result.title,
        metadata: {
          source: result.url,
          title: result.title,
          searchQuery: query,
          searchEngine: result.engine,
          rank: result.rank
        }
      }))
    }
  }
}

// Advanced Query Engine with Zapserp
import { VectorStoreIndex, ServiceContext } from 'llamaindex'
import { OpenAIEmbedding, OpenAI } from 'llamaindex'

class ZapserpQueryEngine {
  private index: VectorStoreIndex
  private zapserp: Zapserp
  
  constructor(zapserp: Zapserp) {
    this.zapserp = zapserp
  }
  
  async initialize(searchQueries: string[]) {
    // Create service context
    const serviceContext = ServiceContext.fromDefaults({
      llm: new OpenAI({ model: 'gpt-4-turbo-preview' }),
      embedModel: new OpenAIEmbedding()
    })
    
    // Load documents
    const reader = new ZapserpReader({
      zapserp: this.zapserp,
      searchQueries,
      maxDocumentsPerQuery: 8,
      contentExtraction: true,
      filterDomains: [
        'wikipedia.org', 'arxiv.org', 'nature.com',
        'techcrunch.com', 'wired.com', 'arstechnica.com'
      ]
    })
    
    const documents = await reader.loadData()
    
    // Create index
    this.index = await VectorStoreIndex.fromDocuments(documents, { serviceContext })
  }
  
  async query(question: string, options: {
    includeRealTimeData?: boolean
    searchStrategy?: 'web' | 'news' | 'academic'
  } = {}) {
    // Query the existing index
    const queryEngine = this.index.asQueryEngine()
    const indexResponse = await queryEngine.query(question)
    
    // Optionally include real-time data
    if (options.includeRealTimeData) {
      const realtimeData = await this.getRealTimeData(question, options.searchStrategy)
      
      // Combine responses
      return {
        indexedResponse: indexResponse.response,
        realtimeData: realtimeData,
        sources: [
          ...this.extractSources(indexResponse),
          ...realtimeData.sources
        ],
        timestamp: new Date().toISOString()
      }
    }
    
    return {
      response: indexResponse.response,
      sources: this.extractSources(indexResponse),
      timestamp: new Date().toISOString()
    }
  }
  
  private async getRealTimeData(question: string, strategy: string = 'web') {
    const searchQuery = this.generateSearchQuery(question, strategy)
    
    const searchResponse = await this.zapserp.search({
      query: searchQuery,
      engines: [SearchEngine.GOOGLE],
      limit: 3
    })
    
    if (!searchResponse.results.length) {
      return { content: 'No real-time data found', sources: [] }
    }
    
    // Extract content from top results
    const urls = searchResponse.results.map(r => r.url)
    const contentResponse = await this.zapserp.readerBatch({ urls })
    
    const realtimeContent = contentResponse.results
      .filter(content => content && content.content)
      .map(content => ({
        title: content.title,
        content: content.content.substring(0, 500) + '...',
        url: content.url,
        publishedTime: content.metadata?.publishedTime
      }))
    
    return {
      content: realtimeContent.map(c => `${c.title}: ${c.content}`).join('\n\n'),
      sources: realtimeContent.map(c => ({ title: c.title, url: c.url }))
    }
  }
  
  private generateSearchQuery(question: string, strategy: string): string {
    const baseQuery = question.replace(/[?]/g, '').trim()
    
    switch (strategy) {
      case 'news':
        return `${baseQuery} latest news today`
      case 'academic':
        return `${baseQuery} research paper study`
      default:
        return `${baseQuery} 2024 latest`
    }
  }
  
  private extractSources(response: any): Array<{ title: string, url: string }> {
    // Extract sources from LlamaIndex response
    // This would depend on the specific response format
    return response.sourceNodes?.map((node: any) => ({
      title: node.metadata?.title || 'Unknown',
      url: node.metadata?.source || '#'
    })) || []
  }
}

// Usage example
const zapserp = new Zapserp({ apiKey: 'YOUR_ZAPSERP_API_KEY' })
const queryEngine = new ZapserpQueryEngine(zapserp)

// Initialize with domain-specific queries
await queryEngine.initialize([
  'artificial intelligence safety research',
  'machine learning interpretability',
  'AI alignment techniques',
  'large language model risks'
])

// Query with real-time data
const result = await queryEngine.query(
  'What are the current concerns about AI safety?',
  { includeRealTimeData: true, searchStrategy: 'news' }
)

console.log('Query Result:', result)

interface ResearchTask {
  topic: string
  depth: 'basic' | 'intermediate' | 'comprehensive'
  domains?: string[]
  timeframe?: string
  sources?: 'web' | 'news' | 'academic' | 'mixed'
}

interface ResearchResult {
  topic: string
  summary: string
  keyFindings: string[]
  sources: Array<{
    title: string
    url: string
    relevance: number
    content: string
  }>
  relatedTopics: string[]
  confidence: number
  timestamp: string
}

class AutonomousResearchAgent {
  private zapserp: Zapserp
  private openai: OpenAI
  private maxIterations: number = 3
  
  constructor(apiKeys: { zapserp: string, openai: string }) {
    this.zapserp = new Zapserp({ apiKey: apiKeys.zapserp })
    this.openai = new OpenAI({ apiKey: apiKeys.openai })
  }
  
  async conductResearch(task: ResearchTask): Promise<ResearchResult> {
    console.log(`Starting research on: ${task.topic}`)
    
    // Phase 1: Initial discovery
    const initialQueries = this.generateInitialQueries(task)
    const discoveryResults = await this.performDiscoveryPhase(initialQueries, task)
    
    // Phase 2: Deep dive into promising areas
    const refinedQueries = await this.generateRefinedQueries(task.topic, discoveryResults)
    const deepResults = await this.performDeepDive(refinedQueries, task)
    
    // Phase 3: Synthesis and analysis
    const allResults = [...discoveryResults, ...deepResults]
    const research = await this.synthesizeFindings(task.topic, allResults)
    
    return research
  }
  
  private generateInitialQueries(task: ResearchTask): string[] {
    const baseQueries = [
      task.topic,
      `${task.topic} overview`,
      `${task.topic} latest developments`,
      `${task.topic} research`
    ]
    
    if (task.timeframe) {
      baseQueries.push(`${task.topic} ${task.timeframe}`)
    }
    
    return baseQueries
  }
  
  private async performDiscoveryPhase(queries: string[], task: ResearchTask) {
    const results = []
    
    for (const query of queries) {
      try {
        const searchResponse = await this.zapserp.search({
          query,
          engines: this.getEnginesForSource(task.sources),
          limit: 8,
          language: 'en'
        })
        
        // Filter URLs based on domains if specified
        let urls = searchResponse.results.map(r => r.url)
        if (task.domains) {
          urls = urls.filter(url => 
            task.domains!.some(domain => url.includes(domain))
          )
        }
        
        if (urls.length > 0) {
          const contentResponse = await this.zapserp.readerBatch({ 
            urls: urls.slice(0, 4) 
          })
          
          results.push(...contentResponse.results.filter(r => r && r.content))
        }
      } catch (error) {
        console.error(`Discovery failed for query: ${query}`, error)
      }
    }
    
    return results
  }
  
  private async generateRefinedQueries(topic: string, discoveryResults: any[]): Promise<string[]> {
    // Use LLM to analyze discovery results and generate refined queries
    const contentSummary = discoveryResults
      .slice(0, 5)
      .map(r => `${r.title}: ${r.content.substring(0, 200)}`)
      .join('\n\n')
    
    const prompt = `Based on this initial research about "${topic}", suggest 3-4 specific search queries to dive deeper into the most important aspects:

Initial findings:
${contentSummary}

Generate focused search queries that would help understand:
1. Key subtopics and areas of focus
2. Recent developments and trends
3. Expert opinions and analysis
4. Practical applications or implications

Return only the search queries, one per line:`
    
    try {
      const completion = await this.openai.chat.completions.create({
        model: 'gpt-4-turbo-preview',
        messages: [{ role: 'user', content: prompt }],
        temperature: 0.3,
        max_tokens: 300
      })
      
      const queries = completion.choices[0]?.message?.content
        ?.split('\n')
        .map(q => q.trim())
        .filter(q => q.length > 0 && !q.startsWith('#'))
        || []
      
      return queries.slice(0, 4) // Limit to 4 refined queries
      
    } catch (error) {
      console.error('Failed to generate refined queries:', error)
      return [`${topic} detailed analysis`, `${topic} expert opinion`]
    }
  }
  
  private async performDeepDive(queries: string[], task: ResearchTask) {
    const results = []
    
    for (const query of queries) {
      try {
        const searchResponse = await this.zapserp.search({
          query,
          engines: this.getEnginesForSource(task.sources),
          limit: 6
        })
        
        const urls = searchResponse.results.map(r => r.url).slice(0, 3)
        const contentResponse = await this.zapserp.readerBatch({ urls })
        
        results.push(...contentResponse.results.filter(r => r && r.content))
      } catch (error) {
        console.error(`Deep dive failed for query: ${query}`, error)
      }
    }
    
    return results
  }
  
  private async synthesizeFindings(topic: string, allResults: any[]): Promise<ResearchResult> {
    // Prepare content for analysis
    const sourceMaterial = allResults
      .slice(0, 15) // Limit to prevent token overflow
      .map((result, index) => `
Source ${index + 1}: ${result.title}
URL: ${result.url}
Author: ${result.metadata?.author || 'Unknown'}
Published: ${result.metadata?.publishedTime || 'Unknown'}
Content: ${result.content.substring(0, 1000)}...
---`)
      .join('\n')
    
    const analysisPrompt = `Analyze the following research material about "${topic}" and provide a comprehensive synthesis:

${sourceMaterial}

Please provide:
1. A concise but comprehensive summary (2-3 paragraphs)
2. 5-7 key findings or insights
3. 3-4 related topics for further research
4. A confidence score (1-10) for the completeness of this research

Format your response as JSON with the following structure:
{
  "summary": "...",
  "keyFindings": ["...", "..."],
  "relatedTopics": ["...", "..."],
  "confidence": 8
}`
    
    try {
      const completion = await this.openai.chat.completions.create({
        model: 'gpt-4-turbo-preview',
        messages: [{ role: 'user', content: analysisPrompt }],
        temperature: 0.2,
        max_tokens: 2000
      })
      
      const analysisText = completion.choices[0]?.message?.content || ''
      const analysis = JSON.parse(analysisText)
      
      return {
        topic,
        summary: analysis.summary,
        keyFindings: analysis.keyFindings,
        sources: allResults.map(r => ({
          title: r.title,
          url: r.url,
          relevance: this.calculateRelevance(r.content, topic),
          content: r.content.substring(0, 300) + '...'
        })),
        relatedTopics: analysis.relatedTopics,
        confidence: analysis.confidence,
        timestamp: new Date().toISOString()
      }
      
    } catch (error) {
      console.error('Synthesis failed:', error)
      
      // Fallback synthesis
      return {
        topic,
        summary: `Research conducted on ${topic} yielding ${allResults.length} sources.`,
        keyFindings: allResults.slice(0, 5).map(r => r.title),
        sources: allResults.map(r => ({
          title: r.title,
          url: r.url,
          relevance: 0.5,
          content: r.content.substring(0, 300) + '...'
        })),
        relatedTopics: [],
        confidence: 5,
        timestamp: new Date().toISOString()
      }
    }
  }
  
  private getEnginesForSource(sources?: string): SearchEngine[] {
    switch (sources) {
      case 'news':
        return [SearchEngine.GOOGLE, SearchEngine.BING]
      case 'academic':
        return [SearchEngine.GOOGLE]
      default:
        return [SearchEngine.GOOGLE, SearchEngine.BING]
    }
  }
  
  private calculateRelevance(content: string, topic: string): number {
    // Simple relevance calculation based on keyword frequency
    const topicWords = topic.toLowerCase().split(' ')
    const contentWords = content.toLowerCase().split(' ')
    
    let matches = 0
    for (const word of topicWords) {
      matches += contentWords.filter(w => w.includes(word)).length
    }
    
    return Math.min(matches / contentWords.length * 100, 1)
  }
}

// Usage example
const researchAgent = new AutonomousResearchAgent({
  zapserp: 'YOUR_ZAPSERP_API_KEY',
  openai: 'YOUR_OPENAI_API_KEY'
})

const research = await researchAgent.conductResearch({
  topic: 'quantum computing recent breakthroughs',
  depth: 'comprehensive',
  domains: ['nature.com', 'science.org', 'arxiv.org', 'techcrunch.com'],
  timeframe: '2024',
  sources: 'mixed'
})

console.log('Research Complete:', research)

import { Queue } from 'bull'
import Redis from 'ioredis'

interface AIJobData {
  type: 'rag' | 'research' | 'chat'
  input: any
  userId: string
  priority: number
}

class ProductionAIService {
  private zapserp: Zapserp
  private ragQueue: Queue<AIJobData>
  private redis: Redis
  private rateLimiter: Map<string, number[]> = new Map()
  
  constructor(config: {
    zapserp: string
    openai: string
    redisUrl: string
  }) {
    this.zapserp = new Zapserp({ apiKey: config.zapserp })
    this.redis = new Redis(config.redisUrl)
    
    // Initialize job queue
    this.ragQueue = new Queue<AIJobData>('AI processing', {
      redis: { port: 6379, host: 'localhost' }
    })
    
    this.setupQueueProcessors()
    this.startRateLimitCleanup()
  }
  
  private setupQueueProcessors() {
    // RAG processing
    this.ragQueue.process('rag', 5, async (job) => {
      const { input, userId } = job.data
      
      try {
        const ragSystem = new ZapserpRAGSystem({
          zapserp: this.zapserp.apiKey,
          openai: process.env.OPENAI_API_KEY!
        })
        
        const result = await ragSystem.generateRAGResponse(input.query, input.options)
        
        // Cache result
        await this.cacheResult(`rag:${userId}:${input.query}`, result, 300) // 5 minutes
        
        return result
        
      } catch (error) {
        console.error('RAG processing failed:', error)
        throw error
      }
    })
    
    // Research processing
    this.ragQueue.process('research', 3, async (job) => {
      const { input, userId } = job.data
      
      try {
        const researchAgent = new AutonomousResearchAgent({
          zapserp: this.zapserp.apiKey,
          openai: process.env.OPENAI_API_KEY!
        })
        
        const result = await researchAgent.conductResearch(input.task)
        
        // Cache result
        await this.cacheResult(`research:${userId}:${input.task.topic}`, result, 3600) // 1 hour
        
        return result
        
      } catch (error) {
        console.error('Research processing failed:', error)
        throw error
      }
    })
  }
  
  async processRAGRequest(userId: string, query: string, options: any = {}) {
    // Check rate limits
    if (!this.checkRateLimit(userId, 'rag', 10, 60)) { // 10 per minute
      throw new Error('Rate limit exceeded')
    }
    
    // Check cache first
    const cacheKey = `rag:${userId}:${query}`
    const cached = await this.getCachedResult(cacheKey)
    if (cached) return cached
    
    // Add to queue
    const job = await this.ragQueue.add('rag', {
      type: 'rag',
      input: { query, options },
      userId,
      priority: options.priority || 0
    }, {
      attempts: 3,
      backoff: 'exponential',
      removeOnComplete: 100,
      removeOnFail: 50
    })
    
    return job.finished()
  }
  
  async processResearchRequest(userId: string, task: ResearchTask) {
    // Check rate limits for research (more restrictive)
    if (!this.checkRateLimit(userId, 'research', 3, 3600)) { // 3 per hour
      throw new Error('Research rate limit exceeded')
    }
    
    // Check cache
    const cacheKey = `research:${userId}:${task.topic}`
    const cached = await this.getCachedResult(cacheKey)
    if (cached) return cached
    
    // Add to queue with higher priority for research
    const job = await this.ragQueue.add('research', {
      type: 'research',
      input: { task },
      userId,
      priority: 10
    }, {
      attempts: 2,
      backoff: 'exponential',
      removeOnComplete: 50,
      removeOnFail: 25,
      delay: 1000 // Small delay to manage load
    })
    
    return job.finished()
  }
  
  private checkRateLimit(userId: string, operation: string, limit: number, windowSeconds: number): boolean {
    const key = `${userId}:${operation}`
    const now = Date.now()
    const windowStart = now - (windowSeconds * 1000)
    
    // Get current requests
    const requests = this.rateLimiter.get(key) || []
    
    // Filter to current window
    const currentRequests = requests.filter(time => time > windowStart)
    
    if (currentRequests.length >= limit) {
      return false
    }
    
    // Add current request
    currentRequests.push(now)
    this.rateLimiter.set(key, currentRequests)
    
    return true
  }
  
  private async cacheResult(key: string, result: any, ttlSeconds: number) {
    try {
      await this.redis.setex(key, ttlSeconds, JSON.stringify(result))
    } catch (error) {
      console.error('Cache write failed:', error)
    }
  }
  
  private async getCachedResult(key: string): Promise<any | null> {
    try {
      const cached = await this.redis.get(key)
      return cached ? JSON.parse(cached) : null
    } catch (error) {
      console.error('Cache read failed:', error)
      return null
    }
  }
  
  private startRateLimitCleanup() {
    setInterval(() => {
      const now = Date.now()
      const oneHourAgo = now - (3600 * 1000)
      
      for (const [key, requests] of this.rateLimiter.entries()) {
        const validRequests = requests.filter(time => time > oneHourAgo)
        if (validRequests.length === 0) {
          this.rateLimiter.delete(key)
        } else {
          this.rateLimiter.set(key, validRequests)
        }
      }
    }, 300000) // Every 5 minutes
  }
}

// Express API integration
import express from 'express'

const app = express()
const aiService = new ProductionAIService({
  zapserp: process.env.ZAPSERP_API_KEY!,
  openai: process.env.OPENAI_API_KEY!,
  redisUrl: process.env.REDIS_URL!
})

app.post('/api/rag', async (req, res) => {
  try {
    const { query, options } = req.body
    const userId = req.user?.id || 'anonymous'
    
    const result = await aiService.processRAGRequest(userId, query, options)
    res.json(result)
    
  } catch (error) {
    res.status(error.message.includes('rate limit') ? 429 : 500)
       .json({ error: error.message })
  }
})

app.post('/api/research', async (req, res) => {
  try {
    const { task } = req.body
    const userId = req.user?.id || 'anonymous'
    
    const result = await aiService.processResearchRequest(userId, task)
    res.json(result)
    
  } catch (error) {
    res.status(error.message.includes('rate limit') ? 429 : 500)
       .json({ error: error.message })
  }
})