Merge pull request #10 from Dynamite2003/bugfix/search

bugfix: fix searching errors and revise pdf selection; add new recommendations and revise recommendation logic

Author: Dynamite2003

backend/app/api/v1/arxiv.py

@@ -61,9 +61,9 @@
 # multi-batch candidate expansion tuning (keep modest to reduce arXiv timeouts)
 DEFAULT_BATCH_SIZE = 20
 MAX_BATCH_SIZE = 50
-CANDIDATE_MULTIPLIER = 2
 MAX_TOTAL_CANDIDATES = 150
-
+CANDIDATE_MULTIPLIER = 4
+CANDIDATE_MULTIPLIER = 2
 
 def fetch_arxiv_batch(
     query: str,

backend/app/core/config.py

@@ -29,12 +29,27 @@ class Settings(BaseSettings):
 
     # paper search APIs
     SEMANTIC_SCHOLAR_API_KEY: str = ""
+    SEMANTIC_SCHOLAR_CA_BUNDLE: str | None = None
+    SEMANTIC_SCHOLAR_VERIFY_SSL: bool = True
+    SEMANTIC_SCHOLAR_TRUST_ENV: bool = True
+    SEMANTIC_SCHOLAR_TIMEOUT: int = 30
+    SEMANTIC_SCHOLAR_USER_AGENT: str = "InsightReading/1.0 (mailto:tech@insightreading.local)"
     ARXIV_API_BASE: str = "http://export.arxiv.org/api/query"
     ARXIV_CA_BUNDLE: str | None = None
     ARXIV_VERIFY_SSL: bool = True
     ARXIV_TRUST_ENV: bool = True
-    ARXIV_TIMEOUT: int = 12
+    ARXIV_TIMEOUT: int = 30
+    ARXIV_MAX_RETRIES: int = 3
+    ARXIV_RETRY_BACKOFF_SECONDS: float = 1.5
+    ARXIV_USER_AGENT: str = "InsightReading/1.0 (mailto:tech@insightreading.local)"
     ARXIV_TIMEOUT_FALLBACK: int = 25
+    OPENALEX_BASE_URL: str = "https://api.openalex.org/works"
+    OPENALEX_TIMEOUT: int = 30
+    OPENALEX_PER_PAGE: int = 25
+    OPENALEX_MAX_PER_PAGE: int = 100
+    OPENALEX_TRUST_ENV: bool = True
+    OPENALEX_MAILTO: str = "insightreading@example.com"
+
 
     DEEPSEEK_API_KEY: str = "sk-7b3e938c92fb4a5ca601a35becdee4e2"
     DEEPSEEK_MODEL: str = "deepseek-chat"

backend/app/schemas/academic.py

@@ -32,11 +32,14 @@ class Paper(BaseModel):
     tldr: Optional[str] = None
     published_date: Optional[str] = None
     external_ids: Optional[dict] = None
+    score: Optional[float] = None
 
 class PaperSearchResponse(BaseModel):
     papers: List[Paper]
     total: int
     query_interpretation: Optional[dict] = None
+    sources_used: Optional[List[str]] = None
+    source_errors: Optional[dict[str, str]] = None
 
 
 class QueryInterpretation(BaseModel):
@@ -91,4 +94,4 @@ class IntelligentSearchResponse(BaseModel):
     conversation: List[ChatMessage] = Field(default_factory=list)
     search_performed: bool = False
     search_reason: Optional[str] = None
-    conversation_id: Optional[int] = None  # 关联的对话ID
+    conversation_id: Optional[int] = None  # 关联的对话ID

backend/app/services/academic/providers/arxiv.py

@@ -4,18 +4,26 @@
 查询示例: http://export.arxiv.org/api/query?search_query=all:electron&start=0&max_results=10
 """
 
+import asyncio
+import logging
+import re
 import aiohttp
+import ssl
 import xml.etree.ElementTree as ET
-from typing import List, Dict, Optional
 from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
 from urllib.parse import urlencode
+
 from app.core.config import settings
 
+logger = logging.getLogger(__name__)
 
 class ArxivProvider:
     """arXiv API 提供商"""
 
     BASE_URL = settings.ARXIV_API_BASE
+    RETRYABLE_STATUS = {429, 500, 502, 503, 504}
 
     # arXiv 命名空间
     NAMESPACES = {
@@ -68,14 +76,8 @@ async def search(
         }
 
         url = f"{self.BASE_URL}?{urlencode(params)}"
-        
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url) as response:
-                if response.status != 200:
-                    raise Exception(f"arXiv API 错误: {response.status}")
-                
-                xml_content = await response.text()
-                return self._parse_xml_response(xml_content)
+        xml_content = await self._fetch_feed(url)
+        return self._parse_xml_response(xml_content)
 
     async def get_by_id(self, arxiv_id: str) -> Optional[Dict]:
         """
@@ -96,14 +98,15 @@ async def get_by_id(self, arxiv_id: str) -> Optional[Dict]:
 
         url = f"{self.BASE_URL}?{urlencode(params)}"
 
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url) as response:
-                if response.status != 200:
-                    return None
-                
-                xml_content = await response.text()
-                results = self._parse_xml_response(xml_content)
-                return results[0] if results else None
+        try:
+            xml_content = await self._fetch_feed(url)
+        except Exception as exc:
+            if "404" in str(exc):
+                return None
+            raise
+
+        results = self._parse_xml_response(xml_content)
+        return results[0] if results else None
 
     async def search_by_author(
         self,
@@ -140,13 +143,132 @@ def _build_query(self, query: str, category: Optional[str] = None) -> str:
         
         布尔运算符: AND, OR, ANDNOT
         """
-        # 默认在标题和摘要中搜索
-        base_query = f'all:{query}'
-        
+        terms = self._tokenize_terms(query)
+        if not terms:
+            base_query = 'all:*'
+        else:
+            formatted_terms: List[str] = []
+            for term, is_phrase in terms:
+                formatted = self._format_term(term, is_phrase)
+                if formatted:
+                    formatted_terms.append(formatted)
+            base_query = '+AND+'.join(formatted_terms) if formatted_terms else 'all:*'
+
         if category:
             base_query = f'({base_query})+AND+cat:{category}'
 
         return base_query
+
+    def _tokenize_terms(self, query: str) -> List[Tuple[str, bool]]:
+        tokens = re.findall(r'"[^"]+"|\S+', query or '')
+        parsed: List[Tuple[str, bool]] = []
+        for token in tokens:
+            stripped = token.strip()
+            if not stripped:
+                continue
+            if stripped.startswith('"') and stripped.endswith('"') and len(stripped) >= 2:
+                parsed.append((stripped[1:-1].strip(), True))
+            else:
+                parsed.append((stripped, False))
+        return parsed
+
+    def _format_term(self, term: str, is_phrase: bool) -> Optional[str]:
+        sanitized = term.replace('"', '').strip()
+        if not sanitized:
+            return None
+        if is_phrase or ' ' in sanitized:
+            return f'all:"{sanitized}"'
+        return f'all:{sanitized}'
+
+    def _build_session_kwargs(self) -> Dict:
+        """
+        Construct aiohttp session kwargs honoring SSL-related settings.
+        """
+        connector = self._build_connector()
+        kwargs: Dict = {
+            "trust_env": settings.ARXIV_TRUST_ENV,
+            "timeout": aiohttp.ClientTimeout(total=settings.ARXIV_TIMEOUT),
+        }
+        if connector is not None:
+            kwargs["connector"] = connector
+        return kwargs
+
+    def _build_connector(self) -> Optional[aiohttp.TCPConnector]:
+        """
+        Build a TCPConnector that respects custom CA bundles or disables SSL validation.
+        """
+        if not settings.ARXIV_VERIFY_SSL:
+            return aiohttp.TCPConnector(ssl=False)
+
+        if settings.ARXIV_CA_BUNDLE:
+            bundle_path = Path(settings.ARXIV_CA_BUNDLE)
+            if bundle_path.is_file():
+                ssl_context = ssl.create_default_context(cafile=str(bundle_path))
+                return aiohttp.TCPConnector(ssl=ssl_context)
+            logger.warning(
+                "Configured ARXIV_CA_BUNDLE does not exist: %s. Falling back to default cert store.",
+                settings.ARXIV_CA_BUNDLE,
+            )
+
+        return None
+
+    def _build_headers(self) -> Dict[str, str]:
+        return {
+            "User-Agent": settings.ARXIV_USER_AGENT,
+        }
+
+    async def _fetch_feed(self, url: str) -> str:
+        """
+        执行带有重试的 arXiv 请求，处理速率限制和暂时性错误。
+        """
+        session_kwargs = self._build_session_kwargs()
+        headers = self._build_headers()
+        retries = max(1, settings.ARXIV_MAX_RETRIES)
+        backoff = max(0.5, settings.ARXIV_RETRY_BACKOFF_SECONDS)
+
+        async with aiohttp.ClientSession(**session_kwargs) as session:
+            for attempt in range(retries):
+                try:
+                    async with session.get(url, headers=headers) as response:
+                        if response.status == 200:
+                            return await response.text()
+
+                        if (
+                            response.status in self.RETRYABLE_STATUS
+                            and attempt < retries - 1
+                        ):
+                            await self._respect_retry_after(response, attempt, backoff)
+                            continue
+
+                        raise Exception(f"arXiv API 错误: {response.status}")
+
+                except asyncio.TimeoutError as exc:
+                    if attempt >= retries - 1:
+                        raise Exception("arXiv 请求超时") from exc
+                    await asyncio.sleep(backoff * (2 ** attempt))
+
+                except aiohttp.ClientError as exc:
+                    if attempt >= retries - 1:
+                        raise Exception(f"arXiv 网络错误: {exc}") from exc
+                    await asyncio.sleep(backoff * (2 ** attempt))
+
+        raise Exception("arXiv API 错误: 请求失败")
+
+    async def _respect_retry_after(
+        self,
+        response: aiohttp.ClientResponse,
+        attempt: int,
+        backoff: float,
+    ) -> None:
+        retry_after = response.headers.get("Retry-After")
+        if retry_after:
+            try:
+                delay = float(retry_after)
+            except ValueError:
+                delay = backoff * (2 ** attempt)
+        else:
+            delay = backoff * (2 ** attempt)
+        await asyncio.sleep(delay)
 
     def _parse_xml_response(self, xml_content: str) -> List[Dict]:
         """
@@ -352,4 +474,4 @@ async def get_arxiv_paper(arxiv_id: str) -> Optional[Dict]:
     >>> paper = await get_arxiv_paper("2103.00020")
     """
     provider = ArxivProvider()
-    return await provider.get_by_id(arxiv_id)
+    return await provider.get_by_id(arxiv_id)