Merge pull request #70 from ZimoLiao/issue-65

Fix macOS semantic search crash in FAISS-backed paths

Author: ZimoLiao

.github/workflows/macos-semantic-smoke.yml

@@ -0,0 +1,94 @@
+name: macOS Semantic Smoke
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main, master]
+    paths:
+      - ".github/workflows/macos-semantic-smoke.yml"
+      - "pyproject.toml"
+      - "scholaraio/cli.py"
+      - "scholaraio/explore.py"
+      - "scholaraio/index.py"
+      - "scholaraio/vectors.py"
+      - "tests/test_cli_messages.py"
+      - "tests/test_index.py"
+      - "tests/test_vectors.py"
+  push:
+    branches: [main, master]
+    paths:
+      - ".github/workflows/macos-semantic-smoke.yml"
+      - "pyproject.toml"
+      - "scholaraio/cli.py"
+      - "scholaraio/explore.py"
+      - "scholaraio/index.py"
+      - "scholaraio/vectors.py"
+      - "tests/test_cli_messages.py"
+      - "tests/test_index.py"
+      - "tests/test_vectors.py"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: macos-semantic-smoke-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  semantic-smoke:
+    runs-on: macos-14
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install semantic-search dependencies
+        run: pip install -e ".[dev,embed]"
+
+      - name: Smoke test issue-65 explore search flow
+        env:
+          HF_HUB_DISABLE_TELEMETRY: "1"
+          TOKENIZERS_PARALLELISM: "false"
+        run: |
+          export SMOKE_ROOT="$RUNNER_TEMP/issue-65-smoke"
+          mkdir -p "$SMOKE_ROOT/data/explore/issue-65-smoke"
+
+          cat > "$SMOKE_ROOT/config.yaml" <<'YAML'
+          embed:
+            provider: local
+            model: sentence-transformers/all-MiniLM-L6-v2
+            source: huggingface
+            device: cpu
+          YAML
+
+          cat > "$SMOKE_ROOT/data/explore/issue-65-smoke/papers.jsonl" <<'JSONL'
+          {"openalex_id":"https://openalex.org/W1","doi":"10.1234/issue65.1","title":"Boundary-layer turbulence transfer learning","abstract":"We study semantic retrieval for boundary layer turbulence with transformer embeddings on CPU systems and compare keyword and unified search behavior.","authors":["Alice Example"],"year":2024,"cited_by_count":7,"type":"article"}
+          {"openalex_id":"https://openalex.org/W2","doi":"10.1234/issue65.2","title":"Wake mixing in bluff-body flows","abstract":"This paper focuses on wake mixing and provides a distractor document for ranking checks.","authors":["Bob Example"],"year":2023,"cited_by_count":3,"type":"article"}
+          JSONL
+
+          export SCHOLARAIO_CONFIG="$SMOKE_ROOT/config.yaml"
+
+          scholaraio explore embed --name issue-65-smoke
+
+          scholaraio explore search --name issue-65-smoke "boundary layer turbulence" --mode semantic > "$SMOKE_ROOT/semantic.out"
+          scholaraio explore search --name issue-65-smoke "boundary layer turbulence" --mode unified > "$SMOKE_ROOT/unified.out"
+
+          grep -q "Boundary-layer turbulence transfer learning" "$SMOKE_ROOT/semantic.out"
+          grep -q "Boundary-layer turbulence transfer learning" "$SMOKE_ROOT/unified.out"
+          grep -q "分数:" "$SMOKE_ROOT/semantic.out"
+          grep -q "分数:" "$SMOKE_ROOT/unified.out"
+
+          echo "issue-65 macOS explore semantic/unified smoke passed"
+
+      - name: Run semantic-search regression tests
+        run: >
+          python -m pytest -q -p no:cacheprovider
+          tests/test_ci_workflows.py
+          tests/test_vectors.py
+          tests/test_index.py
+          tests/test_cli_messages.py
+          -k "macos_semantic_smoke_workflow_runs_issue_search_commands or embeds_query_before_loading_faiss_index or does_not_embed_when_vector_table_is_empty or unified_search or explore_search_semantic_reports_embedding_disabled_cleanly"

CHANGELOG.md

@@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
+- **macOS semantic-search smoke workflow**: Added a dedicated GitHub Actions job on `macos-14` to exercise the `sentence-transformers` -> `faiss-cpu` semantic-search stack and run targeted regression tests for vector-search paths
 - **Academic writing router** ([#55](https://github.com/ZimoLiao/scholaraio/issues/55)): Added the `academic-writing` skill as a stable top-level entry point that routes users by deliverable and writing stage instead of forcing them to guess among multiple writing skills
 - **Deliverable-first writing workflows** ([#55](https://github.com/ZimoLiao/scholaraio/issues/55)): Added lightweight `poster` and `technical-report` skills so conference posters, poster-style summaries, topic reports, and research briefings are first-class workflows rather than implicit combinations of lower-level skills
 - **Writing workflow regression coverage**: Added tests for skill frontmatter validity, router references, approximate host-style skill selection, and 11 rounds of documentation-alignment checks across docs, agent instructions, and marketplace metadata
 
 ### Fixed
 
+- **macOS semantic/unified search crash** ([#65](https://github.com/ZimoLiao/scholaraio/issues/65)): main-library and explore semantic search now embed and normalize the query before loading or searching FAISS indexes, avoiding a known `faiss` / `sentence-transformers` import-order segfault pattern on macOS while preserving existing ranking behavior
 - **Academic writing docs alignment** ([#55](https://github.com/ZimoLiao/scholaraio/issues/55)): Synchronized `docs/guide/writing.md`, `README.md`, `README_CN.md`, `docs/index.md`, `AGENTS.md`, `AGENTS_CN.md`, `CLAUDE.md`, and `clawhub.yaml` around a router-first writing model so poster/report workflows and the academic-writing entry point are discoverable consistently across user and agent surfaces
 
 ## [1.3.1] — 2026-04-14

scholaraio/explore.py

@@ -700,10 +700,18 @@ def explore_vsearch(name: str, query: str, *, top_k: int = 10, cfg: Config | Non
     Returns:
         论文列表，按 cosine similarity 降序。
     """
-    from scholaraio.vectors import _vsearch_faiss
+    from scholaraio.vectors import _embed_query_vector, _ensure_vector_search_ready, _vsearch_faiss
 
+    db_path = _db_path(name, cfg)
+    _ensure_vector_search_ready(
+        db_path,
+        missing_table_msg=f"向量库为空: {db_path}",
+        empty_msg=f"向量库为空: {db_path}",
+    )
+    # Prepare the query embedding before FAISS import/load to avoid macOS crashes.
+    q_vec = _embed_query_vector(query, cfg)
     index, paper_ids = _build_faiss_index(name, cfg)
-    hits = _vsearch_faiss(query, index, paper_ids, top_k, cfg=cfg)
+    hits = _vsearch_faiss(q_vec, index, paper_ids, top_k, cfg=cfg)
 
     paper_map = {}
     for p in iter_papers(name, cfg):

scholaraio/vectors.py

@@ -717,6 +717,40 @@ def _embed_batch(texts: list[str], cfg: Config | None = None) -> list[list[float
     raise ValueError(f"未知 embedding provider: {provider}")
 
 
+def _embed_query_vector(query: str, cfg: Config | None = None):
+    """Embed and L2-normalize a single query before touching FAISS."""
+    import numpy as np
+
+    q_vec = np.array([_embed_text(query, cfg)], dtype="float32")
+    norms = np.linalg.norm(q_vec, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    return q_vec / norms
+
+
+def _ensure_vector_search_ready(
+    db_path: Path,
+    *,
+    missing_table_msg: str,
+    empty_msg: str,
+    table_name: str = "paper_vectors",
+) -> None:
+    """Verify the vector table exists and has at least one row before embedding queries."""
+    conn = sqlite3.connect(db_path)
+    try:
+        has_vectors = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,),
+        ).fetchone()
+        if not has_vectors:
+            raise FileNotFoundError(missing_table_msg)
+
+        has_rows = conn.execute(f"SELECT 1 FROM {table_name} LIMIT 1").fetchone()
+        if not has_rows:
+            raise FileNotFoundError(empty_msg)
+    finally:
+        conn.close()
+
+
 class QwenEmbedder:
     """BERTopic-compatible embedder wrapping the configured embedding backend.
 
@@ -1028,7 +1062,7 @@ def _build_faiss_index(db_path: Path) -> tuple[faiss.Index, list[str]]:
 
 
 def _vsearch_faiss(
-    query: str,
+    query: str | object,
     index: faiss.Index,
     paper_ids: list[str],
     top_k: int,
@@ -1046,14 +1080,10 @@ def _vsearch_faiss(
     Returns:
         List of ``(paper_id, score)`` sorted by descending similarity.
     """
-    import faiss
-    import numpy as np
-
     if _embed_provider(cfg) == "none":
         raise FileNotFoundError("当前 embed.provider=none，已禁用语义向量检索")
 
-    q_vec = np.array([_embed_text(query, cfg)], dtype="float32")
-    faiss.normalize_L2(q_vec)
+    q_vec = _embed_query_vector(query, cfg) if isinstance(query, str) else query
 
     fetch_k = min(top_k, index.ntotal)
     scores, indices = index.search(q_vec, fetch_k)
@@ -1099,9 +1129,6 @@ def vsearch(
     Raises:
         FileNotFoundError: 索引文件或 ``paper_vectors`` 表不存在。
     """
-    import faiss
-    import numpy as np
-
     if _embed_provider(cfg) == "none":
         raise FileNotFoundError("当前 embed.provider=none，已禁用语义向量检索")
 
@@ -1111,21 +1138,16 @@ def vsearch(
     if not db_path.exists():
         raise FileNotFoundError(f"索引文件不存在：{db_path}\n请先运行 `scholaraio index`")
 
-    conn = sqlite3.connect(db_path)
-    try:
-        has_vectors = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='paper_vectors'"
-        ).fetchone()
-        if not has_vectors:
-            raise FileNotFoundError("向量索引不存在，请先运行 `scholaraio embed`")
-    finally:
-        conn.close()
+    _ensure_vector_search_ready(
+        db_path,
+        missing_table_msg="向量索引不存在，请先运行 `scholaraio embed`",
+        empty_msg="向量索引为空，请先运行 `scholaraio embed`",
+    )
 
+    # Load the embedding model before FAISS to avoid known faiss/torch crashes on macOS.
+    q_vec = _embed_query_vector(query, cfg)
     index, faiss_ids = _build_faiss_index(db_path)
 
-    q_vec = np.array([_embed_text(query, cfg)], dtype="float32")
-    faiss.normalize_L2(q_vec)
-
     # Fetch more candidates when post-filtering is needed
     fetch_k = top_k * 5 if (year or journal or paper_type or paper_ids) else top_k
     fetch_k = min(fetch_k, index.ntotal)

tests/test_ci_workflows.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+
+def test_macos_semantic_smoke_workflow_runs_issue_search_commands() -> None:
+    workflow = yaml.safe_load(Path(".github/workflows/macos-semantic-smoke.yml").read_text(encoding="utf-8"))
+    steps = workflow["jobs"]["semantic-smoke"]["steps"]
+
+    smoke_step = next(step for step in steps if step.get("name") == "Smoke test issue-65 explore search flow")
+    run_script = smoke_step["run"]
+
+    assert "scholaraio explore embed --name issue-65-smoke" in run_script
+    assert 'scholaraio explore search --name issue-65-smoke "boundary layer turbulence" --mode semantic' in run_script
+    assert 'scholaraio explore search --name issue-65-smoke "boundary layer turbulence" --mode unified' in run_script