diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9383fca37..82ea5c615 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -254,7 +254,7 @@ jobs: - name: Install dependencies run: | - uv pip install -e ".[dev,semantic]" + uv pip install -e ".[dev]" - name: Run tests (Semantic) run: | @@ -296,7 +296,7 @@ jobs: - name: Install dependencies run: | - uv pip install -e ".[dev,semantic]" + uv pip install -e ".[dev]" - name: Run combined coverage (SQLite + Postgres) run: | diff --git a/docs/post-v0.18.0-test-plan.md b/docs/post-v0.18.0-test-plan.md index b1a18c938..f8972c0a4 100644 --- a/docs/post-v0.18.0-test-plan.md +++ b/docs/post-v0.18.0-test-plan.md @@ -79,7 +79,7 @@ These are the most important post-`v0.18.0` feature modules currently under-cove ### Acceptance criteria - `search_type=text|vector|hybrid` returns expected ranked results on canonical semantic corpus. -- Missing semantic extras fail fast with actionable install guidance. +- Missing semantic dependencies fail fast with actionable install guidance. - Reindex and provider/model changes produce valid vectors without dimension mismatch. - SQLite and Postgres produce equivalent behavior for semantic modes on the same dataset. - Generated-column migration path is valid on SQLite environments in use. diff --git a/docs/semantic-search.md b/docs/semantic-search.md index be77776e0..92ed027a5 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -1,26 +1,26 @@ # Semantic Search -This guide covers Basic Memory's optional semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search. +This guide covers Basic Memory's semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search. ## Overview -Basic Memory's default search uses full-text search (FTS) — keyword matching with boolean operators. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling: +Basic Memory's search supports both full-text search (FTS) and semantic retrieval. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling: - **Paraphrase matching**: Find "authentication flow" when searching for "login process" - **Conceptual queries**: Search for "ways to improve performance" and find notes about caching, indexing, and optimization - **Hybrid retrieval**: Combine the precision of keyword search with the recall of semantic similarity -Semantic search is **opt-in** — existing behavior is completely unchanged unless you enable it. It works on both SQLite (local) and Postgres (cloud) backends. +Semantic search is enabled by default when semantic dependencies are available at runtime. It works on both SQLite (local) and Postgres (cloud) backends. ## Installation -Semantic search dependencies (fastembed, sqlite-vec, openai) are **optional extras** — they are not installed with the base `basic-memory` package. Install them with: +Semantic search dependencies (fastembed, sqlite-vec, openai) are included in the default `basic-memory` install. ```bash -pip install 'basic-memory[semantic]' +pip install basic-memory ``` -This keeps the base install lightweight and avoids platform-specific issues with ONNX Runtime wheels. +You can always override with `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true|false`. ### Platform Compatibility @@ -34,36 +34,40 @@ This keeps the base install lightweight and avoids platform-specific issues with #### Intel Mac Workaround -The default FastEmbed provider uses ONNX Runtime, which dropped Intel Mac (x86_64) wheels starting in v1.24. Intel Mac users have two options: +The default install includes FastEmbed, which depends on ONNX Runtime. ONNX Runtime dropped Intel Mac (x86_64) wheels starting in v1.24, so install with a compatible ONNX Runtime pin first: -**Option 1: Use OpenAI embeddings (recommended)** +```bash +pip install basic-memory 'onnxruntime<1.24' +``` -Install only the OpenAI dependency manually — no ONNX Runtime or FastEmbed needed: +After installation, Intel Mac users have two runtime options: + +**Option 1: Use OpenAI embeddings (recommended)** ```bash -pip install openai sqlite-vec export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai export OPENAI_API_KEY=sk-... ``` -**Option 2: Pin an older ONNX Runtime** +**Option 2: Use FastEmbed locally** -FastEmbed's ONNX Runtime dependency is unpinned, so you can constrain it to an older version that still ships Intel Mac wheels by passing both requirements in the same install command: +Keep the same pinned installation and use FastEmbed (default provider): ```bash -pip install 'basic-memory[semantic]' 'onnxruntime<1.24' +export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true +export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=fastembed ``` ## Quick Start -1. Install semantic extras: +1. Install Basic Memory: ```bash -pip install 'basic-memory[semantic]' +pip install basic-memory ``` -2. Enable semantic search: +2. (Optional) Explicitly enable semantic search: ```bash export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true @@ -84,7 +88,7 @@ search_notes("login process", search_type="vector") # Hybrid: combines FTS precision with vector recall (recommended) search_notes("login process", search_type="hybrid") -# Traditional full-text search (still the default) +# Explicit full-text search search_notes("login process", search_type="text") ``` @@ -94,7 +98,7 @@ All settings are fields on `BasicMemoryConfig` and can be set via environment va | Config Field | Env Var | Default | Description | |---|---|---|---| -| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | `false` | Enable semantic search. Required before vector/hybrid modes work. | +| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | Auto (`true` when semantic deps are available) | Enable semantic search. Required before vector/hybrid modes work. | | `semantic_embedding_provider` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER` | `"fastembed"` | Embedding provider: `"fastembed"` (local) or `"openai"` (API). | | `semantic_embedding_model` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_MODEL` | `"bge-small-en-v1.5"` | Model identifier. Auto-adjusted per provider if left at default. | | `semantic_embedding_dimensions` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_DIMENSIONS` | Auto-detected | Vector dimensions. 384 for FastEmbed, 1536 for OpenAI. Override only if using a non-default model. | @@ -112,8 +116,8 @@ FastEmbed runs entirely locally using ONNX models — no API key, no network cal - **Tradeoff**: Smaller model, fast inference, good quality for most use cases ```bash -# Install semantic extras and enable -pip install 'basic-memory[semantic]' +# Install basic-memory and enable semantic search +pip install basic-memory export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true ``` @@ -197,7 +201,8 @@ bm reindex -p my-project ### When You Need to Reindex -- **First enable**: After turning on `semantic_search_enabled` for the first time +- **Upgrade note**: Migration now performs a one-time automatic embedding backfill on upgrade. +- **Manual enable case**: If you explicitly had `semantic_search_enabled=false` and then turn it on - **Provider change**: After switching between `fastembed` and `openai` - **Model change**: After changing `semantic_embedding_model` - **Dimension change**: After changing `semantic_embedding_dimensions` diff --git a/justfile b/justfile index 148ff2bea..229c74be0 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ # Install dependencies install: - uv sync --extra semantic + uv sync @echo "" @echo "💡 Remember to activate the virtual environment by running: source .venv/bin/activate" diff --git a/pyproject.toml b/pyproject.toml index d44c061bf..bfbd5de73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,10 +44,6 @@ dependencies = [ "sniffio>=1.3.1", "anyio>=4.10.0", "httpx>=0.28.0", -] - -[project.optional-dependencies] -semantic = [ "fastembed>=0.7.4", "sqlite-vec>=0.1.6", "openai>=1.100.2", @@ -78,7 +74,7 @@ markers = [ "postgres: Tests that run against Postgres backend (deselect with '-m \"not postgres\"')", "windows: Windows-specific tests (deselect with '-m \"not windows\"')", "smoke: Fast end-to-end smoke tests for MCP flows", - "semantic: Tests requiring [semantic] extras (fastembed, sqlite-vec, openai)", + "semantic: Tests requiring semantic dependencies (fastembed, sqlite-vec, openai)", ] [tool.ruff] diff --git a/src/basic_memory/alembic/versions/i2c3d4e5f6g7_auto_backfill_semantic_embeddings.py b/src/basic_memory/alembic/versions/i2c3d4e5f6g7_auto_backfill_semantic_embeddings.py new file mode 100644 index 000000000..75c8694ff --- /dev/null +++ b/src/basic_memory/alembic/versions/i2c3d4e5f6g7_auto_backfill_semantic_embeddings.py @@ -0,0 +1,29 @@ +"""Trigger automatic semantic embedding backfill during migration. + +Revision ID: i2c3d4e5f6g7 +Revises: h1b2c3d4e5f6 +Create Date: 2026-02-19 00:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "i2c3d4e5f6g7" +down_revision: Union[str, None] = "h1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """No schema change. + + Trigger: this revision is newly applied. + Why: db.run_migrations() detects this revision transition and runs the existing + sync_entity_vectors() pipeline to backfill semantic embeddings automatically. + Outcome: users no longer need to run `bm reindex --embeddings` after upgrading. + """ + + +def downgrade() -> None: + """No-op downgrade.""" diff --git a/src/basic_memory/cli/commands/tool.py b/src/basic_memory/cli/commands/tool.py index 7ef289f42..875bd0d92 100644 --- a/src/basic_memory/cli/commands/tool.py +++ b/src/basic_memory/cli/commands/tool.py @@ -847,8 +847,8 @@ def search_notes( if not metadata_filters: metadata_filters = None - # set search type - search_type = "text" + # set search type (None delegates to MCP tool default selection) + search_type: str | None = None if permalink: search_type = "permalink" if query and "*" in query: diff --git a/src/basic_memory/config.py b/src/basic_memory/config.py index d48c9b055..d5f849815 100644 --- a/src/basic_memory/config.py +++ b/src/basic_memory/config.py @@ -40,8 +40,9 @@ class DatabaseBackend(str, Enum): def _default_semantic_search_enabled() -> bool: - """Enable semantic search by default when semantic extras are installed.""" - return importlib.util.find_spec("fastembed") is not None + """Enable semantic search by default when required local semantic dependencies exist.""" + required_modules = ("fastembed", "sqlite_vec") + return all(importlib.util.find_spec(module_name) is not None for module_name in required_modules) @dataclass @@ -145,7 +146,7 @@ class BasicMemoryConfig(BaseSettings): # Semantic search configuration semantic_search_enabled: bool = Field( default_factory=_default_semantic_search_enabled, - description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic extras.", + description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic dependencies (included by default).", ) semantic_embedding_provider: str = Field( default="fastembed", diff --git a/src/basic_memory/db.py b/src/basic_memory/db.py index 8247345c5..e12b8a76b 100644 --- a/src/basic_memory/db.py +++ b/src/basic_memory/db.py @@ -43,6 +43,99 @@ _engine: Optional[AsyncEngine] = None _session_maker: Optional[async_sessionmaker[AsyncSession]] = None +# Alembic revision that enables one-time automatic embedding backfill. +SEMANTIC_EMBEDDING_BACKFILL_REVISION = "i2c3d4e5f6g7" + + +async def _load_applied_alembic_revisions( + session_maker: async_sessionmaker[AsyncSession], +) -> set[str]: + """Load applied Alembic revisions from alembic_version. + + Returns an empty set when the version table does not exist yet + (fresh database before first migration). + """ + try: + async with scoped_session(session_maker) as session: + result = await session.execute(text("SELECT version_num FROM alembic_version")) + return {str(row[0]) for row in result.fetchall() if row[0]} + except Exception as exc: + error_message = str(exc).lower() + if "alembic_version" in error_message and ( + "no such table" in error_message or "does not exist" in error_message + ): + return set() + raise + + +def _should_run_semantic_embedding_backfill( + revisions_before_upgrade: set[str], + revisions_after_upgrade: set[str], +) -> bool: + """Check if this migration run newly applied the backfill-trigger revision.""" + return ( + SEMANTIC_EMBEDDING_BACKFILL_REVISION in revisions_after_upgrade + and SEMANTIC_EMBEDDING_BACKFILL_REVISION not in revisions_before_upgrade + ) + + +async def _run_semantic_embedding_backfill( + app_config: BasicMemoryConfig, + session_maker: async_sessionmaker[AsyncSession], +) -> None: + """Backfill semantic embeddings for all active projects/entities.""" + if not app_config.semantic_search_enabled: + logger.info("Skipping automatic semantic embedding backfill: semantic search is disabled.") + return + + async with scoped_session(session_maker) as session: + project_result = await session.execute( + text("SELECT id, name FROM project WHERE is_active = :is_active ORDER BY id"), + {"is_active": True}, + ) + projects = [(int(row[0]), str(row[1])) for row in project_result.fetchall()] + + if not projects: + logger.info("Skipping automatic semantic embedding backfill: no active projects found.") + return + + repository_class = ( + PostgresSearchRepository + if app_config.database_backend == DatabaseBackend.POSTGRES + else SQLiteSearchRepository + ) + + total_entities = 0 + for project_id, project_name in projects: + async with scoped_session(session_maker) as session: + entity_result = await session.execute( + text("SELECT id FROM entity WHERE project_id = :project_id ORDER BY id"), + {"project_id": project_id}, + ) + entity_ids = [int(row[0]) for row in entity_result.fetchall()] + + if not entity_ids: + continue + + total_entities += len(entity_ids) + logger.info( + "Automatic semantic embedding backfill: " + f"project={project_name}, entities={len(entity_ids)}" + ) + + search_repository = repository_class( + session_maker, + project_id=project_id, + app_config=app_config, + ) + for entity_id in entity_ids: + await search_repository.sync_entity_vectors(entity_id) + + logger.info( + "Automatic semantic embedding backfill complete: " + f"projects={len(projects)}, entities={total_entities}" + ) + class DatabaseType(Enum): """Types of supported databases.""" @@ -384,6 +477,23 @@ async def run_migrations( """ logger.info("Running database migrations...") try: + revisions_before_upgrade: set[str] = set() + # Trigger: run_migrations() can be invoked before module-level session maker is set. + # Why: we still need reliable before/after revision detection for one-time backfill. + # Outcome: create a short-lived session maker when needed, then dispose it immediately. + if _session_maker is None: + temp_engine, temp_session_maker = _create_engine_and_session( + app_config.database_path, + database_type, + app_config, + ) + try: + revisions_before_upgrade = await _load_applied_alembic_revisions(temp_session_maker) + finally: + await temp_engine.dispose() + else: + revisions_before_upgrade = await _load_applied_alembic_revisions(_session_maker) + # Get the absolute path to the alembic directory relative to this file alembic_dir = Path(__file__).parent / "alembic" config = Config() @@ -422,6 +532,13 @@ async def run_migrations( await PostgresSearchRepository(session_maker, 1).init_search_index() else: await SQLiteSearchRepository(session_maker, 1).init_search_index() + + revisions_after_upgrade = await _load_applied_alembic_revisions(session_maker) + if _should_run_semantic_embedding_backfill( + revisions_before_upgrade, + revisions_after_upgrade, + ): + await _run_semantic_embedding_backfill(app_config, session_maker) except Exception as e: # pragma: no cover logger.error(f"Error running migrations: {e}") raise diff --git a/src/basic_memory/mcp/tools/chatgpt_tools.py b/src/basic_memory/mcp/tools/chatgpt_tools.py index baad43047..f9040153b 100644 --- a/src/basic_memory/mcp/tools/chatgpt_tools.py +++ b/src/basic_memory/mcp/tools/chatgpt_tools.py @@ -120,7 +120,6 @@ async def search( project=default_project, # Use default project for ChatGPT page=1, page_size=10, # Reasonable default for ChatGPT consumption - search_type="text", # Default to full-text search output_format="json", context=context, ) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 1c3e5e841..cfae2f3b2 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -29,6 +29,11 @@ def _semantic_search_enabled_for_text_search() -> bool: return ConfigManager().config.semantic_search_enabled +def _default_search_type() -> str: + """Pick default search mode from semantic-search config.""" + return "hybrid" if _semantic_search_enabled_for_text_search() else "text" + + def _format_search_error_response( project: str, error_message: str, query: str, search_type: str = "text" ) -> str: @@ -57,7 +62,7 @@ def _format_search_error_response( Semantic retrieval is enabled but required packages are not installed. ## Fix - 1. Install semantic extras: `pip install 'basic-memory[semantic]'` + 1. Install/update Basic Memory: `pip install -U basic-memory` 2. Restart Basic Memory 3. Retry your query: `search_notes("{project}", "{query}", search_type="{search_type}")` @@ -252,7 +257,7 @@ async def search_notes( workspace: Optional[str] = None, page: int = 1, page_size: int = 10, - search_type: str = "text", + search_type: str | None = None, output_format: Literal["text", "json"] = "text", types: List[str] | None = None, entity_types: List[str] | None = None, @@ -295,8 +300,8 @@ async def search_notes( ### Search Type Examples - `search_notes("my-project", "Meeting", search_type="title")` - Search only in titles - `search_notes("work-docs", "docs/meeting-*", search_type="permalink")` - Pattern match permalinks - - `search_notes("research", "keyword", search_type="text")` - Text search (default; auto-upgrades - to hybrid when semantic search is enabled) + - `search_notes("research", "keyword")` - Default search (hybrid when semantic is enabled, + text when disabled) ### Filtering Options - `search_notes("my-project", "query", types=["entity"])` - Search only entities @@ -339,8 +344,8 @@ async def search_notes( page: The page number of results to return (default 1) page_size: The number of results to return per page (default 10) search_type: Type of search to perform, one of: - "text", "title", "permalink", "vector", "semantic", "hybrid" (default: "text"; - text mode auto-upgrades to hybrid when semantic search is enabled) + "text", "title", "permalink", "vector", "semantic", "hybrid". + Default is dynamic: "hybrid" when semantic search is enabled, otherwise "text". output_format: "text" preserves existing structured search response behavior. "json" returns a machine-readable dictionary payload. types: Optional list of note types to search (e.g., ["note", "person"]) @@ -426,9 +431,10 @@ async def search_notes( _, resolved_query, is_memory_url = await resolve_project_and_path( client, query, project, context ) + effective_search_type = search_type or _default_search_type() if is_memory_url: query = resolved_query - search_type = "permalink" + effective_search_type = "permalink" try: # Create a SearchQuery object based on the parameters @@ -436,27 +442,24 @@ async def search_notes( # Map search_type to the appropriate query field and retrieval mode valid_search_types = {"text", "title", "permalink", "vector", "semantic", "hybrid"} - if search_type == "text": + if effective_search_type == "text": search_query.text = query - # Upgrade to hybrid when semantic search is available — - # combines FTS keyword matching with vector similarity for better results - if _semantic_search_enabled_for_text_search(): - search_query.retrieval_mode = SearchRetrievalMode.HYBRID - elif search_type in ("vector", "semantic"): + search_query.retrieval_mode = SearchRetrievalMode.FTS + elif effective_search_type in ("vector", "semantic"): search_query.text = query search_query.retrieval_mode = SearchRetrievalMode.VECTOR - elif search_type == "hybrid": + elif effective_search_type == "hybrid": search_query.text = query search_query.retrieval_mode = SearchRetrievalMode.HYBRID - elif search_type == "title": + elif effective_search_type == "title": search_query.title = query - elif search_type == "permalink" and "*" in query: + elif effective_search_type == "permalink" and "*" in query: search_query.permalink_match = query - elif search_type == "permalink": + elif effective_search_type == "permalink": search_query.permalink = query else: raise ValueError( - f"Invalid search_type '{search_type}'. " + f"Invalid search_type '{effective_search_type}'. " f"Valid options: {', '.join(sorted(valid_search_types))}" ) @@ -504,7 +507,9 @@ async def search_notes( except Exception as e: logger.error(f"Search failed for query '{query}': {e}, project: {active_project.name}") # Return formatted error message as string for better user experience - return _format_search_error_response(active_project.name, str(e), query, search_type) + return _format_search_error_response( + active_project.name, str(e), query, effective_search_type + ) @mcp.tool( diff --git a/src/basic_memory/mcp/tools/ui_sdk.py b/src/basic_memory/mcp/tools/ui_sdk.py index 985a84a4d..6c4da5d04 100644 --- a/src/basic_memory/mcp/tools/ui_sdk.py +++ b/src/basic_memory/mcp/tools/ui_sdk.py @@ -26,7 +26,7 @@ async def search_notes_ui( project: Optional[str] = None, page: int = 1, page_size: int = 10, - search_type: str = "text", + search_type: Optional[str] = None, types: List[str] | None = None, entity_types: List[str] | None = None, after_date: Optional[str] = None, diff --git a/src/basic_memory/repository/fastembed_provider.py b/src/basic_memory/repository/fastembed_provider.py index b59673b54..b8c2bde39 100644 --- a/src/basic_memory/repository/fastembed_provider.py +++ b/src/basic_memory/repository/fastembed_provider.py @@ -48,7 +48,8 @@ def _create_model() -> "TextEmbedding": ) as exc: # pragma: no cover - exercised via tests with monkeypatch raise SemanticDependenciesMissingError( "fastembed package is missing. " - "Install semantic extras: pip install 'basic-memory[semantic]'" + "Install/update basic-memory to include semantic dependencies: " + "pip install -U basic-memory" ) from exc resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name) return TextEmbedding(model_name=resolved_model_name) diff --git a/src/basic_memory/repository/openai_provider.py b/src/basic_memory/repository/openai_provider.py index 65a6021bc..69c4acfa6 100644 --- a/src/basic_memory/repository/openai_provider.py +++ b/src/basic_memory/repository/openai_provider.py @@ -45,7 +45,8 @@ async def _get_client(self) -> Any: except ImportError as exc: # pragma: no cover - covered via monkeypatch tests raise SemanticDependenciesMissingError( "OpenAI dependency is missing. " - "Install semantic extras: pip install 'basic-memory[semantic]'" + "Install/update basic-memory to include semantic dependencies: " + "pip install -U basic-memory" ) from exc api_key = self._api_key or os.getenv("OPENAI_API_KEY") diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 4919a68b5..0f573a04d 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -355,7 +355,8 @@ def _assert_semantic_available(self) -> None: if self._embedding_provider is None: raise SemanticDependenciesMissingError( "No embedding provider configured. " - "Install semantic extras: pip install 'basic-memory[semantic]' " + "Install/update basic-memory to include semantic dependencies " + "(pip install -U basic-memory) " "and set semantic_search_enabled=true." ) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index ecb540524..a0bdb20d7 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -347,7 +347,8 @@ async def _ensure_sqlite_vec_loaded(self, session) -> None: except ImportError as exc: raise SemanticDependenciesMissingError( "sqlite-vec package is missing. " - "Install semantic extras: pip install 'basic-memory[semantic]'" + "Install/update basic-memory to include semantic dependencies: " + "pip install -U basic-memory" ) from exc async with self._sqlite_vec_lock: diff --git a/test-int/semantic/conftest.py b/test-int/semantic/conftest.py index 75709dbb4..4369906b0 100644 --- a/test-int/semantic/conftest.py +++ b/test-int/semantic/conftest.py @@ -95,11 +95,11 @@ def skip_if_needed(combo: SearchCombo) -> None: pytest.skip("Docker not available for Postgres testcontainer") if combo.provider_name == "fastembed" and not _fastembed_available(): - pytest.skip("fastembed not installed (install basic-memory[semantic])") + pytest.skip("fastembed not installed (install/update basic-memory)") if combo.provider_name == "openai": if not _fastembed_available(): - pytest.skip("semantic extras not installed") + pytest.skip("semantic dependencies not installed") if not _openai_key_available(): pytest.skip("OPENAI_API_KEY not set") diff --git a/tests/mcp/test_tool_search.py b/tests/mcp/test_tool_search.py index cf20b4df0..e0261cf5a 100644 --- a/tests/mcp/test_tool_search.py +++ b/tests/mcp/test_tool_search.py @@ -6,7 +6,7 @@ from basic_memory.mcp.tools import write_note from basic_memory.mcp.tools.search import search_notes, _format_search_error_response -from basic_memory.schemas.search import SearchResponse +from basic_memory.schemas.search import SearchItemType, SearchResponse @pytest.mark.asyncio @@ -331,13 +331,13 @@ def test_format_search_error_semantic_dependencies_missing(self): """Test formatting for missing semantic dependencies.""" result = _format_search_error_response( "test-project", - "fastembed package is missing. Install semantic extras: pip install 'basic-memory[semantic]'", + "fastembed package is missing. Install/update basic-memory to include semantic dependencies: pip install -U basic-memory", "semantic query", "hybrid", ) assert "# Search Failed - Semantic Dependencies Missing" in result - assert "pip install 'basic-memory[semantic]'" in result + assert "pip install -U basic-memory" in result def test_format_search_error_generic(self): """Test formatting for generic errors.""" @@ -615,7 +615,7 @@ async def fake_get_project_client(*args, **kwargs): title=f"Item {i}", permalink=f"item-{i}", file_path=f"item-{i}.md", - type="entity", + type=SearchItemType.ENTITY, score=1.0 - i * 0.1, ) for i in range(5) @@ -775,8 +775,8 @@ async def search(self, payload, page, page_size): @pytest.mark.asyncio -async def test_search_notes_text_upgrades_to_hybrid_when_semantic_enabled(monkeypatch): - """Default text search should auto-upgrade to hybrid when semantic search is enabled.""" +async def test_search_notes_defaults_to_hybrid_when_semantic_enabled(monkeypatch): + """When search_type is omitted, semantic-enabled configs should default to hybrid.""" import importlib from dataclasses import dataclass @@ -817,7 +817,7 @@ class StubConfig: @dataclass class StubContainer: - config: StubConfig = None + config: StubConfig | None = None def __post_init__(self): if self.config is None: @@ -828,17 +828,16 @@ def __post_init__(self): await search_mod.search_notes.fn( project="test-project", query="test query", - search_type="text", ) - # Default text search should have been upgraded to hybrid + # Default mode should be hybrid when semantic search is enabled assert captured_payload["retrieval_mode"] == "hybrid" assert captured_payload["text"] == "test query" @pytest.mark.asyncio -async def test_search_notes_text_stays_fts_when_semantic_disabled(monkeypatch): - """Default text search should stay FTS when semantic search is disabled.""" +async def test_search_notes_defaults_to_fts_when_semantic_disabled(monkeypatch): + """When search_type is omitted, semantic-disabled configs should default to FTS.""" import importlib from dataclasses import dataclass @@ -879,7 +878,67 @@ class StubConfig: @dataclass class StubContainer: - config: StubConfig = None + config: StubConfig | None = None + + def __post_init__(self): + if self.config is None: + self.config = StubConfig() + + monkeypatch.setattr(search_mod, "get_container", lambda: StubContainer()) + + await search_mod.search_notes.fn( + project="test-project", + query="test query", + ) + + # Default mode should be FTS when semantic search is disabled + assert captured_payload["retrieval_mode"] == "fts" + assert captured_payload["text"] == "test query" + + +@pytest.mark.asyncio +async def test_search_notes_explicit_text_stays_fts_when_semantic_enabled(monkeypatch): + """Explicit text mode should preserve FTS behavior even when semantic is enabled.""" + import importlib + from dataclasses import dataclass + + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + clients_mod = importlib.import_module("basic_memory.mcp.clients") + + class StubProject: + name = "test-project" + external_id = "test-external-id" + + @asynccontextmanager + async def fake_get_project_client(*args, **kwargs): + yield (object(), StubProject()) + + async def fake_resolve_project_and_path( + client, identifier, project=None, context=None, headers=None + ): + return StubProject(), identifier, False + + captured_payload: dict = {} + + class MockSearchClient: + def __init__(self, *args, **kwargs): + pass + + async def search(self, payload, page, page_size): + captured_payload.update(payload) + return SearchResponse(results=[], current_page=page, page_size=page_size) + + monkeypatch.setattr(search_mod, "get_project_client", fake_get_project_client) + monkeypatch.setattr(search_mod, "resolve_project_and_path", fake_resolve_project_and_path) + monkeypatch.setattr(clients_mod, "SearchClient", MockSearchClient) + + @dataclass + class StubConfig: + semantic_search_enabled: bool = True + + @dataclass + class StubContainer: + config: StubConfig | None = None def __post_init__(self): if self.config is None: @@ -893,14 +952,13 @@ def __post_init__(self): search_type="text", ) - # Should stay as default FTS (no retrieval_mode override) assert captured_payload["retrieval_mode"] == "fts" assert captured_payload["text"] == "test query" @pytest.mark.asyncio -async def test_search_notes_text_upgrades_to_hybrid_when_container_not_initialized(monkeypatch): - """Default text search should upgrade to hybrid in CLI contexts when semantic is enabled.""" +async def test_search_notes_defaults_to_hybrid_when_container_not_initialized(monkeypatch): + """CLI fallback config should still default omitted search_type to hybrid.""" import importlib search_mod = importlib.import_module("basic_memory.mcp.tools.search") @@ -951,7 +1009,6 @@ def raise_runtime_error(): await search_mod.search_notes.fn( project="test-project", query="test query", - search_type="text", ) # Should upgrade using ConfigManager fallback @@ -960,10 +1017,10 @@ def raise_runtime_error(): @pytest.mark.asyncio -async def test_search_notes_text_stays_fts_when_container_not_initialized_and_semantic_disabled( +async def test_search_notes_defaults_to_fts_when_container_not_initialized_and_semantic_disabled( monkeypatch, ): - """Default text search should remain FTS in CLI contexts when semantic is disabled.""" + """CLI fallback config should default omitted search_type to FTS when semantic is disabled.""" import importlib search_mod = importlib.import_module("basic_memory.mcp.tools.search") @@ -1013,7 +1070,6 @@ def raise_runtime_error(): await search_mod.search_notes.fn( project="test-project", query="test query", - search_type="text", ) assert captured_payload["retrieval_mode"] == "fts" diff --git a/tests/mcp/tools/test_chatgpt_tools.py b/tests/mcp/tools/test_chatgpt_tools.py index d3f952696..15c9edbe5 100644 --- a/tests/mcp/tools/test_chatgpt_tools.py +++ b/tests/mcp/tools/test_chatgpt_tools.py @@ -66,6 +66,25 @@ async def fake_search_notes_fn(*args, **kwargs): assert "error_details" in content +@pytest.mark.asyncio +async def test_search_uses_dynamic_default_search_type(monkeypatch, client, test_project): + """ChatGPT adapter should not hardcode search_type so search_notes can pick defaults.""" + import basic_memory.mcp.tools.chatgpt_tools as chatgpt_tools + + captured_kwargs: dict = {} + + async def fake_search_notes_fn(*args, **kwargs): + captured_kwargs.update(kwargs) + return {"results": []} + + monkeypatch.setattr(chatgpt_tools.search_notes, "fn", fake_search_notes_fn) + + result = await chatgpt_tools.search.fn("default search mode query") + + assert isinstance(result, list) + assert "search_type" not in captured_kwargs + + @pytest.mark.asyncio async def test_fetch_successful_document(client, test_project): """Test fetch with successful document retrieval.""" diff --git a/tests/repository/test_fastembed_provider.py b/tests/repository/test_fastembed_provider.py index a10c5aa71..79762c72b 100644 --- a/tests/repository/test_fastembed_provider.py +++ b/tests/repository/test_fastembed_provider.py @@ -84,4 +84,4 @@ def _raising_import(name, globals=None, locals=None, fromlist=(), level=0): with pytest.raises(SemanticDependenciesMissingError) as error: await provider.embed_query("test") - assert "pip install 'basic-memory[semantic]'" in str(error.value) + assert "pip install -U basic-memory" in str(error.value) diff --git a/tests/repository/test_openai_provider.py b/tests/repository/test_openai_provider.py index 8fb8dafb3..50a3d52b6 100644 --- a/tests/repository/test_openai_provider.py +++ b/tests/repository/test_openai_provider.py @@ -92,7 +92,7 @@ def _raising_import(name, globals=None, locals=None, fromlist=(), level=0): with pytest.raises(SemanticDependenciesMissingError) as error: await provider.embed_query("test") - assert "pip install 'basic-memory[semantic]'" in str(error.value) + assert "pip install -U basic-memory" in str(error.value) @pytest.mark.asyncio diff --git a/tests/services/test_initialization.py b/tests/services/test_initialization.py index 0ebfc84ed..469c6970e 100644 --- a/tests/services/test_initialization.py +++ b/tests/services/test_initialization.py @@ -6,6 +6,7 @@ from __future__ import annotations +from datetime import datetime from unittest.mock import AsyncMock import pytest @@ -196,3 +197,162 @@ def capture_warning(message: str) -> None: "ensure_frontmatter_on_sync=True overrides disable_permalinks=True" in message for message in warnings ) + + +@pytest.mark.asyncio +async def test_run_migrations_triggers_embedding_backfill_on_new_revision( + monkeypatch, app_config: BasicMemoryConfig +): + """When the trigger revision is newly applied, run automatic embedding backfill once.""" + + class StubSearchRepository: + def __init__(self, *args, **kwargs): + pass + + async def init_search_index(self): + return None + + original_session_maker = db._session_maker # pyright: ignore [reportPrivateUsage] + try: + session_marker = object() + db._session_maker = session_marker # pyright: ignore [reportPrivateUsage] + + monkeypatch.setattr( + "basic_memory.db.command.upgrade", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr("basic_memory.db.SQLiteSearchRepository", StubSearchRepository) + monkeypatch.setattr("basic_memory.db.PostgresSearchRepository", StubSearchRepository) + + load_revisions_mock = AsyncMock( + side_effect=[ + set(), + {db.SEMANTIC_EMBEDDING_BACKFILL_REVISION}, + ] + ) + backfill_mock = AsyncMock() + monkeypatch.setattr("basic_memory.db._load_applied_alembic_revisions", load_revisions_mock) + monkeypatch.setattr("basic_memory.db._run_semantic_embedding_backfill", backfill_mock) + + await db.run_migrations(app_config) + + assert load_revisions_mock.await_count == 2 + backfill_mock.assert_awaited_once_with(app_config, session_marker) + finally: + db._session_maker = original_session_maker # pyright: ignore [reportPrivateUsage] + + +@pytest.mark.asyncio +async def test_run_migrations_skips_embedding_backfill_when_revision_already_applied( + monkeypatch, app_config: BasicMemoryConfig +): + """If the trigger revision was already present before upgrade, skip backfill.""" + + class StubSearchRepository: + def __init__(self, *args, **kwargs): + pass + + async def init_search_index(self): + return None + + original_session_maker = db._session_maker # pyright: ignore [reportPrivateUsage] + try: + session_marker = object() + db._session_maker = session_marker # pyright: ignore [reportPrivateUsage] + + monkeypatch.setattr( + "basic_memory.db.command.upgrade", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr("basic_memory.db.SQLiteSearchRepository", StubSearchRepository) + monkeypatch.setattr("basic_memory.db.PostgresSearchRepository", StubSearchRepository) + + load_revisions_mock = AsyncMock( + side_effect=[ + {db.SEMANTIC_EMBEDDING_BACKFILL_REVISION}, + {db.SEMANTIC_EMBEDDING_BACKFILL_REVISION}, + ] + ) + backfill_mock = AsyncMock() + monkeypatch.setattr("basic_memory.db._load_applied_alembic_revisions", load_revisions_mock) + monkeypatch.setattr("basic_memory.db._run_semantic_embedding_backfill", backfill_mock) + + await db.run_migrations(app_config) + + assert load_revisions_mock.await_count == 2 + assert backfill_mock.await_count == 0 + finally: + db._session_maker = original_session_maker # pyright: ignore [reportPrivateUsage] + + +@pytest.mark.asyncio +async def test_semantic_embedding_backfill_syncs_each_entity( + monkeypatch, + app_config: BasicMemoryConfig, + session_maker, + test_project, +): + """Automatic backfill should run sync_entity_vectors for every entity in active projects.""" + from basic_memory.repository.entity_repository import EntityRepository + + entity_repository = EntityRepository(session_maker, project_id=test_project.id) + created_entity_ids: list[int] = [] + for i in range(3): + entity = await entity_repository.create( + { + "title": f"Backfill Entity {i}", + "entity_type": "note", + "entity_metadata": {}, + "content_type": "text/markdown", + "file_path": f"test/backfill-{i}.md", + "permalink": f"test/backfill-{i}", + "project_id": test_project.id, + "created_at": datetime.now(), + "updated_at": datetime.now(), + } + ) + created_entity_ids.append(entity.id) + + synced_pairs: list[tuple[int, int]] = [] + + class StubSearchRepository: + def __init__(self, _session_maker, project_id: int, app_config=None): + self.project_id = project_id + + async def sync_entity_vectors(self, entity_id: int) -> None: + synced_pairs.append((self.project_id, entity_id)) + + monkeypatch.setattr("basic_memory.db.SQLiteSearchRepository", StubSearchRepository) + monkeypatch.setattr("basic_memory.db.PostgresSearchRepository", StubSearchRepository) + + app_config.semantic_search_enabled = True + + await db._run_semantic_embedding_backfill(app_config, session_maker) # pyright: ignore [reportPrivateUsage] + + expected_pairs = {(test_project.id, entity_id) for entity_id in created_entity_ids} + assert expected_pairs.issubset(set(synced_pairs)) + + +@pytest.mark.asyncio +async def test_semantic_embedding_backfill_skips_when_semantic_disabled( + monkeypatch, + app_config: BasicMemoryConfig, + session_maker, +): + """Automatic backfill should no-op when semantic search is disabled.""" + called = False + + class StubSearchRepository: + def __init__(self, *args, **kwargs): + nonlocal called + called = True + + async def sync_entity_vectors(self, entity_id: int) -> None: # pragma: no cover + return None + + monkeypatch.setattr("basic_memory.db.SQLiteSearchRepository", StubSearchRepository) + monkeypatch.setattr("basic_memory.db.PostgresSearchRepository", StubSearchRepository) + + app_config.semantic_search_enabled = False + await db._run_semantic_embedding_backfill(app_config, session_maker) # pyright: ignore [reportPrivateUsage] + assert called is False diff --git a/tests/test_config.py b/tests/test_config.py index a8ea430bc..63961b5ef 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -603,29 +603,33 @@ def test_model_post_init_uses_platform_native_separators(self, config_home, monk class TestSemanticSearchConfig: """Test semantic search configuration options.""" - def test_semantic_search_enabled_defaults_to_true_when_fastembed_is_available( + def test_semantic_search_enabled_defaults_to_true_when_semantic_modules_are_available( self, monkeypatch ): - """Semantic search defaults on when fastembed is importable.""" + """Semantic search defaults on when fastembed and sqlite_vec are importable.""" import basic_memory.config as config_module monkeypatch.delenv("BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED", raising=False) monkeypatch.setattr( config_module.importlib.util, "find_spec", - lambda name: object() if name == "fastembed" else None, + lambda name: object() if name in {"fastembed", "sqlite_vec"} else None, ) config = BasicMemoryConfig() assert config.semantic_search_enabled is True - def test_semantic_search_enabled_defaults_to_false_when_fastembed_is_unavailable( + def test_semantic_search_enabled_defaults_to_false_when_any_semantic_module_is_unavailable( self, monkeypatch ): - """Semantic search defaults off when fastembed is not importable.""" + """Semantic search defaults off when required semantic modules are missing.""" import basic_memory.config as config_module monkeypatch.delenv("BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED", raising=False) - monkeypatch.setattr(config_module.importlib.util, "find_spec", lambda name: None) + monkeypatch.setattr( + config_module.importlib.util, + "find_spec", + lambda name: object() if name == "fastembed" else None, + ) config = BasicMemoryConfig() assert config.semantic_search_enabled is False diff --git a/uv.lock b/uv.lock index db1eef26b..44e8d1467 100644 --- a/uv.lock +++ b/uv.lock @@ -151,6 +151,7 @@ dependencies = [ { name = "asyncpg" }, { name = "dateparser" }, { name = "fastapi", extra = ["standard"] }, + { name = "fastembed" }, { name = "fastmcp" }, { name = "greenlet" }, { name = "httpx" }, @@ -161,6 +162,7 @@ dependencies = [ { name = "mdformat-frontmatter" }, { name = "mdformat-gfm" }, { name = "nest-asyncio" }, + { name = "openai" }, { name = "pillow" }, { name = "psycopg" }, { name = "pybars3" }, @@ -176,18 +178,12 @@ dependencies = [ { name = "rich" }, { name = "sniffio" }, { name = "sqlalchemy" }, + { name = "sqlite-vec" }, { name = "typer" }, { name = "unidecode" }, { name = "watchfiles" }, ] -[package.optional-dependencies] -semantic = [ - { name = "fastembed" }, - { name = "openai" }, - { name = "sqlite-vec" }, -] - [package.dev-dependencies] dev = [ { name = "freezegun" }, @@ -214,7 +210,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.30.0" }, { name = "dateparser", specifier = ">=1.2.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.115.8" }, - { name = "fastembed", marker = "extra == 'semantic'", specifier = ">=0.7.4" }, + { name = "fastembed", specifier = ">=0.7.4" }, { name = "fastmcp", specifier = "==2.12.3" }, { name = "greenlet", specifier = ">=3.1.1" }, { name = "httpx", specifier = ">=0.28.0" }, @@ -225,7 +221,7 @@ requires-dist = [ { name = "mdformat-frontmatter", specifier = ">=2.0.8" }, { name = "mdformat-gfm", specifier = ">=0.3.7" }, { name = "nest-asyncio", specifier = ">=1.6.0" }, - { name = "openai", marker = "extra == 'semantic'", specifier = ">=1.100.2" }, + { name = "openai", specifier = ">=1.100.2" }, { name = "pillow", specifier = ">=11.1.0" }, { name = "psycopg", specifier = "==3.3.1" }, { name = "pybars3", specifier = ">=0.9.7" }, @@ -241,12 +237,11 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "sniffio", specifier = ">=1.3.1" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, - { name = "sqlite-vec", marker = "extra == 'semantic'", specifier = ">=0.1.6" }, + { name = "sqlite-vec", specifier = ">=0.1.6" }, { name = "typer", specifier = ">=0.9.0" }, { name = "unidecode", specifier = ">=1.3.8" }, { name = "watchfiles", specifier = ">=1.0.4" }, ] -provides-extras = ["semantic"] [package.metadata.requires-dev] dev = [