diff --git a/src/services/sqlite/SessionSearch.ts b/src/services/sqlite/SessionSearch.ts index 08f704719..0aabe298a 100644 --- a/src/services/sqlite/SessionSearch.ts +++ b/src/services/sqlite/SessionSearch.ts @@ -17,8 +17,8 @@ import { /** * Search interface for session-based memory - * Provides filter-only structured queries for sessions, observations, and user prompts - * Vector search is handled by ChromaDB - this class only supports filtering without query text + * Provides structured queries for sessions, observations, and user prompts. + * Supports filter-only queries and FTS5/LIKE keyword search as a fallback when Chroma is unavailable. */ export class SessionSearch { private db: Database; @@ -178,6 +178,39 @@ export class SessionSearch { } } + /** + * Check if FTS5 tables exist for text search. + * Result is cached after first check. + */ + private _hasFtsTables: boolean | null = null; + private hasFtsTables(): boolean { + if (this._hasFtsTables !== null) return this._hasFtsTables; + try { + const tables = this.db.prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('observations_fts', 'session_summaries_fts')" + ).all() as TableNameRow[]; + this._hasFtsTables = tables.length >= 2; + } catch { + this._hasFtsTables = false; + } + return this._hasFtsTables; + } + + /** + * Sanitize a query string for safe use in FTS5 MATCH expressions. + * Strips FTS5 operators and wraps each token in double quotes. + */ + private sanitizeFtsQuery(query: string): string { + // Remove FTS5 special characters/operators, keep alphanumeric and whitespace + const tokens = query + .replace(/[^\w\s]/g, ' ') + .split(/\s+/) + .filter(t => t.length > 0); + if (tokens.length === 0) return ''; + // Join with spaces (implicit AND in FTS5) + return tokens.map(t => `"${t}"`).join(' '); + } + /** * Build WHERE clause for structured filters @@ -271,15 +304,14 @@ export class SessionSearch { } /** - * Search observations using filter-only direct SQLite query. - * Vector search is handled by ChromaDB - this only supports filtering without query text. + * Search observations using direct SQLite query. + * Supports filter-only queries and FTS5 keyword search as a fallback when Chroma is unavailable. */ searchObservations(query: string | undefined, options: SearchOptions = {}): ObservationSearchResult[] { const params: any[] = []; const { limit = 50, offset = 0, orderBy = 'relevance', ...filters } = options; // FILTER-ONLY PATH: When no query text, query table directly - // This enables date filtering which Chroma cannot do (requires direct SQLite access) if (!query) { const filterClause = this.buildFilterClause(filters, params, 'o'); if (!filterClause) { @@ -300,15 +332,57 @@ export class SessionSearch { return this.db.prepare(sql).all(...params) as ObservationSearchResult[]; } - // Vector search with query text should be handled by ChromaDB - // This method only supports filter-only queries (query=undefined) - logger.warn('DB', 'Text search not supported - use ChromaDB for vector search'); - return []; + // TEXT QUERY PATH: Use FTS5 keyword matching when available + if (this.hasFtsTables()) { + const ftsQuery = this.sanitizeFtsQuery(query); + if (ftsQuery) { + logger.debug('DB', 'Using FTS5 keyword search for observations', { query: ftsQuery }); + params.push(ftsQuery); + const filterClause = this.buildFilterClause(filters, params, 'o'); + const whereExtra = filterClause ? ` AND ${filterClause}` : ''; + const orderClause = this.buildOrderClause(orderBy, true, 'observations_fts'); + + const sql = ` + SELECT o.*, o.discovery_tokens + FROM observations o + JOIN observations_fts ON observations_fts.rowid = o.id + WHERE observations_fts MATCH ?${whereExtra} + ${orderClause} + LIMIT ? OFFSET ? + `; + + params.push(limit, offset); + try { + return this.db.prepare(sql).all(...params) as ObservationSearchResult[]; + } catch (error) { + logger.warn('DB', 'FTS5 query failed, falling back to LIKE search', {}, error as Error); + } + } + } + + // LIKE FALLBACK: When FTS5 is not available, use LIKE-based text search + logger.debug('DB', 'Using LIKE fallback for observation text search', { query }); + const likeParam = `%${query}%`; + params.push(likeParam, likeParam, likeParam, likeParam); + const filterClause = this.buildFilterClause(filters, params, 'o'); + const whereExtra = filterClause ? ` AND ${filterClause}` : ''; + const orderClause = this.buildOrderClause(orderBy, false); + + const sql = ` + SELECT o.*, o.discovery_tokens + FROM observations o + WHERE (o.title LIKE ? OR o.narrative LIKE ? OR o.text LIKE ? OR o.facts LIKE ?)${whereExtra} + ${orderClause} + LIMIT ? OFFSET ? + `; + + params.push(limit, offset); + return this.db.prepare(sql).all(...params) as ObservationSearchResult[]; } /** - * Search session summaries using filter-only direct SQLite query. - * Vector search is handled by ChromaDB - this only supports filtering without query text. + * Search session summaries using direct SQLite query. + * Supports filter-only queries and FTS5 keyword search as a fallback when Chroma is unavailable. */ searchSessions(query: string | undefined, options: SearchOptions = {}): SessionSummarySearchResult[] { const params: any[] = []; @@ -339,10 +413,62 @@ export class SessionSearch { return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[]; } - // Vector search with query text should be handled by ChromaDB - // This method only supports filter-only queries (query=undefined) - logger.warn('DB', 'Text search not supported - use ChromaDB for vector search'); - return []; + // TEXT QUERY PATH: Use FTS5 keyword matching when available + if (this.hasFtsTables()) { + const ftsQuery = this.sanitizeFtsQuery(query); + if (ftsQuery) { + logger.debug('DB', 'Using FTS5 keyword search for sessions', { query: ftsQuery }); + params.push(ftsQuery); + const filterOptions = { ...filters }; + delete filterOptions.type; + const filterClause = this.buildFilterClause(filterOptions, params, 's'); + const whereExtra = filterClause ? ` AND ${filterClause}` : ''; + const orderClause = orderBy === 'relevance' + ? 'ORDER BY session_summaries_fts.rank ASC' + : orderBy === 'date_asc' + ? 'ORDER BY s.created_at_epoch ASC' + : 'ORDER BY s.created_at_epoch DESC'; + + const sql = ` + SELECT s.*, s.discovery_tokens + FROM session_summaries s + JOIN session_summaries_fts ON session_summaries_fts.rowid = s.id + WHERE session_summaries_fts MATCH ?${whereExtra} + ${orderClause} + LIMIT ? OFFSET ? + `; + + params.push(limit, offset); + try { + return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[]; + } catch (error) { + logger.warn('DB', 'FTS5 query failed for sessions, falling back to LIKE search', {}, error as Error); + } + } + } + + // LIKE FALLBACK: When FTS5 is not available, use LIKE-based text search + logger.debug('DB', 'Using LIKE fallback for session text search', { query }); + const likeParam = `%${query}%`; + const filterOptions = { ...filters }; + delete filterOptions.type; + params.push(likeParam, likeParam, likeParam, likeParam); + const filterClause = this.buildFilterClause(filterOptions, params, 's'); + const whereExtra = filterClause ? ` AND ${filterClause}` : ''; + const orderClause = orderBy === 'date_asc' + ? 'ORDER BY s.created_at_epoch ASC' + : 'ORDER BY s.created_at_epoch DESC'; + + const sql = ` + SELECT s.*, s.discovery_tokens + FROM session_summaries s + WHERE (s.request LIKE ? OR s.investigated LIKE ? OR s.learned LIKE ? OR s.completed LIKE ?)${whereExtra} + ${orderClause} + LIMIT ? OFFSET ? + `; + + params.push(limit, offset); + return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[]; } /** @@ -523,8 +649,8 @@ export class SessionSearch { } /** - * Search user prompts using filter-only direct SQLite query. - * Vector search is handled by ChromaDB - this only supports filtering without query text. + * Search user prompts using direct SQLite query. + * Supports filter-only queries and LIKE-based keyword search as a fallback when Chroma is unavailable. */ searchUserPrompts(query: string | undefined, options: SearchOptions = {}): UserPromptSearchResult[] { const params: any[] = []; @@ -575,10 +701,28 @@ export class SessionSearch { return this.db.prepare(sql).all(...params) as UserPromptSearchResult[]; } - // Vector search with query text should be handled by ChromaDB - // This method only supports filter-only queries (query=undefined) - logger.warn('DB', 'Text search not supported - use ChromaDB for vector search'); - return []; + // TEXT QUERY PATH: Use LIKE-based search for user prompts (no FTS5 table) + logger.debug('DB', 'Using LIKE search for user prompts', { query }); + const likeParam = `%${query}%`; + baseConditions.push('up.prompt_text LIKE ?'); + params.push(likeParam); + + const whereClause = `WHERE ${baseConditions.join(' AND ')}`; + const orderClause = orderBy === 'date_asc' + ? 'ORDER BY up.created_at_epoch ASC' + : 'ORDER BY up.created_at_epoch DESC'; + + const sql = ` + SELECT up.* + FROM user_prompts up + JOIN sdk_sessions s ON up.content_session_id = s.content_session_id + ${whereClause} + ${orderClause} + LIMIT ? OFFSET ? + `; + + params.push(limit, offset); + return this.db.prepare(sql).all(...params) as UserPromptSearchResult[]; } /** diff --git a/src/services/worker/SearchManager.ts b/src/services/worker/SearchManager.ts index 337b2df92..5ab1207a2 100644 --- a/src/services/worker/SearchManager.ts +++ b/src/services/worker/SearchManager.ts @@ -252,14 +252,19 @@ export class SearchManager { logger.debug('SEARCH', 'ChromaDB found no matches (final result, no FTS5 fallback)', {}); } } - // ChromaDB not initialized - mark as failed to show proper error message + // PATH 3: ChromaDB not initialized - fall back to FTS5/LIKE keyword search else if (query) { - chromaFailed = true; - logger.debug('SEARCH', 'ChromaDB not initialized - semantic search unavailable', {}); - logger.debug('SEARCH', 'Install UVX/Python to enable vector search', { url: 'https://docs.astral.sh/uv/getting-started/installation/' }); - observations = []; - sessions = []; - prompts = []; + logger.debug('SEARCH', 'ChromaDB not initialized - falling back to FTS5/LIKE keyword search', {}); + const obsOptions = { ...options, type: obs_type, concepts, files }; + if (searchObservations) { + observations = this.sessionSearch.searchObservations(query, obsOptions); + } + if (searchSessions) { + sessions = this.sessionSearch.searchSessions(query, options); + } + if (searchPrompts) { + prompts = this.sessionSearch.searchUserPrompts(query, options); + } } const totalResults = observations.length + sessions.length + prompts.length; diff --git a/src/services/worker/search/SearchOrchestrator.ts b/src/services/worker/search/SearchOrchestrator.ts index d0c82c629..3b4ffb4cb 100644 --- a/src/services/worker/search/SearchOrchestrator.ts +++ b/src/services/worker/search/SearchOrchestrator.ts @@ -110,13 +110,12 @@ export class SearchOrchestrator { }; } - // PATH 3: No Chroma available - logger.debug('SEARCH', 'Orchestrator: Chroma not available', {}); + // PATH 3: No Chroma available - fall back to SQLite FTS5/LIKE keyword search + logger.debug('SEARCH', 'Orchestrator: Chroma not available, falling back to SQLite keyword search', {}); + const fallbackResult = await this.sqliteStrategy.search(options); return { - results: { observations: [], sessions: [], prompts: [] }, - usedChroma: false, - fellBack: false, - strategy: 'sqlite' + ...fallbackResult, + fellBack: true }; } diff --git a/src/services/worker/search/strategies/SQLiteSearchStrategy.ts b/src/services/worker/search/strategies/SQLiteSearchStrategy.ts index ba9ffc3b9..5b83635d7 100644 --- a/src/services/worker/search/strategies/SQLiteSearchStrategy.ts +++ b/src/services/worker/search/strategies/SQLiteSearchStrategy.ts @@ -37,6 +37,7 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt async search(options: StrategySearchOptions): Promise { const { + query, searchType = 'all', obsType, concepts, @@ -58,7 +59,8 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt const baseOptions = { limit, offset, orderBy, project, dateRange }; - logger.debug('SEARCH', 'SQLiteSearchStrategy: Filter-only query', { + logger.debug('SEARCH', 'SQLiteSearchStrategy: query', { + hasQuery: !!query, searchType, hasDateRange: !!dateRange, hasProject: !!project @@ -72,15 +74,15 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt concepts, files }; - observations = this.sessionSearch.searchObservations(undefined, obsOptions); + observations = this.sessionSearch.searchObservations(query, obsOptions); } if (searchSessions) { - sessions = this.sessionSearch.searchSessions(undefined, baseOptions); + sessions = this.sessionSearch.searchSessions(query, baseOptions); } if (searchPrompts) { - prompts = this.sessionSearch.searchUserPrompts(undefined, baseOptions); + prompts = this.sessionSearch.searchUserPrompts(query, baseOptions); } logger.debug('SEARCH', 'SQLiteSearchStrategy: Results', { diff --git a/tests/worker/search/search-orchestrator.test.ts b/tests/worker/search/search-orchestrator.test.ts index 745c50ab3..c089a8b2b 100644 --- a/tests/worker/search/search-orchestrator.test.ts +++ b/tests/worker/search/search-orchestrator.test.ts @@ -313,14 +313,17 @@ describe('SearchOrchestrator', () => { }); describe('search', () => { - it('should return empty results for query search without Chroma', async () => { + it('should fall back to SQLite FTS5/LIKE search for query without Chroma', async () => { const result = await orchestrator.search({ query: 'semantic query' }); - // No Chroma available, can't do semantic search - expect(result.results.observations).toHaveLength(0); + // No Chroma available, falls back to SQLite keyword search + expect(result.results.observations).toHaveLength(1); expect(result.usedChroma).toBe(false); + expect(result.fellBack).toBe(true); + expect(result.strategy).toBe('sqlite'); + expect(mockSessionSearch.searchObservations).toHaveBeenCalledWith('semantic query', expect.any(Object)); }); it('should still work for filter-only queries', async () => { diff --git a/tests/worker/search/strategies/sqlite-search-strategy.test.ts b/tests/worker/search/strategies/sqlite-search-strategy.test.ts index a3269dff9..e4fc89ac9 100644 --- a/tests/worker/search/strategies/sqlite-search-strategy.test.ts +++ b/tests/worker/search/strategies/sqlite-search-strategy.test.ts @@ -226,6 +226,39 @@ describe('SQLiteSearchStrategy', () => { expect(result.results.prompts).toHaveLength(0); expect(result.usedChroma).toBe(false); }); + + it('should pass query text through to SessionSearch for FTS5/LIKE fallback', async () => { + const options: StrategySearchOptions = { + query: 'authentication bug', + limit: 10 + }; + + const result = await strategy.search(options); + + expect(result.usedChroma).toBe(false); + expect(result.strategy).toBe('sqlite'); + expect(result.results.observations).toHaveLength(1); + // Verify query was passed to search methods + expect(mockSessionSearch.searchObservations).toHaveBeenCalledWith('authentication bug', expect.any(Object)); + expect(mockSessionSearch.searchSessions).toHaveBeenCalledWith('authentication bug', expect.any(Object)); + expect(mockSessionSearch.searchUserPrompts).toHaveBeenCalledWith('authentication bug', expect.any(Object)); + }); + + it('should pass query with filters for combined text + filter search', async () => { + const options: StrategySearchOptions = { + query: 'test query', + project: 'my-project', + dateRange: { start: '2025-01-01', end: '2025-01-31' }, + limit: 10 + }; + + await strategy.search(options); + + const callArgs = mockSessionSearch.searchObservations.mock.calls[0]; + expect(callArgs[0]).toBe('test query'); + expect(callArgs[1].project).toBe('my-project'); + expect(callArgs[1].dateRange).toEqual({ start: '2025-01-01', end: '2025-01-31' }); + }); }); describe('findByConcept', () => {