Fall back to FTS5/LIKE keyword search when Chroma is disabled

src/services/sqlite/SessionSearch.ts

@@ -17,8 +17,8 @@ import {
 
 /**
  * Search interface for session-based memory
- * Provides filter-only structured queries for sessions, observations, and user prompts
- * Vector search is handled by ChromaDB - this class only supports filtering without query text
+ * Provides structured queries for sessions, observations, and user prompts.
+ * Supports filter-only queries and FTS5/LIKE keyword search as a fallback when Chroma is unavailable.
  */
 export class SessionSearch {
   private db: Database;
@@ -178,6 +178,39 @@ export class SessionSearch {
     }
   }
 
+  /**
+   * Check if FTS5 tables exist for text search.
+   * Result is cached after first check.
+   */
+  private _hasFtsTables: boolean | null = null;
+  private hasFtsTables(): boolean {
+    if (this._hasFtsTables !== null) return this._hasFtsTables;
+    try {
+      const tables = this.db.prepare(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('observations_fts', 'session_summaries_fts')"
+      ).all() as TableNameRow[];
+      this._hasFtsTables = tables.length >= 2;
+    } catch {
+      this._hasFtsTables = false;
+    }
+    return this._hasFtsTables;
+  }
+
+  /**
+   * Sanitize a query string for safe use in FTS5 MATCH expressions.
+   * Strips FTS5 operators and wraps each token in double quotes.
+   */
+  private sanitizeFtsQuery(query: string): string {
+    // Remove FTS5 special characters/operators, keep alphanumeric and whitespace
+    const tokens = query
+      .replace(/[^\w\s]/g, ' ')
+      .split(/\s+/)
+      .filter(t => t.length > 0);
+    if (tokens.length === 0) return '';
+    // Join with spaces (implicit AND in FTS5)
+    return tokens.map(t => `"${t}"`).join(' ');
+  }
+
 
   /**
    * Build WHERE clause for structured filters
@@ -271,15 +304,14 @@ export class SessionSearch {
   }
 
   /**
-   * Search observations using filter-only direct SQLite query.
-   * Vector search is handled by ChromaDB - this only supports filtering without query text.
+   * Search observations using direct SQLite query.
+   * Supports filter-only queries and FTS5 keyword search as a fallback when Chroma is unavailable.
    */
   searchObservations(query: string | undefined, options: SearchOptions = {}): ObservationSearchResult[] {
     const params: any[] = [];
     const { limit = 50, offset = 0, orderBy = 'relevance', ...filters } = options;
 
     // FILTER-ONLY PATH: When no query text, query table directly
-    // This enables date filtering which Chroma cannot do (requires direct SQLite access)
     if (!query) {
       const filterClause = this.buildFilterClause(filters, params, 'o');
       if (!filterClause) {
@@ -300,15 +332,57 @@ export class SessionSearch {
       return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
     }
 
-    // Vector search with query text should be handled by ChromaDB
-    // This method only supports filter-only queries (query=undefined)
-    logger.warn('DB', 'Text search not supported - use ChromaDB for vector search');
-    return [];
+    // TEXT QUERY PATH: Use FTS5 keyword matching when available
+    if (this.hasFtsTables()) {
+      const ftsQuery = this.sanitizeFtsQuery(query);
+      if (ftsQuery) {
+        logger.debug('DB', 'Using FTS5 keyword search for observations', { query: ftsQuery });
+        params.push(ftsQuery);
+        const filterClause = this.buildFilterClause(filters, params, 'o');
+        const whereExtra = filterClause ? ` AND ${filterClause}` : '';
+        const orderClause = this.buildOrderClause(orderBy, true, 'observations_fts');
+
+        const sql = `
+          SELECT o.*, o.discovery_tokens
+          FROM observations o
+          JOIN observations_fts ON observations_fts.rowid = o.id
+          WHERE observations_fts MATCH ?${whereExtra}
+          ${orderClause}
+          LIMIT ? OFFSET ?
+        `;
+
+        params.push(limit, offset);
+        try {
+          return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
+        } catch (error) {
+          logger.warn('DB', 'FTS5 query failed, falling back to LIKE search', {}, error as Error);
+        }
+      }
+    }
+
+    // LIKE FALLBACK: When FTS5 is not available, use LIKE-based text search
+    logger.debug('DB', 'Using LIKE fallback for observation text search', { query });
+    const likeParam = `%${query}%`;
+    params.push(likeParam, likeParam, likeParam, likeParam);
+    const filterClause = this.buildFilterClause(filters, params, 'o');
+    const whereExtra = filterClause ? ` AND ${filterClause}` : '';
+    const orderClause = this.buildOrderClause(orderBy, false);
+
+    const sql = `
+      SELECT o.*, o.discovery_tokens
+      FROM observations o
+      WHERE (o.title LIKE ? OR o.narrative LIKE ? OR o.text LIKE ? OR o.facts LIKE ?)${whereExtra}
+      ${orderClause}
+      LIMIT ? OFFSET ?
+    `;
+
+    params.push(limit, offset);
+    return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
   }
 
   /**
-   * Search session summaries using filter-only direct SQLite query.
-   * Vector search is handled by ChromaDB - this only supports filtering without query text.
+   * Search session summaries using direct SQLite query.
+   * Supports filter-only queries and FTS5 keyword search as a fallback when Chroma is unavailable.
    */
   searchSessions(query: string | undefined, options: SearchOptions = {}): SessionSummarySearchResult[] {
     const params: any[] = [];
@@ -339,10 +413,62 @@ export class SessionSearch {
       return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[];
     }
 
-    // Vector search with query text should be handled by ChromaDB
-    // This method only supports filter-only queries (query=undefined)
-    logger.warn('DB', 'Text search not supported - use ChromaDB for vector search');
-    return [];
+    // TEXT QUERY PATH: Use FTS5 keyword matching when available
+    if (this.hasFtsTables()) {
+      const ftsQuery = this.sanitizeFtsQuery(query);
+      if (ftsQuery) {
+        logger.debug('DB', 'Using FTS5 keyword search for sessions', { query: ftsQuery });
+        params.push(ftsQuery);
+        const filterOptions = { ...filters };
+        delete filterOptions.type;
+        const filterClause = this.buildFilterClause(filterOptions, params, 's');
+        const whereExtra = filterClause ? ` AND ${filterClause}` : '';
+        const orderClause = orderBy === 'relevance'
+          ? 'ORDER BY session_summaries_fts.rank ASC'
+          : orderBy === 'date_asc'
+            ? 'ORDER BY s.created_at_epoch ASC'
+            : 'ORDER BY s.created_at_epoch DESC';
+
+        const sql = `
+          SELECT s.*, s.discovery_tokens
+          FROM session_summaries s
+          JOIN session_summaries_fts ON session_summaries_fts.rowid = s.id
+          WHERE session_summaries_fts MATCH ?${whereExtra}
+          ${orderClause}
+          LIMIT ? OFFSET ?
+        `;
+
+        params.push(limit, offset);
+        try {
+          return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[];
+        } catch (error) {
+          logger.warn('DB', 'FTS5 query failed for sessions, falling back to LIKE search', {}, error as Error);
+        }
+      }
+    }
+
+    // LIKE FALLBACK: When FTS5 is not available, use LIKE-based text search
+    logger.debug('DB', 'Using LIKE fallback for session text search', { query });
+    const likeParam = `%${query}%`;
+    const filterOptions = { ...filters };
+    delete filterOptions.type;
+    params.push(likeParam, likeParam, likeParam, likeParam);
+    const filterClause = this.buildFilterClause(filterOptions, params, 's');
+    const whereExtra = filterClause ? ` AND ${filterClause}` : '';
+    const orderClause = orderBy === 'date_asc'
+      ? 'ORDER BY s.created_at_epoch ASC'
+      : 'ORDER BY s.created_at_epoch DESC';
+
+    const sql = `
+      SELECT s.*, s.discovery_tokens
+      FROM session_summaries s
+      WHERE (s.request LIKE ? OR s.investigated LIKE ? OR s.learned LIKE ? OR s.completed LIKE ?)${whereExtra}
+      ${orderClause}
+      LIMIT ? OFFSET ?
+    `;
+
+    params.push(limit, offset);
+    return this.db.prepare(sql).all(...params) as SessionSummarySearchResult[];
   }
 
   /**
@@ -523,8 +649,8 @@ export class SessionSearch {
   }
 
   /**
-   * Search user prompts using filter-only direct SQLite query.
-   * Vector search is handled by ChromaDB - this only supports filtering without query text.
+   * Search user prompts using direct SQLite query.
+   * Supports filter-only queries and LIKE-based keyword search as a fallback when Chroma is unavailable.
    */
   searchUserPrompts(query: string | undefined, options: SearchOptions = {}): UserPromptSearchResult[] {
     const params: any[] = [];
@@ -575,10 +701,28 @@ export class SessionSearch {
       return this.db.prepare(sql).all(...params) as UserPromptSearchResult[];
     }
 
-    // Vector search with query text should be handled by ChromaDB
-    // This method only supports filter-only queries (query=undefined)
-    logger.warn('DB', 'Text search not supported - use ChromaDB for vector search');
-    return [];
+    // TEXT QUERY PATH: Use LIKE-based search for user prompts (no FTS5 table)
+    logger.debug('DB', 'Using LIKE search for user prompts', { query });
+    const likeParam = `%${query}%`;
+    baseConditions.push('up.prompt_text LIKE ?');
+    params.push(likeParam);
+
+    const whereClause = `WHERE ${baseConditions.join(' AND ')}`;
+    const orderClause = orderBy === 'date_asc'
+      ? 'ORDER BY up.created_at_epoch ASC'
+      : 'ORDER BY up.created_at_epoch DESC';
+
+    const sql = `
+      SELECT up.*
+      FROM user_prompts up
+      JOIN sdk_sessions s ON up.content_session_id = s.content_session_id
+      ${whereClause}
+      ${orderClause}
+      LIMIT ? OFFSET ?
+    `;
+
+    params.push(limit, offset);
+    return this.db.prepare(sql).all(...params) as UserPromptSearchResult[];
   }
 
   /**

src/services/worker/SearchManager.ts

@@ -252,14 +252,19 @@ export class SearchManager {
         logger.debug('SEARCH', 'ChromaDB found no matches (final result, no FTS5 fallback)', {});
       }
     }
-    // ChromaDB not initialized - mark as failed to show proper error message
+    // PATH 3: ChromaDB not initialized - fall back to FTS5/LIKE keyword search
     else if (query) {
-      chromaFailed = true;
-      logger.debug('SEARCH', 'ChromaDB not initialized - semantic search unavailable', {});
-      logger.debug('SEARCH', 'Install UVX/Python to enable vector search', { url: 'https://docs.astral.sh/uv/getting-started/installation/' });
-      observations = [];
-      sessions = [];
-      prompts = [];
+      logger.debug('SEARCH', 'ChromaDB not initialized - falling back to FTS5/LIKE keyword search', {});
+      const obsOptions = { ...options, type: obs_type, concepts, files };
+      if (searchObservations) {
+        observations = this.sessionSearch.searchObservations(query, obsOptions);
+      }
+      if (searchSessions) {
+        sessions = this.sessionSearch.searchSessions(query, options);
+      }
+      if (searchPrompts) {
+        prompts = this.sessionSearch.searchUserPrompts(query, options);
+      }
     }
 
     const totalResults = observations.length + sessions.length + prompts.length;

src/services/worker/search/SearchOrchestrator.ts

@@ -110,13 +110,12 @@ export class SearchOrchestrator {
       };
     }
 
-    // PATH 3: No Chroma available
-    logger.debug('SEARCH', 'Orchestrator: Chroma not available', {});
+    // PATH 3: No Chroma available - fall back to SQLite FTS5/LIKE keyword search
+    logger.debug('SEARCH', 'Orchestrator: Chroma not available, falling back to SQLite keyword search', {});
+    const fallbackResult = await this.sqliteStrategy.search(options);
     return {
-      results: { observations: [], sessions: [], prompts: [] },
-      usedChroma: false,
-      fellBack: false,
-      strategy: 'sqlite'
+      ...fallbackResult,
+      fellBack: true
     };
   }
 

src/services/worker/search/strategies/SQLiteSearchStrategy.ts

@@ -37,6 +37,7 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt
 
   async search(options: StrategySearchOptions): Promise<StrategySearchResult> {
     const {
+      query,
       searchType = 'all',
       obsType,
       concepts,
@@ -58,7 +59,8 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt
 
     const baseOptions = { limit, offset, orderBy, project, dateRange };
 
-    logger.debug('SEARCH', 'SQLiteSearchStrategy: Filter-only query', {
+    logger.debug('SEARCH', 'SQLiteSearchStrategy: query', {
+      hasQuery: !!query,
       searchType,
       hasDateRange: !!dateRange,
       hasProject: !!project
@@ -72,15 +74,15 @@ export class SQLiteSearchStrategy extends BaseSearchStrategy implements SearchSt
           concepts,
           files
         };
-        observations = this.sessionSearch.searchObservations(undefined, obsOptions);
+        observations = this.sessionSearch.searchObservations(query, obsOptions);
       }
 
       if (searchSessions) {
-        sessions = this.sessionSearch.searchSessions(undefined, baseOptions);
+        sessions = this.sessionSearch.searchSessions(query, baseOptions);
       }
 
       if (searchPrompts) {
-        prompts = this.sessionSearch.searchUserPrompts(undefined, baseOptions);
+        prompts = this.sessionSearch.searchUserPrompts(query, baseOptions);
       }
 
       logger.debug('SEARCH', 'SQLiteSearchStrategy: Results', {

tests/worker/search/search-orchestrator.test.ts

@@ -313,14 +313,17 @@ describe('SearchOrchestrator', () => {
     });
 
     describe('search', () => {
-      it('should return empty results for query search without Chroma', async () => {
+      it('should fall back to SQLite FTS5/LIKE search for query without Chroma', async () => {
         const result = await orchestrator.search({
           query: 'semantic query'
         });
 
-        // No Chroma available, can't do semantic search
-        expect(result.results.observations).toHaveLength(0);
+        // No Chroma available, falls back to SQLite keyword search
+        expect(result.results.observations).toHaveLength(1);
         expect(result.usedChroma).toBe(false);
+        expect(result.fellBack).toBe(true);
+        expect(result.strategy).toBe('sqlite');
+        expect(mockSessionSearch.searchObservations).toHaveBeenCalledWith('semantic query', expect.any(Object));
       });
 
       it('should still work for filter-only queries', async () => {

tests/worker/search/strategies/sqlite-search-strategy.test.ts

@@ -226,6 +226,39 @@ describe('SQLiteSearchStrategy', () => {
       expect(result.results.prompts).toHaveLength(0);
       expect(result.usedChroma).toBe(false);
     });
+
+    it('should pass query text through to SessionSearch for FTS5/LIKE fallback', async () => {
+      const options: StrategySearchOptions = {
+        query: 'authentication bug',
+        limit: 10
+      };
+
+      const result = await strategy.search(options);
+
+      expect(result.usedChroma).toBe(false);
+      expect(result.strategy).toBe('sqlite');
+      expect(result.results.observations).toHaveLength(1);
+      // Verify query was passed to search methods
+      expect(mockSessionSearch.searchObservations).toHaveBeenCalledWith('authentication bug', expect.any(Object));
+      expect(mockSessionSearch.searchSessions).toHaveBeenCalledWith('authentication bug', expect.any(Object));
+      expect(mockSessionSearch.searchUserPrompts).toHaveBeenCalledWith('authentication bug', expect.any(Object));
+    });
+
+    it('should pass query with filters for combined text + filter search', async () => {
+      const options: StrategySearchOptions = {
+        query: 'test query',
+        project: 'my-project',
+        dateRange: { start: '2025-01-01', end: '2025-01-31' },
+        limit: 10
+      };
+
+      await strategy.search(options);
+
+      const callArgs = mockSessionSearch.searchObservations.mock.calls[0];
+      expect(callArgs[0]).toBe('test query');
+      expect(callArgs[1].project).toBe('my-project');
+      expect(callArgs[1].dateRange).toEqual({ start: '2025-01-01', end: '2025-01-31' });
+    });
   });
 
   describe('findByConcept', () => {