add support for SQLite Full Text Search

CHANGELOG.md

@@ -23,6 +23,9 @@ All notable changes to this project will be documented in this file.
   This is called when underlying IndexedDB encountered a quota exceeded error (ran out of allotted disk space for app)
   This means that app can't save more data or that it will fall back to using in-memory database only
   Note that this only works when `useWebWorker: false`
+- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter:
+  Add `isSearchable` boolean flag to schema column descriptor for creating Full Text Search-able columns
+  Add `Q.textMatches(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater
 
 ### Changes
 

src/QueryDescription/index.d.ts

@@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
     | 'oneOf'
     | 'notIn'
     | 'between'
+    | 'match'
 
   export interface ColumnDescription {
     column: ColumnName
@@ -69,6 +70,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
   export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription
   export function and(...conditions: Where[]): And
   export function or(...conditions: Where[]): Or
+  export function textMatches(value: string): Comparison
   export function like(value: string): Comparison
   export function notLike(value: string): Comparison
   export function sanitizeLikeString(value: string): string

src/QueryDescription/index.js

@@ -27,6 +27,7 @@ export type Operator =
   | 'between'
   | 'like'
   | 'notLike'
+  | 'match'
 
 export type ColumnDescription = $RE<{ column: ColumnName }>
 export type ComparisonRight =
@@ -174,6 +175,10 @@ export function sanitizeLikeString(value: string): string {
   return value.replace(nonLikeSafeRegexp, '_')
 }
 
+export function textMatches(value: string): Comparison {
+  return { operator: 'match', right: { value } }
+}
+
 export function column(name: ColumnName): ColumnDescription {
   return { column: name }
 }

src/QueryDescription/test.js

@@ -481,4 +481,21 @@ describe('QueryDescription', () => {
       ],
     })
   })
+
+  it('supports textMatches as fts join', () => {
+    const query = Q.buildQueryDescription([
+      Q.textMatches('searchable', 'hello world'),
+    ])
+    expect(query).toEqual({
+      'where': [
+        {
+          'operator': 'match',
+          'right': {
+            'value': 'searchable',
+          },
+        },
+      ],
+      'join': [],
+    })
+  })
 })

src/Schema/index.d.ts

@@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' {
     type: ColumnType
     isOptional?: boolean
     isIndexed?: boolean
+    isSearchable?: boolean
   }
 
   interface ColumnMap {

src/Schema/index.js

@@ -14,6 +14,7 @@ export type ColumnSchema = $RE<{
   type: ColumnType,
   isOptional?: boolean,
   isIndexed?: boolean,
+  isSearchable?: boolean,
 }>
 
 export type ColumnMap = { [name: ColumnName]: ColumnSchema }

src/adapters/lokijs/worker/executor.js

@@ -5,7 +5,7 @@ import { prop, forEach, values } from 'rambdax'
 import { logger } from '../../../utils/common'
 
 import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type'
-import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema'
+import type { TableName, AppSchema, SchemaVersion, TableSchema, ColumnSchema } from '../../../Schema'
 import type {
   SchemaMigrations,
   CreateTableMigrationStep,
@@ -276,6 +276,8 @@ export default class LokiExecutor {
       [],
     )
 
+    this._warnAboutLackingFTSSupport(values(columns))
+
     this.loki.addCollection(name, {
       unique: ['id'],
       indices: ['_status', ...indexedColumns],
@@ -380,6 +382,8 @@ export default class LokiExecutor {
         collection.ensureIndex(column.name)
       }
     })
+
+    this._warnAboutLackingFTSSupport(columns)
   }
 
   // Maps records to their IDs if the record is already cached on JS side
@@ -404,4 +408,14 @@ export default class LokiExecutor {
     const localStorage = this._localStorage
     return localStorage && localStorage.by('key', key)
   }
+
+  _warnAboutLackingFTSSupport(columns: Array<ColumnSchema>): void {
+    const searchableColumns = columns.filter(column => column.isSearchable)
+    if (searchableColumns.length > 0) {
+      // Warn the user about missing FTS support for the LokiJS adapter
+      // Please contribute! Here are some pointers:
+      // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts
+      logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented')
+    }
+  }
 }

src/adapters/sqlite/encodeQuery/index.js

@@ -53,6 +53,7 @@ const operators: { [Operator]: string } = {
   between: 'between',
   like: 'like',
   notLike: 'not like',
+  match: 'match',
 }
 
 const encodeComparison = (table: TableName<any>, comparison: Comparison) => {
@@ -92,6 +93,20 @@ const encodeWhereCondition = (
     )
   }
 
+  if (comparison.operator === 'match') {
+    const srcTable = encodeName(table)
+    const ftsTable = encodeName(`${table}_fts`)
+    const rowid = encodeName('rowid')
+    const ftsColumn = encodeName(left)
+    const matchValue = getComparisonRight(table, comparison.right)
+    return (
+      `${srcTable}.${rowid} in (` +
+      `select ${ftsTable}.${rowid} from ${ftsTable} ` +
+      `where ${ftsTable}.${ftsColumn} match ${matchValue}` +
+      `)`
+    )
+  }
+
   return `${encodeName(table)}.${encodeName(left)} ${encodeComparison(table, comparison)}`
 }
 

src/adapters/sqlite/encodeQuery/test.js

@@ -111,4 +111,16 @@ describe('SQLite encodeQuery', () => {
       `select "tasks".* from "tasks" where "tasks"."col1" like '%abc' and "tasks"."col2" not like 'def%' and "tasks"."_status" is not 'deleted'`,
     )
   })
+  it('encodes JOIN over FTS table', () => {
+    const query = new Query(mockCollection, [
+      Q.where('searchable', Q.textMatches('hello world')),
+    ])
+    expect(encodeQuery(query)).toBe(
+      `select "tasks".* from "tasks" ` +
+      `where "tasks"."rowid" in (` +
+      `select "tasks_fts"."rowid" from "tasks_fts" ` +
+      `where "tasks_fts"."searchable" match 'hello world'` +
+      `) and "tasks"."_status" is not 'deleted'`
+    )
+  })
 })

src/adapters/sqlite/encodeSchema/index.js

@@ -9,6 +9,7 @@ import type {
   AddColumnsMigrationStep,
 } from '../../../Schema/migrations'
 import type { SQL } from '../index'
+import { logger } from '../../../utils/common'
 
 import encodeName from '../encodeName'
 import encodeValue from '../encodeValue'
@@ -35,8 +36,103 @@ const encodeTableIndicies: TableSchema => SQL = ({ name: tableName, columns }) =
     .concat([`create index ${tableName}__status on ${encodeName(tableName)} ("_status");`])
     .join('')
 
+const encodeFTSTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  event: 'delete' | 'insert' | 'update',
+  action: SQL,
+}) => SQL = ({ tableName, ftsTableName, event, action }) => {
+  const triggerName = `${ftsTableName}_${event}`
+  return `create trigger ${encodeName(triggerName)} after ${event} on ${encodeName(
+    tableName,
+  )} begin ${action} end;`
+}
+
+const encodeFTSDeleteTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+}) => SQL = ({ tableName, ftsTableName }) =>
+  encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'delete',
+    action: `delete from ${encodeName(ftsTableName)} where "rowid" = OLD.rowid;`,
+  })
+
+const encodeFTSInsertTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)]
+  const columns = rawColumnNames.map(encodeName)
+  const valueColumns = rawColumnNames.map(column => `NEW.${encodeName(column)}`)
+
+  const columnsSQL = columns.join(', ')
+  const valueColumnsSQL = valueColumns.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'insert',
+    action: `insert into ${encodeName(ftsTableName)} (${columnsSQL}) values (${valueColumnsSQL});`,
+  })
+}
+
+const encodeFTSUpdateTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ftsColumns.map(column => column.name)
+  const assignments = rawColumnNames.map(
+    column => `${encodeName(column)} = NEW.${encodeName(column)}`,
+  )
+
+  const assignmentsSQL = assignments.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'update',
+    action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = NEW."rowid";`,
+  })
+}
+
+const encodeFTSTriggers: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  return (
+    encodeFTSDeleteTrigger({ tableName, ftsTableName }) +
+    encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) +
+    encodeFTSUpdateTrigger({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
+const encodeFTSTable: ({
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ ftsTableName, ftsColumns }) => {
+  const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ')
+  return `create virtual table ${encodeName(`${ftsTableName}`)} using fts4(${columnsSQL});`
+}
+
+const encodeFTSSearch: TableSchema => SQL = ({ name: tableName, columns }) => {
+  const ftsColumns = values(columns).filter(c => c.isSearchable)
+  if (ftsColumns.length === 0) {
+    return ''
+  }
+  const ftsTableName = `${tableName}_fts`
+  return (
+    encodeFTSTable({ ftsTableName, ftsColumns }) +
+    encodeFTSTriggers({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
 const encodeTable: TableSchema => SQL = table =>
-  encodeCreateTable(table) + encodeTableIndicies(table)
+  encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table)
 
 export const encodeSchema: AppSchema => SQL = ({ tables }) =>
   values(tables)
@@ -55,6 +151,12 @@ const encodeAddColumnsMigrationStep: AddColumnsMigrationStep => SQL = ({ table,
       )} = ${encodeValue(nullValue(column))};`
       const addIndex = encodeIndex(column, table)
 
+      if (column.isSearchable) {
+        logger.warn(
+          '[DB][Worker] Support for migrations and isSearchable is still to be implemented',
+        )
+      }
+
       return addColumn + setDefaultValue + addIndex
     })
     .join('')

src/adapters/sqlite/encodeSchema/test.js

@@ -33,6 +33,34 @@ describe('encodeSchema', () => {
 
     expect(encodeSchema(testSchema)).toBe(expectedSchema)
   })
+  it('encodes schema with FTS', () => {
+    const testSchema = appSchema({
+      version: 1,
+      tables: [
+        tableSchema({
+          name: 'tasks',
+          columns: [
+            { name: 'author_id', type: 'string', isIndexed: true },
+            { name: 'author_name', type: 'string', isSearchable: true },
+            { name: 'author_title', type: 'string', isSearchable: true },
+            { name: 'created_at', type: 'number' },
+          ],
+        }),
+      ],
+    })
+
+    const expectedSchema =
+      'create table "tasks" ("id" primary key, "_changed", "_status", "author_id", "author_name", "author_title", "created_at");' +
+      'create index tasks_author_id on "tasks" ("author_id");' +
+      'create index tasks__status on "tasks" ("_status");' +
+      'create virtual table "tasks_fts" using fts4("author_name", "author_title");' +
+      'create trigger "tasks_fts_delete" after delete on "tasks" begin delete from "tasks_fts" where "rowid" = OLD.rowid; end;' +
+      'create trigger "tasks_fts_insert" after insert on "tasks" begin insert into "tasks_fts" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' +
+      'create trigger "tasks_fts_update" after update on "tasks" begin update "tasks_fts" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' +
+      ''
+
+    expect(encodeSchema(testSchema)).toBe(expectedSchema)
+  })
   it('encodes migrations', () => {
     const migrationSteps = [
       addColumns({

src/sync/impl/applyRemote.js

@@ -167,8 +167,8 @@ const getAllRecordsToApply = (
       const collection = db.collections.get((tableName: any))
 
       if (!collection) {
-        return Promise.reject(`You are trying to sync a collection named ${tableName}, but currently this collection does not exist.` + 
-        `Have you remembered to add it to your Database constructor\'s modelClasses property?`)
+        return Promise.reject(new Error(`You are trying to sync a collection named ${tableName}, but currently this collection does not exist.` + 
+        `Have you remembered to add it to your Database constructor\'s modelClasses property?`))
       }
 
       return recordsToApplyRemoteChangesTo(collection, changes)