add support for SQLite Full Text Search

CHANGELOG-Unreleased.md

@@ -14,6 +14,11 @@
 
 ### New features
 
+
+- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter:
+  Add `isFTS` boolean flag to schema column descriptor for creating Full Text Search-able columns
+  Add `Q.ftsMatch(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater
+
 ### Performance
 
 ### Changes

src/QueryDescription/index.d.ts

@@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
     | 'oneOf'
     | 'notIn'
     | 'between'
+    | 'match'
 
   export interface ColumnDescription {
     column: ColumnName
@@ -105,6 +106,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
   export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription
   export function and(...conditions: Condition[]): And
   export function or(...conditions: Condition[]): Or
+  export function ftsMatch(value: string): Comparison
   export function like(value: string): Comparison
   export function notLike(value: string): Comparison
   export function experimentalSortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy

src/QueryDescription/index.js

@@ -29,6 +29,7 @@ export type Operator =
   | 'between'
   | 'like'
   | 'notLike'
+  | 'ftsMatch'
 
 export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }>
 export type ComparisonRight =
@@ -233,6 +234,11 @@ export function sanitizeLikeString(value: string): string {
   return value.replace(nonLikeSafeRegexp, '_')
 }
 
+export function ftsMatch(value: string): Comparison {
+  invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string')
+  return { operator: 'ftsMatch', right: { value }, type: comparisonSymbol }
+}
+
 export function column(name: ColumnName): ColumnDescription {
   invariant(typeof name === 'string', 'Name passed to Q.column() is not a string')
   return { column: checkName(name), type: columnSymbol }

src/QueryDescription/test.js

@@ -448,6 +448,26 @@ describe('buildQueryDescription', () => {
       process.env.NODE_ENV = env
     }
   })
+  it('supports ftsMatch as fts join', () => {
+    const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))])
+    expect(query).toEqual({
+      where: [
+        {
+          type: 'where',
+          left: 'searchable',
+          comparison: {
+            operator: 'ftsMatch',
+            right: {
+              value: 'hello world',
+            },
+          },
+        },
+      ],
+      joinTables: [],
+      nestedJoinTables: [],
+      sortBy: [],
+    })
+  })
   it('catches bad types', () => {
     expect(() => Q.eq({})).toThrow('Invalid value passed to query')
     // TODO: oneOf/notIn values?
@@ -458,6 +478,7 @@ describe('buildQueryDescription', () => {
     expect(() => Q.notLike(null)).toThrow('not a string')
     expect(() => Q.notLike({})).toThrow('not a string')
     expect(() => Q.sanitizeLikeString(null)).toThrow('not a string')
+    expect(() => Q.ftsMatch(null)).toThrow('not a string')
     expect(() => Q.column({})).toThrow('not a string')
     expect(() => Q.experimentalTake('0')).toThrow('not a number')
     expect(() => Q.experimentalSkip('0')).toThrow('not a number')

src/Schema/index.d.ts

@@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' {
     type: ColumnType
     isOptional?: boolean
     isIndexed?: boolean
+    isFTS?: boolean
   }
 
   interface ColumnMap {

src/Schema/index.js

@@ -15,6 +15,7 @@ export type ColumnSchema = $RE<{
   type: ColumnType,
   isOptional?: boolean,
   isIndexed?: boolean,
+  isFTS?: boolean,
 }>
 
 export type ColumnMap = { [name: ColumnName]: ColumnSchema }

src/adapters/lokijs/worker/executor.js

@@ -4,7 +4,13 @@
 import logger from '../../../utils/common/logger'
 
 import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type'
-import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema'
+import type {
+  TableName,
+  AppSchema,
+  SchemaVersion,
+  TableSchema,
+  ColumnSchema,
+} from '../../../Schema'
 import type {
   SchemaMigrations,
   CreateTableMigrationStep,
@@ -312,6 +318,8 @@ export default class LokiExecutor {
       [],
     )
 
+    this._warnAboutLackingFTSSupport(columnArray)
+
     this.loki.addCollection(name, {
       unique: ['id'],
       indices: ['_status', ...indexedColumns],
@@ -415,6 +423,8 @@ export default class LokiExecutor {
         collection.ensureIndex(column.name)
       }
     })
+
+    this._warnAboutLackingFTSSupport(columns)
   }
 
   // Maps records to their IDs if the record is already cached on JS side
@@ -470,4 +480,13 @@ export default class LokiExecutor {
     // Rethrow error
     throw error
   }
+
+  _warnAboutLackingFTSSupport(columns: Array<ColumnSchema>): void {
+    if (columns.some((column) => column.isFTS)) {
+      // Warn the user about missing FTS support for the LokiJS adapter
+      // Please contribute! Here are some pointers:
+      // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts
+      logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented')
+    }
+  }
 }

src/adapters/sqlite/encodeQuery/index.js

@@ -52,6 +52,7 @@ const operators: { [Operator]: string } = {
   between: 'between',
   like: 'like',
   notLike: 'not like',
+  ftsMatch: 'match',
 }
 
 const encodeComparison = (table: TableName<any>, comparison: Comparison) => {
@@ -109,6 +110,20 @@ const encodeWhereCondition = (
     )
   }
 
+  if (comparison.operator === 'ftsMatch') {
+    const srcTable = encodeName(table)
+    const ftsTable = encodeName(`_fts_${table}`)
+    const rowid = encodeName('rowid')
+    const ftsColumn = encodeName(left)
+    const matchValue = getComparisonRight(table, comparison.right)
+    return (
+      `${srcTable}.${rowid} in (` +
+      `select ${ftsTable}.${rowid} from ${ftsTable} ` +
+      `where ${ftsTable}.${ftsColumn} match ${matchValue}` +
+      `)`
+    )
+  }
+
   return `${encodeName(table)}.${encodeName(left)} ${encodeComparison(table, comparison)}`
 }
 

src/adapters/sqlite/encodeQuery/test.js

@@ -65,7 +65,7 @@ describe('SQLite encodeQuery', () => {
         Q.where('col5', Q.lte(5)),
         Q.where('col6', Q.notEq(null)),
         Q.where('col7', Q.oneOf([1, 2, 3])),
-        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])),
+        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), // eslint-disable-line quotes
         Q.where('col9', Q.between(10, 11)),
         Q.where('col10', Q.like('%abc')),
         Q.where('col11', Q.notLike('def%')),
@@ -163,6 +163,48 @@ describe('SQLite encodeQuery', () => {
         ` and "tasks"."_status" is not 'deleted'`,
     )
   })
+  it('encodes ftsMatch', () => {
+    const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))])
+    expect(encodeQuery(query)).toBe(
+      `select "tasks".* from "tasks" ` +
+        `where "tasks"."rowid" in (` +
+        `select "_fts_tasks"."rowid" from "_fts_tasks" ` +
+        `where "_fts_tasks"."searchable" match 'hello world'` +
+        `) and "tasks"."_status" is not 'deleted'`,
+    )
+  })
+  it('encodes ftsMatch with other joins', () => {
+    const query = [
+      Q.on('projects', 'team_id', 'abcdef'),
+      Q.on('projects', 'is_active', true),
+      Q.on('projects', 'left_column', Q.lte(Q.column('right_column'))),
+      Q.on('projects', 'left2', Q.weakGt(Q.column('right2'))),
+      Q.where('left_column', 'right_value'),
+      Q.on('tag_assignments', 'tag_id', Q.oneOf(['a', 'b', 'c'])),
+      Q.where('searchable', Q.ftsMatch('hello world')),
+    ]
+    const expectedQuery =
+      `join "projects" on "projects"."id" = "tasks"."project_id"` +
+      ` join "tag_assignments" on "tag_assignments"."task_id" = "tasks"."id"` +
+      ` where ("projects"."team_id" is 'abcdef'` +
+      ` and "projects"."is_active" is 1` +
+      ` and "projects"."left_column" <= "projects"."right_column"` +
+      ` and ("projects"."left2" > "projects"."right2"` +
+      ` or ("projects"."left2" is not null` +
+      ` and "projects"."right2" is null))` +
+      ` and "projects"."_status" is not 'deleted')` +
+      ` and ("tag_assignments"."tag_id" in ('a', 'b', 'c')` +
+      ` and "tag_assignments"."_status" is not 'deleted')` +
+      ` and "tasks"."left_column" is 'right_value'` +
+      ` and "tasks"."rowid" in` +
+      ` (select "_fts_tasks"."rowid" from "_fts_tasks" where` +
+      ` "_fts_tasks"."searchable" match 'hello world')` +
+      ` and "tasks"."_status" is not 'deleted'`
+    expect(encoded(query)).toBe(`select distinct "tasks".* from "tasks" ${expectedQuery}`)
+    expect(encoded(query, true)).toBe(
+      `select count(distinct "tasks"."id") as "count" from "tasks" ${expectedQuery}`,
+    )
+  })
   it(`encodes on nested in and/or`, () => {
     expect(
       encoded([

src/adapters/sqlite/encodeSchema/index.js

@@ -8,6 +8,7 @@ import type {
   AddColumnsMigrationStep,
 } from '../../../Schema/migrations'
 import type { SQL } from '../index'
+import { logger } from '../../../utils/common'
 
 import encodeName from '../encodeName'
 import encodeValue from '../encodeValue'
@@ -38,8 +39,112 @@ const encodeTableIndicies: (TableSchema) => SQL = ({ name: tableName, columns })
 const transform = (sql: string, transformer: ?(string) => string) =>
   transformer ? transformer(sql) : sql
 
-const encodeTable: (TableSchema) => SQL = (table) =>
-  transform(encodeCreateTable(table) + encodeTableIndicies(table), table.unsafeSql)
+const encodeTable: TableSchema => SQL = table =>
+  transform(
+    // eslint-disable-next-line no-use-before-define
+    encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table),
+    table.unsafeSql,
+  )
+
+/** FTS Full Text Search */
+
+const encodeFTSTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  event: 'delete' | 'insert' | 'update',
+  action: SQL,
+}) => SQL = ({ tableName, ftsTableName, event, action }) => {
+  const triggerName = `${ftsTableName}_${event}`
+  return `create trigger ${encodeName(triggerName)} after ${event} on ${encodeName(
+    tableName,
+  )} begin ${action} end;`
+}
+
+const encodeFTSDeleteTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+}) => SQL = ({ tableName, ftsTableName }) =>
+  encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'delete',
+    action: `delete from ${encodeName(ftsTableName)} where "rowid" = OLD.rowid;`,
+  })
+
+const encodeFTSInsertTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)]
+  const columns = rawColumnNames.map(encodeName)
+  const valueColumns = rawColumnNames.map(column => `NEW.${encodeName(column)}`)
+
+  const columnsSQL = columns.join(', ')
+  const valueColumnsSQL = valueColumns.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'insert',
+    action: `insert into ${encodeName(ftsTableName)} (${columnsSQL}) values (${valueColumnsSQL});`,
+  })
+}
+
+const encodeFTSUpdateTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ftsColumns.map(column => column.name)
+  const assignments = rawColumnNames.map(
+    column => `${encodeName(column)} = NEW.${encodeName(column)}`,
+  )
+
+  const assignmentsSQL = assignments.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'update',
+    action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = NEW."rowid";`,
+  })
+}
+
+const encodeFTSTriggers: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  return (
+    encodeFTSDeleteTrigger({ tableName, ftsTableName }) +
+    encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) +
+    encodeFTSUpdateTrigger({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
+const encodeFTSTable: ({
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ ftsTableName, ftsColumns }) => {
+  const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ')
+  return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});`
+}
+
+const encodeFTSSearch: TableSchema => SQL = tableSchema => {
+  const { name: tableName, columnArray } = tableSchema
+  const ftsColumns = columnArray.filter(column => column.isFTS)
+  if (ftsColumns.length === 0) {
+    return ''
+  }
+  const ftsTableName = `_fts_${tableName}`
+  return (
+    encodeFTSTable({ ftsTableName, ftsColumns }) +
+    encodeFTSTriggers({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
+/** FTS END */
 
 export const encodeSchema: (AppSchema) => SQL = ({ tables, unsafeSql }) => {
   const sql = Object.values(tables)
@@ -65,6 +170,10 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({
       )} = ${encodeValue(nullValue(column))};`
       const addIndex = encodeIndex(column, table)
 
+      if (column.isFTS) {
+        logger.warn('[DB][Worker] Support for migrations and isFTS is still to be implemented')
+      }
+
       return transform(addColumn + setDefaultValue + addIndex, unsafeSql)
     })
     .join('')