From e83d2904d6cf52954e438392a91d088797280b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Fri, 21 Jun 2019 20:53:54 +0200 Subject: [PATCH 01/13] Adding `isSearchable` to column schema --- src/Schema/index.d.ts | 1 + src/Schema/index.js | 1 + src/adapters/lokijs/worker/executor.js | 16 +++- src/adapters/sqlite/encodeSchema/index.js | 105 ++++++++++++++++++++++ src/adapters/sqlite/encodeSchema/test.js | 28 ++++++ 5 files changed, 150 insertions(+), 1 deletion(-) diff --git a/src/Schema/index.d.ts b/src/Schema/index.d.ts index 296bc68db..ced17534a 100644 --- a/src/Schema/index.d.ts +++ b/src/Schema/index.d.ts @@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' { type: ColumnType isOptional?: boolean isIndexed?: boolean + isSearchable?: boolean } interface ColumnMap { diff --git a/src/Schema/index.js b/src/Schema/index.js index 018d4e4a4..f9c632853 100644 --- a/src/Schema/index.js +++ b/src/Schema/index.js @@ -15,6 +15,7 @@ export type ColumnSchema = $RE<{ type: ColumnType, isOptional?: boolean, isIndexed?: boolean, + isSearchable?: boolean, }> export type ColumnMap = { [name: ColumnName]: ColumnSchema } diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index f46c19075..04fe67908 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -4,7 +4,7 @@ import logger from '../../../utils/common/logger' import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type' -import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema' +import type { TableName, AppSchema, SchemaVersion, TableSchema, ColumnSchema } from '../../../Schema' import type { SchemaMigrations, CreateTableMigrationStep, @@ -312,6 +312,8 @@ export default class LokiExecutor { [], ) + this._warnAboutLackingFTSSupport(values(columns)) + this.loki.addCollection(name, { unique: ['id'], indices: ['_status', ...indexedColumns], @@ -415,6 +417,8 @@ export default class LokiExecutor { collection.ensureIndex(column.name) } }) + + this._warnAboutLackingFTSSupport(columns) } // Maps records to their IDs if the record is already cached on JS side @@ -470,4 +474,14 @@ export default class LokiExecutor { // Rethrow error throw error } + + _warnAboutLackingFTSSupport(columns: Array) { + const searchableColumns = columns.filter(column => column.isSearchable) + if (searchableColumns.length > 0) { + // Warn the user about missing FTS support for the LokiJS adapter + // Please contribute! Here are some pointers: + // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts + logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented') + } + } } diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index 7b08913ce..eb733ccba 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -8,6 +8,7 @@ import type { AddColumnsMigrationStep, } from '../../../Schema/migrations' import type { SQL } from '../index' +import { logger } from '../../../utils/common' import encodeName from '../encodeName' import encodeValue from '../encodeValue' @@ -40,6 +41,104 @@ const transform = (sql: string, transformer: ?(string) => string) => const encodeTable: (TableSchema) => SQL = (table) => transform(encodeCreateTable(table) + encodeTableIndicies(table), table.unsafeSql) +const encodeFTSTrigger: ({ + tableName: string, + ftsTableName: string, + event: 'delete' | 'insert' | 'update', + action: SQL, +}) => SQL = ({ tableName, ftsTableName, event, action }) => { + const triggerName = `${ftsTableName}_${event}` + return `create trigger ${encodeName(triggerName)} after ${event} on ${encodeName( + tableName, + )} begin ${action} end;` +} + +const encodeFTSDeleteTrigger: ({ + tableName: string, + ftsTableName: string, +}) => SQL = ({ tableName, ftsTableName }) => + encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'delete', + action: `delete from ${encodeName(ftsTableName)} where "rowid" = "OLD.rowid";`, + }) + +const encodeFTSInsertTrigger: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)] + const columns = rawColumnNames.map(encodeName) + const valueColumns = rawColumnNames.map(column => encodeName(`NEW.${column}`)) + + const columnsSQL = columns.join(', ') + const valueColumnsSQL = valueColumns.join(', ') + + return encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'insert', + action: `insert into ${encodeName(ftsTableName)} (${columnsSQL}) values (${valueColumnsSQL});`, + }) +} + +const encodeFTSUpdateTrigger: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + const rawColumnNames = ftsColumns.map(column => column.name) + const assignments = rawColumnNames.map( + column => `${encodeName(column)}=${encodeName(`NEW.${column}`)}`, + ) + + const assignmentsSQL = assignments.join(', ') + + return encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'update', + action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = "NEW.rowid";`, + }) +} + +const encodeFTSTriggers: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + const updateTrigger = '' + return ( + encodeFTSDeleteTrigger({ tableName, ftsTableName }) + + encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) + + encodeFTSUpdateTrigger({ tableName, ftsTableName, ftsColumns }) + ) +} + +const encodeFTSTable: ({ + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ ftsTableName, ftsColumns }) => { + const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ') + return `create virtual table ${encodeName(`${ftsTableName}`)} using fts4(${columnsSQL});` +} + +const encodeFTSSearch: TableSchema => SQL = ({ name: tableName, columns }) => { + const ftsColumns = values(columns).filter(c => c.isSearchable) + if (ftsColumns.length === 0) { + return '' + } + const ftsTableName = `${tableName}_fts` + return ( + encodeFTSTable({ ftsTableName, ftsColumns }) + + encodeFTSTriggers({ tableName, ftsTableName, ftsColumns }) + ) +} + +const encodeTable: TableSchema => SQL = table => + encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table) export const encodeSchema: (AppSchema) => SQL = ({ tables, unsafeSql }) => { const sql = Object.values(tables) @@ -65,6 +164,12 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({ )} = ${encodeValue(nullValue(column))};` const addIndex = encodeIndex(column, table) + if (column.isSearchable) { + logger.warn( + '[DB][Worker] Support for migrations and isSearchable is still to be implemented', + ) + } + return transform(addColumn + setDefaultValue + addIndex, unsafeSql) }) .join('') diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index ea13e41f5..285171a7a 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -57,6 +57,34 @@ describe('encodeSchema', () => { expect(encodeSchema(testSchema)).toBe(expectedSchema) }) + fit('encodes schema with FTS', () => { + const testSchema = appSchema({ + version: 1, + tables: [ + tableSchema({ + name: 'tasks', + columns: [ + { name: 'author_id', type: 'string', isIndexed: true }, + { name: 'author_name', type: 'string', isSearchable: true }, + { name: 'author_title', type: 'string', isSearchable: true }, + { name: 'created_at', type: 'number' }, + ], + }), + ], + }) + + const expectedSchema = + 'create table "tasks" ("id" primary key, "_changed", "_status", "author_id", "author_name", "author_title", "created_at");' + + 'create index tasks_author_id on "tasks" ("author_id");' + + 'create index tasks__status on "tasks" ("_status");' + + 'create virtual table "tasks_fts" using fts4("author_name", "author_title");' + + 'create trigger "tasks_fts_delete" after delete on "tasks" begin delete from "tasks_fts" where "rowid" = "OLD.rowid"; end;' + + 'create trigger "tasks_fts_insert" after insert on "tasks" begin insert into "tasks_fts" ("rowid", "author_name", "author_title") values ("NEW.rowid", "NEW.author_name", "NEW.author_title"); end;' + + 'create trigger "tasks_fts_update" after update on "tasks" begin update "tasks_fts" set "author_name"="NEW.author_name", "author_title"="NEW.author_title" where "rowid" = "NEW.rowid"; end;' + + '' + + expect(encodeSchema(testSchema)).toBe(expectedSchema) + }) it('encodes migrations', () => { const migrationSteps = [ addColumns({ From 869900b1026baf0be6edf5c5aebd41b37f2a8879 Mon Sep 17 00:00:00 2001 From: Kenneth Law Date: Fri, 14 Feb 2020 02:09:11 +0800 Subject: [PATCH 02/13] :sparkles: add support for Full Text Search match operator and implement support for Sqlite adapter --- src/QueryDescription/index.d.ts | 2 ++ src/QueryDescription/index.js | 5 +++++ src/QueryDescription/test.js | 17 +++++++++++++++++ src/adapters/sqlite/encodeQuery/index.js | 15 +++++++++++++++ src/adapters/sqlite/encodeQuery/test.js | 12 ++++++++++++ src/adapters/sqlite/encodeSchema/test.js | 2 +- 6 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/QueryDescription/index.d.ts b/src/QueryDescription/index.d.ts index 7a0187032..6c4003352 100644 --- a/src/QueryDescription/index.d.ts +++ b/src/QueryDescription/index.d.ts @@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' { | 'oneOf' | 'notIn' | 'between' + | 'match' export interface ColumnDescription { column: ColumnName @@ -105,6 +106,7 @@ declare module '@nozbe/watermelondb/QueryDescription' { export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription export function and(...conditions: Condition[]): And export function or(...conditions: Condition[]): Or + export function textMatches(value: string): Comparison export function like(value: string): Comparison export function notLike(value: string): Comparison export function experimentalSortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy diff --git a/src/QueryDescription/index.js b/src/QueryDescription/index.js index 2c8a806c8..59cbf5f2b 100644 --- a/src/QueryDescription/index.js +++ b/src/QueryDescription/index.js @@ -29,6 +29,7 @@ export type Operator = | 'between' | 'like' | 'notLike' + | 'match' export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }> export type ComparisonRight = @@ -233,6 +234,10 @@ export function sanitizeLikeString(value: string): string { return value.replace(nonLikeSafeRegexp, '_') } +export function textMatches(value: string): Comparison { + return { operator: 'match', right: { value } } +} + export function column(name: ColumnName): ColumnDescription { invariant(typeof name === 'string', 'Name passed to Q.column() is not a string') return { column: checkName(name), type: columnSymbol } diff --git a/src/QueryDescription/test.js b/src/QueryDescription/test.js index 66646af10..7601316da 100644 --- a/src/QueryDescription/test.js +++ b/src/QueryDescription/test.js @@ -635,4 +635,21 @@ describe('queryWithoutDeleted', () => { ]), ) }) + + it('supports textMatches as fts join', () => { + const query = Q.buildQueryDescription([ + Q.textMatches('searchable', 'hello world'), + ]) + expect(query).toEqual({ + 'where': [ + { + 'operator': 'match', + 'right': { + 'value': 'searchable', + }, + }, + ], + 'join': [], + }) + }) }) diff --git a/src/adapters/sqlite/encodeQuery/index.js b/src/adapters/sqlite/encodeQuery/index.js index 0cebcdeca..08788997d 100644 --- a/src/adapters/sqlite/encodeQuery/index.js +++ b/src/adapters/sqlite/encodeQuery/index.js @@ -52,6 +52,7 @@ const operators: { [Operator]: string } = { between: 'between', like: 'like', notLike: 'not like', + match: 'match', } const encodeComparison = (table: TableName, comparison: Comparison) => { @@ -109,6 +110,20 @@ const encodeWhereCondition = ( ) } + if (comparison.operator === 'match') { + const srcTable = encodeName(table) + const ftsTable = encodeName(`${table}_fts`) + const rowid = encodeName('rowid') + const ftsColumn = encodeName(left) + const matchValue = getComparisonRight(table, comparison.right) + return ( + `${srcTable}.${rowid} in (` + + `select ${ftsTable}.${rowid} from ${ftsTable} ` + + `where ${ftsTable}.${ftsColumn} match ${matchValue}` + + `)` + ) + } + return `${encodeName(table)}.${encodeName(left)} ${encodeComparison(table, comparison)}` } diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index 89b151bf4..1e6f37540 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -270,4 +270,16 @@ describe('SQLite encodeQuery', () => { expect(() => encoded([Q.unsafeLokiExpr({ hi: true })])).toThrow('Unknown clause') expect(() => encoded([Q.unsafeLokiTransform(() => {})])).toThrow('not supported') }) + it('encodes JOIN over FTS table', () => { + const query = new Query(mockCollection, [ + Q.where('searchable', Q.textMatches('hello world')), + ]) + expect(encodeQuery(query)).toBe( + `select "tasks".* from "tasks" ` + + `where "tasks"."rowid" in (` + + `select "tasks_fts"."rowid" from "tasks_fts" ` + + `where "tasks_fts"."searchable" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'` + ) + }) }) diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index 285171a7a..b2b67df8a 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -57,7 +57,7 @@ describe('encodeSchema', () => { expect(encodeSchema(testSchema)).toBe(expectedSchema) }) - fit('encodes schema with FTS', () => { + it('encodes schema with FTS', () => { const testSchema = appSchema({ version: 1, tables: [ From be6549f058f9e77a06247d10a7bad55648e8efeb Mon Sep 17 00:00:00 2001 From: Kenneth Law Date: Fri, 14 Feb 2020 04:57:57 +0800 Subject: [PATCH 03/13] :bug: fix incorrectly quoted SQL NEW and OLD trigger references in Sqlite FTS trigger code added in d3fc571 --- src/adapters/sqlite/encodeSchema/index.js | 8 ++++---- src/adapters/sqlite/encodeSchema/test.js | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index eb733ccba..c5b1a2eb3 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -61,7 +61,7 @@ const encodeFTSDeleteTrigger: ({ tableName, ftsTableName, event: 'delete', - action: `delete from ${encodeName(ftsTableName)} where "rowid" = "OLD.rowid";`, + action: `delete from ${encodeName(ftsTableName)} where "rowid" = OLD.rowid;`, }) const encodeFTSInsertTrigger: ({ @@ -71,7 +71,7 @@ const encodeFTSInsertTrigger: ({ }) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)] const columns = rawColumnNames.map(encodeName) - const valueColumns = rawColumnNames.map(column => encodeName(`NEW.${column}`)) + const valueColumns = rawColumnNames.map(column => `NEW.${encodeName(column)}`) const columnsSQL = columns.join(', ') const valueColumnsSQL = valueColumns.join(', ') @@ -91,7 +91,7 @@ const encodeFTSUpdateTrigger: ({ }) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { const rawColumnNames = ftsColumns.map(column => column.name) const assignments = rawColumnNames.map( - column => `${encodeName(column)}=${encodeName(`NEW.${column}`)}`, + column => `${encodeName(column)} = NEW.${encodeName(column)}`, ) const assignmentsSQL = assignments.join(', ') @@ -100,7 +100,7 @@ const encodeFTSUpdateTrigger: ({ tableName, ftsTableName, event: 'update', - action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = "NEW.rowid";`, + action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = NEW."rowid";`, }) } diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index b2b67df8a..4496eaa4d 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -78,9 +78,9 @@ describe('encodeSchema', () => { 'create index tasks_author_id on "tasks" ("author_id");' + 'create index tasks__status on "tasks" ("_status");' + 'create virtual table "tasks_fts" using fts4("author_name", "author_title");' + - 'create trigger "tasks_fts_delete" after delete on "tasks" begin delete from "tasks_fts" where "rowid" = "OLD.rowid"; end;' + - 'create trigger "tasks_fts_insert" after insert on "tasks" begin insert into "tasks_fts" ("rowid", "author_name", "author_title") values ("NEW.rowid", "NEW.author_name", "NEW.author_title"); end;' + - 'create trigger "tasks_fts_update" after update on "tasks" begin update "tasks_fts" set "author_name"="NEW.author_name", "author_title"="NEW.author_title" where "rowid" = "NEW.rowid"; end;' + + 'create trigger "tasks_fts_delete" after delete on "tasks" begin delete from "tasks_fts" where "rowid" = OLD.rowid; end;' + + 'create trigger "tasks_fts_insert" after insert on "tasks" begin insert into "tasks_fts" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' + + 'create trigger "tasks_fts_update" after update on "tasks" begin update "tasks_fts" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' + '' expect(encodeSchema(testSchema)).toBe(expectedSchema) From be8113deb79d0304122d54f869baca074570f163 Mon Sep 17 00:00:00 2001 From: Kenneth Law Date: Sat, 15 Feb 2020 12:43:43 +0800 Subject: [PATCH 04/13] :art: fix flow & lint issues in d3fc571 --- src/adapters/lokijs/worker/executor.js | 2 +- src/adapters/sqlite/encodeSchema/index.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index 04fe67908..e2fd2a9e7 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -475,7 +475,7 @@ export default class LokiExecutor { throw error } - _warnAboutLackingFTSSupport(columns: Array) { + _warnAboutLackingFTSSupport(columns: Array): void { const searchableColumns = columns.filter(column => column.isSearchable) if (searchableColumns.length > 0) { // Warn the user about missing FTS support for the LokiJS adapter diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index c5b1a2eb3..599f75ac6 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -109,7 +109,6 @@ const encodeFTSTriggers: ({ ftsTableName: string, ftsColumns: ColumnSchema[], }) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { - const updateTrigger = '' return ( encodeFTSDeleteTrigger({ tableName, ftsTableName }) + encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) + From 3d6353d371ba24bd22dd17c8b3b9234fb0ee5b2b Mon Sep 17 00:00:00 2001 From: Kenneth Law Date: Sat, 15 Feb 2020 13:03:14 +0800 Subject: [PATCH 05/13] :pencil: keep a changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 094331998..b0fc0e24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -274,6 +274,9 @@ cause all or some of user's data to be deleted. More information available via G This is called when underlying IndexedDB encountered a quota exceeded error (ran out of allotted disk space for app) This means that app can't save more data or that it will fall back to using in-memory database only Note that this only works when `useWebWorker: false` +- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter: + Add `isSearchable` boolean flag to schema column descriptor for creating Full Text Search-able columns + Add `Q.textMatches(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater ### Changes From 06d4166b6643f950b4e5c381c8c2b9291e85a30c Mon Sep 17 00:00:00 2001 From: Radek Pietruszewski Date: Thu, 18 Jun 2020 11:10:11 +0200 Subject: [PATCH 06/13] Update src/adapters/sqlite/encodeSchema/index.js --- src/adapters/sqlite/encodeSchema/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index 599f75ac6..fb436700b 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -121,7 +121,7 @@ const encodeFTSTable: ({ ftsColumns: ColumnSchema[], }) => SQL = ({ ftsTableName, ftsColumns }) => { const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ') - return `create virtual table ${encodeName(`${ftsTableName}`)} using fts4(${columnsSQL});` + return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});` } const encodeFTSSearch: TableSchema => SQL = ({ name: tableName, columns }) => { From c8d7ff3d4ccd54f4f7275e368bd39c238407c26e Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Sun, 4 Apr 2021 22:03:06 -0700 Subject: [PATCH 07/13] FTS: Rebase + fix tests --- src/QueryDescription/index.js | 3 ++- src/QueryDescription/test.js | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/QueryDescription/index.js b/src/QueryDescription/index.js index 59cbf5f2b..b2eccc5fa 100644 --- a/src/QueryDescription/index.js +++ b/src/QueryDescription/index.js @@ -235,7 +235,8 @@ export function sanitizeLikeString(value: string): string { } export function textMatches(value: string): Comparison { - return { operator: 'match', right: { value } } + invariant(typeof value === 'string', 'Value passed to Q.textMatches() is not a string') + return { operator: 'match', right: { value }, type: comparisonSymbol } } export function column(name: ColumnName): ColumnDescription { diff --git a/src/QueryDescription/test.js b/src/QueryDescription/test.js index 7601316da..65cfe4e54 100644 --- a/src/QueryDescription/test.js +++ b/src/QueryDescription/test.js @@ -637,19 +637,23 @@ describe('queryWithoutDeleted', () => { }) it('supports textMatches as fts join', () => { - const query = Q.buildQueryDescription([ - Q.textMatches('searchable', 'hello world'), - ]) + const query = Q.buildQueryDescription([Q.where('searchable', Q.textMatches('hello world'))]) expect(query).toEqual({ - 'where': [ + where: [ { - 'operator': 'match', - 'right': { - 'value': 'searchable', + type: 'where', + left: 'searchable', + comparison: { + operator: 'match', + right: { + value: 'hello world', + }, }, }, ], - 'join': [], + joinTables: [], + nestedJoinTables: [], + sortBy: [], }) }) }) From ae7c8234119c48aada66b5af2ceb0f0cd175520d Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Thu, 22 Apr 2021 10:45:26 -0700 Subject: [PATCH 08/13] FTS improvements (uses `isFTS` and the tables are `_fts_${tableName}`) --- src/Schema/index.d.ts | 2 +- src/Schema/index.js | 2 +- src/adapters/lokijs/worker/executor.js | 12 +++++++--- src/adapters/sqlite/encodeQuery/index.js | 2 +- src/adapters/sqlite/encodeQuery/test.js | 12 +++++----- src/adapters/sqlite/encodeSchema/index.js | 27 ++++++++++++++--------- src/adapters/sqlite/encodeSchema/test.js | 16 +++++++------- 7 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/Schema/index.d.ts b/src/Schema/index.d.ts index ced17534a..5b546d5fb 100644 --- a/src/Schema/index.d.ts +++ b/src/Schema/index.d.ts @@ -17,7 +17,7 @@ declare module '@nozbe/watermelondb/Schema' { type: ColumnType isOptional?: boolean isIndexed?: boolean - isSearchable?: boolean + isFTS?: boolean } interface ColumnMap { diff --git a/src/Schema/index.js b/src/Schema/index.js index f9c632853..975a055d0 100644 --- a/src/Schema/index.js +++ b/src/Schema/index.js @@ -15,7 +15,7 @@ export type ColumnSchema = $RE<{ type: ColumnType, isOptional?: boolean, isIndexed?: boolean, - isSearchable?: boolean, + isFTS?: boolean, }> export type ColumnMap = { [name: ColumnName]: ColumnSchema } diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index e2fd2a9e7..3753d3f49 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -4,7 +4,13 @@ import logger from '../../../utils/common/logger' import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type' -import type { TableName, AppSchema, SchemaVersion, TableSchema, ColumnSchema } from '../../../Schema' +import type { + TableName, + AppSchema, + SchemaVersion, + TableSchema, + ColumnSchema, +} from '../../../Schema' import type { SchemaMigrations, CreateTableMigrationStep, @@ -474,9 +480,9 @@ export default class LokiExecutor { // Rethrow error throw error } - + _warnAboutLackingFTSSupport(columns: Array): void { - const searchableColumns = columns.filter(column => column.isSearchable) + const searchableColumns = columns.filter(column => column.isFTS) if (searchableColumns.length > 0) { // Warn the user about missing FTS support for the LokiJS adapter // Please contribute! Here are some pointers: diff --git a/src/adapters/sqlite/encodeQuery/index.js b/src/adapters/sqlite/encodeQuery/index.js index 08788997d..b351dcd7a 100644 --- a/src/adapters/sqlite/encodeQuery/index.js +++ b/src/adapters/sqlite/encodeQuery/index.js @@ -112,7 +112,7 @@ const encodeWhereCondition = ( if (comparison.operator === 'match') { const srcTable = encodeName(table) - const ftsTable = encodeName(`${table}_fts`) + const ftsTable = encodeName(`_fts_${table}`) const rowid = encodeName('rowid') const ftsColumn = encodeName(left) const matchValue = getComparisonRight(table, comparison.right) diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index 1e6f37540..9d9cdf923 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -271,15 +271,13 @@ describe('SQLite encodeQuery', () => { expect(() => encoded([Q.unsafeLokiTransform(() => {})])).toThrow('not supported') }) it('encodes JOIN over FTS table', () => { - const query = new Query(mockCollection, [ - Q.where('searchable', Q.textMatches('hello world')), - ]) + const query = new Query(mockCollection, [Q.where('searchable', Q.textMatches('hello world'))]) expect(encodeQuery(query)).toBe( `select "tasks".* from "tasks" ` + - `where "tasks"."rowid" in (` + - `select "tasks_fts"."rowid" from "tasks_fts" ` + - `where "tasks_fts"."searchable" match 'hello world'` + - `) and "tasks"."_status" is not 'deleted'` + `where "tasks"."rowid" in (` + + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + + `where "_fts_tasks"."searchable" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'`, ) }) }) diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index fb436700b..47e8f00f3 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -39,8 +39,15 @@ const encodeTableIndicies: (TableSchema) => SQL = ({ name: tableName, columns }) const transform = (sql: string, transformer: ?(string) => string) => transformer ? transformer(sql) : sql -const encodeTable: (TableSchema) => SQL = (table) => - transform(encodeCreateTable(table) + encodeTableIndicies(table), table.unsafeSql) +const encodeTable: TableSchema => SQL = table => + transform( + // eslint-disable-next-line no-use-before-define + encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table), + table.unsafeSql, + ) + +/** FTS Full Text Search */ + const encodeFTSTrigger: ({ tableName: string, ftsTableName: string, @@ -124,20 +131,20 @@ const encodeFTSTable: ({ return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});` } -const encodeFTSSearch: TableSchema => SQL = ({ name: tableName, columns }) => { - const ftsColumns = values(columns).filter(c => c.isSearchable) +const encodeFTSSearch: TableSchema => SQL = tableSchema => { + const { name: tableName, columnArray } = tableSchema + const ftsColumns = columnArray.filter(column => column.isFTS) if (ftsColumns.length === 0) { return '' } - const ftsTableName = `${tableName}_fts` + const ftsTableName = `_fts_${tableName}` return ( encodeFTSTable({ ftsTableName, ftsColumns }) + encodeFTSTriggers({ tableName, ftsTableName, ftsColumns }) ) } -const encodeTable: TableSchema => SQL = table => - encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table) +/** FTS END */ export const encodeSchema: (AppSchema) => SQL = ({ tables, unsafeSql }) => { const sql = Object.values(tables) @@ -163,10 +170,8 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({ )} = ${encodeValue(nullValue(column))};` const addIndex = encodeIndex(column, table) - if (column.isSearchable) { - logger.warn( - '[DB][Worker] Support for migrations and isSearchable is still to be implemented', - ) + if (column.isFTS) { + logger.warn('[DB][Worker] Support for migrations and isFTS is still to be implemented') } return transform(addColumn + setDefaultValue + addIndex, unsafeSql) diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index 4496eaa4d..6c9beb9d8 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -65,8 +65,8 @@ describe('encodeSchema', () => { name: 'tasks', columns: [ { name: 'author_id', type: 'string', isIndexed: true }, - { name: 'author_name', type: 'string', isSearchable: true }, - { name: 'author_title', type: 'string', isSearchable: true }, + { name: 'author_name', type: 'string', isFTS: true }, + { name: 'author_title', type: 'string', isFTS: true }, { name: 'created_at', type: 'number' }, ], }), @@ -75,12 +75,12 @@ describe('encodeSchema', () => { const expectedSchema = 'create table "tasks" ("id" primary key, "_changed", "_status", "author_id", "author_name", "author_title", "created_at");' + - 'create index tasks_author_id on "tasks" ("author_id");' + - 'create index tasks__status on "tasks" ("_status");' + - 'create virtual table "tasks_fts" using fts4("author_name", "author_title");' + - 'create trigger "tasks_fts_delete" after delete on "tasks" begin delete from "tasks_fts" where "rowid" = OLD.rowid; end;' + - 'create trigger "tasks_fts_insert" after insert on "tasks" begin insert into "tasks_fts" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' + - 'create trigger "tasks_fts_update" after update on "tasks" begin update "tasks_fts" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' + + 'create index "tasks_author_id" on "tasks" ("author_id");' + + 'create index "tasks__status" on "tasks" ("_status");' + + 'create virtual table "_fts_tasks" using fts4("author_name", "author_title");' + + 'create trigger "_fts_tasks_delete" after delete on "tasks" begin delete from "_fts_tasks" where "rowid" = OLD.rowid; end;' + + 'create trigger "_fts_tasks_insert" after insert on "tasks" begin insert into "_fts_tasks" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' + + 'create trigger "_fts_tasks_update" after update on "tasks" begin update "_fts_tasks" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' + '' expect(encodeSchema(testSchema)).toBe(expectedSchema) From 140d4f4a8cc89aad13239e25f07489977090969f Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Thu, 22 Apr 2021 11:07:28 -0700 Subject: [PATCH 09/13] FTS improvements (uses `isFTS` and the tables are `_fts_${tableName}`) + rebase --- CHANGELOG-Unreleased.md | 5 +++++ CHANGELOG.md | 3 --- src/QueryDescription/index.d.ts | 2 +- src/QueryDescription/index.js | 4 ++-- src/QueryDescription/test.js | 4 ++-- src/adapters/lokijs/worker/executor.js | 2 +- src/adapters/sqlite/encodeQuery/test.js | 4 ++-- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG-Unreleased.md b/CHANGELOG-Unreleased.md index cd03d4c60..cb50ff9ac 100644 --- a/CHANGELOG-Unreleased.md +++ b/CHANGELOG-Unreleased.md @@ -14,6 +14,11 @@ ### New features + +- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter: + Add `isFTS` boolean flag to schema column descriptor for creating Full Text Search-able columns + Add `Q.ftsMatch(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater + ### Performance ### Changes diff --git a/CHANGELOG.md b/CHANGELOG.md index b0fc0e24d..094331998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -274,9 +274,6 @@ cause all or some of user's data to be deleted. More information available via G This is called when underlying IndexedDB encountered a quota exceeded error (ran out of allotted disk space for app) This means that app can't save more data or that it will fall back to using in-memory database only Note that this only works when `useWebWorker: false` -- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter: - Add `isSearchable` boolean flag to schema column descriptor for creating Full Text Search-able columns - Add `Q.textMatches(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater ### Changes diff --git a/src/QueryDescription/index.d.ts b/src/QueryDescription/index.d.ts index 6c4003352..f5915f8a3 100644 --- a/src/QueryDescription/index.d.ts +++ b/src/QueryDescription/index.d.ts @@ -106,7 +106,7 @@ declare module '@nozbe/watermelondb/QueryDescription' { export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription export function and(...conditions: Condition[]): And export function or(...conditions: Condition[]): Or - export function textMatches(value: string): Comparison + export function ftsMatch(value: string): Comparison export function like(value: string): Comparison export function notLike(value: string): Comparison export function experimentalSortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy diff --git a/src/QueryDescription/index.js b/src/QueryDescription/index.js index b2eccc5fa..91c84410b 100644 --- a/src/QueryDescription/index.js +++ b/src/QueryDescription/index.js @@ -234,8 +234,8 @@ export function sanitizeLikeString(value: string): string { return value.replace(nonLikeSafeRegexp, '_') } -export function textMatches(value: string): Comparison { - invariant(typeof value === 'string', 'Value passed to Q.textMatches() is not a string') +export function ftsMatch(value: string): Comparison { + invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string') return { operator: 'match', right: { value }, type: comparisonSymbol } } diff --git a/src/QueryDescription/test.js b/src/QueryDescription/test.js index 65cfe4e54..f451aa17b 100644 --- a/src/QueryDescription/test.js +++ b/src/QueryDescription/test.js @@ -636,8 +636,8 @@ describe('queryWithoutDeleted', () => { ) }) - it('supports textMatches as fts join', () => { - const query = Q.buildQueryDescription([Q.where('searchable', Q.textMatches('hello world'))]) + it('supports ftsMatch as fts join', () => { + const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))]) expect(query).toEqual({ where: [ { diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index 3753d3f49..e7b9488b0 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -318,7 +318,7 @@ export default class LokiExecutor { [], ) - this._warnAboutLackingFTSSupport(values(columns)) + this._warnAboutLackingFTSSupport(columnArray) this.loki.addCollection(name, { unique: ['id'], diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index 9d9cdf923..5fa17acc2 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -65,7 +65,7 @@ describe('SQLite encodeQuery', () => { Q.where('col5', Q.lte(5)), Q.where('col6', Q.notEq(null)), Q.where('col7', Q.oneOf([1, 2, 3])), - Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), + Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), // eslint-disable-line quotes Q.where('col9', Q.between(10, 11)), Q.where('col10', Q.like('%abc')), Q.where('col11', Q.notLike('def%')), @@ -271,7 +271,7 @@ describe('SQLite encodeQuery', () => { expect(() => encoded([Q.unsafeLokiTransform(() => {})])).toThrow('not supported') }) it('encodes JOIN over FTS table', () => { - const query = new Query(mockCollection, [Q.where('searchable', Q.textMatches('hello world'))]) + const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))]) expect(encodeQuery(query)).toBe( `select "tasks".* from "tasks" ` + `where "tasks"."rowid" in (` + From 1371b4e62d4ca7761914f98d833f4d5e316243ce Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Fri, 14 May 2021 12:50:39 -0700 Subject: [PATCH 10/13] FTS improvements --- src/QueryDescription/index.js | 4 +- src/QueryDescription/test.js | 42 +++++++++---------- src/adapters/lokijs/worker/executor.js | 3 +- src/adapters/sqlite/encodeQuery/index.js | 4 +- src/adapters/sqlite/encodeQuery/test.js | 52 +++++++++++++++++++----- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/QueryDescription/index.js b/src/QueryDescription/index.js index 91c84410b..b51fa3be2 100644 --- a/src/QueryDescription/index.js +++ b/src/QueryDescription/index.js @@ -29,7 +29,7 @@ export type Operator = | 'between' | 'like' | 'notLike' - | 'match' + | 'ftsMatch' export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }> export type ComparisonRight = @@ -236,7 +236,7 @@ export function sanitizeLikeString(value: string): string { export function ftsMatch(value: string): Comparison { invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string') - return { operator: 'match', right: { value }, type: comparisonSymbol } + return { operator: 'ftsMatch', right: { value }, type: comparisonSymbol } } export function column(name: ColumnName): ColumnDescription { diff --git a/src/QueryDescription/test.js b/src/QueryDescription/test.js index f451aa17b..406ea06e2 100644 --- a/src/QueryDescription/test.js +++ b/src/QueryDescription/test.js @@ -448,6 +448,26 @@ describe('buildQueryDescription', () => { process.env.NODE_ENV = env } }) + it('supports ftsMatch as fts join', () => { + const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))]) + expect(query).toEqual({ + where: [ + { + type: 'where', + left: 'searchable', + comparison: { + operator: 'ftsMatch', + right: { + value: 'hello world', + }, + }, + }, + ], + joinTables: [], + nestedJoinTables: [], + sortBy: [], + }) + }) it('catches bad types', () => { expect(() => Q.eq({})).toThrow('Invalid value passed to query') // TODO: oneOf/notIn values? @@ -458,6 +478,7 @@ describe('buildQueryDescription', () => { expect(() => Q.notLike(null)).toThrow('not a string') expect(() => Q.notLike({})).toThrow('not a string') expect(() => Q.sanitizeLikeString(null)).toThrow('not a string') + expect(() => Q.ftsMatch(null)).toThrow('not a string') expect(() => Q.column({})).toThrow('not a string') expect(() => Q.experimentalTake('0')).toThrow('not a number') expect(() => Q.experimentalSkip('0')).toThrow('not a number') @@ -635,25 +656,4 @@ describe('queryWithoutDeleted', () => { ]), ) }) - - it('supports ftsMatch as fts join', () => { - const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))]) - expect(query).toEqual({ - where: [ - { - type: 'where', - left: 'searchable', - comparison: { - operator: 'match', - right: { - value: 'hello world', - }, - }, - }, - ], - joinTables: [], - nestedJoinTables: [], - sortBy: [], - }) - }) }) diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index e7b9488b0..522f7ac6b 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -482,8 +482,7 @@ export default class LokiExecutor { } _warnAboutLackingFTSSupport(columns: Array): void { - const searchableColumns = columns.filter(column => column.isFTS) - if (searchableColumns.length > 0) { + if (columns.some((column) => column.isFTS)) { // Warn the user about missing FTS support for the LokiJS adapter // Please contribute! Here are some pointers: // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts diff --git a/src/adapters/sqlite/encodeQuery/index.js b/src/adapters/sqlite/encodeQuery/index.js index b351dcd7a..6d0bfa170 100644 --- a/src/adapters/sqlite/encodeQuery/index.js +++ b/src/adapters/sqlite/encodeQuery/index.js @@ -52,7 +52,7 @@ const operators: { [Operator]: string } = { between: 'between', like: 'like', notLike: 'not like', - match: 'match', + ftsMatch: 'match', } const encodeComparison = (table: TableName, comparison: Comparison) => { @@ -110,7 +110,7 @@ const encodeWhereCondition = ( ) } - if (comparison.operator === 'match') { + if (comparison.operator === 'ftsMatch') { const srcTable = encodeName(table) const ftsTable = encodeName(`_fts_${table}`) const rowid = encodeName('rowid') diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index 5fa17acc2..06c5dbcf8 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -163,6 +163,48 @@ describe('SQLite encodeQuery', () => { ` and "tasks"."_status" is not 'deleted'`, ) }) + it('encodes ftsMatch', () => { + const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))]) + expect(encodeQuery(query)).toBe( + `select "tasks".* from "tasks" ` + + `where "tasks"."rowid" in (` + + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + + `where "_fts_tasks"."searchable" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'`, + ) + }) + it('encodes ftsMatch with other joins', () => { + const query = [ + Q.on('projects', 'team_id', 'abcdef'), + Q.on('projects', 'is_active', true), + Q.on('projects', 'left_column', Q.lte(Q.column('right_column'))), + Q.on('projects', 'left2', Q.weakGt(Q.column('right2'))), + Q.where('left_column', 'right_value'), + Q.on('tag_assignments', 'tag_id', Q.oneOf(['a', 'b', 'c'])), + Q.where('searchable', Q.ftsMatch('hello world')), + ] + const expectedQuery = + `join "projects" on "projects"."id" = "tasks"."project_id"` + + ` join "tag_assignments" on "tag_assignments"."task_id" = "tasks"."id"` + + ` where ("projects"."team_id" is 'abcdef'` + + ` and "projects"."is_active" is 1` + + ` and "projects"."left_column" <= "projects"."right_column"` + + ` and ("projects"."left2" > "projects"."right2"` + + ` or ("projects"."left2" is not null` + + ` and "projects"."right2" is null))` + + ` and "projects"."_status" is not 'deleted')` + + ` and ("tag_assignments"."tag_id" in ('a', 'b', 'c')` + + ` and "tag_assignments"."_status" is not 'deleted')` + + ` and "tasks"."left_column" is 'right_value'` + + ` and "tasks"."rowid" in` + + ` (select "_fts_tasks"."rowid" from "_fts_tasks" where` + + ` "_fts_tasks"."searchable" match 'hello world')` + + ` and "tasks"."_status" is not 'deleted'` + expect(encoded(query)).toBe(`select distinct "tasks".* from "tasks" ${expectedQuery}`) + expect(encoded(query, true)).toBe( + `select count(distinct "tasks"."id") as "count" from "tasks" ${expectedQuery}`, + ) + }) it(`encodes on nested in and/or`, () => { expect( encoded([ @@ -270,14 +312,4 @@ describe('SQLite encodeQuery', () => { expect(() => encoded([Q.unsafeLokiExpr({ hi: true })])).toThrow('Unknown clause') expect(() => encoded([Q.unsafeLokiTransform(() => {})])).toThrow('not supported') }) - it('encodes JOIN over FTS table', () => { - const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))]) - expect(encodeQuery(query)).toBe( - `select "tasks".* from "tasks" ` + - `where "tasks"."rowid" in (` + - `select "_fts_tasks"."rowid" from "_fts_tasks" ` + - `where "_fts_tasks"."searchable" match 'hello world'` + - `) and "tasks"."_status" is not 'deleted'`, - ) - }) }) From aa3d9b11a364aa4257a9a14254e4303704bea0fd Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Tue, 18 May 2021 09:14:07 -0700 Subject: [PATCH 11/13] Update src/adapters/sqlite/encodeSchema/index.js Co-authored-by: Radek Pietruszewski --- src/adapters/sqlite/encodeSchema/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index 47e8f00f3..d439716d2 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -170,9 +170,7 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({ )} = ${encodeValue(nullValue(column))};` const addIndex = encodeIndex(column, table) - if (column.isFTS) { - logger.warn('[DB][Worker] Support for migrations and isFTS is still to be implemented') - } + invariant(!column.isFTS, '[DB][Worker] Support for migrations with isFTS is still to be implemented') return transform(addColumn + setDefaultValue + addIndex, unsafeSql) }) From 183ac16bcea8e5821929f1761b418e87d960dc65 Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Wed, 26 May 2021 10:20:42 -0700 Subject: [PATCH 12/13] FTS: improves tests and documentation --- docs-master/Query.md | 5 +++ src/__tests__/databaseTests.js | 49 ++++++++++++++++++++++++ src/adapters/__tests__/commonTests.js | 22 ++++++++++- src/adapters/__tests__/helpers.js | 10 ++++- src/adapters/sqlite/encodeQuery/index.js | 3 +- 5 files changed, 85 insertions(+), 4 deletions(-) diff --git a/docs-master/Query.md b/docs-master/Query.md index cfb61460c..4704bb89a 100644 --- a/docs-master/Query.md +++ b/docs-master/Query.md @@ -234,6 +234,11 @@ tasksCollection.query( ) ``` + +#### Full Text Search with `Q.ftsMatch` + +If you are using `SQLite` and used `isFTS` in one or more of your text columns, you can use `Q.where(fieldName, Q.ftsMatch(searchText))`. If you have more than one column with `isFTS`, you can either use `tableName` instead of `fieldName` to search in all fields, or specify `fieldName` to focus the search in only one field. + ## Advanced Queries ### Advanced observing diff --git a/src/__tests__/databaseTests.js b/src/__tests__/databaseTests.js index 768741588..2ed4257b8 100644 --- a/src/__tests__/databaseTests.js +++ b/src/__tests__/databaseTests.js @@ -1253,3 +1253,52 @@ export const joinTests = [ skipSqlite: true, }, ] + +export const ftsMatchTests = [ + { + name: 'Can ftsMatch - text1', + query: [Q.where('text1', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + ], + nonMatching: [ + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }, + ], + skipLoki: true, + }, + { + name: 'Can ftsMatch - text2', + query: [Q.where('text2', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + ], + nonMatching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }, + ], + skipLoki: true, + }, + { + name: 'Can ftsMatch - text1 and text2', + query: [Q.where('tasks', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + ], + nonMatching: [{ id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }], + skipLoki: true, + }, +] diff --git a/src/adapters/__tests__/commonTests.js b/src/adapters/__tests__/commonTests.js index 744681231..e93e72ae7 100644 --- a/src/adapters/__tests__/commonTests.js +++ b/src/adapters/__tests__/commonTests.js @@ -9,7 +9,12 @@ import * as Q from '../../QueryDescription' import { appSchema, tableSchema } from '../../Schema' import { schemaMigrations, createTable, addColumns } from '../../Schema/migrations' -import { matchTests, naughtyMatchTests, joinTests } from '../../__tests__/databaseTests' +import { + matchTests, + naughtyMatchTests, + joinTests, + ftsMatchTests, +} from '../../__tests__/databaseTests' import DatabaseAdapterCompat from '../compat' import { testSchema, @@ -17,6 +22,7 @@ import { mockTaskRaw, performMatchTest, performJoinTest, + performFtsMatchTest, expectSortedEqual, MockTask, mockProjectRaw, @@ -1137,6 +1143,20 @@ export default () => [ } }, ], + ...ftsMatchTests.map((testCase) => [ + `[shared ftsMatch test] ${testCase.name}`, + async (adapter, AdapterClass) => { + const perform = () => performFtsMatchTest(adapter, testCase) + const shouldSkip = + (AdapterClass.name === 'LokiJSAdapter' && testCase.skipLoki) || + (AdapterClass.name === 'SQLiteAdapter' && testCase.skipSqlite) + if (shouldSkip) { + await expect(perform()).rejects.toBeInstanceOf(Error) + } else { + await perform() + } + }, + ]), [ 'can store and retrieve large numbers (regression test)', async (_adapter) => { diff --git a/src/adapters/__tests__/helpers.js b/src/adapters/__tests__/helpers.js index 70d7a6d97..837fd2199 100644 --- a/src/adapters/__tests__/helpers.js +++ b/src/adapters/__tests__/helpers.js @@ -56,8 +56,8 @@ export const testSchema = appSchema({ { name: 'num3', type: 'number' }, { name: 'float1', type: 'number' }, // TODO: Remove me? { name: 'float2', type: 'number' }, - { name: 'text1', type: 'string' }, - { name: 'text2', type: 'string' }, + { name: 'text1', type: 'string', isFTS: true }, + { name: 'text2', type: 'string', isFTS: true }, { name: 'bool1', type: 'boolean' }, { name: 'bool2', type: 'boolean' }, { name: 'order', type: 'number' }, @@ -177,3 +177,9 @@ export const performJoinTest = async (adapter, testCase) => { await allPromises(([table, records]) => insertAll(adapter, table, records), pairs) await performMatchTest(adapter, testCase) } + +export const performFtsMatchTest = async (adapter, testCase) => { + const pairs = toPairs(testCase.extraRecords) + await allPromises(([table, records]) => insertAll(adapter, table, records), pairs) + await performMatchTest(adapter, testCase) +} diff --git a/src/adapters/sqlite/encodeQuery/index.js b/src/adapters/sqlite/encodeQuery/index.js index 6d0bfa170..a422d1941 100644 --- a/src/adapters/sqlite/encodeQuery/index.js +++ b/src/adapters/sqlite/encodeQuery/index.js @@ -116,10 +116,11 @@ const encodeWhereCondition = ( const rowid = encodeName('rowid') const ftsColumn = encodeName(left) const matchValue = getComparisonRight(table, comparison.right) + const ftsTableColumn = table === left ? `${ftsTable}` : `${ftsTable}.${ftsColumn}` return ( `${srcTable}.${rowid} in (` + `select ${ftsTable}.${rowid} from ${ftsTable} ` + - `where ${ftsTable}.${ftsColumn} match ${matchValue}` + + `where ${ftsTableColumn} match ${matchValue}` + `)` ) } From b6fca2329cc0fa81cf0fffb6112c2e5952e3c8cb Mon Sep 17 00:00:00 2001 From: Sid Ferreira <143615+sidferreira@users.noreply.github.com> Date: Wed, 26 May 2021 10:52:30 -0700 Subject: [PATCH 13/13] FTS: lint and tests --- src/adapters/sqlite/encodeQuery/test.js | 21 +++++++++++++++------ src/adapters/sqlite/encodeSchema/index.js | 23 +++++++++++++---------- src/adapters/sqlite/encodeSchema/test.js | 2 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index bbab20d2e..107002bb2 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -172,14 +172,20 @@ describe('SQLite encodeQuery', () => { ) }) it('encodes ftsMatch', () => { - const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))]) - expect(encodeQuery(query)).toBe( + expect(encoded([Q.where('searchable', Q.ftsMatch('hello world'))])).toBe( `select "tasks".* from "tasks" ` + `where "tasks"."rowid" in (` + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + `where "_fts_tasks"."searchable" match 'hello world'` + `) and "tasks"."_status" is not 'deleted'`, ) + expect(encoded([Q.where('tasks', Q.ftsMatch('hello world'))])).toBe( + `select "tasks".* from "tasks" ` + + `where "tasks"."rowid" in (` + + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + + `where "_fts_tasks" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'`, + ) }) it('encodes ftsMatch with other joins', () => { const query = [ @@ -195,15 +201,18 @@ describe('SQLite encodeQuery', () => { `join "projects" on "projects"."id" = "tasks"."project_id"` + ` join "tag_assignments" on "tag_assignments"."task_id" = "tasks"."id"` + ` where ("projects"."team_id" is 'abcdef'` + - ` and "projects"."is_active" is 1` + - ` and "projects"."left_column" <= "projects"."right_column"` + - ` and ("projects"."left2" > "projects"."right2"` + + ` and "projects"."_status" is not 'deleted')` + + ` and ("projects"."is_active" is 1` + + ` and "projects"."_status" is not 'deleted')` + + ` and ("projects"."left_column" <= "projects"."right_column"` + + ` and "projects"."_status" is not 'deleted')` + + ` and (("projects"."left2" > "projects"."right2"` + ` or ("projects"."left2" is not null` + ` and "projects"."right2" is null))` + ` and "projects"."_status" is not 'deleted')` + + ` and "tasks"."left_column" is 'right_value'` + ` and ("tag_assignments"."tag_id" in ('a', 'b', 'c')` + ` and "tag_assignments"."_status" is not 'deleted')` + - ` and "tasks"."left_column" is 'right_value'` + ` and "tasks"."rowid" in` + ` (select "_fts_tasks"."rowid" from "_fts_tasks" where` + ` "_fts_tasks"."searchable" match 'hello world')` + diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index c5f703eb4..dd734b5f4 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -8,7 +8,7 @@ import type { AddColumnsMigrationStep, } from '../../../Schema/migrations' import type { SQL } from '../index' -import { logger } from '../../../utils/common' +import { invariant } from '../../../utils/common' import encodeName from '../encodeName' import encodeValue from '../encodeValue' @@ -42,7 +42,7 @@ const encodeTableIndicies: (TableSchema) => SQL = ({ name: tableName, columns }) const transform = (sql: string, transformer: ?(string) => string) => transformer ? transformer(sql) : sql -const encodeTable: TableSchema => SQL = table => +const encodeTable: (TableSchema) => SQL = (table) => transform( // eslint-disable-next-line no-use-before-define encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table), @@ -79,9 +79,9 @@ const encodeFTSInsertTrigger: ({ ftsTableName: string, ftsColumns: ColumnSchema[], }) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { - const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)] + const rawColumnNames = ['rowid', ...ftsColumns.map((column) => column.name)] const columns = rawColumnNames.map(encodeName) - const valueColumns = rawColumnNames.map(column => `NEW.${encodeName(column)}`) + const valueColumns = rawColumnNames.map((column) => `NEW.${encodeName(column)}`) const columnsSQL = columns.join(', ') const valueColumnsSQL = valueColumns.join(', ') @@ -99,9 +99,9 @@ const encodeFTSUpdateTrigger: ({ ftsTableName: string, ftsColumns: ColumnSchema[], }) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { - const rawColumnNames = ftsColumns.map(column => column.name) + const rawColumnNames = ftsColumns.map((column) => column.name) const assignments = rawColumnNames.map( - column => `${encodeName(column)} = NEW.${encodeName(column)}`, + (column) => `${encodeName(column)} = NEW.${encodeName(column)}`, ) const assignmentsSQL = assignments.join(', ') @@ -130,13 +130,13 @@ const encodeFTSTable: ({ ftsTableName: string, ftsColumns: ColumnSchema[], }) => SQL = ({ ftsTableName, ftsColumns }) => { - const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ') + const columnsSQL = ftsColumns.map((column) => encodeName(column.name)).join(', ') return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});` } -const encodeFTSSearch: TableSchema => SQL = tableSchema => { +const encodeFTSSearch: (TableSchema) => SQL = (tableSchema) => { const { name: tableName, columnArray } = tableSchema - const ftsColumns = columnArray.filter(column => column.isFTS) + const ftsColumns = columnArray.filter((column) => column.isFTS) if (ftsColumns.length === 0) { return '' } @@ -173,7 +173,10 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({ )} = ${encodeValue(nullValue(column))};` const addIndex = encodeIndex(column, table) - invariant(!column.isFTS, '[DB][Worker] Support for migrations with isFTS is still to be implemented') + invariant( + !column.isFTS, + '[DB][Worker] Support for migrations with isFTS is still to be implemented', + ) return transform(addColumn + setDefaultValue + addIndex, unsafeSql) }) diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index a76536584..7060cdaed 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -88,7 +88,7 @@ describe('encodeSchema', () => { 'create trigger "_fts_tasks_delete" after delete on "tasks" begin delete from "_fts_tasks" where "rowid" = OLD.rowid; end;' + 'create trigger "_fts_tasks_insert" after insert on "tasks" begin insert into "_fts_tasks" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' + 'create trigger "_fts_tasks_update" after update on "tasks" begin update "_fts_tasks" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' + - '' + 'create table "local_storage" ("key" varchar(16) primary key not null, "value" text not null);create index "local_storage_key_index" on "local_storage" ("key");' expect(encodeSchema(testSchema)).toBe(expectedSchema) })