diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d2564e18..d09232f60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,10 @@ jobs: permissions: packages: write + env: + cds_features_pool: true # TODO: make it work with Postgres test setup + FORCE_COLOR: true + strategy: fail-fast: true matrix: @@ -41,7 +45,13 @@ jobs: # testing - run: npm test -ws env: - cds_features_pool: true - FORCE_COLOR: true TAG: ${{ steps.hxe.outputs.TAG }} IMAGE_ID: ${{ steps.hxe.outputs.IMAGE_ID }} + - name: sqlite driver (node:sqlite) + run: npm test -w sqlite + env: + CDS_REQUIRES_DB_DRIVER: 'node' + - name: sqlite driver (sql.js) + run: npm test -w sqlite + env: + CDS_REQUIRES_DB_DRIVER: 'sql.js' diff --git a/package-lock.json b/package-lock.json index f46279efe..ff7c1e376 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1806,6 +1806,13 @@ "node": ">= 10.x" } }, + "node_modules/sql.js": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", + "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", + "license": "MIT", + "optional": true + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -1976,6 +1983,9 @@ "@cap-js/db-service": "^2.8.2", "better-sqlite3": "^12.0.0" }, + "optionalDependencies": { + "sql.js": "^1.13.0" + }, "peerDependencies": { "@sap/cds": ">=9" } diff --git a/package.json b/package.json index b647e5a49..69689862d 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ ], "devDependencies": { "@cap-js/cds-test": ">=0.2.0", + "sql.js": "^1.13.0", "axios": "^1" }, "scripts": { diff --git a/postgres/test/ql.test.js b/postgres/test/ql.test.js index 4ad6d1489..f75015e98 100644 --- a/postgres/test/ql.test.js +++ b/postgres/test/ql.test.js @@ -1,6 +1,4 @@ const cds = require('../../test/cds.js') -if (cds.env.fiori) cds.env.fiori.lean_draft = true -else cds.env.features.lean_draft = true const project = require('path').resolve(__dirname, 'beershop') diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 15c007f28..8ae90d8be 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -1,6 +1,7 @@ const { SQLService } = require('@cap-js/db-service') -const cds = require('@sap/cds') -const sqlite = require('better-sqlite3') +const cds = require('@sap/cds/lib') +let sqlite // sqlite driver is loaded on connect + const $session = Symbol('dbc.session') const sessionVariableMap = require('./session.json') // Adjust the path as necessary for your project const convStrm = require('stream/consumers') @@ -28,9 +29,12 @@ class SQLiteService extends SQLService { get factory() { return { options: this.options.pool || {}, - create: tenant => { + create: async tenant => { + if (!sqlite) loadSQLite(this.options.driver || this.options.credentials?.driver) const database = this.url4(tenant) - const dbc = new sqlite(database, this.options.client) + const dbc = new sqlite(database, this.options.client || {}) + await dbc.ready + const deterministic = { deterministic: true } dbc.function('session_context', key => dbc[$session][key]) dbc.function('regexp', deterministic, (re, x) => (RegExp(re).test(x) ? 1 : 0)) @@ -41,7 +45,7 @@ class SQLiteService extends SQLService { dbc.function('hour', deterministic, d => d === null ? null : toDate(d, true).getUTCHours()) dbc.function('minute', deterministic, d => d === null ? null : toDate(d, true).getUTCMinutes()) dbc.function('second', deterministic, d => d === null ? null : toDate(d, true).getUTCSeconds()) - if (!dbc.memory) dbc.pragma('journal_mode = WAL') + if (database !== ':memory:') dbc.pragma?.('journal_mode = WAL') || dbc.exec('PRAGMA journal_mode = WAL') return dbc }, destroy: dbc => dbc.close(), @@ -134,10 +138,8 @@ class SQLiteService extends SQLService { } async _allStream(stmt, binding_params, one, objectMode) { - stmt = stmt.constructor.name === 'Statement' ? stmt : stmt.__proto__ - stmt.raw(true) - const get = stmt.get(binding_params) - if (!get) return [] + stmt = stmt.iterate ? stmt : stmt.__proto__ + stmt.raw?.(true) const rs = stmt.iterate(binding_params) const stream = Readable.from(objectMode ? this._iteratorObjectMode(rs) : this._iteratorRaw(rs, one), { objectMode }) const close = () => rs.return() // finish result set when closed early @@ -293,4 +295,28 @@ class SQLiteService extends SQLService { } } +function loadSQLite(driver) { + const drivers = { + node: './node-sqlite.js', + 'better-sqlite3': 'better-sqlite3', + 'sql.js': './sql.js.js', + } + + if (driver) { + sqlite = require(drivers[driver]) + return + } + + try { + sqlite = require(drivers['better-sqlite3']) + } catch { + try { + sqlite = require(drivers.node) + } catch { + // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) + sqlite = require(drivers['sql.js']) + } + } +} + module.exports = SQLiteService diff --git a/sqlite/lib/node-sqlite.js b/sqlite/lib/node-sqlite.js new file mode 100644 index 000000000..bcd739dd1 --- /dev/null +++ b/sqlite/lib/node-sqlite.js @@ -0,0 +1,36 @@ +const { DatabaseSync } = require('node:sqlite'); + +class NodeSqlite extends DatabaseSync { + prepare(sql) { + const stmt = super.prepare(sql) + const ret = { + run(params) { + try { + params = Array.isArray(params) ? params : [params] + return stmt.run(...params) + } catch (err) { + if (err.message.indexOf('NOT NULL constraint failed:') === 0) { + err.code = 'SQLITE_CONSTRAINT_NOTNULL' + } + throw err + } + }, + get(params) { + params = Array.isArray(params) ? params : [params] + return stmt.get(...params) + }, + all(params) { + params = Array.isArray(params) ? params : [params] + return stmt.all(...params) + }, + iterate(params) { + stmt.setReturnArrays(true) + params = Array.isArray(params) ? params : [params] + return stmt.iterate(...params) + } + } + return ret + } +} + +module.exports = NodeSqlite diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js new file mode 100644 index 000000000..fd3c62aa8 --- /dev/null +++ b/sqlite/lib/sql.js.js @@ -0,0 +1,97 @@ +const initSqlJs = require('sql.js'); + +const init = initSqlJs({}) + +class WasmSqlite { + constructor(/*database*/) { + // TODO: load / store database file contents + this.ready = init + .then(SQL => { + this.db = new SQL.Database() + // polyfill for missing or mismatched default sqlite3 math functions + this.db.create_function('ln', x => Math.log(x)) + this.db.create_function('log', (x) => Math.log10(x)) + this.db.create_function('log', (x, y) => Math.log(y) / Math.log(x)) + this.db.create_function('mod', (x, y) => x % y) + }) + + this.memory = true + this.gc = new FinalizationRegistry(stmt => { stmt.free() }) + } + + prepare(sql) { + const stmt = this.db.prepare(sql) + const ret = { + run(params) { + try { + stmt.bind(params) + stmt.step() + return { changes: stmt.db.getRowsModified(stmt) } + } catch (err) { + if (err.message.indexOf('NOT NULL constraint failed:') === 0) { + err.code = 'SQLITE_CONSTRAINT_NOTNULL' + } + throw err + } + }, + get(params) { + const columns = stmt.getColumnNames() + stmt.bind(params) + stmt.step() + const row = stmt.get() + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = row[i] + } + return ret + }, + all(params) { + const columns = stmt.getColumnNames() + const ret = [] + stmt.bind(params) + while (stmt.step()) { + const row = stmt.get() + const obj = {} + for (let i = 0; i < columns.length; i++) { + obj[columns[i]] = row[i] + } + ret.push(obj) + } + return ret + }, + *iterate(params) { + stmt.bind(params) + while (stmt.step()) { + yield stmt.get() + } + } + } + this.gc.register(ret, stmt) + return ret + } + + exec(sql) { + try { + const { columns, values } = this.db.exec(sql) + return !Array.isArray(values) ? values : values.map(val => { + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = val[i] + } + return ret + }) + } catch (err) { + // REVISIT: address transaction errors + if (sql === 'BEGIN' || sql === 'ROLLBACK') { return } + throw err + } + } + + function(name, config, func) { + this.db.create_function(name, func || config) + } + + close() { this.db.close() } +} + +module.exports = WasmSqlite diff --git a/sqlite/package.json b/sqlite/package.json index 70400fcf4..784c612db 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -26,11 +26,17 @@ "test": "cds-test" }, "dependencies": { - "@cap-js/db-service": "^2.8.2", - "better-sqlite3": "^12.0.0" + "better-sqlite3": "^12.0.0", + "@cap-js/db-service": "^2.8.2" }, "peerDependencies": { - "@sap/cds": ">=9" + "@sap/cds": ">=9", + "sql.js": "^1.13.0" + }, + "peerDependenciesMeta": { + "sql.js": { + "optional": true + } }, "cds": { "requires": { @@ -54,4 +60,4 @@ } }, "license": "Apache-2.0" -} +} \ No newline at end of file diff --git a/test/cds.js b/test/cds.js index c6d773f4d..4bc287dc2 100644 --- a/test/cds.js +++ b/test/cds.js @@ -49,7 +49,9 @@ cds.test = Object.setPrototypeOf(function () { const serviceDefinitionPath = `${testSource}/test/service` // Overwrite default cds.requires.db with test config - process.env.CDS_REQUIRES_DB = JSON.stringify(require(serviceDefinitionPath)) + const config = require(serviceDefinitionPath) + config.driver = process.env.CDS_REQUIRES_DB_DRIVER ?? config.driver + process.env.CDS_REQUIRES_DB = JSON.stringify(config) } catch { // Default to sqlite for packages without their own service process.env.CDS_REQUIRES_DB = JSON.stringify(require('@cap-js/sqlite/test/service'))