feat: support for node:sqlite and sql.js

.github/workflows/test.yml

@@ -45,3 +45,15 @@ jobs:
           FORCE_COLOR: true
           TAG: ${{ steps.hxe.outputs.TAG }}
           IMAGE_ID: ${{ steps.hxe.outputs.IMAGE_ID }}
+      - name: sqlite driver (node:sqlite)
+        run: npm test -w sqlite
+        env:
+          CDS_REQUIRES_DB_DRIVER: 'node'
+          cds_features_pool: true
+          FORCE_COLOR: true
+      - name: sqlite driver (sql.js)
+        run: npm test -w sqlite
+        env:
+          CDS_REQUIRES_DB_DRIVER: 'sql.js'
+          cds_features_pool: true
+          FORCE_COLOR: true

package.json

@@ -11,6 +11,7 @@
   ],
   "devDependencies": {
     "@cap-js/cds-test": ">=0.2.0",
+    "sql.js": "^1.13.0",
     "axios": "^1"
   },
   "scripts": {

sqlite/lib/SQLiteService.js

@@ -1,6 +1,7 @@
 const { SQLService } = require('@cap-js/db-service')
-const cds = require('@sap/cds')
-const sqlite = require('better-sqlite3')
+const cds = require('@sap/cds/lib')
+let sqlite // sqlite driver is loaded on connect
+
 const $session = Symbol('dbc.session')
 const sessionVariableMap = require('./session.json')  // Adjust the path as necessary for your project
 const convStrm = require('stream/consumers')
@@ -28,9 +29,12 @@ class SQLiteService extends SQLService {
   get factory() {
     return {
       options: this.options.pool || {},
-      create: tenant => {
+      create: async tenant => {
+        if (!sqlite) loadSQLite(this.options.driver || this.options.credentials?.driver)
         const database = this.url4(tenant)
-        const dbc = new sqlite(database, this.options.client)
+        const dbc = new sqlite(database, this.options.client || {})
+        await dbc.ready
+
         const deterministic = { deterministic: true }
         dbc.function('session_context', key => dbc[$session][key])
         dbc.function('regexp', deterministic, (re, x) => (RegExp(re).test(x) ? 1 : 0))
@@ -41,7 +45,7 @@ class SQLiteService extends SQLService {
         dbc.function('hour', deterministic, d => d === null ? null : toDate(d, true).getUTCHours())
         dbc.function('minute', deterministic, d => d === null ? null : toDate(d, true).getUTCMinutes())
         dbc.function('second', deterministic, d => d === null ? null : toDate(d, true).getUTCSeconds())
-        if (!dbc.memory) dbc.pragma('journal_mode = WAL')
+        if (database !== ':memory:') dbc.pragma?.('journal_mode = WAL') || dbc.exec('PRAGMA journal_mode = WAL')
         return dbc
       },
       destroy: dbc => dbc.close(),
@@ -134,10 +138,8 @@ class SQLiteService extends SQLService {
   }
 
   async _allStream(stmt, binding_params, one, objectMode) {
-    stmt = stmt.constructor.name === 'Statement' ? stmt : stmt.__proto__
-    stmt.raw(true)
-    const get = stmt.get(binding_params)
-    if (!get) return []
+    stmt = stmt.iterate ? stmt : stmt.__proto__
+    stmt.raw?.(true)
     const rs = stmt.iterate(binding_params)
     const stream = Readable.from(objectMode ? this._iteratorObjectMode(rs) : this._iteratorRaw(rs, one), { objectMode })
     const close = () => rs.return() // finish result set when closed early
@@ -293,4 +295,28 @@ class SQLiteService extends SQLService {
   }
 }
 
+function loadSQLite(driver) {
+  const drivers = {
+    node: './node-sqlite.js',
+    'better-sqlite3': 'better-sqlite3',
+    'sql.js': './sql.js.js',
+  }
+
+  if (driver) {
+    sqlite = require(drivers[driver])
+    return
+  }
+
+  try {
+    sqlite = require(drivers['better-sqlite3'])
+  } catch {
+    try {
+      sqlite = require(drivers.node)
+    } catch {
+      // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite)
+      sqlite = require(drivers['sql.js'])
+    }
+  }
+}
+
 module.exports = SQLiteService

sqlite/lib/node-sqlite.js

@@ -0,0 +1,36 @@
+const { DatabaseSync } = require('node:sqlite');
+
+class NodeSqlite extends DatabaseSync {
+  prepare(sql) {
+    const stmt = super.prepare(sql)
+    const ret = {
+      run(params) {
+        try {
+          params = Array.isArray(params) ? params : [params]
+          return stmt.run(...params)
+        } catch (err) {
+          if (err.message.indexOf('NOT NULL constraint failed:') === 0) {
+            err.code = 'SQLITE_CONSTRAINT_NOTNULL'
+          }
+          throw err
+        }
+      },
+      get(params) {
+        params = Array.isArray(params) ? params : [params]
+        return stmt.get(...params)
+      },
+      all(params) {
+        params = Array.isArray(params) ? params : [params]
+        return stmt.all(...params)
+      },
+      iterate(params) {
+        stmt.setReturnArrays(true)
+        params = Array.isArray(params) ? params : [params]
+        return stmt.iterate(...params)
+      }
+    }
+    return ret
+  }
+}
+
+module.exports = NodeSqlite

sqlite/lib/sql.js.js

@@ -0,0 +1,97 @@
+const initSqlJs = require('sql.js');
+
+const init = initSqlJs({})
+
+class WasmSqlite {
+  constructor(/*database*/) {
+    // TODO: load / store database file contents
+    this.ready = init
+      .then(SQL => {
+        this.db = new SQL.Database()
+        // polyfill for missing or mismatched default sqlite3 math functions
+        this.db.create_function('ln', x => Math.log(x))
+        this.db.create_function('log', (x) => Math.log10(x))
+        this.db.create_function('log', (x, y) => Math.log(y) / Math.log(x))
+        this.db.create_function('mod', (x, y) => x % y)
+      })
+
+    this.memory = true
+    this.gc = new FinalizationRegistry(stmt => { stmt.free() })
+  }
+
+  prepare(sql) {
+    const stmt = this.db.prepare(sql)
+    const ret = {
+      run(params) {
+        try {
+          stmt.bind(params)
+          stmt.step()
+          return { changes: stmt.db.getRowsModified(stmt) }
+        } catch (err) {
+          if (err.message.indexOf('NOT NULL constraint failed:') === 0) {
+            err.code = 'SQLITE_CONSTRAINT_NOTNULL'
+          }
+          throw err
+        }
+      },
+      get(params) {
+        const columns = stmt.getColumnNames()
+        stmt.bind(params)
+        stmt.step()
+        const row = stmt.get()
+        const ret = {}
+        for (let i = 0; i < columns.length; i++) {
+          ret[columns[i]] = row[i]
+        }
+        return ret
+      },
+      all(params) {
+        const columns = stmt.getColumnNames()
+        const ret = []
+        stmt.bind(params)
+        while (stmt.step()) {
+          const row = stmt.get()
+          const obj = {}
+          for (let i = 0; i < columns.length; i++) {
+            obj[columns[i]] = row[i]
+          }
+          ret.push(obj)
+        }
+        return ret
+      },
+      *iterate(params) {
+        stmt.bind(params)
+        while (stmt.step()) {
+          yield stmt.get()
+        }
+      }
+    }
+    this.gc.register(ret, stmt)
+    return ret
+  }
+
+  exec(sql) {
+    try {
+      const { columns, values } = this.db.exec(sql)
+      return !Array.isArray(values) ? values : values.map(val => {
+        const ret = {}
+        for (let i = 0; i < columns.length; i++) {
+          ret[columns[i]] = val[i]
+        }
+        return ret
+      })
+    } catch (err) {
+      // REVISIT: address transaction errors
+      if (sql === 'BEGIN' || sql === 'ROLLBACK') { return }
+      throw err
+    }
+  }
+
+  function(name, config, func) {
+    this.db.create_function(name, func || config)
+  }
+
+  close() { this.db.close() }
+}
+
+module.exports = WasmSqlite

sqlite/package.json

@@ -26,11 +26,17 @@
     "test": "cds-test"
   },
   "dependencies": {
-    "@cap-js/db-service": "^2.8.2",
-    "better-sqlite3": "^12.0.0"
+    "better-sqlite3": "^12.0.0",
+    "@cap-js/db-service": "^2.8.2"
   },
   "peerDependencies": {
-    "@sap/cds": ">=9"
+    "@sap/cds": ">=9",
+    "sql.js": "^1.13.0"
+  },
+  "peerDependenciesMeta": {
+    "sql.js": {
+      "optional": true
+    }
   },
   "cds": {
     "requires": {
@@ -54,4 +60,4 @@
     }
   },
   "license": "Apache-2.0"
-}
+}

test/cds.js

@@ -49,7 +49,9 @@ cds.test = Object.setPrototypeOf(function () {
       const serviceDefinitionPath = `${testSource}/test/service`
 
       // Overwrite default cds.requires.db with test config
-      process.env.CDS_REQUIRES_DB = JSON.stringify(require(serviceDefinitionPath))
+      const config = require(serviceDefinitionPath)
+      config.driver = process.env.CDS_REQUIRES_DB_DRIVER ?? config.driver
+      process.env.CDS_REQUIRES_DB = JSON.stringify(config)
     } catch {
       // Default to sqlite for packages without their own service
       process.env.CDS_REQUIRES_DB = JSON.stringify(require('@cap-js/sqlite/test/service'))