diff --git a/packages/php-wasm/universal/src/lib/fs-helpers.ts b/packages/php-wasm/universal/src/lib/fs-helpers.ts index baae4967ee8..47c3df465f1 100644 --- a/packages/php-wasm/universal/src/lib/fs-helpers.ts +++ b/packages/php-wasm/universal/src/lib/fs-helpers.ts @@ -59,7 +59,7 @@ export class FSHelpers { static writeFile( FS: Emscripten.RootFS, path: string, - data: string | Uint8Array + data: string | Uint8Array | Buffer ) { FS.writeFile(path, data); } diff --git a/packages/php-wasm/universal/src/lib/php.ts b/packages/php-wasm/universal/src/lib/php.ts index c063209673e..844ca402bc1 100644 --- a/packages/php-wasm/universal/src/lib/php.ts +++ b/packages/php-wasm/universal/src/lib/php.ts @@ -285,7 +285,7 @@ export class PHP implements Disposable { // Always enable the file cache. 'opcache.file_cache_only = 1', 'opcache.file_cache_consistency_checks = 1', - ] + ] : []; /*if ( @@ -508,9 +508,8 @@ export class PHP implements Disposable { */ async run(request: PHPRunOptions): Promise { const streamedResponse = await this.runStream(request); - const syncResponse = await PHPResponse.fromStreamedResponse( - streamedResponse - ); + const syncResponse = + await PHPResponse.fromStreamedResponse(streamedResponse); if (syncResponse.exitCode !== 0) { // Legacy run() behavior: throw if PHP exited with a non-zero exit code. @@ -1189,7 +1188,7 @@ export class PHP implements Disposable { * @param path - The file path to write to. * @param data - The data to write to the file. */ - writeFile(path: string, data: string | Uint8Array) { + writeFile(path: string, data: string | Uint8Array | Buffer) { const result = FSHelpers.writeFile( this[__private__dont__use].FS, path, @@ -1754,9 +1753,9 @@ const getNodeType = (fs: Emscripten.FileSystemInstance, path: string) => { return 'contents' in target.node ? 'memfs' : /** - * Could be NODEFS, PROXYFS, etc. - */ - 'not-memfs'; + * Could be NODEFS, PROXYFS, etc. + */ + 'not-memfs'; } catch { return 'missing'; } diff --git a/packages/playground/blueprints/src/lib/steps/WP_MySQL_Naive_Query_Stream.php b/packages/playground/blueprints/src/lib/steps/WP_MySQL_Naive_Query_Stream.php new file mode 100644 index 00000000000..10ce6a67433 --- /dev/null +++ b/packages/playground/blueprints/src/lib/steps/WP_MySQL_Naive_Query_Stream.php @@ -0,0 +1,164 @@ +append_sql( 'SELECT id FROM users; SELECT * FROM posts;' ); + * while ( $stream->next_query() ) { + * $sql_string = $stream->get_query(); + * // Process the query. + * } + * $stream->append_sql( 'CREATE TABLE users (id INT, name VARCHAR(255));' ); + * while ( $stream->next_query() ) { + * $sql_string = $stream->get_query(); + * // Process the query. + * } + * $stream->mark_input_complete(); + * $stream->next_query(); // returns false + */ +class WP_MySQL_Naive_Query_Stream { + + private $sql_buffer = ''; + private $input_complete = false; + private $state = true; + private $last_query = false; + + const STATE_QUERY = 'valid'; + const STATE_SYNTAX_ERROR = 'syntax_error'; + const STATE_PAUSED_ON_INCOMPLETE_INPUT = 'paused_on_incomplete_input'; + const STATE_FINISHED = 'finished'; + + /** + * The maximum size of the buffer to store the SQL input. We don't + * have enough information from the lexer to distinguish between + * an incomplete input and a syntax error so we use a heuristic – + * if we've accumulated more than this amount of SQL input, we assume + * it's a syntax error. That's why this class is called a "naive" query + * stream. + */ + const MAX_SQL_BUFFER_SIZE = 1024 * 1024 * 15; + + public function __construct() {} + + public function append_sql( string $sql ) { + if($this->input_complete) { + return false; + } + $this->sql_buffer .= $sql; + $this->state = self::STATE_QUERY; + return true; + } + + public function is_paused_on_incomplete_input(): bool { + return $this->state === self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + + public function mark_input_complete() { + $this->input_complete = true; + } + + public function next_query() { + $this->last_query = false; + if($this->state === self::STATE_PAUSED_ON_INCOMPLETE_INPUT) { + return false; + } + + $result = $this->do_next_query(); + if(!$result && strlen($this->sql_buffer) > self::MAX_SQL_BUFFER_SIZE) { + $this->state = self::STATE_SYNTAX_ERROR; + return false; + } + return $result; + } + + private function do_next_query() { + $query = []; + $lexer = new WP_MySQL_Lexer( $this->sql_buffer ); + while ( $lexer->next_token() ) { + $token = $lexer->get_token(); + $query[] = $token; + if ( $token->id === WP_MySQL_Lexer::SEMICOLON_SYMBOL ) { + // Got a complete query! + break; + } + } + + // @TODO: expose this method from the lexer + // if($lexer->get_state() === WP_MySQL_Lexer::STATE_SYNTAX_ERROR) { + // return false; + // } + + if(!count($query)) { + if ( $this->input_complete ) { + $this->state = self::STATE_FINISHED; + } else { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + return false; + } + + // The last token either needs to end with a semicolon, or be the + // last token in the input. + $last_token = $query[count($query) - 1]; + if ( + $last_token->id !== WP_MySQL_Lexer::SEMICOLON_SYMBOL && + ! $this->input_complete + ) { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + return false; + } + + // See if the query has any meaningful tokens. We don't want to return + // to give the caller a comment disguised as a query. + $has_meaningful_tokens = false; + foreach($query as $token) { + if ( + $token->id !== WP_MySQL_Lexer::WHITESPACE && + $token->id !== WP_MySQL_Lexer::COMMENT && + $token->id !== WP_MySQL_Lexer::MYSQL_COMMENT_START && + $token->id !== WP_MySQL_Lexer::MYSQL_COMMENT_END && + $token->id !== WP_MySQL_Lexer::EOF + ) { + $has_meaningful_tokens = true; + break; + } + } + if(!$has_meaningful_tokens) { + if ( $this->input_complete ) { + $this->state = self::STATE_FINISHED; + } else { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + return false; + } + + // Remove the query from the input buffer and return it. + $last_byte = $last_token->start + $last_token->length; + $query = substr($this->sql_buffer, 0, $last_byte); + $this->sql_buffer = substr($this->sql_buffer, $last_byte); + $this->last_query = $query; + $this->state = self::STATE_QUERY; + return true; + } + + public function get_query() { + return $this->last_query; + } + + public function get_state() { + return $this->state; + } + +} \ No newline at end of file diff --git a/packages/playground/blueprints/src/lib/steps/run-sql.spec.ts b/packages/playground/blueprints/src/lib/steps/run-sql.spec.ts index 2601182a0f6..9b7037a3d04 100644 --- a/packages/playground/blueprints/src/lib/steps/run-sql.spec.ts +++ b/packages/playground/blueprints/src/lib/steps/run-sql.spec.ts @@ -1,11 +1,15 @@ -import { PHP } from '@php-wasm/universal'; +import type { PHP } from '@php-wasm/universal'; import { phpVars } from '@php-wasm/util'; import { runSql } from './run-sql'; -import { PHPRequestHandler } from '@php-wasm/universal'; +import type { PHPRequestHandler } from '@php-wasm/universal'; import { loadNodeRuntime } from '@php-wasm/node'; import { RecommendedPHPVersion } from '@wp-playground/common'; +import { bootWordPressAndRequestHandler } from '@wp-playground/wordpress'; +import { + getSqliteDriverModule, + getWordPressModule, +} from '@wp-playground/wordpress-builds'; -const phpVersion = RecommendedPHPVersion; describe('Blueprint step runSql', () => { let php: PHP; let handler: PHPRequestHandler; @@ -13,38 +17,44 @@ describe('Blueprint step runSql', () => { const outputLogPath = `/tmp/sql-execution-log.json`; beforeEach(async () => { - handler = new PHPRequestHandler({ - phpFactory: async () => new PHP(await loadNodeRuntime(phpVersion)), - documentRoot, + handler = await bootWordPressAndRequestHandler({ + createPhpRuntime: async () => + await loadNodeRuntime(RecommendedPHPVersion), + siteUrl: 'http://playground-domain/', + + wordPressZip: await getWordPressModule(), + sqliteIntegrationPluginZip: await getSqliteDriverModule(), }); php = await handler.getPrimaryPhp(); - php.mkdir(documentRoot); + // Create an object that will log all function calls const js = phpVars({ documentRoot, outputLogPath }); + /** * The run-sql step loads WordPress by including wp-load.php. * We don't need the rest of WordPress for this test, so we * create a minimal wp-load.php that just logs the sql queries. */ + php.mkdir(`${documentRoot}/wp-content/mu-plugins`); php.writeFile( - `${documentRoot}/wp-load.php`, + `${documentRoot}/wp-content/mu-plugins/logger.php`, ` 'CALL', - 'function' => $function, - 'args' => $args, + 'type' => 'SQL_QUERY', + 'query' => $query, ]; file_put_contents(${js.outputLogPath}, json_encode($entry) . "\n", FILE_APPEND); - } - } - - global $wpdb; - $wpdb = new MockLogger(); + return $query; + }); + }); + file_put_contents(${js.outputLogPath}, ''); ` ); @@ -57,13 +67,21 @@ describe('Blueprint step runSql', () => { it('should split and "run" sql queries', async () => { // Test a single query - await runSql(php, { + const sqlResult = await runSql(php, { sql: new File(['SELECT * FROM wp_users;'], 'single-query.sql'), }); + // Debug: Check if there were any errors + if (sqlResult.exitCode !== 0) { + console.log('SQL execution failed:'); + console.log('Exit code:', sqlResult.exitCode); + console.log('Stdout:', sqlResult.text); + console.log('Stderr:', sqlResult.errors); + } + const result = php.readFileAsText(outputLogPath); expect(result).toBe( - `{"type":"CALL","function":"query","args":["SELECT * FROM wp_users;"]}\n` + `{"type":"SQL_QUERY","query":"SELECT * FROM wp_users;"}\n` ); }); @@ -81,7 +99,7 @@ describe('Blueprint step runSql', () => { const result = php.readFileAsText(outputLogPath); expect(result).toBe( - `{"type":"CALL","function":"query","args":["SELECT * FROM wp_users;\\n"]}\n{"type":"CALL","function":"query","args":["SELECT * FROM wp_posts;"]}\n` + `{"type":"SQL_QUERY","query":"SELECT * FROM wp_users;"}\n{"type":"SQL_QUERY","query":"\\nSELECT * FROM wp_posts;"}\n` ); }); @@ -103,7 +121,84 @@ describe('Blueprint step runSql', () => { const result = php.readFileAsText(outputLogPath); expect(result).toBe( - `{"type":"CALL","function":"query","args":["SELECT * FROM wp_users;\\n"]}\n{"type":"CALL","function":"query","args":["SELECT * FROM wp_posts;\\n"]}\n` + `{"type":"SQL_QUERY","query":"SELECT * FROM wp_users;"}\n{"type":"SQL_QUERY","query":"\\n;"}\n{"type":"SQL_QUERY","query":"\\n\\nSELECT * FROM wp_posts;"}\n` + ); + }); + + it('should handle multiline queries', async () => { + await runSql(php, { + sql: new File( + [ + [ + 'CREATE TABLE test_table (', + ' id INT PRIMARY KEY,', + ' name VARCHAR(255),', + ' created_at TIMESTAMP', + ');', + '', + 'INSERT INTO test_table', + ' (id, name, created_at)', + 'VALUES', + ' (1, "John Doe", NOW());', + ].join('\n'), + ], + 'multiline-queries.sql' + ), + }); + + const result = php.readFileAsText(outputLogPath); + expect(result).toBe( + `{"type":"SQL_QUERY","query":"CREATE TABLE test_table (\\n id INT PRIMARY KEY,\\n name VARCHAR(255),\\n created_at TIMESTAMP\\n);"}\n{"type":"SQL_QUERY","query":"\\n\\nINSERT INTO test_table\\n (id, name, created_at)\\nVALUES\\n (1, \\"John Doe\\", NOW());"}\n` + ); + }); + + it('should handle queries with SQL comments', async () => { + await runSql(php, { + sql: new File( + [ + [ + '-- This is a comment', + 'SELECT * FROM wp_users;', + '', + '/* This is a', + ' multiline comment */', + 'SELECT * FROM wp_posts;', + ].join('\n'), + ], + 'queries-with-comments.sql' + ), + }); + + const result = php.readFileAsText(outputLogPath); + expect(result).toBe( + `{"type":"SQL_QUERY","query":"-- This is a comment\\nSELECT * FROM wp_users;"}\n{"type":"SQL_QUERY","query":"\\n\\n\\/* This is a\\n multiline comment *\\/\\nSELECT * FROM wp_posts;"}\n` + ); + }); + + it('should handle complex multiline query with subquery', async () => { + await runSql(php, { + sql: new File( + [ + [ + 'SELECT', + ' u.id,', + ' u.name,', + ' (SELECT COUNT(*) FROM wp_posts WHERE author_id = u.id) as post_count', + 'FROM', + ' wp_users u', + 'WHERE', + ' u.status = "active"', + 'ORDER BY', + ' u.name ASC;', + ].join('\n'), + ], + 'complex-multiline-query.sql' + ), + }); + + const result = php.readFileAsText(outputLogPath); + expect(result).toBe( + `{"type":"SQL_QUERY","query":"SELECT\\n u.id,\\n u.name,\\n (SELECT COUNT(*) FROM wp_posts WHERE author_id = u.id) as post_count\\nFROM\\n wp_users u\\nWHERE\\n u.status = \\"active\\"\\nORDER BY\\n u.name ASC;"}\n` ); }); }); diff --git a/packages/playground/blueprints/src/lib/steps/run-sql.ts b/packages/playground/blueprints/src/lib/steps/run-sql.ts index 1b47aaa9a05..ae1baba1710 100644 --- a/packages/playground/blueprints/src/lib/steps/run-sql.ts +++ b/packages/playground/blueprints/src/lib/steps/run-sql.ts @@ -1,6 +1,8 @@ import type { StepHandler } from '.'; import { rm } from './rm'; import { phpVars, randomFilename } from '@php-wasm/util'; +/** @ts-ignore */ +import streamClassContent from './WP_MySQL_Naive_Query_Stream.php?raw'; /** * @inheritDoc runSql @@ -32,9 +34,11 @@ export interface RunSqlStep { /** * Run one or more SQL queries. * - * This step will treat each non-empty line in the input SQL as a query and - * try to execute it using `$wpdb`. Queries spanning multiple lines are not - * yet supported. + * This step uses WP_MySQL_Naive_Query_Stream to parse and execute SQL queries using + * streaming semantics. It supports multiline queries, comments, and queries + * separated by semicolons. Each query is executed using `$wpdb`. This step assumes + * a presence of the `sqlite-database-integration` plugin that ships the required + * query tokenizer classes. */ export const runSql: StepHandler> = async ( playground, @@ -44,35 +48,72 @@ export const runSql: StepHandler> = async ( progress?.tracker.setCaption(`Executing SQL Queries`); const sqlFilename = `/tmp/${randomFilename()}.sql`; + const streamClassFilename = `/tmp/${randomFilename()}.php`; await playground.writeFile( sqlFilename, new Uint8Array(await sql.arrayBuffer()) ); + await playground.writeFile( + streamClassFilename, + new TextEncoder().encode(streamClassContent) + ); + const docroot = await playground.documentRoot; - const js = phpVars({ docroot, sqlFilename }); + const js = phpVars({ docroot, sqlFilename, streamClassFilename }); const runPhp = await playground.run({ code: `query($line); + $stream->append_sql($chunk); + + // Process any complete queries in the stream + while ($stream->next_query()) { + $query = $stream->get_query(); + $wpdb->query($query); + } + } + + fclose($handle); + + // Mark input as complete and process any remaining queries + $stream->mark_input_complete(); + while ($stream->next_query()) { + $query = $stream->get_query(); + $wpdb->query($query); } `, }); await rm(playground, { path: sqlFilename }); + await rm(playground, { path: streamClassFilename }); return runPhp; }; diff --git a/packages/playground/blueprints/vite.config.ts b/packages/playground/blueprints/vite.config.ts index 6800ad9174f..9c8d20271d0 100644 --- a/packages/playground/blueprints/vite.config.ts +++ b/packages/playground/blueprints/vite.config.ts @@ -12,7 +12,7 @@ import { getExternalModules } from '../../vite-extensions/vite-external-modules' import viteGlobalExtensions from '../../vite-extensions/vite-global-extensions'; export default defineConfig({ - assetsInclude: ['**/*.phar'], + assetsInclude: ['**/*.phar', '**/*.php'], cacheDir: '../../../node_modules/.vite/playground-blueprints', plugins: [ diff --git a/packages/playground/cli/src/run-cli.ts b/packages/playground/cli/src/run-cli.ts index 0a79672b130..f5701bdbb23 100644 --- a/packages/playground/cli/src/run-cli.ts +++ b/packages/playground/cli/src/run-cli.ts @@ -3,6 +3,7 @@ import type { PHPRequest, RemoteAPI, SupportedPHPVersion, + UniversalPHP, } from '@php-wasm/universal'; import { PHPResponse, @@ -966,7 +967,7 @@ export async function runCLI(args: RunCLIArgs): Promise { logger.log(`Running the Blueprint...`); await runBlueprintV1Steps( compiledBlueprint, - initialPlayground + initialPlayground as UniversalPHP ); logger.log(`Finished running the blueprint`); }