advanced lazy sqlpage functions

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## unreleased
 
+- **SQLPage function evaluation is now lazier.** A `sqlpage.*` call that is the whole value of a selected column now runs after the database query, once for each returned row. If the query returns no rows, the function is not called. Other SQLPage function calls still run before the query when their arguments do not depend on database columns. `SET x = (SELECT ...)` now has explicit scalar-query semantics: zero rows set `x` to `NULL`, one row with one column sets the value, and multiple rows or multiple columns return a clear SQLPage error. Migration notes:
+  -  `SELECT sqlpage.fetch('https://api.example.com') AS body FROM many_rows` will now make one HTTP request per row; use `SET body = sqlpage.fetch(...)` first for one request per page.
+  - `SELECT sqlpage.exec('command') AS result FROM many_rows` will now run once per row; use `SET` first for once-per-page execution
+  - `SELECT sqlpage.random_string(8) AS token FROM many_rows` now produces one token per row instead of one token reused across rows
+  - `SET result = (SELECT sqlpage.fetch($url) WHERE NOT $cached)` now works as expected and does not re-fetch a cached result. Apps relying on sqlite-specific `SET x = (SELECT y FROM t)` first-row behavior should add `LIMIT 1` and select exactly one column.
 - **Access logs now go to stdout.** SQLPage now writes the single per-request completion log line to stdout with the target `sqlpage::access`, matching common application-server and container logging conventions. Diagnostic logs, warnings, and internal errors still go to stderr. If your `LOG_LEVEL` or `RUST_LOG` filter is scoped to a specific old target such as `sqlpage::webserver::http=info`, add `sqlpage::access=info` so request-completion logs are still emitted. If your log pipeline only collects stderr, update it to collect stdout too.
 
 ## v0.44.1

examples/official-site/functions.sql

@@ -18,12 +18,20 @@ select '
 In addition to normal SQL functions supported by your database,
 SQLPage provides a few special functions to help you extract data from user requests.
 
-These functions are special, because they are not executed inside your database,
-but by SQLPage itself before sending the query to your database.
-Thus, they require all the parameters to be known at the time the query is sent to your database.
-Function parameters cannot reference columns from the rest of your query.
-The only case when you can call a SQLPage function with a parameter that is not a constant is when it appears at the top level of a `SELECT` statement.
-For example, `SELECT sqlpage.url_encode(url) FROM t` is allowed because SQLPage can execute `SELECT url FROM t` and then apply the `url_encode` function to each value.
+These functions are special because they are not database functions.
+SQLPage evaluates them itself, either before or after the database query.
+
+When a SQLPage function call is the whole value of a selected column, SQLPage first lets the database decide which rows exist.
+It selects the function arguments as hidden columns, then calls the SQLPage function once for each returned row.
+For example, `SELECT sqlpage.url_encode(url) AS encoded FROM t` runs the database query first, then applies `url_encode` to every returned `url` value.
+If the query returns no row, the function is not called.
+This also applies inside scalar `SET` subqueries, for instance `SET body = (SELECT sqlpage.fetch(url) FROM cache_misses WHERE enabled)`.
+
+In other positions, SQLPage functions run before the query, but only when their arguments can be evaluated without reading database columns.
+For instance, `WHERE sqlpage.cookie(''x'') = ''1''` can run before the query, but `WHERE sqlpage.fetch(url) IS NOT NULL` cannot because `url` is a database column.
+
+If a SQLPage function is expensive or has side effects and should run only once, store its result with `SET` first and reuse the variable.
+If a function should run only when a database row exists, put it as a standalone selected column in that row-producing query.
 
 For more information about how SQLPage functions are evaluated, and data types in SQLPage, read [the SQLPage data model documentation](/extensions-to-sql).
 ' as contents_md where $function IS NULL;

examples/official-site/sqlpage/migrations/08_functions.sql

@@ -204,6 +204,9 @@ VALUES (
         'arrows-shuffle',
         'Returns a cryptographically secure random string of the given length.
 
+When used as a standalone selected column in a query that returns several rows, `random_string` runs once per returned row.
+Use `SET token = sqlpage.random_string(32)` first if you want one token reused later in the page.
+
 ### Example
 
 Generate a random string of 32 characters and use it as a session ID stored in a cookie:
@@ -414,9 +417,10 @@ from json_each(sqlpage.exec(''curl'', ''https://jsonplaceholder.typicode.com/use
    This means that the SQLPage server will not be blocked while the command is running, it will be able to serve other requests, but it will not be able to serve the current request until the command has finished.
    You should generally avoid long running commands.
  - If the program name is NULL, the result will be NULL.
- - If any argument is NULL, it will be passed to the command as an empty string.
- - If the command exits with a non-zero exit code, the function will raise an error.
- - Arbitrary SQL operations are not allowed as sqlpage function arguments. Use `SET` to assign the result of a SQL query to a variable, and then use that variable as an argument to `sqlpage.exec`.
+  - If any argument is NULL, it will be passed to the command as an empty string.
+  - If the command exits with a non-zero exit code, the function will raise an error.
+  - Arbitrary SQL operations are not allowed as sqlpage function arguments. Use `SET` to assign the result of a SQL query to a variable, and then use that variable as an argument to `sqlpage.exec`.
+  - When `sqlpage.exec(...)` is a standalone selected column, the command runs once per returned row. Use `SET command_result = sqlpage.exec(...)` first if the command should run only once for the page.
 '
     );
 INSERT INTO sqlpage_function_parameters (

examples/official-site/sqlpage/migrations/38_run_sql.sql

@@ -56,6 +56,8 @@ for help with manipulating the json array returned by `run_sql`.
  - **variables**: the included file will have access to the same variables (URL parameters, POST variables, etc.)
    as the calling file.
    If the included file changes the value of a variable or creates a new variable, the change will not be visible in the calling file.
+ - **per-row execution**: when `sqlpage.run_sql(...)` is a standalone selected column in a query that returns several rows, the included file runs once per returned row.
+   Use `SET included = sqlpage.run_sql(...)` first if the included file should run only once for the page.
 
 ### Parameters
 

examples/official-site/sqlpage/migrations/40_fetch.sql

@@ -106,6 +106,20 @@ set api_value = sqlpage.fetch($target_url); -- no http request made if the field
 update my_table set field = $api_value where id = 1 and $api_value is not null; -- update the field only if it was not present before
 ```
 
+You can also make the HTTP request depend on whether a database query returns a row.
+When `fetch` is a standalone selected column, SQLPage runs it only for rows returned by the database:
+
+```sql
+set api_value = (
+    select sqlpage.fetch(url)
+    from cache_misses
+    where key = $key
+);
+```
+
+If `cache_misses` has no matching row, no HTTP request is made and `$api_value` is set to `NULL`.
+If the query returns more than one row or more than one column, SQLPage returns a clear scalar `SET` error.
+
 ## Advanced usage
 
 If you need to handle errors or inspect the response headers or the status code,

examples/official-site/sqlpage/migrations/72_set_variable.sql

@@ -29,6 +29,10 @@ select
 from categories;
 ```
 
+When `sqlpage.set_variable(...)` is used as a standalone selected column in a query that returns several rows, it runs once per returned row.
+This is useful for generating one link per row, as in the example above.
+If you need a single link reused later in the page, store it with `SET` first.
+
 ### Parameters
  - `name` (TEXT): The name of the variable to set.
  - `value` (TEXT): The value to set the variable to. If `NULL` is passed, the variable is removed from the URL.

src/lib.rs

@@ -85,7 +85,7 @@ pub mod webserver;
 
 use crate::app_config::AppConfig;
 use crate::filesystem::FileSystem;
-use crate::webserver::database::ParsedSqlFile;
+use crate::webserver::database::SqlFile;
 use crate::webserver::oidc::OidcState;
 use file_cache::FileCache;
 use std::path::{Path, PathBuf};
@@ -106,7 +106,7 @@ pub const DEFAULT_404_FILE: &str = "default_404.sql";
 pub struct AppState {
     pub db: Database,
     all_templates: AllTemplates,
-    sql_file_cache: FileCache<ParsedSqlFile>,
+    sql_file_cache: FileCache<SqlFile>,
     file_system: FileSystem,
     config: AppConfig,
     pub oidc_state: Option<Arc<OidcState>>,
@@ -124,11 +124,11 @@ impl AppState {
         let file_system = FileSystem::init(&config.web_root, &db).await;
         sql_file_cache.add_static(
             PathBuf::from("index.sql"),
-            ParsedSqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")),
+            SqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")),
         );
         sql_file_cache.add_static(
             PathBuf::from(DEFAULT_404_FILE),
-            ParsedSqlFile::new(
+            SqlFile::new(
                 &db,
                 include_str!("default_404.sql"),
                 Path::new(DEFAULT_404_FILE),

src/webserver/database/error_highlighting.rs

@@ -3,7 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use super::sql::{SourceSpan, StmtWithParams};
+use super::sql::SourceSpan;
 
 #[derive(Debug)]
 struct NiceDatabaseError {
@@ -114,14 +114,15 @@ pub fn display_db_error(
 #[must_use]
 pub fn display_stmt_db_error(
     source_file: &Path,
-    stmt: &StmtWithParams,
+    query: &str,
+    query_position: SourceSpan,
     db_err: sqlx::error::Error,
 ) -> anyhow::Error {
     anyhow::Error::new(NiceDatabaseError {
         source_file: source_file.to_path_buf(),
         db_err,
-        query: stmt.query.clone(),
-        query_position: Some(stmt.query_position),
+        query: query.to_owned(),
+        query_position: Some(query_position),
     })
 }