From 62a57f6c20750e63c656db2b4bdb322b18490d46 Mon Sep 17 00:00:00 2001 From: Alan Stephensen Date: Fri, 15 Mar 2024 18:26:20 +1100 Subject: [PATCH 01/12] Wrap included file with include comment --- README.md | 2 ++ __tests__/helpers.ts | 12 ++++++++++ __tests__/include.test.ts | 13 ++++++++++- __tests__/readCurrentMigration.test.ts | 5 +++- __tests__/uncommit.test.ts | 32 ++++++++++++++++++++++++++ src/actions.ts | 4 +++- src/commands/_common.ts | 4 +++- src/commands/migrate.ts | 4 ++-- src/commands/reset.ts | 4 +++- src/commands/run.ts | 4 ++-- src/commands/uncommit.ts | 9 +++++++- src/commands/watch.ts | 4 +++- src/current.ts | 4 +++- src/migration.ts | 14 +++++++---- tsconfig.json | 4 ++-- 15 files changed, 101 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a1f5dec6..40022f89 100644 --- a/README.md +++ b/README.md @@ -769,10 +769,12 @@ and when the migration is committed or watched, the contents of `myfunction.sql` will be included in the result, such that the following SQL is executed: ```sql +--! Included functions/myfunction.sql create or replace function myfunction(a int, b int) returns int as $$ select a + b; $$ language sql stable; +--! EndIncluded functions/myfunction.sql drop policy if exists access_by_numbers on mytable; create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42); ``` diff --git a/__tests__/helpers.ts b/__tests__/helpers.ts index 1a5af27b..31e3c90f 100644 --- a/__tests__/helpers.ts +++ b/__tests__/helpers.ts @@ -266,6 +266,15 @@ export const makeMigrations = (commitMessage?: string) => { commitMessage ? `\n--! Message: ${commitMessage}` : `` }\n\n${MIGRATION_NOTRX_TEXT.trim()}\n`; + const MIGRATION_INCLUDE_TEXT = `--!include foo.sql`; + const MIGRATION_INCLUDE_COMPILED = `${MIGRATION_INCLUDE_TEXT}\n${MIGRATION_1_TEXT}\n${MIGRATION_INCLUDE_TEXT}`; + const MIGRATION_INCLUDE_HASH = createHash("sha1") + .update(`${MIGRATION_INCLUDE_COMPILED.trim()}` + "\n") + .digest("hex"); + const MIGRATION_INCLUDE_COMMITTED = `--! Previous: -\n--! Hash: sha1:${MIGRATION_INCLUDE_HASH}${ + commitMessage ? `\n--! Message: ${commitMessage}` : `` + }\n\n${MIGRATION_INCLUDE_COMPILED}\n`; + const MIGRATION_MULTIFILE_FILES = { "migrations/links/two.sql": "select 2;", "migrations/current": { @@ -308,6 +317,9 @@ select 3; MIGRATION_NOTRX_TEXT, MIGRATION_NOTRX_HASH, MIGRATION_NOTRX_COMMITTED, + MIGRATION_INCLUDE_TEXT, + MIGRATION_INCLUDE_HASH, + MIGRATION_INCLUDE_COMMITTED, MIGRATION_MULTIFILE_TEXT, MIGRATION_MULTIFILE_HASH, MIGRATION_MULTIFILE_COMMITTED, diff --git a/__tests__/include.test.ts b/__tests__/include.test.ts index adafe553..60c7c8fb 100644 --- a/__tests__/include.test.ts +++ b/__tests__/include.test.ts @@ -42,7 +42,9 @@ it("compiles an included file", async () => { FAKE_VISITED, ), ).toEqual(`\ +--! Include foo.sql select * from foo; +--! EndInclude foo.sql `); }); @@ -64,9 +66,17 @@ it("compiles multiple included files", async () => { FAKE_VISITED, ), ).toEqual(`\ +--! Include dir1/foo.sql select * from foo; +--! EndInclude dir1/foo.sql +--! Include dir2/bar.sql select * from bar; +--! EndInclude dir2/bar.sql +--! Include dir3/baz.sql +--! Include dir4/qux.sql select * from qux; +--! EndInclude dir4/qux.sql +--! EndInclude dir3/baz.sql `); }); @@ -129,6 +139,7 @@ commit; FAKE_VISITED, ), ).toEqual(`\ +--! Include foo.sql begin; create or replace function current_user_id() returns uuid as $$ @@ -140,6 +151,6 @@ comment on function current_user_id is E'The ID of the current user.'; grant all on function current_user_id to :DATABASE_USER; commit; - +--! EndInclude foo.sql `); }); diff --git a/__tests__/readCurrentMigration.test.ts b/__tests__/readCurrentMigration.test.ts index 6e0efb5c..e6736a55 100644 --- a/__tests__/readCurrentMigration.test.ts +++ b/__tests__/readCurrentMigration.test.ts @@ -111,5 +111,8 @@ it("reads from current.sql, and processes included files", async () => { const currentLocation = await getCurrentMigrationLocation(parsedSettings); const content = await readCurrentMigration(parsedSettings, currentLocation); - expect(content).toEqual("-- TEST from foo"); + expect(content).toEqual(`\ +--! Included foo_current.sql +-- TEST from foo +--! EndIncluded foo_current.sql`); }); diff --git a/__tests__/uncommit.test.ts b/__tests__/uncommit.test.ts index a57031c2..10dbe33f 100644 --- a/__tests__/uncommit.test.ts +++ b/__tests__/uncommit.test.ts @@ -55,6 +55,8 @@ describe.each([[undefined], ["My Commit Message"]])( const { MIGRATION_1_TEXT, MIGRATION_1_COMMITTED, + MIGRATION_INCLUDE_TEXT, + MIGRATION_INCLUDE_COMMITTED, MIGRATION_MULTIFILE_COMMITTED, MIGRATION_MULTIFILE_FILES, } = makeMigrations(commitMessage); @@ -88,6 +90,36 @@ describe.each([[undefined], ["My Commit Message"]])( ).toEqual(MIGRATION_1_COMMITTED); }); + it("rolls back a migration that has included another file", async () => { + mockFs({ + [`migrations/committed/000001${commitMessageSlug}.sql`]: + MIGRATION_INCLUDE_COMMITTED, + "migrations/current.sql": "-- JUST A COMMENT\n", + "migrations/fixtures/foo.sql": MIGRATION_1_TEXT, + }); + await migrate(settings); + await uncommit(settings); + + await expect( + fsp.stat("migrations/committed/000001.sql"), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await fsp.readFile("migrations/current.sql", "utf8")).toEqual( + (commitMessage ? `--! Message: ${commitMessage}\n\n` : "") + + MIGRATION_INCLUDE_TEXT.trim() + + "\n", + ); + + await commit(settings); + expect( + await fsp.readFile( + `migrations/committed/000001${commitMessageSlug}.sql`, + "utf8", + ), + ).toEqual(MIGRATION_INCLUDE_COMMITTED); + }); + it("rolls back multifile migration", async () => { mockFs({ [`migrations/committed/000001${commitMessageSlug}.sql`]: diff --git a/src/actions.ts b/src/actions.ts index bbb1351c..a011757e 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -183,7 +183,9 @@ export function makeValidateActionCallback(logger: Logger, allowRoot = false) { specs.push(rawSpec); } else { throw new Error( - `Action spec '${inspect(rawSpec)}' not supported; perhaps you need to upgrade?`, + `Action spec '${inspect( + rawSpec, + )}' not supported; perhaps you need to upgrade?`, ); } } else { diff --git a/src/commands/_common.ts b/src/commands/_common.ts index 958a9ce6..3a6fd1f0 100644 --- a/src/commands/_common.ts +++ b/src/commands/_common.ts @@ -44,7 +44,9 @@ export async function getSettingsFromJSON(path: string): Promise { return JSON5.parse(data); } catch (e) { throw new Error( - `Failed to parse '${path}': ${e instanceof Error ? e.message : String(e)}`, + `Failed to parse '${path}': ${ + e instanceof Error ? e.message : String(e) + }`, ); } } diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts index 4ca540f3..e349da51 100644 --- a/src/commands/migrate.ts +++ b/src/commands/migrate.ts @@ -69,8 +69,8 @@ export async function _migrate( remainingMigrations.length > 0 ? `${remainingMigrations.length} committed migrations executed` : lastMigration - ? "Already up to date" - : `Up to date — no committed migrations to run` + ? "Already up to date" + : `Up to date — no committed migrations to run` }`, ); }); diff --git a/src/commands/reset.ts b/src/commands/reset.ts index e70d69db..90494f10 100644 --- a/src/commands/reset.ts +++ b/src/commands/reset.ts @@ -48,7 +48,9 @@ export async function _reset( ); } catch (e) { throw new Error( - `Failed to create database '${databaseName}' with owner '${databaseOwner}': ${e instanceof Error ? e.message : String(e)}`, + `Failed to create database '${databaseName}' with owner '${databaseOwner}': ${ + e instanceof Error ? e.message : String(e) + }`, ); } await pgClient.query( diff --git a/src/commands/run.ts b/src/commands/run.ts index 7d963c2b..bb7ccbce 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -39,8 +39,8 @@ export async function run( const baseConnectionString = rootDatabase ? parsedSettings.rootConnectionString : shadow - ? parsedSettings.shadowConnectionString - : parsedSettings.connectionString; + ? parsedSettings.shadowConnectionString + : parsedSettings.connectionString; if (!baseConnectionString) { throw new Error("Could not determine connection string to use."); } diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 01fd188f..18b46586 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -42,9 +42,16 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { const contents = await fsp.readFile(lastMigrationFilepath, "utf8"); const { headers, body } = parseMigrationText(lastMigrationFilepath, contents); + // Remove included migrations + const includeRegex = + /^--![ \t]*Included[ \t]+(?.*?\.sql)[ \t]*$.*?^--![ \t]*EndIncluded[ \t]*\k[ \t]*$/gms; + const decompiledBody = body.replace(includeRegex, (match) => { + return match.split("\n")[0].replace(" Included", "include"); + }); + // Drop Hash, Previous and AllowInvalidHash from headers; then write out const { Hash, Previous, AllowInvalidHash, ...otherHeaders } = headers; - const completeBody = serializeMigration(body, otherHeaders); + const completeBody = serializeMigration(decompiledBody, otherHeaders); await writeCurrentMigration(parsedSettings, currentLocation, completeBody); // Delete the migration from committed and from the DB diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 0ce32772..79632329 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -204,7 +204,9 @@ export async function _watch( .catch((error: unknown) => { if (!isLoggedError(error)) { parsedSettings.logger.error( - `Error occurred whilst processing migration: ${error instanceof Error ? error.message : String(error)}`, + `Error occurred whilst processing migration: ${ + error instanceof Error ? error.message : String(error) + }`, { error }, ); } diff --git a/src/current.ts b/src/current.ts index 748598b1..284a148e 100644 --- a/src/current.ts +++ b/src/current.ts @@ -38,7 +38,9 @@ async function readFileOrError(path: string): Promise { return await fsp.readFile(path, "utf8"); } catch (e) { throw new Error( - `Failed to read file at '${path}': ${e instanceof Error ? e.message : String(e)}`, + `Failed to read file at '${path}': ${ + e instanceof Error ? e.message : String(e) + }`, ); } } diff --git a/src/migration.ts b/src/migration.ts index 86ad7e61..4f5429de 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -132,7 +132,7 @@ export async function compileIncludes( content: string, processedFiles: ReadonlySet, ): Promise { - const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; + const regex = /^--![ \t]*[iI]nclude[ \t]+(.*\.sql)[ \t]*$/gm; // Find all includes in this `content` const matches = [...content.matchAll(regex)]; @@ -165,7 +165,11 @@ export async function compileIncludes( if (processedFiles.has(sqlPath)) { throw new Error( - `Circular include detected - '${sqlPath}' is included again! Import statement: \`${line}\`; trace:\n ${[...processedFiles].reverse().join("\n ")}`, + `Circular include detected - '${sqlPath}' is included again! Import statement: \`${line}\`; trace:\n ${[ + ...processedFiles, + ] + .reverse() + .join("\n ")}`, ); } @@ -205,10 +209,12 @@ export async function compileIncludes( // Simple string replacement for each path matched const compiledContent = content.replace( regex, - (_match, rawSqlPath: string) => { + (match, rawSqlPath: string) => { const sqlPath = sqlPathByRawSqlPath[rawSqlPath]; const content = contentBySqlPath[sqlPath]; - return content; + const included = match.replace(/^--![ \t]*include/, "--! Included"); + const endIncluded = included.replace("Included", "EndIncluded"); + return `${included}\n${content.trim()}\n${endIncluded}`; }, ); diff --git a/tsconfig.json b/tsconfig.json index c880e7f8..5b9a9415 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,6 @@ "include": ["src/**/*", "__tests__/**/*", "*.js", "./.*.js"], "exclude": [], "compilerOptions": { - "noEmit": true, - }, + "noEmit": true + } } From 82b2dda9101f079286ca5d0b566685dd48bc3336 Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Tue, 14 May 2024 10:20:37 +0100 Subject: [PATCH 02/12] yarn lint:fix --- src/commands/migrate.ts | 4 ++-- src/commands/run.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts index e349da51..4ca540f3 100644 --- a/src/commands/migrate.ts +++ b/src/commands/migrate.ts @@ -69,8 +69,8 @@ export async function _migrate( remainingMigrations.length > 0 ? `${remainingMigrations.length} committed migrations executed` : lastMigration - ? "Already up to date" - : `Up to date — no committed migrations to run` + ? "Already up to date" + : `Up to date — no committed migrations to run` }`, ); }); diff --git a/src/commands/run.ts b/src/commands/run.ts index bb7ccbce..7d963c2b 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -39,8 +39,8 @@ export async function run( const baseConnectionString = rootDatabase ? parsedSettings.rootConnectionString : shadow - ? parsedSettings.shadowConnectionString - : parsedSettings.connectionString; + ? parsedSettings.shadowConnectionString + : parsedSettings.connectionString; if (!baseConnectionString) { throw new Error("Could not determine connection string to use."); } From df0135951dd1289a76d251da1d69fb2a697a9abe Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Tue, 14 May 2024 10:29:03 +0100 Subject: [PATCH 03/12] Revert formatting changes --- src/actions.ts | 4 +--- src/commands/_common.ts | 4 +--- src/commands/reset.ts | 4 +--- src/commands/watch.ts | 4 +--- src/current.ts | 4 +--- src/migration.ts | 6 +----- 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/actions.ts b/src/actions.ts index a011757e..bbb1351c 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -183,9 +183,7 @@ export function makeValidateActionCallback(logger: Logger, allowRoot = false) { specs.push(rawSpec); } else { throw new Error( - `Action spec '${inspect( - rawSpec, - )}' not supported; perhaps you need to upgrade?`, + `Action spec '${inspect(rawSpec)}' not supported; perhaps you need to upgrade?`, ); } } else { diff --git a/src/commands/_common.ts b/src/commands/_common.ts index 3a6fd1f0..958a9ce6 100644 --- a/src/commands/_common.ts +++ b/src/commands/_common.ts @@ -44,9 +44,7 @@ export async function getSettingsFromJSON(path: string): Promise { return JSON5.parse(data); } catch (e) { throw new Error( - `Failed to parse '${path}': ${ - e instanceof Error ? e.message : String(e) - }`, + `Failed to parse '${path}': ${e instanceof Error ? e.message : String(e)}`, ); } } diff --git a/src/commands/reset.ts b/src/commands/reset.ts index 90494f10..e70d69db 100644 --- a/src/commands/reset.ts +++ b/src/commands/reset.ts @@ -48,9 +48,7 @@ export async function _reset( ); } catch (e) { throw new Error( - `Failed to create database '${databaseName}' with owner '${databaseOwner}': ${ - e instanceof Error ? e.message : String(e) - }`, + `Failed to create database '${databaseName}' with owner '${databaseOwner}': ${e instanceof Error ? e.message : String(e)}`, ); } await pgClient.query( diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 79632329..0ce32772 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -204,9 +204,7 @@ export async function _watch( .catch((error: unknown) => { if (!isLoggedError(error)) { parsedSettings.logger.error( - `Error occurred whilst processing migration: ${ - error instanceof Error ? error.message : String(error) - }`, + `Error occurred whilst processing migration: ${error instanceof Error ? error.message : String(error)}`, { error }, ); } diff --git a/src/current.ts b/src/current.ts index 284a148e..748598b1 100644 --- a/src/current.ts +++ b/src/current.ts @@ -38,9 +38,7 @@ async function readFileOrError(path: string): Promise { return await fsp.readFile(path, "utf8"); } catch (e) { throw new Error( - `Failed to read file at '${path}': ${ - e instanceof Error ? e.message : String(e) - }`, + `Failed to read file at '${path}': ${e instanceof Error ? e.message : String(e)}`, ); } } diff --git a/src/migration.ts b/src/migration.ts index 4f5429de..95fdb78b 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -165,11 +165,7 @@ export async function compileIncludes( if (processedFiles.has(sqlPath)) { throw new Error( - `Circular include detected - '${sqlPath}' is included again! Import statement: \`${line}\`; trace:\n ${[ - ...processedFiles, - ] - .reverse() - .join("\n ")}`, + `Circular include detected - '${sqlPath}' is included again! Import statement: \`${line}\`; trace:\n ${[...processedFiles].reverse().join("\n ")}`, ); } From 355f45bbb855d88a6e50709b383156fed22520c8 Mon Sep 17 00:00:00 2001 From: Benjie Date: Wed, 22 Apr 2026 16:16:36 +0100 Subject: [PATCH 04/12] Apply suggestions from code review Co-authored-by: Benjie --- src/commands/uncommit.ts | 9 +++++---- src/migration.ts | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 18b46586..97302900 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -44,10 +44,11 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { // Remove included migrations const includeRegex = - /^--![ \t]*Included[ \t]+(?.*?\.sql)[ \t]*$.*?^--![ \t]*EndIncluded[ \t]*\k[ \t]*$/gms; - const decompiledBody = body.replace(includeRegex, (match) => { - return match.split("\n")[0].replace(" Included", "include"); - }); + /^--![ \t]*Included[ \t]+(?\S+)[ \t]*$[\s\S]*?^--![ \t]*EndIncluded[ \t]*\k[ \t]*$/gm; + const decompiledBody = body.replace( + includeRegex, + (_, filename) => `--! include ${filename}`, + ); // Drop Hash, Previous and AllowInvalidHash from headers; then write out const { Hash, Previous, AllowInvalidHash, ...otherHeaders } = headers; diff --git a/src/migration.ts b/src/migration.ts index 95fdb78b..21298e89 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -132,7 +132,7 @@ export async function compileIncludes( content: string, processedFiles: ReadonlySet, ): Promise { - const regex = /^--![ \t]*[iI]nclude[ \t]+(.*\.sql)[ \t]*$/gm; + const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; // Find all includes in this `content` const matches = [...content.matchAll(regex)]; From fd05f8dff82ceaddeeb54110976ee026fc6b307d Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 16:31:26 +0100 Subject: [PATCH 05/12] Simplify/stricter --- src/commands/uncommit.ts | 12 +++++------- src/migration.ts | 6 ++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 97302900..f382a783 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -42,13 +42,11 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { const contents = await fsp.readFile(lastMigrationFilepath, "utf8"); const { headers, body } = parseMigrationText(lastMigrationFilepath, contents); - // Remove included migrations - const includeRegex = - /^--![ \t]*Included[ \t]+(?\S+)[ \t]*$[\s\S]*?^--![ \t]*EndIncluded[ \t]*\k[ \t]*$/gm; - const decompiledBody = body.replace( - includeRegex, - (_, filename) => `--! include ${filename}`, - ); + // Replace included migrations with their `--! include` equivalent + const decompiledBody = body.replace( + /^--! Included (?\S+)$[\s\S]*?^--! EndIncluded \k$/gm, + (_, filename) => `--! include ${filename}`, + ); // Drop Hash, Previous and AllowInvalidHash from headers; then write out const { Hash, Previous, AllowInvalidHash, ...otherHeaders } = headers; diff --git a/src/migration.ts b/src/migration.ts index 21298e89..e1de2b2e 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -205,12 +205,10 @@ export async function compileIncludes( // Simple string replacement for each path matched const compiledContent = content.replace( regex, - (match, rawSqlPath: string) => { + (_match, rawSqlPath: string) => { const sqlPath = sqlPathByRawSqlPath[rawSqlPath]; const content = contentBySqlPath[sqlPath]; - const included = match.replace(/^--![ \t]*include/, "--! Included"); - const endIncluded = included.replace("Included", "EndIncluded"); - return `${included}\n${content.trim()}\n${endIncluded}`; + return `--! Included ${rawSqlPath}\n${content.trim()}\n--! EndIncluded ${rawSqlPath}`; }, ); From a557fbe695107fd522491bcb86388ee58e507a5c Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 16:34:05 +0100 Subject: [PATCH 06/12] Remove whitespace --- src/migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migration.ts b/src/migration.ts index e1de2b2e..2b3ece4b 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -132,7 +132,7 @@ export async function compileIncludes( content: string, processedFiles: ReadonlySet, ): Promise { - const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; + const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; // Find all includes in this `content` const matches = [...content.matchAll(regex)]; From da1ed44887b8d4c44156124f2085365dbae7d176 Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 16:40:00 +0100 Subject: [PATCH 07/12] Include -> Included --- __tests__/include.test.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/__tests__/include.test.ts b/__tests__/include.test.ts index 60c7c8fb..f0929d86 100644 --- a/__tests__/include.test.ts +++ b/__tests__/include.test.ts @@ -42,9 +42,9 @@ it("compiles an included file", async () => { FAKE_VISITED, ), ).toEqual(`\ ---! Include foo.sql +--! Included foo.sql select * from foo; ---! EndInclude foo.sql +--! EndIncluded foo.sql `); }); @@ -66,17 +66,17 @@ it("compiles multiple included files", async () => { FAKE_VISITED, ), ).toEqual(`\ ---! Include dir1/foo.sql +--! Included dir1/foo.sql select * from foo; ---! EndInclude dir1/foo.sql ---! Include dir2/bar.sql +--! EndIncluded dir1/foo.sql +--! Included dir2/bar.sql select * from bar; ---! EndInclude dir2/bar.sql ---! Include dir3/baz.sql ---! Include dir4/qux.sql +--! EndIncluded dir2/bar.sql +--! Included dir3/baz.sql +--! Included dir4/qux.sql select * from qux; ---! EndInclude dir4/qux.sql ---! EndInclude dir3/baz.sql +--! EndIncluded dir4/qux.sql +--! EndIncluded dir3/baz.sql `); }); @@ -139,7 +139,7 @@ commit; FAKE_VISITED, ), ).toEqual(`\ ---! Include foo.sql +--! Included foo.sql begin; create or replace function current_user_id() returns uuid as $$ @@ -151,6 +151,6 @@ comment on function current_user_id is E'The ID of the current user.'; grant all on function current_user_id to :DATABASE_USER; commit; ---! EndInclude foo.sql +--! EndIncluded foo.sql `); }); From 918888bb9100e3848509e3f4418c16c734bce70b Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 16:51:55 +0100 Subject: [PATCH 08/12] Fix tests --- __tests__/helpers.ts | 7 +++++-- __tests__/uncommit.test.ts | 3 ++- src/commands/uncommit.ts | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/__tests__/helpers.ts b/__tests__/helpers.ts index 31e3c90f..91324258 100644 --- a/__tests__/helpers.ts +++ b/__tests__/helpers.ts @@ -266,14 +266,16 @@ export const makeMigrations = (commitMessage?: string) => { commitMessage ? `\n--! Message: ${commitMessage}` : `` }\n\n${MIGRATION_NOTRX_TEXT.trim()}\n`; + const MIGRATION_INCLUDED_FIXTURE = "select 42;\n"; + const MIGRATION_INCLUDE_TEXT = `--!include foo.sql`; - const MIGRATION_INCLUDE_COMPILED = `${MIGRATION_INCLUDE_TEXT}\n${MIGRATION_1_TEXT}\n${MIGRATION_INCLUDE_TEXT}`; + const MIGRATION_INCLUDE_COMPILED = `--! Included foo.sql\n${MIGRATION_INCLUDED_FIXTURE.trim()}\n--! EndIncluded foo.sql\n`; const MIGRATION_INCLUDE_HASH = createHash("sha1") .update(`${MIGRATION_INCLUDE_COMPILED.trim()}` + "\n") .digest("hex"); const MIGRATION_INCLUDE_COMMITTED = `--! Previous: -\n--! Hash: sha1:${MIGRATION_INCLUDE_HASH}${ commitMessage ? `\n--! Message: ${commitMessage}` : `` - }\n\n${MIGRATION_INCLUDE_COMPILED}\n`; + }\n\n${MIGRATION_INCLUDE_COMPILED.trim()}\n`; const MIGRATION_MULTIFILE_FILES = { "migrations/links/two.sql": "select 2;", @@ -320,6 +322,7 @@ select 3; MIGRATION_INCLUDE_TEXT, MIGRATION_INCLUDE_HASH, MIGRATION_INCLUDE_COMMITTED, + MIGRATION_INCLUDED_FIXTURE, MIGRATION_MULTIFILE_TEXT, MIGRATION_MULTIFILE_HASH, MIGRATION_MULTIFILE_COMMITTED, diff --git a/__tests__/uncommit.test.ts b/__tests__/uncommit.test.ts index 10dbe33f..12c30aa8 100644 --- a/__tests__/uncommit.test.ts +++ b/__tests__/uncommit.test.ts @@ -59,6 +59,7 @@ describe.each([[undefined], ["My Commit Message"]])( MIGRATION_INCLUDE_COMMITTED, MIGRATION_MULTIFILE_COMMITTED, MIGRATION_MULTIFILE_FILES, + MIGRATION_INCLUDED_FIXTURE, } = makeMigrations(commitMessage); it("rolls back migration", async () => { @@ -95,7 +96,7 @@ describe.each([[undefined], ["My Commit Message"]])( [`migrations/committed/000001${commitMessageSlug}.sql`]: MIGRATION_INCLUDE_COMMITTED, "migrations/current.sql": "-- JUST A COMMENT\n", - "migrations/fixtures/foo.sql": MIGRATION_1_TEXT, + "migrations/fixtures/foo.sql": MIGRATION_INCLUDED_FIXTURE, }); await migrate(settings); await uncommit(settings); diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index f382a783..176b2f66 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -42,10 +42,10 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { const contents = await fsp.readFile(lastMigrationFilepath, "utf8"); const { headers, body } = parseMigrationText(lastMigrationFilepath, contents); - // Replace included migrations with their `--! include` equivalent + // Replace included migrations with their `--!include` equivalent const decompiledBody = body.replace( /^--! Included (?\S+)$[\s\S]*?^--! EndIncluded \k$/gm, - (_, filename) => `--! include ${filename}`, + (_, filename) => `--!include ${filename}`, ); // Drop Hash, Previous and AllowInvalidHash from headers; then write out From cd260ea92d4e19d0b8c7457ad8c3d4836196984e Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 16:56:49 +0100 Subject: [PATCH 09/12] Make it clearer which bit we need to be careful with --- src/commands/uncommit.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 176b2f66..b2e8e2e0 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -40,17 +40,20 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { // Restore current.sql from migration const lastMigrationFilepath = lastMigration.fullPath; const contents = await fsp.readFile(lastMigrationFilepath, "utf8"); - const { headers, body } = parseMigrationText(lastMigrationFilepath, contents); + const { headers, body: committedBody } = parseMigrationText( + lastMigrationFilepath, + contents, + ); // Replace included migrations with their `--!include` equivalent - const decompiledBody = body.replace( + const body = committedBody.replace( /^--! Included (?\S+)$[\s\S]*?^--! EndIncluded \k$/gm, (_, filename) => `--!include ${filename}`, ); // Drop Hash, Previous and AllowInvalidHash from headers; then write out const { Hash, Previous, AllowInvalidHash, ...otherHeaders } = headers; - const completeBody = serializeMigration(decompiledBody, otherHeaders); + const completeBody = serializeMigration(body, otherHeaders); await writeCurrentMigration(parsedSettings, currentLocation, completeBody); // Delete the migration from committed and from the DB From 7a099a866ada0985e05e6add2b13bb78ea1710b8 Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 17:17:16 +0100 Subject: [PATCH 10/12] Test duplicate and nested fixtures --- __tests__/uncommit.test.ts | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/__tests__/uncommit.test.ts b/__tests__/uncommit.test.ts index 12c30aa8..020c149c 100644 --- a/__tests__/uncommit.test.ts +++ b/__tests__/uncommit.test.ts @@ -172,5 +172,54 @@ describe.each([[undefined], ["My Commit Message"]])( ), ).toEqual(MIGRATION_MULTIFILE_COMMITTED); }); + + it("supports the same fixture twice", async () => { + const current = `\ +--!include fixture2.sql +select 22; +--!include fixture2.sql +`; + mockFs({ + "migrations/fixtures/fixture1.sql": "select 'fixture1';", + "migrations/fixtures/fixture2.sql": + "select 1;\n--!include fixture1.sql\nselect 2;", + [`migrations/committed/000001${commitMessageSlug}.sql`]: + MIGRATION_1_COMMITTED, + [`migrations/committed/000002${commitMessageSlug}.sql`]: + MIGRATION_MULTIFILE_COMMITTED, + "migrations/current/1.sql": current, + }); + await migrate(settings); + await commit(settings, commitMessage); + expect( + await fsp.readFile( + `migrations/committed/000003${commitMessageSlug}.sql`, + "utf8", + ), + ).toContain( + `\ +--! Included fixture2.sql +select 1; +--! Included fixture1.sql +select 'fixture1'; +--! EndIncluded fixture1.sql +select 2; +--! EndIncluded fixture2.sql +select 22; +--! Included fixture2.sql +select 1; +--! Included fixture1.sql +select 'fixture1'; +--! EndIncluded fixture1.sql +select 2; +--! EndIncluded fixture2.sql +`, + ); + await uncommit(settings); + + expect(await fsp.readFile(`migrations/current/1.sql`, "utf8")).toEqual( + (commitMessage ? `--! Message: ${commitMessage}\n\n` : "") + current, + ); + }); }, ); From 964b5725f3214f863f39f9f32840513b3e1b38cf Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 17:26:07 +0100 Subject: [PATCH 11/12] Ensure user migrations don't include Included --- src/migration.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/migration.ts b/src/migration.ts index 2b3ece4b..377edbeb 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -132,6 +132,11 @@ export async function compileIncludes( content: string, processedFiles: ReadonlySet, ): Promise { + if (/--!\s*(End)?Included?/.test(content)) { + throw new Error( + "`--! Included` / `--! EndIncluded` comments not allowed in user migrations. Use `--!include` instead.", + ); + } const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; // Find all includes in this `content` From 639ee91234d391249ef389e7d3aab05ff1d0d24a Mon Sep 17 00:00:00 2001 From: Benjie Gillam Date: Wed, 22 Apr 2026 17:28:30 +0100 Subject: [PATCH 12/12] Missing markup --- __tests__/compile.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/__tests__/compile.test.ts b/__tests__/compile.test.ts index e361bca2..c660a9bd 100644 --- a/__tests__/compile.test.ts +++ b/__tests__/compile.test.ts @@ -71,7 +71,9 @@ select 2; ), ).toEqual(`\ select 1; +--! Included foo.sql select * from foo; +--! EndIncluded foo.sql select 2; `); });