From a8ac982b0ee2d98e7e71d5c0de6a5620471a7084 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Sat, 28 Mar 2026 20:40:01 +0200 Subject: [PATCH 01/13] Fix #1180 Add the ability to build filters completely locally --- README.md | 48 ++++++++++++++- package.json | 2 + scripts/build/build.js | 137 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 174 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3abf5c0be65f3..c008e9beb6549 100644 --- a/README.md +++ b/README.md @@ -297,9 +297,12 @@ can be found in [its documentation][gh-compiler-include-directive]. yarn build ``` - The `yarn build` command accepts two parameters: + The `yarn build` command accepts the following parameters: - `-i`: Filters to build. - `-s`: Filters to skip. + - `--no-patches-prepare`: Skip copying `platforms/` to `temp/platforms/`, used to build patches. + Speeds up the build when patch generation (`yarn build:patches`) is not needed afterwards. + Works with both `yarn build` and `yarn build:local`. For example, to build only AdGuard filters: @@ -386,6 +389,49 @@ can be found in [its documentation][gh-compiler-include-directive]. yarn validate ``` +1. **Generate filter cache** + + To update the cached `filter.txt` files in `filters` dir, used for testing/reproducible builds, run a build + with the `--generate-cache` flag: + + ```bash + yarn generate-cache + ``` + + This compiles every filter from its `template.txt` and updates the corresponding + `filter.txt` inside `filters/`. Platform-specific filters and patches are **not** generated. + The resulting `filter.txt` files contain the fully resolved filter content + (all `@include` and `!#include` directives expanded) and can be used to build filters from + cache using `--use-cache` parameter. + +1. **Build from cache** + + To build filters from the previously cached `filter.txt` files without downloading + filters, run: + + ```bash + yarn build:local + ``` + + Under the hood this copies `filters/` to `temp/filters_cached/`, + replaces every `template.txt` with a single `@include "./filter.txt"` directive, + and compiles from that copy. The original `filters/` directory is never modified. + + The `-i` / `-s` / `--no-patches-prepare` parameters can be combined: + + ```bash + yarn build:local -i=1,2,3 --no-patches-prepare + ``` + + > **Typical workflow — comparing two compiler versions:** + > + > 1. `yarn generate-cache` — build `filter.txt` files with downloading filters from sources. + > 2. Switch to compiler version A, run `yarn build:local`, save the output. + > 3. Switch to compiler version B, run `yarn build:local`, diff the results. + > + > Both runs use the exact same raw filter content, so any difference comes + > solely from the compiler. + ## Use Cases ### Building *only AdGuard* filters and updating filters and patches diff --git a/package.json b/package.json index 9082f25be0c22..6b3cdb236fc37 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,9 @@ "lint:md": "markdownlint **/*.md", "test": "vitest run", "build": "tsx scripts/build/build.js", + "build:local": "tsx scripts/build/build.js --use-cache", "build:patches": "node scripts/build/patches.js", + "generate-cache": "tsx scripts/build/build.js --generate-cache", "validate": "yarn validate:platforms && yarn validate:locales", "validate:platforms": "tsx scripts/validation/validate_platforms.js", "validate:locales": "tsx scripts/validation/validate_locales.js", diff --git a/scripts/build/build.js b/scripts/build/build.js index 5954c0e2a8e0d..a0a747549df3c 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -13,11 +13,14 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); /** - * Parse command-cli parameters -i|--include and -s|--skip + * Parse command-cli parameters -i|--include, -s|--skip, --use-cache, --generate-cache, --no-patches-prepare */ let includedFilterIDs = []; let excludedFilterIDs = []; let rawReportPath = ''; +let useCache = false; +let generateCache = false; +let noPatchesPrepare = false; const args = process.argv.slice(2); args.forEach((val) => { @@ -40,8 +43,25 @@ args.forEach((val) => { if (val.startsWith('--report=')) { rawReportPath = val.slice(val.indexOf('=') + 1).trim(); } + + if (val === '--use-cache') { + useCache = true; + } + + if (val === '--generate-cache') { + generateCache = true; + } + + if (val === '--no-patches-prepare') { + noPatchesPrepare = true; + } }); +if (useCache && generateCache) { + console.error('Error: --use-cache and --generate-cache are mutually exclusive.'); + process.exit(1); +} + /** * Set all relative paths needed for compiler */ @@ -49,6 +69,7 @@ const filtersDir = path.join(__dirname, '../../filters'); const logPath = path.join(__dirname, '../../log.txt'); const platformsPath = path.join(__dirname, '../..', FOLDER_WITH_NEW_FILTERS); const copyPlatformsPath = path.join(__dirname, '../..', FOLDER_WITH_OLD_FILTERS); +const cachedFiltersDir = path.join(__dirname, '../../temp/filters_cached'); const reportPath = rawReportPath !== '' // report-adguard.txt OR report-third-party.txt @@ -56,10 +77,88 @@ const reportPath = rawReportPath !== '' // report_partial_DD-MM-YYYY_HH-MM-SS.txt : path.join(__dirname, '../..', `report_partial_${formatDate(new Date())}.txt`); +const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n'; + +/** + * Prepare a temporary copy of the filters directory with shadow templates. + * + * Copies `filters/` → `temp/filters_cached/`, then replaces the content of every + * `template.txt` with a single-line local include pointing to the cached `filter.txt`. + * Validates that every filter directory with a `template.txt` also has a `filter.txt`. + * + * @returns {Promise} + */ +const prepareCachedFiltersDir = async () => { + // Remove stale copy if exists + if (fs.existsSync(cachedFiltersDir)) { + await fs.promises.rm(cachedFiltersDir, { recursive: true }); + } + + // Full recursive copy + await fs.promises.cp(filtersDir, cachedFiltersDir, { recursive: true }); + + // Find all directories containing template.txt and replace with shadow templates + const templatePaths = await findTemplatePaths(cachedFiltersDir); + + for (const templatePath of templatePaths) { + const dir = path.dirname(templatePath); + const filterTxtPath = path.join(dir, 'filter.txt'); + + if (!fs.existsSync(filterTxtPath)) { + throw new Error( + `--use-cache: missing filter.txt in ${path.relative(cachedFiltersDir, dir)}. ` + + 'Run "yarn generate-cache" first to generate cached filter files.', + ); + } + + await fs.promises.writeFile(templatePath, SHADOW_TEMPLATE_CONTENT, 'utf8'); + } + + console.log(`Prepared cached filters directory with ${templatePaths.length} shadow templates.`); +}; + +/** + * Recursively find all `template.txt` files under the given directory. + * + * @param {string} dir - Root directory to search. + * @returns {Promise} Array of absolute paths to `template.txt` files. + */ +const findTemplatePaths = async (dir) => { + const results = []; + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + const nested = await findTemplatePaths(fullPath); + results.push(...nested); + } else if (entry.name === 'template.txt') { + results.push(fullPath); + } + } + + return results; +}; + /** * Compiler entry point. */ const buildFilters = async () => { + // When --generate-cache we only need to compile filters (which updates filter.txt), + // skip platform generation, patches preparation, and temp/platforms copying. + if (generateCache) { + await compile( + filtersDir, + logPath, + reportPath, + null, // null ⇒ generate() inside compiler returns early, no platform files + includedFilterIDs, + excludedFilterIDs, + CUSTOM_PLATFORMS_CONFIG, + ); + return; + } + // Clean temporary folder if (fs.existsSync(copyPlatformsPath)) { await fs.promises.rm(copyPlatformsPath, { recursive: true }); @@ -70,24 +169,38 @@ const buildFilters = async () => { let initialRun = false; if (!fs.existsSync(platformsPath)) { initialRun = true; - } else { + } else if (!noPatchesPrepare) { // Make copy for future patches generation await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true }); } - await compile( - filtersDir, - logPath, - reportPath, - platformsPath, - includedFilterIDs, - excludedFilterIDs, - CUSTOM_PLATFORMS_CONFIG, - ); + // Determine which filtersDir to pass to the compiler + const effectiveFiltersDir = useCache ? cachedFiltersDir : filtersDir; + + if (useCache) { + await prepareCachedFiltersDir(); + } + + try { + await compile( + effectiveFiltersDir, + logPath, + reportPath, + platformsPath, + includedFilterIDs, + excludedFilterIDs, + CUSTOM_PLATFORMS_CONFIG, + ); + } finally { + // Clean up temp filters copy + if (useCache && fs.existsSync(cachedFiltersDir)) { + await fs.promises.rm(cachedFiltersDir, { recursive: true }); + } + } // For the very first run, we should copy the built platforms into // the temp folder to create the first empty patches for future versions - if (initialRun) { + if (initialRun && !noPatchesPrepare) { // Make copy for future patches generation await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true }); } From dce899db645156886ac9874538a078daaa7f30d4 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Sun, 29 Mar 2026 00:03:23 +0200 Subject: [PATCH 02/13] fix/lint errors in build.js --- scripts/build/build.js | 52 ++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/scripts/build/build.js b/scripts/build/build.js index a0a747549df3c..758b9b810ec4f 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -58,6 +58,7 @@ args.forEach((val) => { }); if (useCache && generateCache) { + // eslint-disable-next-line no-console console.error('Error: --use-cache and --generate-cache are mutually exclusive.'); process.exit(1); } @@ -79,6 +80,29 @@ const reportPath = rawReportPath !== '' const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n'; +/** + * Recursively find all `template.txt` files under the given directory. + * + * @param {string} dir - Root directory to search. + * @returns {Promise} Array of absolute paths to `template.txt` files. + */ +const findTemplatePaths = async (dir) => { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + + const results = await Promise.all(entries.map(async (entry) => { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + return findTemplatePaths(fullPath); + } + if (entry.name === 'template.txt') { + return [fullPath]; + } + return []; + })); + + return results.flat(); +}; + /** * Prepare a temporary copy of the filters directory with shadow templates. * @@ -100,7 +124,7 @@ const prepareCachedFiltersDir = async () => { // Find all directories containing template.txt and replace with shadow templates const templatePaths = await findTemplatePaths(cachedFiltersDir); - for (const templatePath of templatePaths) { + await Promise.all(templatePaths.map(async (templatePath) => { const dir = path.dirname(templatePath); const filterTxtPath = path.join(dir, 'filter.txt'); @@ -112,34 +136,12 @@ const prepareCachedFiltersDir = async () => { } await fs.promises.writeFile(templatePath, SHADOW_TEMPLATE_CONTENT, 'utf8'); - } + })); + // eslint-disable-next-line no-console console.log(`Prepared cached filters directory with ${templatePaths.length} shadow templates.`); }; -/** - * Recursively find all `template.txt` files under the given directory. - * - * @param {string} dir - Root directory to search. - * @returns {Promise} Array of absolute paths to `template.txt` files. - */ -const findTemplatePaths = async (dir) => { - const results = []; - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - const nested = await findTemplatePaths(fullPath); - results.push(...nested); - } else if (entry.name === 'template.txt') { - results.push(fullPath); - } - } - - return results; -}; - /** * Compiler entry point. */ From 395bd2fca5c51b0afc756abf99078e8b9c968112 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Sun, 29 Mar 2026 17:40:55 +0300 Subject: [PATCH 03/13] fix/Add --strip-generated-meta build flag for debug purposes Strip volatile metadata lines (Checksum, Diff-Path, TimeUpdated, Version) from compiled filter files after build. Reduces noise when comparing outputs between compiler versions. --- README.md | 36 +++++-- package.json | 1 + scripts/build/build.js | 17 +++- scripts/build/strip_generated_meta.js | 130 ++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 scripts/build/strip_generated_meta.js diff --git a/README.md b/README.md index c008e9beb6549..70783e0867554 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,9 @@ can be found in [its documentation][gh-compiler-include-directive]. - `--no-patches-prepare`: Skip copying `platforms/` to `temp/platforms/`, used to build patches. Speeds up the build when patch generation (`yarn build:patches`) is not needed afterwards. Works with both `yarn build` and `yarn build:local`. + - `--strip-generated-meta`: After compilation, remove generated meta lines + (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, `! Version`) from all filter files + in `platforms/`. Useful when comparing outputs between builds, e.g. in Total Commander. For example, to build only AdGuard filters: @@ -417,20 +420,39 @@ can be found in [its documentation][gh-compiler-include-directive]. replaces every `template.txt` with a single `@include "./filter.txt"` directive, and compiles from that copy. The original `filters/` directory is never modified. - The `-i` / `-s` / `--no-patches-prepare` parameters can be combined: + The `-i` / `-s` / `--no-patches-prepare` / `--strip-generated-meta` parameters can be combined: ```bash - yarn build:local -i=1,2,3 --no-patches-prepare + yarn build:local -i=1,2,3 --no-patches-prepare --strip-generated-meta ``` > **Typical workflow — comparing two compiler versions:** > - > 1. `yarn generate-cache` — build `filter.txt` files with downloading filters from sources. - > 2. Switch to compiler version A, run `yarn build:local`, save the output. - > 3. Switch to compiler version B, run `yarn build:local`, diff the results. + > 1. Download and cache fresh source filter content: + + ```bash + `yarn generate-cache` + ``` + > 2. Build from cache with generated metadata lines stripped: + > + > ```bash + > yarn build:local --no-patches-prepare --strip-generated-meta + > ``` + > + > 3. Rename the output to `mv platforms platforms_A`. + > 4. Switch to the other compiler version (e.g. `yarn add @adguard/filters-compiler@...`). + > 5. Build again with the same flags: + > + > ```bash + > yarn build:local --no-patches-prepare --strip-generated-meta + > ``` + > + > 6. Rename the output to `mv platforms platforms_B`. + > 7. Diff the two directories (e.g. in Total Commander or with `diff -r`). > - > Both runs use the exact same raw filter content, so any difference comes - > solely from the compiler. + > Both runs use the exact same cached filter content and strip all + > volatile metadata (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, + > `! Version`), so any difference comes solely from the compiler. ## Use Cases diff --git a/package.json b/package.json index 6b3cdb236fc37..09315af39a73d 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build": "tsx scripts/build/build.js", "build:local": "tsx scripts/build/build.js --use-cache", "build:patches": "node scripts/build/patches.js", + "strip-generated-meta": "tsx scripts/build/strip_generated_meta.js", "generate-cache": "tsx scripts/build/build.js --generate-cache", "validate": "yarn validate:platforms && yarn validate:locales", "validate:platforms": "tsx scripts/validation/validate_platforms.js", diff --git a/scripts/build/build.js b/scripts/build/build.js index 758b9b810ec4f..1abd47a2d9416 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -8,12 +8,14 @@ import { FOLDER_WITH_NEW_FILTERS, FOLDER_WITH_OLD_FILTERS, } from './constants.js'; +import { stripGeneratedMetaFromDir } from './strip_generated_meta.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); /** - * Parse command-cli parameters -i|--include, -s|--skip, --use-cache, --generate-cache, --no-patches-prepare + * Parse command-line parameters -i|--include, -s|--skip, --use-cache, --generate-cache, + * --no-patches-prepare, --strip-generated-meta */ let includedFilterIDs = []; let excludedFilterIDs = []; @@ -21,6 +23,7 @@ let rawReportPath = ''; let useCache = false; let generateCache = false; let noPatchesPrepare = false; +let stripGeneratedMeta = false; const args = process.argv.slice(2); args.forEach((val) => { @@ -55,6 +58,10 @@ args.forEach((val) => { if (val === '--no-patches-prepare') { noPatchesPrepare = true; } + + if (val === '--strip-generated-meta') { + stripGeneratedMeta = true; + } }); if (useCache && generateCache) { @@ -206,6 +213,14 @@ const buildFilters = async () => { // Make copy for future patches generation await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true }); } + + // Strip generated metadata (Checksum, Diff-Path, TimeUpdated, Version) + // from compiled filter files so they don't pollute diff comparisons. + if (stripGeneratedMeta) { + const count = await stripGeneratedMetaFromDir(platformsPath); + // eslint-disable-next-line no-console + console.log(`Stripped generated meta from ${count} file(s).`); + } }; buildFilters(); diff --git a/scripts/build/strip_generated_meta.js b/scripts/build/strip_generated_meta.js new file mode 100644 index 0000000000000..c1e5e31f919b0 --- /dev/null +++ b/scripts/build/strip_generated_meta.js @@ -0,0 +1,130 @@ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const PLATFORMS_DIR = path.join(__dirname, '../../platforms'); + +/** + * Lines starting with these prefixes are generated meta — they change on every build + * regardless of filter content changes, making file comparison noisy. + */ +const GENERATED_META_PREFIXES = [ + '! Checksum:', + '! Diff-Path:', + '! TimeUpdated:', + '! Version:', +]; + +const isGeneratedMetaLine = (line) => GENERATED_META_PREFIXES.some((prefix) => line.startsWith(prefix)); + +/** + * Strip generated metadata lines from a single filter file in-place. + * + * @param {string} filePath - Absolute path to the .txt filter file. + * @returns {Promise} True if the file was modified, false otherwise. + */ +const stripGeneratedMeta = async (filePath) => { + const content = await fs.promises.readFile(filePath, 'utf8'); + const lines = content.split('\n'); + const filtered = lines.filter((line) => !isGeneratedMetaLine(line)); + + if (filtered.length === lines.length) { + return false; + } + + await fs.promises.writeFile(filePath, filtered.join('\n'), 'utf8'); + return true; +}; + +/** + * Recursively find all .txt files inside a given directory. + * + * @param {string} dir - Root directory to search. + * @returns {Promise} Array of absolute paths to .txt files. + */ +const findTxtFiles = async (dir) => { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + + const results = await Promise.all(entries.map(async (entry) => { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + return findTxtFiles(fullPath); + } + if (entry.isFile() && entry.name.endsWith('.txt')) { + return [fullPath]; + } + return []; + })); + + return results.flat(); +}; + +/** + * Recursively find all directories named `filters` under the given root. + * + * @param {string} dir - Root directory to search. + * @returns {Promise} Array of absolute paths to `filters` directories. + */ +const findFiltersDirs = async (dir) => { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + + const results = await Promise.all(entries.map(async (entry) => { + if (!entry.isDirectory()) { + return []; + } + const fullPath = path.join(dir, entry.name); + if (entry.name === 'filters') { + return [fullPath]; + } + return findFiltersDirs(fullPath); + })); + + return results.flat(); +}; + +/** + * Strip generated metadata lines from all .txt files inside all `filters/` + * directories found recursively under the given root. + * + * @param {string} rootDir - Root directory to search (e.g. `platforms/`). + * @returns {Promise} Number of files actually modified. + */ +export const stripGeneratedMetaFromDir = async (rootDir) => { + const filtersDirs = await findFiltersDirs(rootDir); + + let totalModified = 0; + + await Promise.all(filtersDirs.map(async (filtersDir) => { + const files = await findTxtFiles(filtersDir); + const results = await Promise.all(files.map((f) => stripGeneratedMeta(f))); + const modified = results.filter(Boolean).length; + totalModified += modified; + + if (modified > 0) { + // eslint-disable-next-line no-console + console.log(`${path.relative(rootDir, filtersDir)}: stripped headers from ${modified} file(s)`); + } + })); + + return totalModified; +}; + +/** + * Main entry point (standalone execution). + * Recursively finds all `filters/` directories under platforms/ and strips + * generated meta headers from every .txt file found there. + */ +const main = async () => { + const totalModified = await stripGeneratedMetaFromDir(PLATFORMS_DIR); + + // eslint-disable-next-line no-console + console.log(`Done. Total files modified: ${totalModified}`); +}; + +// Run only when executed directly, not when imported as a module. +if (process.argv[1] === __filename) { + main(); +} From b656361cff6124105d937bef43e69a68cbfe2cdf Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Sun, 29 Mar 2026 17:49:01 +0300 Subject: [PATCH 04/13] fix/lint --- README.md | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 70783e0867554..f469efc371e58 100644 --- a/README.md +++ b/README.md @@ -426,33 +426,22 @@ can be found in [its documentation][gh-compiler-include-directive]. yarn build:local -i=1,2,3 --no-patches-prepare --strip-generated-meta ``` - > **Typical workflow — comparing two compiler versions:** - > - > 1. Download and cache fresh source filter content: - - ```bash - `yarn generate-cache` - ``` - > 2. Build from cache with generated metadata lines stripped: - > - > ```bash - > yarn build:local --no-patches-prepare --strip-generated-meta - > ``` - > - > 3. Rename the output to `mv platforms platforms_A`. - > 4. Switch to the other compiler version (e.g. `yarn add @adguard/filters-compiler@...`). - > 5. Build again with the same flags: - > - > ```bash - > yarn build:local --no-patches-prepare --strip-generated-meta - > ``` - > - > 6. Rename the output to `mv platforms platforms_B`. - > 7. Diff the two directories (e.g. in Total Commander or with `diff -r`). - > - > Both runs use the exact same cached filter content and strip all - > volatile metadata (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, - > `! Version`), so any difference comes solely from the compiler. + **Typical workflow — comparing two compiler versions:** + + 1. Download and compile filter content into cache: + `yarn generate-cache` + 1. Build from cache with generated metadata lines stripped: + `yarn build:local --no-patches-prepare --strip-generated-meta` + 1. Rename the output: `mv platforms platforms_A`. + 1. Switch to the other compiler version (e.g. `yarn add @adguard/filters-compiler@...`). + 1. Build again with the same flags: + `yarn build:local --no-patches-prepare --strip-generated-meta` + 1. Rename the output: `mv platforms platforms_B`. + 1. Diff the two directories (e.g. in Total Commander, WinMerge, or with `diff -r`). + + Both runs use the exact same cached filter content and strip all + volatile metadata (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, + `! Version`), so any difference comes solely from the compiler. ## Use Cases From 53c6a5c2886f81df2bafd0662e131affc5cb06fd Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Mon, 6 Apr 2026 22:19:04 +0300 Subject: [PATCH 05/13] - strip_generated_meta.js renamed to kebab-case - one-liner => helper function - .txt extension to the constant in strip_generated_meta.js --- package.json | 2 +- scripts/build/build.js | 2 +- ...erated_meta.js => strip-generated-meta.js} | 27 ++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) rename scripts/build/{strip_generated_meta.js => strip-generated-meta.js} (83%) diff --git a/package.json b/package.json index 09315af39a73d..c807ec7c79a50 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build": "tsx scripts/build/build.js", "build:local": "tsx scripts/build/build.js --use-cache", "build:patches": "node scripts/build/patches.js", - "strip-generated-meta": "tsx scripts/build/strip_generated_meta.js", + "strip-generated-meta": "tsx scripts/build/strip-generated-meta.js", "generate-cache": "tsx scripts/build/build.js --generate-cache", "validate": "yarn validate:platforms && yarn validate:locales", "validate:platforms": "tsx scripts/validation/validate_platforms.js", diff --git a/scripts/build/build.js b/scripts/build/build.js index 1abd47a2d9416..d35694b9f64a1 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -8,7 +8,7 @@ import { FOLDER_WITH_NEW_FILTERS, FOLDER_WITH_OLD_FILTERS, } from './constants.js'; -import { stripGeneratedMetaFromDir } from './strip_generated_meta.js'; +import { stripGeneratedMetaFromDir } from './strip-generated-meta.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/scripts/build/strip_generated_meta.js b/scripts/build/strip-generated-meta.js similarity index 83% rename from scripts/build/strip_generated_meta.js rename to scripts/build/strip-generated-meta.js index c1e5e31f919b0..5eb94aafb6585 100644 --- a/scripts/build/strip_generated_meta.js +++ b/scripts/build/strip-generated-meta.js @@ -6,6 +6,8 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const PLATFORMS_DIR = path.join(__dirname, '../../platforms'); +const FILTERS_DIR_NAME = 'filters'; +const TXT_FILE_EXTENSION = '.txt'; /** * Lines starting with these prefixes are generated meta — they change on every build @@ -18,7 +20,15 @@ const GENERATED_META_PREFIXES = [ '! Version:', ]; -const isGeneratedMetaLine = (line) => GENERATED_META_PREFIXES.some((prefix) => line.startsWith(prefix)); +/** + * Checks whether a line is a generated meta line that should be stripped. + * + * @param {string} line - A single line from a filter file. + * @returns {boolean} True if the line starts with a generated meta prefix. + */ +const isGeneratedMetaLine = (line) => { + return GENERATED_META_PREFIXES.some((prefix) => line.startsWith(prefix)); +}; /** * Strip generated metadata lines from a single filter file in-place. @@ -53,7 +63,7 @@ const findTxtFiles = async (dir) => { if (entry.isDirectory()) { return findTxtFiles(fullPath); } - if (entry.isFile() && entry.name.endsWith('.txt')) { + if (entry.isFile() && entry.name.endsWith(TXT_FILE_EXTENSION)) { return [fullPath]; } return []; @@ -76,7 +86,7 @@ const findFiltersDirs = async (dir) => { return []; } const fullPath = path.join(dir, entry.name); - if (entry.name === 'filters') { + if (entry.name === FILTERS_DIR_NAME) { return [fullPath]; } return findFiltersDirs(fullPath); @@ -95,21 +105,20 @@ const findFiltersDirs = async (dir) => { export const stripGeneratedMetaFromDir = async (rootDir) => { const filtersDirs = await findFiltersDirs(rootDir); - let totalModified = 0; - - await Promise.all(filtersDirs.map(async (filtersDir) => { + const counts = await Promise.all(filtersDirs.map(async (filtersDir) => { const files = await findTxtFiles(filtersDir); const results = await Promise.all(files.map((f) => stripGeneratedMeta(f))); const modified = results.filter(Boolean).length; - totalModified += modified; if (modified > 0) { // eslint-disable-next-line no-console - console.log(`${path.relative(rootDir, filtersDir)}: stripped headers from ${modified} file(s)`); + console.log(`${path.relative(rootDir, filtersDir)}: stripped metadata from ${modified} file(s)`); } + + return modified; })); - return totalModified; + return counts.reduce((sum, count) => sum + count, 0); }; /** From 7af7209541a6052706aa8bbd4f21d5f947d044c8 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Mon, 6 Apr 2026 22:43:21 +0300 Subject: [PATCH 06/13] Reused find_files.js in strip-generated-meta.js and build.js --- scripts/build/build.js | 26 ++------------------------ scripts/build/strip-generated-meta.js | 26 ++------------------------ 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/scripts/build/build.js b/scripts/build/build.js index d35694b9f64a1..48766e76f4656 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -9,6 +9,7 @@ import { FOLDER_WITH_OLD_FILTERS, } from './constants.js'; import { stripGeneratedMetaFromDir } from './strip-generated-meta.js'; +import { findFiles } from '../utils/find_files.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -87,29 +88,6 @@ const reportPath = rawReportPath !== '' const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n'; -/** - * Recursively find all `template.txt` files under the given directory. - * - * @param {string} dir - Root directory to search. - * @returns {Promise} Array of absolute paths to `template.txt` files. - */ -const findTemplatePaths = async (dir) => { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - - const results = await Promise.all(entries.map(async (entry) => { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - return findTemplatePaths(fullPath); - } - if (entry.name === 'template.txt') { - return [fullPath]; - } - return []; - })); - - return results.flat(); -}; - /** * Prepare a temporary copy of the filters directory with shadow templates. * @@ -129,7 +107,7 @@ const prepareCachedFiltersDir = async () => { await fs.promises.cp(filtersDir, cachedFiltersDir, { recursive: true }); // Find all directories containing template.txt and replace with shadow templates - const templatePaths = await findTemplatePaths(cachedFiltersDir); + const templatePaths = await findFiles(cachedFiltersDir, (p) => path.basename(p) === 'template.txt'); await Promise.all(templatePaths.map(async (templatePath) => { const dir = path.dirname(templatePath); diff --git a/scripts/build/strip-generated-meta.js b/scripts/build/strip-generated-meta.js index 5eb94aafb6585..800e2c3f52801 100644 --- a/scripts/build/strip-generated-meta.js +++ b/scripts/build/strip-generated-meta.js @@ -1,6 +1,7 @@ import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; +import { findFiles } from '../utils/find_files.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -49,29 +50,6 @@ const stripGeneratedMeta = async (filePath) => { return true; }; -/** - * Recursively find all .txt files inside a given directory. - * - * @param {string} dir - Root directory to search. - * @returns {Promise} Array of absolute paths to .txt files. - */ -const findTxtFiles = async (dir) => { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - - const results = await Promise.all(entries.map(async (entry) => { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - return findTxtFiles(fullPath); - } - if (entry.isFile() && entry.name.endsWith(TXT_FILE_EXTENSION)) { - return [fullPath]; - } - return []; - })); - - return results.flat(); -}; - /** * Recursively find all directories named `filters` under the given root. * @@ -106,7 +84,7 @@ export const stripGeneratedMetaFromDir = async (rootDir) => { const filtersDirs = await findFiltersDirs(rootDir); const counts = await Promise.all(filtersDirs.map(async (filtersDir) => { - const files = await findTxtFiles(filtersDir); + const files = await findFiles(filtersDir, (p) => p.endsWith(TXT_FILE_EXTENSION)); const results = await Promise.all(files.map((f) => stripGeneratedMeta(f))); const modified = results.filter(Boolean).length; From 576f7efae7b38617f7625c41eed95a8730850e9a Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Mon, 6 Apr 2026 23:52:47 +0300 Subject: [PATCH 07/13] use fs/promises in strip-generated-meta --- scripts/build/strip-generated-meta.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build/strip-generated-meta.js b/scripts/build/strip-generated-meta.js index 800e2c3f52801..8befc97d1285a 100644 --- a/scripts/build/strip-generated-meta.js +++ b/scripts/build/strip-generated-meta.js @@ -1,4 +1,4 @@ -import fs from 'fs'; +import fs from 'fs/promises'; import path from 'path'; import { fileURLToPath } from 'url'; import { findFiles } from '../utils/find_files.js'; @@ -38,7 +38,7 @@ const isGeneratedMetaLine = (line) => { * @returns {Promise} True if the file was modified, false otherwise. */ const stripGeneratedMeta = async (filePath) => { - const content = await fs.promises.readFile(filePath, 'utf8'); + const content = await fs.readFile(filePath, 'utf8'); const lines = content.split('\n'); const filtered = lines.filter((line) => !isGeneratedMetaLine(line)); @@ -46,7 +46,7 @@ const stripGeneratedMeta = async (filePath) => { return false; } - await fs.promises.writeFile(filePath, filtered.join('\n'), 'utf8'); + await fs.writeFile(filePath, filtered.join('\n'), 'utf8'); return true; }; @@ -57,7 +57,7 @@ const stripGeneratedMeta = async (filePath) => { * @returns {Promise} Array of absolute paths to `filters` directories. */ const findFiltersDirs = async (dir) => { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + const entries = await fs.readdir(dir, { withFileTypes: true }); const results = await Promise.all(entries.map(async (entry) => { if (!entry.isDirectory()) { From a12947d6bd83ab16373ce32ecd5edc6238fb4144 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Tue, 7 Apr 2026 15:52:49 +0300 Subject: [PATCH 08/13] convert strip-generated-meta.js to TS --- scripts/build/build.js | 2 +- ...erated-meta.js => strip-generated-meta.ts} | 54 ++++++------------- 2 files changed, 18 insertions(+), 38 deletions(-) rename scripts/build/{strip-generated-meta.js => strip-generated-meta.ts} (57%) diff --git a/scripts/build/build.js b/scripts/build/build.js index 48766e76f4656..3c7f1ab7e38ff 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -8,7 +8,7 @@ import { FOLDER_WITH_NEW_FILTERS, FOLDER_WITH_OLD_FILTERS, } from './constants.js'; -import { stripGeneratedMetaFromDir } from './strip-generated-meta.js'; +import { stripGeneratedMetaFromDir } from './strip-generated-meta.ts'; import { findFiles } from '../utils/find_files.js'; const __filename = fileURLToPath(import.meta.url); diff --git a/scripts/build/strip-generated-meta.js b/scripts/build/strip-generated-meta.ts similarity index 57% rename from scripts/build/strip-generated-meta.js rename to scripts/build/strip-generated-meta.ts index 8befc97d1285a..cc7d300c74bb6 100644 --- a/scripts/build/strip-generated-meta.js +++ b/scripts/build/strip-generated-meta.ts @@ -1,12 +1,7 @@ import fs from 'fs/promises'; import path from 'path'; -import { fileURLToPath } from 'url'; import { findFiles } from '../utils/find_files.js'; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -const PLATFORMS_DIR = path.join(__dirname, '../../platforms'); const FILTERS_DIR_NAME = 'filters'; const TXT_FILE_EXTENSION = '.txt'; @@ -19,25 +14,25 @@ const GENERATED_META_PREFIXES = [ '! Diff-Path:', '! TimeUpdated:', '! Version:', -]; +] as const; /** * Checks whether a line is a generated meta line that should be stripped. * - * @param {string} line - A single line from a filter file. - * @returns {boolean} True if the line starts with a generated meta prefix. + * @param line - A single line from a filter file. + * @returns True if the line starts with a generated meta prefix. */ -const isGeneratedMetaLine = (line) => { +const isGeneratedMetaLine = (line: string): boolean => { return GENERATED_META_PREFIXES.some((prefix) => line.startsWith(prefix)); }; /** * Strip generated metadata lines from a single filter file in-place. * - * @param {string} filePath - Absolute path to the .txt filter file. - * @returns {Promise} True if the file was modified, false otherwise. + * @param filePath - Absolute path to the .txt filter file. + * @returns True if the file was modified, false otherwise. */ -const stripGeneratedMeta = async (filePath) => { +const stripGeneratedMeta = async (filePath: string): Promise => { const content = await fs.readFile(filePath, 'utf8'); const lines = content.split('\n'); const filtered = lines.filter((line) => !isGeneratedMetaLine(line)); @@ -53,10 +48,10 @@ const stripGeneratedMeta = async (filePath) => { /** * Recursively find all directories named `filters` under the given root. * - * @param {string} dir - Root directory to search. - * @returns {Promise} Array of absolute paths to `filters` directories. + * @param dir - Root directory to search. + * @returns Array of absolute paths to `filters` directories. */ -const findFiltersDirs = async (dir) => { +const findFiltersDirs = async (dir: string): Promise => { const entries = await fs.readdir(dir, { withFileTypes: true }); const results = await Promise.all(entries.map(async (entry) => { @@ -73,19 +68,21 @@ const findFiltersDirs = async (dir) => { return results.flat(); }; +const hasTxtExtension = (p: string): boolean => p.endsWith(TXT_FILE_EXTENSION); + /** * Strip generated metadata lines from all .txt files inside all `filters/` * directories found recursively under the given root. * - * @param {string} rootDir - Root directory to search (e.g. `platforms/`). - * @returns {Promise} Number of files actually modified. + * @param rootDir - Root directory to search (e.g. `platforms/`). + * @returns Number of files actually modified. */ -export const stripGeneratedMetaFromDir = async (rootDir) => { +export const stripGeneratedMetaFromDir = async (rootDir: string): Promise => { const filtersDirs = await findFiltersDirs(rootDir); const counts = await Promise.all(filtersDirs.map(async (filtersDir) => { - const files = await findFiles(filtersDir, (p) => p.endsWith(TXT_FILE_EXTENSION)); - const results = await Promise.all(files.map((f) => stripGeneratedMeta(f))); + const files = await findFiles(filtersDir, hasTxtExtension); + const results = await Promise.all(files.map(stripGeneratedMeta)); const modified = results.filter(Boolean).length; if (modified > 0) { @@ -98,20 +95,3 @@ export const stripGeneratedMetaFromDir = async (rootDir) => { return counts.reduce((sum, count) => sum + count, 0); }; - -/** - * Main entry point (standalone execution). - * Recursively finds all `filters/` directories under platforms/ and strips - * generated meta headers from every .txt file found there. - */ -const main = async () => { - const totalModified = await stripGeneratedMetaFromDir(PLATFORMS_DIR); - - // eslint-disable-next-line no-console - console.log(`Done. Total files modified: ${totalModified}`); -}; - -// Run only when executed directly, not when imported as a module. -if (process.argv[1] === __filename) { - main(); -} From 54a6df8bfd650b6d2b54be714799b5a7fef47bef Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Tue, 7 Apr 2026 16:11:12 +0300 Subject: [PATCH 09/13] Upd docs --- AGENTS.md | 4 +++- DEVELOPMENT.md | 8 ++++++++ package.json | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 94342c2d6d58e..2ba7a9b1a2a90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,9 @@ for all supported AdGuard products. ## Technical Context -- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`) +- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`). + New code must be written in TypeScript. The project is gradually migrating all scripts to + TypeScript; existing `.js` files should be converted to `.ts` when touched. - **Primary Dependencies**: - `@adguard/filters-compiler` — compiles filter templates into final filter lists - `@adguard/agtree` — AdGuard filter rule parser / AST diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 428525ba893b8..39a7a0d7967b2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -30,6 +30,11 @@ yarn test ``` + > **Note:** `yarn lint` runs three checks in sequence: ESLint (`yarn lint:code`), + > TypeScript type checking (`yarn lint:types`), and Markdownlint (`yarn lint:md`). + > All three must pass. The build scripts are a mix of JavaScript and TypeScript; + > `tsx` executes both transparently — no manual compilation step is needed. + ## Development Workflow ### Building Filters @@ -307,6 +312,9 @@ All build tooling lives under `scripts/`. After making changes: `scripts/wildcard-domain-processor/__tests__/`. 1. Run `yarn validate` if the change affects filter compilation or platform outputs. +Build scripts under `scripts/` are written in JavaScript and TypeScript, executed +via [tsx](https://github.com/privatenumber/tsx) — no manual compilation step is needed. + ## Troubleshooting ### `yarn build` fails with missing platform files diff --git a/package.json b/package.json index c807ec7c79a50..28ec0337f1d35 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build": "tsx scripts/build/build.js", "build:local": "tsx scripts/build/build.js --use-cache", "build:patches": "node scripts/build/patches.js", - "strip-generated-meta": "tsx scripts/build/strip-generated-meta.js", + "strip-generated-meta": "tsx scripts/build/strip-generated-meta.ts", "generate-cache": "tsx scripts/build/build.js --generate-cache", "validate": "yarn validate:platforms && yarn validate:locales", "validate:platforms": "tsx scripts/validation/validate_platforms.js", From 7a10cdbfd63cd83e0932af3c51348412c2521896 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Tue, 28 Apr 2026 00:02:47 +0300 Subject: [PATCH 10/13] Fixed invoking strip-generated-meta Co-authored-by: Copilot --- .../__tests__/strip-generated-meta.test.ts | 69 +++++++++++++++++++ scripts/build/strip-generated-meta.ts | 9 +++ vitest.config.ts | 5 +- 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 scripts/build/__tests__/strip-generated-meta.test.ts diff --git a/scripts/build/__tests__/strip-generated-meta.test.ts b/scripts/build/__tests__/strip-generated-meta.test.ts new file mode 100644 index 0000000000000..f1b84f3563385 --- /dev/null +++ b/scripts/build/__tests__/strip-generated-meta.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; +import { stripGeneratedMetaFromDir } from '../strip-generated-meta.js'; + +/** + * Test that stripGeneratedMetaFromDir actually strips generated metadata lines + * from .txt files inside `filters/` directories. + */ +describe('stripGeneratedMetaFromDir', () => { + let testDir: string; + let filtersDir: string; + let testFile: string; + + beforeAll(async () => { + // Create a temporary directory structure mimicking the project layout + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'strip-meta-test-')); + filtersDir = path.join(testDir, 'filters'); + await fs.mkdir(filtersDir); + testFile = path.join(filtersDir, 'test.txt'); + + // Create a test file with generated metadata lines + const content = [ + '! Checksum: abc123', + '! Diff-Path: somewhere/diff.txt', + '! TimeUpdated: 2026-04-27T10:00:00Z', + '! Version: 1.0.0', + '', + '! Normal comment', + '||example.com^', + '##.ads-banner', + '', + ].join('\n'); + + await fs.writeFile(testFile, content, 'utf8'); + }); + + afterAll(async () => { + // Clean up the temporary directory + await fs.rm(testDir, { recursive: true, force: true }); + }); + + it('strips generated metadata lines from .txt files', async () => { + const modified = await stripGeneratedMetaFromDir(testDir); + + expect(modified).toBe(1); + + const content = await fs.readFile(testFile, 'utf8'); + const lines = content.split('\n'); + + // Generated meta lines should be removed + expect(lines.some((line) => line.startsWith('! Checksum:'))).toBe(false); + expect(lines.some((line) => line.startsWith('! Diff-Path:'))).toBe(false); + expect(lines.some((line) => line.startsWith('! TimeUpdated:'))).toBe(false); + expect(lines.some((line) => line.startsWith('! Version:'))).toBe(false); + + // Normal content should remain + expect(lines.some((line) => line.includes('Normal comment'))).toBe(true); + expect(lines.some((line) => line.includes('||example.com^'))).toBe(true); + expect(lines.some((line) => line.includes('##.ads-banner'))).toBe(true); + }); + + it('returns 0 when no files need modification', async () => { + // Second run should not modify anything since metadata is already stripped + const modified = await stripGeneratedMetaFromDir(testDir); + expect(modified).toBe(0); + }); +}); diff --git a/scripts/build/strip-generated-meta.ts b/scripts/build/strip-generated-meta.ts index cc7d300c74bb6..d50efc659afe1 100644 --- a/scripts/build/strip-generated-meta.ts +++ b/scripts/build/strip-generated-meta.ts @@ -1,5 +1,6 @@ import fs from 'fs/promises'; import path from 'path'; +import { fileURLToPath } from 'url'; import { findFiles } from '../utils/find_files.js'; const FILTERS_DIR_NAME = 'filters'; @@ -95,3 +96,11 @@ export const stripGeneratedMetaFromDir = async (rootDir: string): Promise sum + count, 0); }; + +// CLI entrypoint: strip generated meta from platform build outputs when run directly +if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + const rootDir = process.argv[2] ?? path.resolve('platforms'); + const modified = await stripGeneratedMetaFromDir(rootDir); + // eslint-disable-next-line no-console + console.log(`Done. ${modified} file(s) modified.`); +} diff --git a/vitest.config.ts b/vitest.config.ts index 49fa6dfd524cc..ae1de3dfcf0cc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,7 +7,10 @@ export default defineConfig({ enabled: true, tsconfig: './tsconfig.json', }, - include: ['scripts/wildcard-domain-processor/__tests__/*.test.ts'], + include: [ + 'scripts/wildcard-domain-processor/__tests__/*.test.ts', + 'scripts/build/__tests__/*.test.ts', + ], watch: false, silent: true, }, From 337d24a2f5a6a4193ace552ef00fcd7ad838167b Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Tue, 28 Apr 2026 00:13:56 +0300 Subject: [PATCH 11/13] Fix lint error Co-authored-by: Copilot --- scripts/build/__tests__/strip-generated-meta.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build/__tests__/strip-generated-meta.test.ts b/scripts/build/__tests__/strip-generated-meta.test.ts index f1b84f3563385..5cc1301faa14e 100644 --- a/scripts/build/__tests__/strip-generated-meta.test.ts +++ b/scripts/build/__tests__/strip-generated-meta.test.ts @@ -1,4 +1,6 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { + describe, it, expect, beforeAll, afterAll, +} from 'vitest'; import fs from 'fs/promises'; import path from 'path'; import os from 'os'; From 9c694a6ac0d25c6e996d0e6674eb007edcfe46ee Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Tue, 28 Apr 2026 00:54:32 +0300 Subject: [PATCH 12/13] strip meta in both platforms/ and temp/platforms/, add tests Co-authored-by: Copilot --- DEVELOPMENT.md | 2 +- .../__tests__/strip-generated-meta.test.ts | 49 +++++++++++++++++++ scripts/build/build.js | 12 ++++- scripts/build/strip-generated-meta.ts | 18 +++++-- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 39a7a0d7967b2..433a49b83f069 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -76,7 +76,7 @@ The `yarn build` command (and `yarn build:local`) also accepts: (`yarn build:patches`) is not needed afterwards. - `--strip-generated-meta` — after compilation, remove generated meta lines (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, `! Version`) from all filter - files in `platforms/`. Useful when comparing outputs between builds. + files in `platforms/` and `temp/platforms/`. Useful when comparing outputs between builds. ### Generating Filter Cache diff --git a/scripts/build/__tests__/strip-generated-meta.test.ts b/scripts/build/__tests__/strip-generated-meta.test.ts index 5cc1301faa14e..725cf4dd5a8ca 100644 --- a/scripts/build/__tests__/strip-generated-meta.test.ts +++ b/scripts/build/__tests__/strip-generated-meta.test.ts @@ -68,4 +68,53 @@ describe('stripGeneratedMetaFromDir', () => { const modified = await stripGeneratedMetaFromDir(testDir); expect(modified).toBe(0); }); + + it('strips filters metadata consistently from two separate directory trees', async () => { + // Simulate the build.js scenario where we have: + // - new build output (platforms/) + // - old baseline copy (temp/platforms/) + // and both need to be stripped consistently. + + const oldRoot = path.join(testDir, 'old_root', 'platforms'); + const newRoot = path.join(testDir, 'new_root', 'platforms'); + const oldDir = path.join(oldRoot, 'filters'); + const newDir = path.join(newRoot, 'filters'); + + await fs.mkdir(oldDir, { recursive: true }); + await fs.mkdir(newDir, { recursive: true }); + + const content = [ + '! Checksum: abc123', + '! Diff-Path: somewhere/diff.txt', + '! TimeUpdated: 2026-04-27T10:00:00Z', + '! Version: 1.0.0', + '', + '||example.com^', + '', + ].join('\n'); + + const oldFile = path.join(oldDir, 'test.txt'); + const newFile = path.join(newDir, 'test.txt'); + + await fs.writeFile(oldFile, content, 'utf8'); + await fs.writeFile(newFile, content, 'utf8'); + + // Strip from both directories as build.js does when --strip-generated-meta is set + const newCount = await stripGeneratedMetaFromDir(newRoot); + const oldCount = await stripGeneratedMetaFromDir(oldRoot); + + expect(newCount).toBe(1); + expect(oldCount).toBe(1); + + // Verify both files are identically stripped + const oldContent = await fs.readFile(oldFile, 'utf8'); + const newContent = await fs.readFile(newFile, 'utf8'); + + expect(oldContent).toBe(newContent); + expect(oldContent.includes('! Checksum:')).toBe(false); + expect(oldContent.includes('! Diff-Path:')).toBe(false); + expect(oldContent.includes('! TimeUpdated:')).toBe(false); + expect(oldContent.includes('! Version:')).toBe(false); + expect(oldContent.includes('||example.com^')).toBe(true); + }); }); diff --git a/scripts/build/build.js b/scripts/build/build.js index 3c7f1ab7e38ff..7b0cd71cc2ec2 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -195,9 +195,17 @@ const buildFilters = async () => { // Strip generated metadata (Checksum, Diff-Path, TimeUpdated, Version) // from compiled filter files so they don't pollute diff comparisons. if (stripGeneratedMeta) { - const count = await stripGeneratedMetaFromDir(platformsPath); + const newCount = await stripGeneratedMetaFromDir(platformsPath); // eslint-disable-next-line no-console - console.log(`Stripped generated meta from ${count} file(s).`); + console.log(`Stripped generated meta from ${newCount} new file(s).`); + + // Also strip the old baseline copy so build:patches diffs + // consistently-stripped content. + if (fs.existsSync(copyPlatformsPath)) { + const oldCount = await stripGeneratedMetaFromDir(copyPlatformsPath); + // eslint-disable-next-line no-console + console.log(`Stripped generated meta from ${oldCount} old baseline file(s).`); + } } }; diff --git a/scripts/build/strip-generated-meta.ts b/scripts/build/strip-generated-meta.ts index d50efc659afe1..5ca401a75e5b5 100644 --- a/scripts/build/strip-generated-meta.ts +++ b/scripts/build/strip-generated-meta.ts @@ -1,4 +1,5 @@ import fs from 'fs/promises'; +import { existsSync } from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; import { findFiles } from '../utils/find_files.js'; @@ -99,8 +100,19 @@ export const stripGeneratedMetaFromDir = async (rootDir: string): Promise Date: Tue, 28 Apr 2026 00:58:56 +0300 Subject: [PATCH 13/13] lint errors Co-authored-by: Copilot --- scripts/build/strip-generated-meta.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/build/strip-generated-meta.ts b/scripts/build/strip-generated-meta.ts index 5ca401a75e5b5..fbbb3de524699 100644 --- a/scripts/build/strip-generated-meta.ts +++ b/scripts/build/strip-generated-meta.ts @@ -104,15 +104,18 @@ if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { ? [path.resolve(process.argv[2])] : [path.resolve('platforms'), path.resolve('temp', 'platforms')]; - let total = 0; - for (const rootDir of rootDirs) { - if (existsSync(rootDir)) { - const count = await stripGeneratedMetaFromDir(rootDir); - // eslint-disable-next-line no-console - console.log(`${path.relative('.', rootDir)}: ${count} file(s) modified.`); - total += count; - } - } + const results = await Promise.all( + rootDirs + .filter((dir) => existsSync(dir)) + .map(async (rootDir) => { + const count = await stripGeneratedMetaFromDir(rootDir); + // eslint-disable-next-line no-console + console.log(`${path.relative('.', rootDir)}: ${count} file(s) modified.`); + return count; + }), + ); + + const total = results.reduce((sum, count) => sum + count, 0); // eslint-disable-next-line no-console console.log(`Done. ${total} file(s) modified in total.`); }