diff --git a/AGENTS.md b/AGENTS.md index 7cc51beeeb12b..2ba7a9b1a2a90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,9 @@ for all supported AdGuard products. ## Technical Context -- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`) +- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`). + New code must be written in TypeScript. The project is gradually migrating all scripts to + TypeScript; existing `.js` files should be converted to `.ts` when touched. - **Primary Dependencies**: - `@adguard/filters-compiler` — compiles filter templates into final filter lists - `@adguard/agtree` — AdGuard filter rule parser / AST @@ -77,8 +79,11 @@ for all supported AdGuard products. | Command | Description | | ------- | ----------- | | `yarn build` | Build all filters (`tsx scripts/build/build.js`) | +| `yarn build:local` | Build filters from cached `filter.txt` files (`tsx scripts/build/build.js --use-cache`) | | `yarn auto-build` | Full automated build via `bash scripts/auto_build.sh` | | `yarn build:patches` | Build incremental patches | +| `yarn generate-cache` | Generate cached `filter.txt` from templates (`tsx scripts/build/build.js --generate-cache`) | +| `yarn strip-generated-meta` | Strip generated meta lines from platform filter files | | `yarn test` | Run unit tests (`vitest run`) | | `yarn lint` | Run all linters (code + types + markdown) | | `yarn lint:code` | ESLint check (`eslint . --ext .js,.ts`) | diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3b5f5ab05123c..39a7a0d7967b2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -30,6 +30,11 @@ yarn test ``` + > **Note:** `yarn lint` runs three checks in sequence: ESLint (`yarn lint:code`), + > TypeScript type checking (`yarn lint:types`), and Markdownlint (`yarn lint:md`). + > All three must pass. The build scripts are a mix of JavaScript and TypeScript; + > `tsx` executes both transparently — no manual compilation step is needed. + ## Development Workflow ### Building Filters @@ -62,6 +67,70 @@ Generate a build report to a custom file: yarn build --report='report-adguard.txt' ``` +#### Additional Build Flags + +The `yarn build` command (and `yarn build:local`) also accepts: + +- `--no-patches-prepare` — skip copying `platforms/` to `temp/platforms/`, used + to build patches. Speeds up the build when patch generation + (`yarn build:patches`) is not needed afterwards. +- `--strip-generated-meta` — after compilation, remove generated meta lines + (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, `! Version`) from all filter + files in `platforms/`. Useful when comparing outputs between builds. + +### Generating Filter Cache + +To update the cached `filter.txt` files in `filters/`, used for +testing/reproducible builds, run: + +```bash +yarn generate-cache +``` + +This compiles every filter from its `template.txt` and updates the corresponding +`filter.txt` inside `filters/`. Platform-specific filters and patches are **not** +generated. The resulting `filter.txt` files contain the fully resolved filter +content (all `@include` and `!#include` directives expanded) and can be used to +build filters from cache with `yarn build:local`. + +### Building From Cache + +To build filters from previously cached `filter.txt` files without downloading +external filters, run: + +```bash +yarn build:local +``` + +Under the hood this copies `filters/` to `temp/filters_cached/`, replaces every +`template.txt` with a single `@include "./filter.txt"` directive, and compiles +from that copy. The original `filters/` directory is never modified. + +The `-i` / `-s` / `--no-patches-prepare` / `--strip-generated-meta` flags can be +combined: + +```bash +yarn build:local -i=1,2,3 --no-patches-prepare --strip-generated-meta +``` + +**Typical workflow — comparing two compiler versions:** + +1. Download and compile filter content into cache: + `yarn generate-cache` +1. Build from cache with generated metadata lines stripped: + `yarn build:local --no-patches-prepare --strip-generated-meta` +1. Rename the output: `mv platforms platforms_A` +1. Switch to the other compiler version + (e.g. `yarn add @adguard/filters-compiler@...`) +1. Build again with the same flags: + `yarn build:local --no-patches-prepare --strip-generated-meta` +1. Rename the output: `mv platforms platforms_B` +1. Diff the two directories (e.g. in Total Commander, WinMerge, or + with `diff -r`) + +Both runs use the exact same cached filter content and strip all volatile +metadata, so any difference comes solely from the compiler. + ### Automated Build The `auto-build` script performs a full build with patches and wildcard domain expansion. @@ -243,6 +312,9 @@ All build tooling lives under `scripts/`. After making changes: `scripts/wildcard-domain-processor/__tests__/`. 1. Run `yarn validate` if the change affects filter compilation or platform outputs. +Build scripts under `scripts/` are written in JavaScript and TypeScript, executed +via [tsx](https://github.com/privatenumber/tsx) — no manual compilation step is needed. + ## Troubleshooting ### `yarn build` fails with missing platform files diff --git a/package.json b/package.json index 9082f25be0c22..28ec0337f1d35 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,10 @@ "lint:md": "markdownlint **/*.md", "test": "vitest run", "build": "tsx scripts/build/build.js", + "build:local": "tsx scripts/build/build.js --use-cache", "build:patches": "node scripts/build/patches.js", + "strip-generated-meta": "tsx scripts/build/strip-generated-meta.ts", + "generate-cache": "tsx scripts/build/build.js --generate-cache", "validate": "yarn validate:platforms && yarn validate:locales", "validate:platforms": "tsx scripts/validation/validate_platforms.js", "validate:locales": "tsx scripts/validation/validate_locales.js", diff --git a/scripts/build/build.js b/scripts/build/build.js index 5954c0e2a8e0d..3c7f1ab7e38ff 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -8,16 +8,23 @@ import { FOLDER_WITH_NEW_FILTERS, FOLDER_WITH_OLD_FILTERS, } from './constants.js'; +import { stripGeneratedMetaFromDir } from './strip-generated-meta.ts'; +import { findFiles } from '../utils/find_files.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); /** - * Parse command-cli parameters -i|--include and -s|--skip + * Parse command-line parameters -i|--include, -s|--skip, --use-cache, --generate-cache, + * --no-patches-prepare, --strip-generated-meta */ let includedFilterIDs = []; let excludedFilterIDs = []; let rawReportPath = ''; +let useCache = false; +let generateCache = false; +let noPatchesPrepare = false; +let stripGeneratedMeta = false; const args = process.argv.slice(2); args.forEach((val) => { @@ -40,8 +47,30 @@ args.forEach((val) => { if (val.startsWith('--report=')) { rawReportPath = val.slice(val.indexOf('=') + 1).trim(); } + + if (val === '--use-cache') { + useCache = true; + } + + if (val === '--generate-cache') { + generateCache = true; + } + + if (val === '--no-patches-prepare') { + noPatchesPrepare = true; + } + + if (val === '--strip-generated-meta') { + stripGeneratedMeta = true; + } }); +if (useCache && generateCache) { + // eslint-disable-next-line no-console + console.error('Error: --use-cache and --generate-cache are mutually exclusive.'); + process.exit(1); +} + /** * Set all relative paths needed for compiler */ @@ -49,6 +78,7 @@ const filtersDir = path.join(__dirname, '../../filters'); const logPath = path.join(__dirname, '../../log.txt'); const platformsPath = path.join(__dirname, '../..', FOLDER_WITH_NEW_FILTERS); const copyPlatformsPath = path.join(__dirname, '../..', FOLDER_WITH_OLD_FILTERS); +const cachedFiltersDir = path.join(__dirname, '../../temp/filters_cached'); const reportPath = rawReportPath !== '' // report-adguard.txt OR report-third-party.txt @@ -56,10 +86,66 @@ const reportPath = rawReportPath !== '' // report_partial_DD-MM-YYYY_HH-MM-SS.txt : path.join(__dirname, '../..', `report_partial_${formatDate(new Date())}.txt`); +const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n'; + +/** + * Prepare a temporary copy of the filters directory with shadow templates. + * + * Copies `filters/` → `temp/filters_cached/`, then replaces the content of every + * `template.txt` with a single-line local include pointing to the cached `filter.txt`. + * Validates that every filter directory with a `template.txt` also has a `filter.txt`. + * + * @returns {Promise} + */ +const prepareCachedFiltersDir = async () => { + // Remove stale copy if exists + if (fs.existsSync(cachedFiltersDir)) { + await fs.promises.rm(cachedFiltersDir, { recursive: true }); + } + + // Full recursive copy + await fs.promises.cp(filtersDir, cachedFiltersDir, { recursive: true }); + + // Find all directories containing template.txt and replace with shadow templates + const templatePaths = await findFiles(cachedFiltersDir, (p) => path.basename(p) === 'template.txt'); + + await Promise.all(templatePaths.map(async (templatePath) => { + const dir = path.dirname(templatePath); + const filterTxtPath = path.join(dir, 'filter.txt'); + + if (!fs.existsSync(filterTxtPath)) { + throw new Error( + `--use-cache: missing filter.txt in ${path.relative(cachedFiltersDir, dir)}. ` + + 'Run "yarn generate-cache" first to generate cached filter files.', + ); + } + + await fs.promises.writeFile(templatePath, SHADOW_TEMPLATE_CONTENT, 'utf8'); + })); + + // eslint-disable-next-line no-console + console.log(`Prepared cached filters directory with ${templatePaths.length} shadow templates.`); +}; + /** * Compiler entry point. */ const buildFilters = async () => { + // When --generate-cache we only need to compile filters (which updates filter.txt), + // skip platform generation, patches preparation, and temp/platforms copying. + if (generateCache) { + await compile( + filtersDir, + logPath, + reportPath, + null, // null ⇒ generate() inside compiler returns early, no platform files + includedFilterIDs, + excludedFilterIDs, + CUSTOM_PLATFORMS_CONFIG, + ); + return; + } + // Clean temporary folder if (fs.existsSync(copyPlatformsPath)) { await fs.promises.rm(copyPlatformsPath, { recursive: true }); @@ -70,27 +156,49 @@ const buildFilters = async () => { let initialRun = false; if (!fs.existsSync(platformsPath)) { initialRun = true; - } else { + } else if (!noPatchesPrepare) { // Make copy for future patches generation await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true }); } - await compile( - filtersDir, - logPath, - reportPath, - platformsPath, - includedFilterIDs, - excludedFilterIDs, - CUSTOM_PLATFORMS_CONFIG, - ); + // Determine which filtersDir to pass to the compiler + const effectiveFiltersDir = useCache ? cachedFiltersDir : filtersDir; + + if (useCache) { + await prepareCachedFiltersDir(); + } + + try { + await compile( + effectiveFiltersDir, + logPath, + reportPath, + platformsPath, + includedFilterIDs, + excludedFilterIDs, + CUSTOM_PLATFORMS_CONFIG, + ); + } finally { + // Clean up temp filters copy + if (useCache && fs.existsSync(cachedFiltersDir)) { + await fs.promises.rm(cachedFiltersDir, { recursive: true }); + } + } // For the very first run, we should copy the built platforms into // the temp folder to create the first empty patches for future versions - if (initialRun) { + if (initialRun && !noPatchesPrepare) { // Make copy for future patches generation await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true }); } + + // Strip generated metadata (Checksum, Diff-Path, TimeUpdated, Version) + // from compiled filter files so they don't pollute diff comparisons. + if (stripGeneratedMeta) { + const count = await stripGeneratedMetaFromDir(platformsPath); + // eslint-disable-next-line no-console + console.log(`Stripped generated meta from ${count} file(s).`); + } }; buildFilters(); diff --git a/scripts/build/strip-generated-meta.ts b/scripts/build/strip-generated-meta.ts new file mode 100644 index 0000000000000..cc7d300c74bb6 --- /dev/null +++ b/scripts/build/strip-generated-meta.ts @@ -0,0 +1,97 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { findFiles } from '../utils/find_files.js'; + +const FILTERS_DIR_NAME = 'filters'; +const TXT_FILE_EXTENSION = '.txt'; + +/** + * Lines starting with these prefixes are generated meta — they change on every build + * regardless of filter content changes, making file comparison noisy. + */ +const GENERATED_META_PREFIXES = [ + '! Checksum:', + '! Diff-Path:', + '! TimeUpdated:', + '! Version:', +] as const; + +/** + * Checks whether a line is a generated meta line that should be stripped. + * + * @param line - A single line from a filter file. + * @returns True if the line starts with a generated meta prefix. + */ +const isGeneratedMetaLine = (line: string): boolean => { + return GENERATED_META_PREFIXES.some((prefix) => line.startsWith(prefix)); +}; + +/** + * Strip generated metadata lines from a single filter file in-place. + * + * @param filePath - Absolute path to the .txt filter file. + * @returns True if the file was modified, false otherwise. + */ +const stripGeneratedMeta = async (filePath: string): Promise => { + const content = await fs.readFile(filePath, 'utf8'); + const lines = content.split('\n'); + const filtered = lines.filter((line) => !isGeneratedMetaLine(line)); + + if (filtered.length === lines.length) { + return false; + } + + await fs.writeFile(filePath, filtered.join('\n'), 'utf8'); + return true; +}; + +/** + * Recursively find all directories named `filters` under the given root. + * + * @param dir - Root directory to search. + * @returns Array of absolute paths to `filters` directories. + */ +const findFiltersDirs = async (dir: string): Promise => { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + const results = await Promise.all(entries.map(async (entry) => { + if (!entry.isDirectory()) { + return []; + } + const fullPath = path.join(dir, entry.name); + if (entry.name === FILTERS_DIR_NAME) { + return [fullPath]; + } + return findFiltersDirs(fullPath); + })); + + return results.flat(); +}; + +const hasTxtExtension = (p: string): boolean => p.endsWith(TXT_FILE_EXTENSION); + +/** + * Strip generated metadata lines from all .txt files inside all `filters/` + * directories found recursively under the given root. + * + * @param rootDir - Root directory to search (e.g. `platforms/`). + * @returns Number of files actually modified. + */ +export const stripGeneratedMetaFromDir = async (rootDir: string): Promise => { + const filtersDirs = await findFiltersDirs(rootDir); + + const counts = await Promise.all(filtersDirs.map(async (filtersDir) => { + const files = await findFiles(filtersDir, hasTxtExtension); + const results = await Promise.all(files.map(stripGeneratedMeta)); + const modified = results.filter(Boolean).length; + + if (modified > 0) { + // eslint-disable-next-line no-console + console.log(`${path.relative(rootDir, filtersDir)}: stripped metadata from ${modified} file(s)`); + } + + return modified; + })); + + return counts.reduce((sum, count) => sum + count, 0); +};