Fix #1180 Add the ability to build filters completely locally

AGENTS.md

@@ -77,8 +77,11 @@ for all supported AdGuard products.
 | Command | Description |
 | ------- | ----------- |
 | `yarn build` | Build all filters (`tsx scripts/build/build.js`) |
+| `yarn build:local` | Build filters from cached `filter.txt` files (`tsx scripts/build/build.js --use-cache`) |
 | `yarn auto-build` | Full automated build via `bash scripts/auto_build.sh` |
 | `yarn build:patches` | Build incremental patches |
+| `yarn generate-cache` | Generate cached `filter.txt` from templates (`tsx scripts/build/build.js --generate-cache`) |
+| `yarn strip-generated-meta` | Strip generated meta lines from platform filter files |
 | `yarn test` | Run unit tests (`vitest run`) |
 | `yarn lint` | Run all linters (code + types + markdown) |
 | `yarn lint:code` | ESLint check (`eslint . --ext .js,.ts`) |

DEVELOPMENT.md

@@ -62,6 +62,70 @@ Generate a build report to a custom file:
 yarn build --report='report-adguard.txt'
 ```
 
+#### Additional Build Flags
+
+The `yarn build` command (and `yarn build:local`) also accepts:
+
+- `--no-patches-prepare` — skip copying `platforms/` to `temp/platforms/`, used
+  to build patches. Speeds up the build when patch generation
+  (`yarn build:patches`) is not needed afterwards.
+- `--strip-generated-meta` — after compilation, remove generated meta lines
+  (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, `! Version`) from all filter
+  files in `platforms/`. Useful when comparing outputs between builds.
+
+### Generating Filter Cache
+
+To update the cached `filter.txt` files in `filters/`, used for
+testing/reproducible builds, run:
+
+```bash
+yarn generate-cache
+```
+
+This compiles every filter from its `template.txt` and updates the corresponding
+`filter.txt` inside `filters/`. Platform-specific filters and patches are **not**
+generated. The resulting `filter.txt` files contain the fully resolved filter
+content (all `@include` and `!#include` directives expanded) and can be used to
+build filters from cache with `yarn build:local`.
+
+### Building From Cache
+
+To build filters from previously cached `filter.txt` files without downloading
+external filters, run:
+
+```bash
+yarn build:local
+```
+
+Under the hood this copies `filters/` to `temp/filters_cached/`, replaces every
+`template.txt` with a single `@include "./filter.txt"` directive, and compiles
+from that copy. The original `filters/` directory is never modified.
+
+The `-i` / `-s` / `--no-patches-prepare` / `--strip-generated-meta` flags can be
+combined:
+
+```bash
+yarn build:local -i=1,2,3 --no-patches-prepare --strip-generated-meta
+```
+
+**Typical workflow — comparing two compiler versions:**
+
+1. Download and compile filter content into cache:
+   `yarn generate-cache`
+1. Build from cache with generated metadata lines stripped:
+   `yarn build:local --no-patches-prepare --strip-generated-meta`
+1. Rename the output: `mv platforms platforms_A`
+1. Switch to the other compiler version
+   (e.g. `yarn add @adguard/filters-compiler@...`)
+1. Build again with the same flags:
+   `yarn build:local --no-patches-prepare --strip-generated-meta`
+1. Rename the output: `mv platforms platforms_B`
+1. Diff the two directories (e.g. in Total Commander, WinMerge, or
+   with `diff -r`)
+
+Both runs use the exact same cached filter content and strip all volatile
+metadata, so any difference comes solely from the compiler.
+
 ### Automated Build
 
 The `auto-build` script performs a full build with patches and wildcard domain expansion.

package.json

@@ -13,7 +13,10 @@
         "lint:md": "markdownlint **/*.md",
         "test": "vitest run",
         "build": "tsx scripts/build/build.js",
+        "build:local": "tsx scripts/build/build.js --use-cache",
         "build:patches": "node scripts/build/patches.js",
+        "strip-generated-meta": "tsx scripts/build/strip_generated_meta.js",
+        "generate-cache": "tsx scripts/build/build.js --generate-cache",
         "validate": "yarn validate:platforms && yarn validate:locales",
         "validate:platforms": "tsx scripts/validation/validate_platforms.js",
         "validate:locales": "tsx scripts/validation/validate_locales.js",

scripts/build/build.js

@@ -8,16 +8,22 @@ import {
     FOLDER_WITH_NEW_FILTERS,
     FOLDER_WITH_OLD_FILTERS,
 } from './constants.js';
+import { stripGeneratedMetaFromDir } from './strip_generated_meta.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 /**
- * Parse command-cli parameters -i|--include and -s|--skip
+ * Parse command-line parameters -i|--include, -s|--skip, --use-cache, --generate-cache,
+ * --no-patches-prepare, --strip-generated-meta
  */
 let includedFilterIDs = [];
 let excludedFilterIDs = [];
 let rawReportPath = '';
+let useCache = false;
+let generateCache = false;
+let noPatchesPrepare = false;
+let stripGeneratedMeta = false;
 
 const args = process.argv.slice(2);
 args.forEach((val) => {
@@ -40,26 +46,128 @@ args.forEach((val) => {
     if (val.startsWith('--report=')) {
         rawReportPath = val.slice(val.indexOf('=') + 1).trim();
     }
+
+    if (val === '--use-cache') {
+        useCache = true;
+    }
+
+    if (val === '--generate-cache') {
+        generateCache = true;
+    }
+
+    if (val === '--no-patches-prepare') {
+        noPatchesPrepare = true;
+    }
+
+    if (val === '--strip-generated-meta') {
+        stripGeneratedMeta = true;
+    }
 });
 
+if (useCache && generateCache) {
+    // eslint-disable-next-line no-console
+    console.error('Error: --use-cache and --generate-cache are mutually exclusive.');
+    process.exit(1);
+}
+
 /**
  * Set all relative paths needed for compiler
  */
 const filtersDir = path.join(__dirname, '../../filters');
 const logPath = path.join(__dirname, '../../log.txt');
 const platformsPath = path.join(__dirname, '../..', FOLDER_WITH_NEW_FILTERS);
 const copyPlatformsPath = path.join(__dirname, '../..', FOLDER_WITH_OLD_FILTERS);
+const cachedFiltersDir = path.join(__dirname, '../../temp/filters_cached');
 
 const reportPath = rawReportPath !== ''
     // report-adguard.txt OR report-third-party.txt
     ? path.join(__dirname, '../..', rawReportPath)
     // report_partial_DD-MM-YYYY_HH-MM-SS.txt
     : path.join(__dirname, '../..', `report_partial_${formatDate(new Date())}.txt`);
 
+const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n';
+
+/**
+ * Recursively find all `template.txt` files under the given directory.
+ *
+ * @param {string} dir - Root directory to search.
+ * @returns {Promise<string[]>} Array of absolute paths to `template.txt` files.
+ */
+const findTemplatePaths = async (dir) => {
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+    const results = await Promise.all(entries.map(async (entry) => {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            return findTemplatePaths(fullPath);
+        }
+        if (entry.name === 'template.txt') {
+            return [fullPath];
+        }
+        return [];
+    }));
+
+    return results.flat();
+};
+
+/**
+ * Prepare a temporary copy of the filters directory with shadow templates.
+ *
+ * Copies `filters/` → `temp/filters_cached/`, then replaces the content of every
+ * `template.txt` with a single-line local include pointing to the cached `filter.txt`.
+ * Validates that every filter directory with a `template.txt` also has a `filter.txt`.
+ *
+ * @returns {Promise<void>}
+ */
+const prepareCachedFiltersDir = async () => {
+    // Remove stale copy if exists
+    if (fs.existsSync(cachedFiltersDir)) {
+        await fs.promises.rm(cachedFiltersDir, { recursive: true });
+    }
+
+    // Full recursive copy
+    await fs.promises.cp(filtersDir, cachedFiltersDir, { recursive: true });
+
+    // Find all directories containing template.txt and replace with shadow templates
+    const templatePaths = await findTemplatePaths(cachedFiltersDir);
+
+    await Promise.all(templatePaths.map(async (templatePath) => {
+        const dir = path.dirname(templatePath);
+        const filterTxtPath = path.join(dir, 'filter.txt');
+
+        if (!fs.existsSync(filterTxtPath)) {
+            throw new Error(
+                `--use-cache: missing filter.txt in ${path.relative(cachedFiltersDir, dir)}. `
+                + 'Run "yarn generate-cache" first to generate cached filter files.',
+            );
+        }
+
+        await fs.promises.writeFile(templatePath, SHADOW_TEMPLATE_CONTENT, 'utf8');
+    }));
+
+    // eslint-disable-next-line no-console
+    console.log(`Prepared cached filters directory with ${templatePaths.length} shadow templates.`);
+};
+
 /**
  * Compiler entry point.
  */
 const buildFilters = async () => {
+    // When --generate-cache we only need to compile filters (which updates filter.txt),
+    // skip platform generation, patches preparation, and temp/platforms copying.
+    if (generateCache) {
+        await compile(
+            filtersDir,
+            logPath,
+            reportPath,
+            null, // null ⇒ generate() inside compiler returns early, no platform files
+            includedFilterIDs,
+            excludedFilterIDs,
+            CUSTOM_PLATFORMS_CONFIG,
+        );
+        return;
+    }
+
     // Clean temporary folder
     if (fs.existsSync(copyPlatformsPath)) {
         await fs.promises.rm(copyPlatformsPath, { recursive: true });
@@ -70,27 +178,49 @@ const buildFilters = async () => {
     let initialRun = false;
     if (!fs.existsSync(platformsPath)) {
         initialRun = true;
-    } else {
+    } else if (!noPatchesPrepare) {
         // Make copy for future patches generation
         await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true });
     }
 
-    await compile(
-        filtersDir,
-        logPath,
-        reportPath,
-        platformsPath,
-        includedFilterIDs,
-        excludedFilterIDs,
-        CUSTOM_PLATFORMS_CONFIG,
-    );
+    // Determine which filtersDir to pass to the compiler
+    const effectiveFiltersDir = useCache ? cachedFiltersDir : filtersDir;
+
+    if (useCache) {
+        await prepareCachedFiltersDir();
+    }
+
+    try {
+        await compile(
+            effectiveFiltersDir,
+            logPath,
+            reportPath,
+            platformsPath,
+            includedFilterIDs,
+            excludedFilterIDs,
+            CUSTOM_PLATFORMS_CONFIG,
+        );
+    } finally {
+        // Clean up temp filters copy
+        if (useCache && fs.existsSync(cachedFiltersDir)) {
+            await fs.promises.rm(cachedFiltersDir, { recursive: true });
+        }
+    }
 
     // For the very first run, we should copy the built platforms into
     // the temp folder to create the first empty patches for future versions
-    if (initialRun) {
+    if (initialRun && !noPatchesPrepare) {
         // Make copy for future patches generation
         await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true });
     }
+
+    // Strip generated metadata (Checksum, Diff-Path, TimeUpdated, Version)
+    // from compiled filter files so they don't pollute diff comparisons.
+    if (stripGeneratedMeta) {
+        const count = await stripGeneratedMetaFromDir(platformsPath);
+        // eslint-disable-next-line no-console
+        console.log(`Stripped generated meta from ${count} file(s).`);
+    }
 };
 
 buildFilters();