Fix #1180 Add the ability to build filters completely locally

AGENTS.md

@@ -12,7 +12,9 @@ for all supported AdGuard products.
 
 ## Technical Context
 
-- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`)
+- **Language / Version**: TypeScript and JavaScript, Node.js >= 22, ESM (`"type": "module"`).
+  New code must be written in TypeScript. The project is gradually migrating all scripts to
+  TypeScript; existing `.js` files should be converted to `.ts` when touched.
 - **Primary Dependencies**:
     - `@adguard/filters-compiler` — compiles filter templates into final filter lists
     - `@adguard/agtree` — AdGuard filter rule parser / AST
@@ -77,8 +79,11 @@ for all supported AdGuard products.
 | Command | Description |
 | ------- | ----------- |
 | `yarn build` | Build all filters (`tsx scripts/build/build.js`) |
+| `yarn build:local` | Build filters from cached `filter.txt` files (`tsx scripts/build/build.js --use-cache`) |
 | `yarn auto-build` | Full automated build via `bash scripts/auto_build.sh` |
 | `yarn build:patches` | Build incremental patches |
+| `yarn generate-cache` | Generate cached `filter.txt` from templates (`tsx scripts/build/build.js --generate-cache`) |
+| `yarn strip-generated-meta` | Strip generated meta lines from platform filter files |
 | `yarn test` | Run unit tests (`vitest run`) |
 | `yarn lint` | Run all linters (code + types + markdown) |
 | `yarn lint:code` | ESLint check (`eslint . --ext .js,.ts`) |

DEVELOPMENT.md

@@ -30,6 +30,11 @@
     yarn test
     ```
 
+    > **Note:** `yarn lint` runs three checks in sequence: ESLint (`yarn lint:code`),
+    > TypeScript type checking (`yarn lint:types`), and Markdownlint (`yarn lint:md`).
+    > All three must pass. The build scripts are a mix of JavaScript and TypeScript;
+    > `tsx` executes both transparently — no manual compilation step is needed.
+
 ## Development Workflow
 
 ### Building Filters
@@ -62,6 +67,70 @@ Generate a build report to a custom file:
 yarn build --report='report-adguard.txt'
 ```
 
+#### Additional Build Flags
+
+The `yarn build` command (and `yarn build:local`) also accepts:
+
+- `--no-patches-prepare` — skip copying `platforms/` to `temp/platforms/`, used
+  to build patches. Speeds up the build when patch generation
+  (`yarn build:patches`) is not needed afterwards.
+- `--strip-generated-meta` — after compilation, remove generated meta lines
+  (`! Checksum`, `! Diff-Path`, `! TimeUpdated`, `! Version`) from all filter
+  files in `platforms/`. Useful when comparing outputs between builds.
+
+### Generating Filter Cache
+
+To update the cached `filter.txt` files in `filters/`, used for
+testing/reproducible builds, run:
+
+```bash
+yarn generate-cache
+```
+
+This compiles every filter from its `template.txt` and updates the corresponding
+`filter.txt` inside `filters/`. Platform-specific filters and patches are **not**
+generated. The resulting `filter.txt` files contain the fully resolved filter
+content (all `@include` and `!#include` directives expanded) and can be used to
+build filters from cache with `yarn build:local`.
+
+### Building From Cache
+
+To build filters from previously cached `filter.txt` files without downloading
+external filters, run:
+
+```bash
+yarn build:local
+```
+
+Under the hood this copies `filters/` to `temp/filters_cached/`, replaces every
+`template.txt` with a single `@include "./filter.txt"` directive, and compiles
+from that copy. The original `filters/` directory is never modified.
+
+The `-i` / `-s` / `--no-patches-prepare` / `--strip-generated-meta` flags can be
+combined:
+
+```bash
+yarn build:local -i=1,2,3 --no-patches-prepare --strip-generated-meta
+```
+
+**Typical workflow — comparing two compiler versions:**
+
+1. Download and compile filter content into cache:
+   `yarn generate-cache`
+1. Build from cache with generated metadata lines stripped:
+   `yarn build:local --no-patches-prepare --strip-generated-meta`
+1. Rename the output: `mv platforms platforms_A`
+1. Switch to the other compiler version
+   (e.g. `yarn add @adguard/filters-compiler@...`)
+1. Build again with the same flags:
+   `yarn build:local --no-patches-prepare --strip-generated-meta`
+1. Rename the output: `mv platforms platforms_B`
+1. Diff the two directories (e.g. in Total Commander, WinMerge, or
+   with `diff -r`)
+
+Both runs use the exact same cached filter content and strip all volatile
+metadata, so any difference comes solely from the compiler.
+
 ### Automated Build
 
 The `auto-build` script performs a full build with patches and wildcard domain expansion.
@@ -243,6 +312,9 @@ All build tooling lives under `scripts/`. After making changes:
    `scripts/wildcard-domain-processor/__tests__/`.
 1. Run `yarn validate` if the change affects filter compilation or platform outputs.
 
+Build scripts under `scripts/` are written in JavaScript and TypeScript, executed
+via [tsx](https://github.com/privatenumber/tsx) — no manual compilation step is needed.
+
 ## Troubleshooting
 
 ### `yarn build` fails with missing platform files

package.json

@@ -13,7 +13,10 @@
         "lint:md": "markdownlint **/*.md",
         "test": "vitest run",
         "build": "tsx scripts/build/build.js",
+        "build:local": "tsx scripts/build/build.js --use-cache",
         "build:patches": "node scripts/build/patches.js",
+        "strip-generated-meta": "tsx scripts/build/strip-generated-meta.ts",
+        "generate-cache": "tsx scripts/build/build.js --generate-cache",
         "validate": "yarn validate:platforms && yarn validate:locales",
         "validate:platforms": "tsx scripts/validation/validate_platforms.js",
         "validate:locales": "tsx scripts/validation/validate_locales.js",

scripts/build/build.js

@@ -8,16 +8,23 @@ import {
     FOLDER_WITH_NEW_FILTERS,
     FOLDER_WITH_OLD_FILTERS,
 } from './constants.js';
+import { stripGeneratedMetaFromDir } from './strip-generated-meta.ts';
+import { findFiles } from '../utils/find_files.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 /**
- * Parse command-cli parameters -i|--include and -s|--skip
+ * Parse command-line parameters -i|--include, -s|--skip, --use-cache, --generate-cache,
+ * --no-patches-prepare, --strip-generated-meta
  */
 let includedFilterIDs = [];
 let excludedFilterIDs = [];
 let rawReportPath = '';
+let useCache = false;
+let generateCache = false;
+let noPatchesPrepare = false;
+let stripGeneratedMeta = false;
 
 const args = process.argv.slice(2);
 args.forEach((val) => {
@@ -40,26 +47,105 @@ args.forEach((val) => {
     if (val.startsWith('--report=')) {
         rawReportPath = val.slice(val.indexOf('=') + 1).trim();
     }
+
+    if (val === '--use-cache') {
+        useCache = true;
+    }
+
+    if (val === '--generate-cache') {
+        generateCache = true;
+    }
+
+    if (val === '--no-patches-prepare') {
+        noPatchesPrepare = true;
+    }
+
+    if (val === '--strip-generated-meta') {
+        stripGeneratedMeta = true;
+    }
 });
 
+if (useCache && generateCache) {
+    // eslint-disable-next-line no-console
+    console.error('Error: --use-cache and --generate-cache are mutually exclusive.');
+    process.exit(1);
+}
+
 /**
  * Set all relative paths needed for compiler
  */
 const filtersDir = path.join(__dirname, '../../filters');
 const logPath = path.join(__dirname, '../../log.txt');
 const platformsPath = path.join(__dirname, '../..', FOLDER_WITH_NEW_FILTERS);
 const copyPlatformsPath = path.join(__dirname, '../..', FOLDER_WITH_OLD_FILTERS);
+const cachedFiltersDir = path.join(__dirname, '../../temp/filters_cached');
 
 const reportPath = rawReportPath !== ''
     // report-adguard.txt OR report-third-party.txt
     ? path.join(__dirname, '../..', rawReportPath)
     // report_partial_DD-MM-YYYY_HH-MM-SS.txt
     : path.join(__dirname, '../..', `report_partial_${formatDate(new Date())}.txt`);
 
+const SHADOW_TEMPLATE_CONTENT = '@include "./filter.txt"\n';
+
+/**
+ * Prepare a temporary copy of the filters directory with shadow templates.
+ *
+ * Copies `filters/` → `temp/filters_cached/`, then replaces the content of every
+ * `template.txt` with a single-line local include pointing to the cached `filter.txt`.
+ * Validates that every filter directory with a `template.txt` also has a `filter.txt`.
+ *
+ * @returns {Promise<void>}
+ */
+const prepareCachedFiltersDir = async () => {
+    // Remove stale copy if exists
+    if (fs.existsSync(cachedFiltersDir)) {
+        await fs.promises.rm(cachedFiltersDir, { recursive: true });
+    }
+
+    // Full recursive copy
+    await fs.promises.cp(filtersDir, cachedFiltersDir, { recursive: true });
+
+    // Find all directories containing template.txt and replace with shadow templates
+    const templatePaths = await findFiles(cachedFiltersDir, (p) => path.basename(p) === 'template.txt');
+
+    await Promise.all(templatePaths.map(async (templatePath) => {
+        const dir = path.dirname(templatePath);
+        const filterTxtPath = path.join(dir, 'filter.txt');
+
+        if (!fs.existsSync(filterTxtPath)) {
+            throw new Error(
+                `--use-cache: missing filter.txt in ${path.relative(cachedFiltersDir, dir)}. `
+                + 'Run "yarn generate-cache" first to generate cached filter files.',
+            );
+        }
+
+        await fs.promises.writeFile(templatePath, SHADOW_TEMPLATE_CONTENT, 'utf8');
+    }));
+
+    // eslint-disable-next-line no-console
+    console.log(`Prepared cached filters directory with ${templatePaths.length} shadow templates.`);
+};
+
 /**
  * Compiler entry point.
  */
 const buildFilters = async () => {
+    // When --generate-cache we only need to compile filters (which updates filter.txt),
+    // skip platform generation, patches preparation, and temp/platforms copying.
+    if (generateCache) {
+        await compile(
+            filtersDir,
+            logPath,
+            reportPath,
+            null, // null ⇒ generate() inside compiler returns early, no platform files
+            includedFilterIDs,
+            excludedFilterIDs,
+            CUSTOM_PLATFORMS_CONFIG,
+        );
+        return;
+    }
+
     // Clean temporary folder
     if (fs.existsSync(copyPlatformsPath)) {
         await fs.promises.rm(copyPlatformsPath, { recursive: true });
@@ -70,27 +156,49 @@ const buildFilters = async () => {
     let initialRun = false;
     if (!fs.existsSync(platformsPath)) {
         initialRun = true;
-    } else {
+    } else if (!noPatchesPrepare) {
         // Make copy for future patches generation
         await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true });
     }
 
-    await compile(
-        filtersDir,
-        logPath,
-        reportPath,
-        platformsPath,
-        includedFilterIDs,
-        excludedFilterIDs,
-        CUSTOM_PLATFORMS_CONFIG,
-    );
+    // Determine which filtersDir to pass to the compiler
+    const effectiveFiltersDir = useCache ? cachedFiltersDir : filtersDir;
+
+    if (useCache) {
+        await prepareCachedFiltersDir();
+    }
+
+    try {
+        await compile(
+            effectiveFiltersDir,
+            logPath,
+            reportPath,
+            platformsPath,
+            includedFilterIDs,
+            excludedFilterIDs,
+            CUSTOM_PLATFORMS_CONFIG,
+        );
+    } finally {
+        // Clean up temp filters copy
+        if (useCache && fs.existsSync(cachedFiltersDir)) {
+            await fs.promises.rm(cachedFiltersDir, { recursive: true });
+        }
+    }
 
     // For the very first run, we should copy the built platforms into
     // the temp folder to create the first empty patches for future versions
-    if (initialRun) {
+    if (initialRun && !noPatchesPrepare) {
         // Make copy for future patches generation
         await fs.promises.cp(platformsPath, copyPlatformsPath, { recursive: true });
     }
+
+    // Strip generated metadata (Checksum, Diff-Path, TimeUpdated, Version)
+    // from compiled filter files so they don't pollute diff comparisons.
+    if (stripGeneratedMeta) {
+        const count = await stripGeneratedMetaFromDir(platformsPath);
+        // eslint-disable-next-line no-console
+        console.log(`Stripped generated meta from ${count} file(s).`);
+    }
 };
 
 buildFilters();