diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000000..c80ab7d18fe3f --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,28 @@ +name: Test + +on: + workflow_dispatch: + pull_request: + +jobs: + test: + runs-on: ubuntu-slim + steps: + - name: Checkout repo + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Use Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Enable corepack + run: corepack enable + + - name: Install dependencies + run: yarn + + - name: Test + run: yarn test diff --git a/AGENTS.md b/AGENTS.md index 8bf94efb7ea4a..41904727dfc1e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,13 +23,15 @@ for all supported AdGuard products. - `zod` — schema validation - `crypto-js` — checksum computation - **Storage**: File-system based; no database -- **Testing**: Vitest (unit tests for wildcard-domain-processor) +- **Testing**: Vitest (unit + integration + e2e tests under `scripts/*/__tests__/`); + local e2e for optimization config (`yarn test:optimization-e2e`) requires `yarn generate-cache` as a prerequisite - **Target Platform**: Node.js CLI tooling; build outputs target 8 top-level AdGuard product platforms (Android, CLI, Extension, iOS, Mac, Mac v2, Mac v3, Windows). The Extension platform has 9 sub-targets: Chromium, Chromium MV3, Edge, Firefox, Opera, Opera MV3, Safari, Android Content Blocker, uBlock. - **Project Type**: Single repository (build tooling + data) -- **CI**: Two GitHub Actions workflows — `build-adguard.yaml` and `build-3p.yaml` +- **CI**: Three GitHub Actions workflows — `build-adguard.yaml`, `build-3p.yaml`, and + `test.yaml` (runs `yarn test` on every pull request) - **Performance Goals**: N/A - **Constraints**: Filter lists must remain compatible with AdGuard's rule syntax; third-party filters follow an acceptance policy documented in README.md @@ -86,6 +88,7 @@ for all supported AdGuard products. | `yarn generate-cache` | Generate cached `filter.txt` from templates (`tsx scripts/build/build.js --generate-cache`) | | `yarn strip-generated-meta` | Strip generated meta lines from platform filter files | | `yarn test` | Run unit tests (`vitest run`) | +| `yarn test:optimization-e2e` | Run local e2e for optimization config (requires `yarn generate-cache`) | | `yarn lint` | Run all linters (code + types + markdown) | | `yarn lint:code` | ESLint check (`eslint . --ext .js,.ts`) | | `yarn lint:types` | TypeScript type check (`tsc --noEmit`) | @@ -118,8 +121,10 @@ After completing any task that modifies code in `scripts/`: yarn test ``` -3. **Update tests for changed code.** If you modify logic in `scripts/wildcard-domain-processor/`, - add or update corresponding tests in `scripts/wildcard-domain-processor/__tests__/`. +3. **Update tests for changed code.** Add or update tests in the `__tests__/` directory + alongside the code you changed: + - `scripts/wildcard-domain-processor/__tests__/` for wildcard-domain-processor changes + - `scripts/build/__tests__/` for build script changes 4. **Validate platform outputs** if filters, templates, or build scripts changed. diff --git a/package.json b/package.json index 28ec0337f1d35..5b42c97e1a131 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "validate:locales": "tsx scripts/validation/validate_locales.js", "update-wildcard-domains": "tsx scripts/wildcard-domain-processor update-wildcard-domains", "expand-wildcard-domains": "tsx scripts/wildcard-domain-processor expand-wildcard-domains", - "compress": "tsx scripts/repository/compress.js" + "compress": "tsx scripts/repository/compress.js", + "test:optimization-e2e": "bash scripts/build/__tests__/optimization-e2e.sh" }, "engines": { "node": ">=22" @@ -31,7 +32,7 @@ "@adguard/agtree": "^4.0.4", "@adguard/dead-domains-linter": "^1.0.22", "@adguard/diff-builder": "1.1.2", - "@adguard/filters-compiler": "3.2.8", + "@adguard/filters-compiler": "AdguardTeam/FiltersCompiler#local_optimization_config", "add": "^2.0.6", "crypto-js": "^4.2.0", "zod": "3" diff --git a/scripts/build/__tests__/optimization-e2e.sh b/scripts/build/__tests__/optimization-e2e.sh new file mode 100755 index 0000000000000..bc3199530dedd --- /dev/null +++ b/scripts/build/__tests__/optimization-e2e.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# E2E test for local optimization config feature. +# +# Prerequisites (must exist before running): +# temp/optimization_config/filters/1/stats.json — generated by 'yarn generate-cache' +# filters/filter_1_Russian/filter.txt — generated by 'yarn generate-cache' +# +# Test steps: +# 1. Pick 2 network rules ('||' prefix) from filter_1_Russian/filter.txt: +# FILTERED_RULE (1st match) — set to hits=0 in stats.json +# KEPT_RULE (2nd match) — set to hits=9999 in stats.json +# 2. Back up stats.json, then rewrite it with threshold hits=1: +# hits < 1 → optimizer removes the rule from compiled output +# hits >= 1 → optimizer keeps the rule in compiled output +# (Node is used instead of sed/heredoc to handle special chars: $, *, ^) +# 3. Run 'yarn build:local --include=1 --no-patches-prepare' using local stats. +# 4. Assert FILTERED_RULE is absent from platforms/**/*_optimized.txt → PASS/FAIL +# 5. Assert KEPT_RULE is present in platforms/**/*_optimized.txt → PASS/FAIL +# (non-optimized 1.txt always contains all rules, so only _optimized files are checked) +# 6. Restore original stats.json on any exit via trap. +set -euo pipefail + +cd "$(dirname "$0")/.." + +STATS="temp/optimization_config/filters/1/stats.json" +FILTER_TXT="filters/filter_1_Russian/filter.txt" + +cleanup() { + if [ -f "${STATS}.bak" ]; then + mv "${STATS}.bak" "$STATS" + echo "Restored stats.json" + fi +} +trap cleanup EXIT + +[ -f "$STATS" ] || { echo "ERROR: $STATS missing. Run 'yarn generate-cache' first."; exit 1; } +[ -f "$FILTER_TXT" ] || { echo "ERROR: $FILTER_TXT missing. Run 'yarn generate-cache' first."; exit 1; } + +# Strip \r: filter.txt has CRLF line endings; grep on macOS returns lines with \r, +# which would cause a mismatch against the trimmed rule text produced by the compiler. +FILTERED_RULE=$(grep -m1 '^||' "$FILTER_TXT" | tr -d '\r') +[ -n "$FILTERED_RULE" ] || { echo "ERROR: No network rules found in $FILTER_TXT"; exit 1; } +KEPT_RULE=$(grep -m2 '^||' "$FILTER_TXT" | tail -n1 | tr -d '\r') +[ -n "$KEPT_RULE" ] || { echo "ERROR: Need at least 2 network rules in $FILTER_TXT"; exit 1; } +[ "$FILTERED_RULE" != "$KEPT_RULE" ] || { echo "ERROR: Only one unique network rule found"; exit 1; } +echo "Rule to filter (hits=0): $FILTERED_RULE" +echo "Rule to keep (hits=9999): $KEPT_RULE" + +cp "$STATS" "${STATS}.bak" + +# Edit stats.json via Node to safely handle special characters in rule text +FILTERED_RULE="$FILTERED_RULE" KEPT_RULE="$KEPT_RULE" STATS_PATH="$STATS" node --input-type=commonjs -e " +const fs = require('fs'); +const filteredRule = process.env.FILTERED_RULE; +const keptRule = process.env.KEPT_RULE; +const statsPath = process.env.STATS_PATH; +const stats = JSON.parse(fs.readFileSync(statsPath, 'utf8')); +stats.groups = [{ + config: { type: 'BASIC', scope: 'GENERIC', hits: 1 }, + rules: { [filteredRule]: 0, [keptRule]: 9999 }, +}]; +// Wide bounds: only 2 rules are in stats out of ~17k in the real filter, +// so ~100% of rules are kept — set [0,100] so the percent check always passes. +stats.percent = 99; +stats.minPercent = 0; +stats.maxPercent = 100; +stats.strict = false; +fs.writeFileSync(statsPath, JSON.stringify(stats, null, 2) + '\n'); +" + +yarn build:local --include=1 --no-patches-prepare + +FAIL=0 +# Check only *_optimized.txt files: the non-optimized 1.txt always contains all rules +if grep -rqF "$FILTERED_RULE" --include='*_optimized.txt' platforms/ 2>/dev/null; then + echo "FAIL: rule still present in optimized output: $FILTERED_RULE" + FAIL=1 +fi +if ! grep -rqF "$KEPT_RULE" --include='*_optimized.txt' platforms/ 2>/dev/null; then + echo "FAIL: rule missing from optimized output: $KEPT_RULE" + FAIL=1 +fi + +[ "$FAIL" -eq 0 ] && echo "PASS: filtered rule absent, kept rule present" +exit "$FAIL" diff --git a/scripts/build/__tests__/optimization-e2e.test.ts b/scripts/build/__tests__/optimization-e2e.test.ts new file mode 100644 index 0000000000000..4b26124213cea --- /dev/null +++ b/scripts/build/__tests__/optimization-e2e.test.ts @@ -0,0 +1,124 @@ +import { + describe, it, expect, beforeAll, afterAll, +} from 'vitest'; +import { compile, optimizationConfigLocal } from '@adguard/filters-compiler'; +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import { fileURLToPath } from 'url'; +import { findFiles } from '../../utils/find_files.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REAL_FILTER_DIR = path.resolve(__dirname, '../../../filters/filter_1_Russian'); + +const RULE_TO_FILTER = '||e2e-optimization-filtered.example^'; +const RULE_TO_KEEP = '||e2e-optimization-kept.example^'; +const FILTER_ID = 1; + +const TEST_PLATFORM_CONFIG: Record = { + TEST: { + platform: 'test', + path: 'test', + configuration: { + ignoreRuleHints: false, + replacements: null, + }, + defines: { + adguard: true, + }, + }, +}; + +describe('optimization config e2e', () => { + let tmpDir: string; + let allOutput: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filters-e2e-')); + + const filtersDir = path.join(tmpDir, 'filters'); + const optimizationDir = path.join(tmpDir, 'optimization_config'); + const platformsDir = path.join(tmpDir, 'platforms'); + const logPath = path.join(tmpDir, 'log.txt'); + const reportPath = path.join(tmpDir, 'report.txt'); + + const filterDir = path.join(filtersDir, 'filter_1_Russian'); + await fs.promises.mkdir(filterDir, { recursive: true }); + + await fs.promises.copyFile( + path.join(REAL_FILTER_DIR, 'metadata.json'), + path.join(filterDir, 'metadata.json'), + ); + await fs.promises.copyFile( + path.join(REAL_FILTER_DIR, 'revision.json'), + path.join(filterDir, 'revision.json'), + ); + + await fs.promises.writeFile( + path.join(filterDir, 'template.txt'), + `! Title: E2E Optimization Test\n${RULE_TO_FILTER}\n${RULE_TO_KEEP}\n`, + 'utf-8', + ); + + const statsDir = path.join(optimizationDir, 'filters', String(FILTER_ID)); + await fs.promises.mkdir(statsDir, { recursive: true }); + + await fs.promises.writeFile( + path.join(optimizationDir, 'percent.json'), + JSON.stringify({ config: [{ filterId: FILTER_ID }] }), + 'utf-8', + ); + + // stats.json: RULE_TO_FILTER hits=0 < threshold=1 → filtered out + // RULE_TO_KEEP hits=9999 ≥ threshold=1 → kept + await fs.promises.writeFile( + path.join(statsDir, 'stats.json'), + JSON.stringify({ + percent: 40, + minPercent: 25, + maxPercent: 50, + strict: true, + groups: [{ + config: { type: 'BASIC', scope: 'GENERIC', hits: 1 }, + rules: { + [RULE_TO_FILTER]: 0, + [RULE_TO_KEEP]: 9999, + }, + }], + }), + 'utf-8', + ); + + optimizationConfigLocal.setPath(optimizationDir); + + await compile( + filtersDir, + logPath, + reportPath, + platformsDir, + [FILTER_ID], + [], + TEST_PLATFORM_CONFIG, + ); + + const outputFiles: string[] = await findFiles(platformsDir, () => true); + const optimizedFiles = outputFiles.filter((f) => f.endsWith('_optimized.txt')); + expect(optimizedFiles.length).toBeGreaterThan(0); + allOutput = (await Promise.all( + optimizedFiles.map((f) => fs.promises.readFile(f, 'utf-8')), + )).join('\n'); + }, 30_000); + + afterAll(async () => { + optimizationConfigLocal.reset(); + await fs.promises.rm(tmpDir, { recursive: true, force: true }); + }); + + it('rule with 0 hits is absent in compiled output', () => { + expect(allOutput).not.toContain(RULE_TO_FILTER); + }); + + it('rule with 9999 hits is present in compiled output', () => { + expect(allOutput).toContain(RULE_TO_KEEP); + }); +}); diff --git a/scripts/build/__tests__/optimization-integration.test.ts b/scripts/build/__tests__/optimization-integration.test.ts new file mode 100644 index 0000000000000..8bfd80d17de5a --- /dev/null +++ b/scripts/build/__tests__/optimization-integration.test.ts @@ -0,0 +1,71 @@ +import { + describe, it, vi, expect, beforeEach, afterEach, +} from 'vitest'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const expectedOptimizationConfigCachePath = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../../temp/optimization_config', +); + +vi.mock('@adguard/filters-compiler', () => ({ + compile: vi.fn().mockResolvedValue(undefined), + optimizationConfigLocal: { + setPath: vi.fn(), + generate: vi.fn().mockResolvedValue(undefined), + }, +})); + +vi.mock('fs', () => ({ + default: { + existsSync: vi.fn().mockReturnValue(false), + promises: { + cp: vi.fn().mockResolvedValue(undefined), + rm: vi.fn().mockResolvedValue(undefined), + writeFile: vi.fn().mockResolvedValue(undefined), + }, + }, +})); + +vi.mock('../../utils/find_files.js', () => ({ + findFiles: vi.fn().mockResolvedValue([]), +})); + +describe('build.js optimization config integration', () => { + const originalArgv = process.argv; + + beforeEach(() => { + vi.resetModules(); + }); + + afterEach(() => { + process.argv = originalArgv; + vi.clearAllMocks(); + }); + + it('setPath is called with optimizationConfigCacheDir when --use-cache', async () => { + process.argv = ['node', 'build.js', '--use-cache']; + await import('../build.js'); + + const { optimizationConfigLocal } = await import('@adguard/filters-compiler'); + await vi.waitFor(() => { + expect(vi.mocked(optimizationConfigLocal.setPath)) + .toHaveBeenCalledWith(expectedOptimizationConfigCachePath); + }); + }); + + it('generate and setPath are called in sequence with optimizationConfigCacheDir ' + + 'when --generate-cache', async () => { + process.argv = ['node', 'build.js', '--generate-cache']; + await import('../build.js'); + + const { optimizationConfigLocal } = await import('@adguard/filters-compiler'); + await vi.waitFor(() => { + expect(vi.mocked(optimizationConfigLocal.generate)) + .toHaveBeenCalledWith(expectedOptimizationConfigCachePath); + expect(vi.mocked(optimizationConfigLocal.setPath)) + .toHaveBeenCalledWith(expectedOptimizationConfigCachePath); + }); + }); +}); diff --git a/scripts/build/build.js b/scripts/build/build.js index 7dbb3192f292c..7ecc435b2bcd1 100755 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; -import { compile } from '@adguard/filters-compiler'; +import { compile, optimizationConfigLocal } from '@adguard/filters-compiler'; import { CUSTOM_PLATFORMS_CONFIG } from './custom_platforms.js'; import { formatDate } from '../utils/strings.js'; import { @@ -63,7 +63,9 @@ const filtersDir = path.join(__dirname, '../../filters'); const logPath = path.join(__dirname, '../../log.txt'); const platformsPath = path.join(__dirname, '../..', FOLDER_WITH_NEW_FILTERS); const copyPlatformsPath = path.join(__dirname, '../..', FOLDER_WITH_OLD_FILTERS); -const cachedFiltersDir = path.join(__dirname, '../../temp/filters_cached'); +const tempDir = path.join(__dirname, '../../temp'); +const cachedFiltersDir = path.join(tempDir, 'filters_cached'); +const optimizationConfigCachePath = path.join(tempDir, 'optimization_config'); const reportPath = rawReportPath !== '' // report-adguard.txt OR report-third-party.txt @@ -119,6 +121,12 @@ const buildFilters = async () => { // When --generate-cache we only need to compile filters (which updates filter.txt), // skip platform generation, patches preparation, and temp/platforms copying. if (generateCache) { + await fs.promises.rm(optimizationConfigCachePath, { recursive: true, force: true }); + await optimizationConfigLocal.generate(optimizationConfigCachePath); + optimizationConfigLocal.setPath(optimizationConfigCachePath); + // eslint-disable-next-line no-console + console.log(`Using local optimization config from: ${optimizationConfigCachePath}`); + await compile( filtersDir, logPath, @@ -151,6 +159,9 @@ const buildFilters = async () => { if (useCache) { await prepareCachedFiltersDir(); + optimizationConfigLocal.setPath(optimizationConfigCachePath); + // eslint-disable-next-line no-console + console.log(`Using local optimization config from: ${optimizationConfigCachePath}`); } try { diff --git a/scripts/build/filters-compiler.d.ts b/scripts/build/filters-compiler.d.ts new file mode 100644 index 0000000000000..18880f20d2cf2 --- /dev/null +++ b/scripts/build/filters-compiler.d.ts @@ -0,0 +1,17 @@ +declare module '@adguard/filters-compiler' { + export function compile( + filtersPath: string, + logPath: string, + reportPath: string, + platformsPath: string, + whitelist: number[], + blacklist: number[], + customPlatformsConfig?: Record, + ): Promise; + + export const optimizationConfigLocal: { + setPath(configPath: string): void; + generate(configPath: string): Promise; + reset(): Promise; + }; +} diff --git a/yarn.lock b/yarn.lock index 1eb7cbb5b7aa0..113da043b0020 100644 --- a/yarn.lock +++ b/yarn.lock @@ -95,19 +95,18 @@ resolved "https://registry.yarnpkg.com/@adguard/extended-css/-/extended-css-2.1.1.tgz#53677a310cf39259f54dfce79b692b0a327a8bb7" integrity sha512-TsHZ20oUWhbtrzQ4w70B96oHkGBIUZg2FdMfkq4StGfLmXQb+kI7fQb6BtXP2D0bvraIHMq6/mtKExglDJ9vsg== -"@adguard/filters-compiler@3.2.8": - version "3.2.8" - resolved "https://registry.yarnpkg.com/@adguard/filters-compiler/-/filters-compiler-3.2.8.tgz#b3b77419ab062d218a5dd7d8bf161493920f1147" - integrity sha512-5BnIQgawIWb6+sOt1RKZwR76G1a8QjkZOw0JtLK53VInSTppEY+XeVanUOmZtzaiifbpu5Ddz8cTgc70tW7ERA== +"@adguard/filters-compiler@AdguardTeam/FiltersCompiler#local_optimization_config": + version "3.2.6" + resolved "https://codeload.github.com/AdguardTeam/FiltersCompiler/tar.gz/2231ebc5c99dd2043869357b46f1c768a075684b" dependencies: - "@adguard/agtree" "^4.0.4" + "@adguard/agtree" "^4.0.3" "@adguard/css-tokenizer" "^1.2.0" "@adguard/ecss-tree" "^2.0.1" "@adguard/extended-css" "^2.1.1" "@adguard/filters-downloader" "^2.4.0" "@adguard/logger" "^2.0.0" "@adguard/scriptlets" "^2.3.1" - "@adguard/tsurlfilter" "^4.0.5" + "@adguard/tsurlfilter" "^4.0.4" "@eslint/css-tree" "3.6.6" ajv "^8.17.1" child_process ">=1.0.2" @@ -148,7 +147,7 @@ "@types/trusted-types" "^2.0.7" js-yaml "^3.14.1" -"@adguard/tsurlfilter@^4.0.5": +"@adguard/tsurlfilter@^4.0.4": version "4.0.5" resolved "https://registry.yarnpkg.com/@adguard/tsurlfilter/-/tsurlfilter-4.0.5.tgz#2a1438d3da7c0c4b0a98c255c4386ce631eb99ae" integrity sha512-s/Iv86M9KUXXAfeyps/hmEbX5tismn19lAVqIkFptBujnrV6uEY2bxthzRsBWxY39Y8ZFwrjhe9nyZ3HILbU3g==