diff --git a/README.md b/README.md index 130cf961..c75e4ea1 100644 --- a/README.md +++ b/README.md @@ -849,10 +849,10 @@ test('resource management', () => { ## Test aliases -Test aliases are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option: +Test aliases (`test.skip()`, `test.fixme()`, `test.fail()`, `test.slow()`) are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option: ``` -TESTOMATIO=11111111 npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myCustomFunction +TESTOMATIO={API_KEY} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myFixture ``` ## Programmatic API diff --git a/example/playwright/annotations-status.ts b/example/playwright/annotations-status.ts new file mode 100644 index 00000000..3e4e7212 --- /dev/null +++ b/example/playwright/annotations-status.ts @@ -0,0 +1,30 @@ +import { test, expect } from '@playwright/test'; + +test('plain test', async () => { + await expect(true).toBe(true); +}); + +// .fail marks a test as expected to fail, but it still runs => not skipped +test.fail('expected to fail test', async () => { + await expect(true).toBe(false); +}); + +// .slow triples the timeout, but the test still runs => not skipped +test.slow('slow test', async () => { + await expect(true).toBe(true); +}); + +// runtime forms without a title declare no separate test +test('runtime annotations have no title', async () => { + test.fail(); + test.skip(); + test.slow(); + await expect(true).toBe(true); +}); + +// .fail inside a skipped suite => skipped (suite wins) +test.describe.skip('skipped suite', () => { + test.fail('fail inside skipped suite', async () => { + await expect(true).toBe(false); + }); +}); diff --git a/example/playwright/custom-fixture-annotations.ts b/example/playwright/custom-fixture-annotations.ts new file mode 100644 index 00000000..ba94f8db --- /dev/null +++ b/example/playwright/custom-fixture-annotations.ts @@ -0,0 +1,29 @@ +import { test as base } from '@playwright/test'; + +const testFixture = base.extend<{ someFixture: any }>({ + someFixture: async ({}, use) => { + await use({ name: 'custom fixture name' }); + }, +}); + +testFixture('plain alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.skip('skipped alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.fixme('fixme alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.fail('failing alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.describe('alias suite', () => { + testFixture.fixme('fixme test inside alias suite', async ({ someFixture }) => { + console.warn(someFixture.name); + }); +}); diff --git a/example/playwright/sibling-after-skipped-suite.ts b/example/playwright/sibling-after-skipped-suite.ts new file mode 100644 index 00000000..74e96325 --- /dev/null +++ b/example/playwright/sibling-after-skipped-suite.ts @@ -0,0 +1,16 @@ +import { test, expect } from '@playwright/test'; + +test.describe.skip('skipped suite', () => { + test('inside skipped suite', async () => { + await expect(true).toBe(true); + }); +}); + +// these are siblings declared AFTER the skipped suite closed - they must not inherit skipped +test('sibling after skipped suite', async () => { + await expect(true).toBe(true); +}); + +test.fail('failing sibling after skipped suite', async () => { + await expect(true).toBe(false); +}); diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index e7a6782e..6dfcc182 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -23,6 +23,9 @@ module.exports = (ast, file = '', source = '', opts = {}) => { let beforeEachCode = ''; let afterCode = ''; + // built-in `test`/`it` plus any custom fixtures/aliases passed via --test-alias + const testNames = ['test', 'it', ...(opts?.testAlias || [])]; + function addSuite(path) { currentSuite = currentSuite.filter(s => s.loc.end.line > path.loc.start.line); path.tags = playwright.getTestProps({ parent: { expression: path } }).tags; @@ -90,31 +93,33 @@ module.exports = (ast, file = '', source = '', opts = {}) => { } } - if (path.isIdentifier({ name: 'skip' })) { + // `.skip`/`.fixme` tests are skipped; `.fail`/`.slow` tests still run; + // runtime forms without a title (e.g. `test.skip()` inside a body) declare no test + if (path.isIdentifier() && ['skip', 'fixme', 'fail', 'slow'].includes(path.node.name)) { if (!path.parent || !path.parent.object) { return; } const name = - path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name; + path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name; - if (name === 'test' || name === 'it') { + if (testNames.includes(name)) { // test or it if (!hasStringOrTemplateArgument(path.parentPath.container)) return; const testName = getStringValue(path.parentPath.container); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); tests.push({ name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), + suites: suites.map(s => getStringValue(s)), line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), + // end line comes from the enclosing call to capture the full test body + code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), file, - skipped: true, + skipped: ['skip', 'fixme'].includes(path.node.name) || suites.some(s => s.skipped), }); } - if (name === 'describe') { + if (name === 'describe' && (path.node.name === 'skip' || path.node.name === 'fixme')) { // suite if (!hasStringOrTemplateArgument(path.parentPath.container)) return; const suite = path.parentPath.container; @@ -125,66 +130,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { // todo: handle "context" } - if (path.isIdentifier({ name: 'fixme' })) { - if (!path.parent || !path.parent.object) { - return; - } - const name = - path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name; - - if (name === 'test' || name === 'it') { - // test or it - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - - const testName = getStringValue(path.parentPath.container); - tests.push({ - name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), - line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), - file, - skipped: true, - }); - } - - if (name === 'describe') { - // suite - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - const suite = path.parentPath.container; - suite.skipped = true; - addSuite(suite); - } - - // todo: handle "context" - } - - if (path.isIdentifier({ name: 'todo' })) { - if (!path.parent || !path.parent.object) { - return; - } - // todo tests => skipped tests - if (path.parent.object.name === 'test') { - // test - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - - const testName = getStringValue(path.parentPath.container); - tests.push({ - name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), - line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), - file, - skipped: true, - }); - } - } - - const fixtureNames = [...['test', 'it'], ...(opts?.testAlias || [])]; - for (const fiixtureName of fixtureNames || []) { + for (const fiixtureName of testNames) { if (path.isIdentifier({ name: fiixtureName })) { if (!hasStringOrTemplateArgument(path.parent)) return; @@ -202,19 +148,19 @@ module.exports = (ast, file = '', source = '', opts = {}) => { afterCode; const testName = getStringValue(path.parent); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); tests.push({ name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), + suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(path), code, file, tags: [...getAllSuiteTags(currentSuite), ...playwright.getTestProps(path.parentPath).tags], annotations: playwright.getTestProps(path.parentPath).annotations, - skipped: !!currentSuite.filter(s => s.skipped).length, + // only suites still enclosing this line can mark it skipped (not closed siblings) + skipped: suites.some(s => s.skipped), }); // stop the loop if the test is found @@ -227,16 +173,15 @@ module.exports = (ast, file = '', source = '', opts = {}) => { if (!hasStringOrTemplateArgument(currentPath.parent)) return; const testName = getStringValue(currentPath.parent); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); tests.push({ name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), + suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(currentPath), code: getCode(source, getLineNumber(currentPath), getEndLineNumber(currentPath), isLineNumber), file, - skipped: !!currentSuite.filter(s => s.skipped).length, + skipped: suites.some(s => s.skipped), }); } }, diff --git a/tests/playwright_test.js b/tests/playwright_test.js index 42bcb5ed..a734b9ec 100644 --- a/tests/playwright_test.js +++ b/tests/playwright_test.js @@ -248,9 +248,11 @@ test.describe.only('my test', () => { ast = jsParser.parse(source, { sourceType: 'unambiguous' }); const tests = playwrightParser(ast, '', source); - expect(tests[1].code.trim()).to.equal("test.skip('my skip test @first', async ({ page }) => {".trim()); expect(tests[1].name).to.equal('my skip test @first'); expect(tests[1].suites.length).to.eql(1); + // code captures the full test body, not just the signature line + expect(tests[1].code).to.include("test.skip('my skip test @first', async ({ page }) => {"); + expect(tests[1].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); it('should parse playwright-js tests with annotation including fixme', () => { @@ -258,9 +260,11 @@ test.describe.only('my test', () => { ast = jsParser.parse(source, { sourceType: 'unambiguous' }); const tests = playwrightParser(ast, '', source); - expect(tests[2].code.trim()).to.equal("test.fixme('my fixme test @third', async ({ page }) => {".trim()); expect(tests[2].name).to.equal('my fixme test @third'); expect(tests[2].suites.length).to.eql(1); + // code captures the full test body, not just the signature line + expect(tests[2].code).to.include("test.fixme('my fixme test @third', async ({ page }) => {"); + expect(tests[2].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); it('should parse playwright-ts tests with annotations', () => { @@ -505,6 +509,90 @@ test.describe.only('my test', () => { expect(tests.length).to.equal(1); }); + it('should parse annotations (.skip/.fixme/.fail) on a custom test alias', () => { + source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source, { testAlias: ['testFixture'] }); + + const byName = name => tests.find(t => t.name === name); + + expect(tests.length).to.equal(5); + expect(byName('plain alias test').skipped).to.be.false; + expect(byName('skipped alias test').skipped).to.be.true; + expect(byName('fixme alias test').skipped).to.be.true; + expect(byName('failing alias test').skipped).to.be.false; // .fail still runs + expect(byName('fixme test inside alias suite').skipped).to.be.true; + expect(byName('fixme test inside alias suite').suites).to.deep.equal(['alias suite']); + }); + + it('should not parse custom alias annotations when the alias is not configured', () => { + source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source); + + expect(tests.length).to.equal(0); + }); + + describe('annotations status (.skip/.fixme/.fail/.slow)', () => { + let tests; + + beforeEach(() => { + source = fs.readFileSync('./example/playwright/annotations-status.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + tests = playwrightParser(ast, '', source); + }); + + const byName = name => tests.find(t => t.name === name); + + it('registers every named test exactly once (runtime no-title forms excluded)', () => { + // 5 named tests; inline `test.fail()` / `test.skip()` / `test.slow()` without a title add nothing + expect(tests.length).to.equal(5); + expect(tests.map(t => t.name)).to.deep.equal([ + 'plain test', + 'expected to fail test', + 'slow test', + 'runtime annotations have no title', + 'fail inside skipped suite', + ]); + }); + + it('keeps .fail tests runnable (not skipped)', () => { + expect(byName('expected to fail test').skipped).to.be.false; + }); + + it('keeps .slow tests runnable (not skipped)', () => { + expect(byName('slow test').skipped).to.be.false; + }); + + it('treats .fail inside a skipped suite as skipped', () => { + const test = byName('fail inside skipped suite'); + expect(test.skipped).to.be.true; + expect(test.suites).to.deep.equal(['skipped suite']); + }); + + it('ignores runtime `test.fail()` / `test.skip()` / `test.slow()` calls without a title', () => { + const test = byName('runtime annotations have no title'); + expect(test).to.not.be.undefined; + expect(test.skipped).to.be.false; + }); + }); + + it('should not leak a skipped suite onto sibling tests declared after it', () => { + source = fs.readFileSync('./example/playwright/sibling-after-skipped-suite.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source); + + const byName = name => tests.find(t => t.name === name); + + expect(byName('inside skipped suite').skipped).to.be.true; + expect(byName('inside skipped suite').suites).to.deep.equal(['skipped suite']); + // siblings declared after the suite closed must not inherit it + expect(byName('sibling after skipped suite').skipped).to.be.false; + expect(byName('sibling after skipped suite').suites).to.deep.equal([]); + expect(byName('failing sibling after skipped suite').skipped).to.be.false; + expect(byName('failing sibling after skipped suite').suites).to.deep.equal([]); + }); + it('should not crash when test is assigned to a variable or inside an array (regression for issue #1)', () => { const source = ` // This works normally