new_audit(webmcp-schema-validity): add audit to check WebMCP schema issues #16973
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import {Audit} from './audit.js'; | ||
| import * as i18n from '../lib/i18n/i18n.js'; | ||
|
|
||
| const UIStrings = { | ||
| /** Title of a Lighthouse audit that evaluates WebMCP schema validity. This descriptive title is shown to users when there are no schema validity issues. "WebMCP" stands for "Web Model Context Protocol" and should not be translated. */ | ||
| title: 'WebMCP schemas are valid', | ||
| /** Title of a Lighthouse audit that provides detail on WebMCP schema validity. This descriptive title is shown to users when there are schema validity issues. "WebMCP" stands for "Web Model Context Protocol" and should not be translated. */ | ||
| failureTitle: 'WebMCP schemas have validity issues. ' + | ||
| 'Fix them to improve accessibility for AI agents.', | ||
|
Member
There was a problem hiding this comment. drop the second sentence. we dont do 2 sentence titles :) |
||
| /** Description of a Lighthouse audit that tells the user why they should ensure WebMCP schemas are valid. This is displayed after a user expands the section to see more. No character length limits. "WebMCP" stands for "Web Model Context Protocol" and should not be translated. */ | ||
| description: 'Valid WebMCP schemas are required for AI agents to ' + | ||
| ' understand and interact with tools correctly. ' + | ||
| 'Please fix any errors or warnings reported by the browser.', | ||
| /** Header of the table column which displays the element. */ | ||
| columnElement: 'Element', | ||
| /** Header of the table column which displays the issue. */ | ||
| columnIssue: 'Issue', | ||
| /** Descriptive reason for why a form fails WebMCP validation due to missing `toolname` attribute. */ | ||
| missingToolName: 'Form level `toolname` attribute is missing. Add it to define the tool name.', | ||
| /** Descriptive reason for why a form fails WebMCP validation due to missing `tooldescription` attribute. */ | ||
| missingToolDescription: 'Form level `tooldescription` attribute is missing. ' + | ||
| 'Add it to describe the tool for AI agents.', | ||
| /** Descriptive reason for why a form field fails WebMCP validation due to missing `name` attribute for a required field. */ | ||
| missingRequiredParamName: 'Missing `name` attribute for a required field. ' + | ||
| 'Add it to define the parameter name.', | ||
| /** Descriptive reason for why a form field fails WebMCP validation due to missing `name` attribute for an optional field. */ | ||
| missingOptionalParamName: 'Missing `name` attribute for an optional field. ' + | ||
| 'Add it to define the parameter name.', | ||
| /** Descriptive reason for why a form field fails WebMCP validation due to missing description. */ | ||
| missingParamDescription: 'Add a description to make this form more accessible for AI agents.', | ||
| }; | ||
|
|
||
| const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings); | ||
|
|
||
| class WebMcpSchemaValidity extends Audit { | ||
| /** | ||
| * @return {LH.Audit.Meta} | ||
| */ | ||
| static get meta() { | ||
| return { | ||
| id: 'webmcp-schema-validity', | ||
| title: str_(UIStrings.title), | ||
| failureTitle: str_(UIStrings.failureTitle), | ||
| description: str_(UIStrings.description), | ||
| requiredArtifacts: ['WebMcpSchemaIssues'], | ||
| supportedModes: ['navigation', 'snapshot'], | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * @param {LH.Artifacts} artifacts | ||
| * @return {Promise<LH.Audit.Product>} | ||
| */ | ||
| static async audit(artifacts) { | ||
| /** @enum {number} */ | ||
| const Severity = { | ||
| ERROR: 1, | ||
| WARNING: 2, | ||
| }; | ||
|
|
||
| /** @type {Record<string, {severity: Severity, description: LH.IcuMessage}>} */ | ||
| const issueConfigs = { | ||
| 'FormModelContextMissingToolName': { | ||
| severity: Severity.ERROR, | ||
| description: str_(UIStrings.missingToolName), | ||
| }, | ||
| 'FormModelContextMissingToolDescription': { | ||
| severity: Severity.ERROR, | ||
| description: str_(UIStrings.missingToolDescription), | ||
| }, | ||
| 'FormModelContextRequiredParameterMissingName': { | ||
| severity: Severity.ERROR, | ||
| description: str_(UIStrings.missingRequiredParamName), | ||
| }, | ||
| 'FormModelContextParameterMissingTitleAndDescription': { | ||
| severity: Severity.WARNING, | ||
| description: str_(UIStrings.missingParamDescription), | ||
| }, | ||
| 'FormModelContextParameterMissingName': { | ||
| severity: Severity.WARNING, | ||
| description: str_(UIStrings.missingOptionalParamName), | ||
| }, | ||
| }; | ||
|
|
||
| const rawIssues = artifacts.WebMcpSchemaIssues; | ||
|
|
||
| const uniqueIssues = [ | ||
| ...new Map( | ||
| rawIssues.map(issue => [`${issue.violatingNodeId}_${issue.errorType}`, issue]) | ||
| ).values(), | ||
| ]; | ||
| const sortedUniqueIssues = uniqueIssues.sort((a, b) => { | ||
| return (issueConfigs[a.errorType]?.severity || Severity.ERROR) - | ||
| (issueConfigs[b.errorType]?.severity || Severity.ERROR); | ||
| }); | ||
| const items = sortedUniqueIssues.map(issue => { | ||
| return { | ||
| element: issue.nodeDetails ? Audit.makeNodeItem(issue.nodeDetails) : undefined, | ||
| issue: issueConfigs[issue.errorType]?.description || '', | ||
| }; | ||
| }); | ||
|
|
||
| /** @type {LH.Audit.Details.Table['headings']} */ | ||
| const headings = [ | ||
| {key: 'element', valueType: 'node', label: str_(UIStrings.columnElement)}, | ||
| {key: 'issue', valueType: 'text', label: str_(UIStrings.columnIssue)}, | ||
| ]; | ||
|
|
||
| const details = Audit.makeTableDetails(headings, items); | ||
|
|
||
| const hasErrors = | ||
| sortedUniqueIssues.some(issue => issueConfigs[issue.errorType]?.severity === Severity.ERROR); | ||
|
|
||
| return { | ||
| score: hasErrors ? 0 : (items.length > 0 ? 0.5 : 1), | ||
| details: items.length > 0 ? details : undefined, | ||
| }; | ||
| } | ||
| } | ||
|
|
||
| export default WebMcpSchemaValidity; | ||
| export {UIStrings}; | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import BaseGatherer from '../base-gatherer.js'; | ||
| import {resolveNodeIdToObjectId} from '../driver/dom.js'; | ||
| import {pageFunctions} from '../../lib/page-functions.js'; | ||
| import {ExecutionContext} from '../driver/execution-context.js'; | ||
|
|
||
| class WebMcpSchemaIssues extends BaseGatherer { | ||
| /** @type {LH.Gatherer.GathererMeta} */ | ||
| meta = { | ||
| supportedModes: ['navigation', 'snapshot'], | ||
| }; | ||
|
|
||
| constructor() { | ||
| super(); | ||
| /** @type {LH.Artifacts.WebMcpSchemaIssue[]} */ | ||
| this._issues = []; | ||
| this._onIssueAdded = this.onIssueAdded.bind(this); | ||
| } | ||
|
|
||
| /** | ||
| * @param {Record<string, any>} event | ||
| */ | ||
| onIssueAdded(event) { | ||
| const issue = event.issue; | ||
| if (!issue || issue.code !== 'GenericIssue') return; | ||
|
|
||
| const details = issue.details?.genericIssueDetails; | ||
| if (!details) return; | ||
|
|
||
| const errorType = details.errorType; | ||
| if (errorType && ( | ||
| errorType === 'FormModelContextMissingToolName' || | ||
| errorType === 'FormModelContextMissingToolDescription' || | ||
| errorType === 'FormModelContextRequiredParameterMissingName' || | ||
| errorType === 'FormModelContextParameterMissingTitleAndDescription' || | ||
| errorType === 'FormModelContextParameterMissingName' | ||
| )) { | ||
| this._issues.push(details); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @param {LH.Gatherer.Context} passContext | ||
| */ | ||
| async startInstrumentation(passContext) { | ||
| const session = passContext.driver.defaultSession; | ||
| session.on('Audits.issueAdded', this._onIssueAdded); | ||
| await session.sendCommand('Audits.enable'); | ||
| } | ||
|
|
||
| /** | ||
| * @param {LH.Gatherer.Context} passContext | ||
| */ | ||
| async stopInstrumentation(passContext) { | ||
| const session = passContext.driver.defaultSession; | ||
| session.off('Audits.issueAdded', this._onIssueAdded); | ||
| } | ||
|
|
||
| /** | ||
| * @param {LH.Gatherer.Context} context | ||
| * @return {Promise<LH.Artifacts.WebMcpSchemaIssue[]>} | ||
| */ | ||
| async getArtifact(context) { | ||
| const session = context.driver.defaultSession; | ||
|
|
||
| const deps = ExecutionContext.serializeDeps([ | ||
| pageFunctions.getNodeDetails, | ||
| pageFunctions.getNodeDetailsData, | ||
| ]); | ||
|
|
||
| const promises = this._issues.map(async (issue) => { | ||
| const processedIssue = {...issue}; | ||
| if (issue.violatingNodeId) { | ||
| try { | ||
| const objectId = await resolveNodeIdToObjectId(session, issue.violatingNodeId); | ||
| if (objectId) { | ||
| const response = await session.sendCommand('Runtime.callFunctionOn', { | ||
| objectId, | ||
| functionDeclaration: `function () { | ||
| ${deps} | ||
| return getNodeDetailsData(this); | ||
| }`, | ||
| returnByValue: true, | ||
| awaitPromise: true, | ||
| }); | ||
| if (response && response.result && response.result.value) { | ||
| processedIssue.nodeDetails = response.result.value; | ||
| } | ||
| } | ||
| } catch (err) { | ||
| // Ignore error | ||
| } | ||
| } | ||
| return processedIssue; | ||
| }); | ||
|
|
||
| return Promise.all(promises); | ||
| } | ||
| } | ||
|
|
||
| export default WebMcpSchemaIssues; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
:)
how about something like
WebMCP schema is invalid