diff --git a/packages/dev/core/src/FlowGraph/Blocks/Data/Math/flowGraphMathBlocks.ts b/packages/dev/core/src/FlowGraph/Blocks/Data/Math/flowGraphMathBlocks.ts index 53a9a032be06..54901ff6e430 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/Data/Math/flowGraphMathBlocks.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/Data/Math/flowGraphMathBlocks.ts @@ -316,6 +316,29 @@ export class FlowGraphRandomBlock extends FlowGraphConstantOperationBlock { + constructor(config?: IFlowGraphBlockConfiguration) { + super(RichTypeAny, RichTypeAny, RichTypeNumber, RichTypeAny, (a, b, c) => Quaternion.Slerp(a, b, c), FlowGraphBlockNames.MathSlerp, config); + } +} +RegisterClass(FlowGraphBlockNames.MathSlerp, FlowGraphMathSlerpBlock); + /** * Equals block. */ @@ -719,14 +759,31 @@ export class FlowGraphEqualityBlock extends FlowGraphBinaryOperationBlock 0 && aStr[0] === "/" && bStr.length > 0 && bStr[0] === "/") { + const ar = aStr.endsWith("/") ? aStr.slice(0, -1) : aStr; + const br = bStr.endsWith("/") ? bStr.slice(0, -1) : bStr; + return ar === br; + } } + return a === b; } } RegisterClass(FlowGraphBlockNames.Equality, FlowGraphEqualityBlock); diff --git a/packages/dev/core/src/FlowGraph/Blocks/Data/Transformers/flowGraphJsonPointerParserBlock.ts b/packages/dev/core/src/FlowGraph/Blocks/Data/Transformers/flowGraphJsonPointerParserBlock.ts index 8885a631bd8a..e64c10c8dfc5 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/Data/Transformers/flowGraphJsonPointerParserBlock.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/Data/Transformers/flowGraphJsonPointerParserBlock.ts @@ -106,6 +106,10 @@ export class FlowGraphJsonPointerParserBlock

{ +export class FlowGraphIntToFloat extends FlowGraphUnaryOperationBlock { constructor(config?: IFlowGraphBlockConfiguration) { - super(RichTypeFlowGraphInteger, RichTypeNumber, (a) => a.value, FlowGraphBlockNames.IntToFloat, config); + super(RichTypeAny, RichTypeNumber, (a) => (typeof a === "number" ? a : a?.value), FlowGraphBlockNames.IntToFloat, config); } } diff --git a/packages/dev/core/src/FlowGraph/Blocks/Data/Utils/flowGraphArrayIndexBlock.ts b/packages/dev/core/src/FlowGraph/Blocks/Data/Utils/flowGraphArrayIndexBlock.ts index 9fab5d1bc1c2..e8dd5acc5124 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/Data/Utils/flowGraphArrayIndexBlock.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/Data/Utils/flowGraphArrayIndexBlock.ts @@ -44,7 +44,27 @@ export class FlowGraphArrayIndexBlock extends FlowGraphBlock { */ public override _updateOutputs(context: FlowGraphContext): void { const array = this.array.getValue(context); - const index = getNumericValue(this.index.getValue(context)); + const rawIndex = this.index.getValue(context); + // KHR_interactivity opaque-reference values feed in here as JSON-Pointer + // ref strings (e.g. "/animations/0/") instead of plain integers. Extract + // the trailing numeric segment as the index in that case. Undefined / + // unconnected inputs short-circuit to a null output instead of crashing + // ``getNumericValue`` on a missing ``.value`` property. + let index: number; + if (rawIndex === undefined || rawIndex === null) { + this.value.setValue(null, context); + return; + } + if (typeof rawIndex === "string") { + const parsed = _ParseRefIndex(rawIndex); + if (parsed === undefined) { + this.value.setValue(null, context); + return; + } + index = parsed; + } else { + index = getNumericValue(rawIndex); + } if (array && index >= 0 && index < array.length) { this.value.setValue(array[index], context); } else { @@ -66,3 +86,28 @@ export class FlowGraphArrayIndexBlock extends FlowGraphBlock { } RegisterClass(FlowGraphBlockNames.ArrayIndex, FlowGraphArrayIndexBlock); + +/** + * Extract the trailing numeric segment from a JSON-Pointer-shaped ref string, + * e.g. ``/animations/0/`` → 0, ``/nodes/12`` → 12. Returns ``undefined`` if the + * string does not match the expected ``//(/?)`` shape. + * @param ref the ref string to parse. + * @returns the trailing integer segment, or ``undefined`` when the input is + * not a JSON-Pointer-shaped ref ending in an integer. + */ +function _ParseRefIndex(ref: string): number | undefined { + if (ref.length === 0 || ref[0] !== "/") { + return undefined; + } + const trimmed = ref.endsWith("/") ? ref.slice(0, -1) : ref; + const lastSlash = trimmed.lastIndexOf("/"); + if (lastSlash < 0) { + return undefined; + } + const tail = trimmed.substring(lastSlash + 1); + if (tail.length === 0) { + return undefined; + } + const parsed = Number(tail); + return Number.isFinite(parsed) && Number.isInteger(parsed) ? parsed : undefined; +} diff --git a/packages/dev/core/src/FlowGraph/Blocks/Event/flowGraphReceiveCustomEventBlock.ts b/packages/dev/core/src/FlowGraph/Blocks/Event/flowGraphReceiveCustomEventBlock.ts index da2678c3f022..e85d65e59e9e 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/Event/flowGraphReceiveCustomEventBlock.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/Event/flowGraphReceiveCustomEventBlock.ts @@ -47,7 +47,8 @@ export class FlowGraphReceiveCustomEventBlock extends FlowGraphEventBlock { const typeKey = typeof entry.type === "string" ? entry.type : entry.type?.typeName; const richType = typeof entry.type?.serialize === "function" ? entry.type : getRichTypeByFlowGraphType(typeKey); entry.type = richType; - this.registerDataOutput(key, richType); + // Pass default value from event data schema so outputs have the correct initial value + this.registerDataOutput(key, richType, (entry as any).value); } } diff --git a/packages/dev/core/src/FlowGraph/Blocks/Execution/Animation/flowGraphPlayAnimationBlock.ts b/packages/dev/core/src/FlowGraph/Blocks/Execution/Animation/flowGraphPlayAnimationBlock.ts index 6d519b05572c..32863c58f5fc 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/Execution/Animation/flowGraphPlayAnimationBlock.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/Execution/Animation/flowGraphPlayAnimationBlock.ts @@ -126,6 +126,31 @@ export class FlowGraphPlayAnimationBlock extends FlowGraphAsyncExecutionBlock { const from = this.from.getValue(context) ?? 0; // not accepting 0 const to = this.to.getValue(context) || animationGroupToUse.to; + + // Validate duration for interpolation animations: non-finite or negative values trigger the error flow. + // Only validate when animation is provided (interpolation case), not for general animation/start + // where the AnimationGroup's default to value may legitimately be negative. + if (!isFinite(to) || !isFinite(from)) { + return this._reportError(context, "Invalid animation duration"); + } + if (animation && to < 0) { + return this._reportError(context, "Invalid animation duration"); + } + + // Validate easing function: NaN bezier control points trigger the error flow + if (animation) { + const animationsArray = Array.isArray(animation) ? animation : [animation]; + for (const anim of animationsArray) { + const easing = anim.getEasingFunction?.(); + if (easing && "x1" in easing) { + const bezier = easing as unknown as { x1: number; y1: number; x2: number; y2: number }; + if (isNaN(bezier.x1) || isNaN(bezier.y1) || isNaN(bezier.x2) || isNaN(bezier.y2)) { + return this._reportError(context, "Invalid bezier curve control points"); + } + } + } + } + const loop = !isFinite(to) || this.loop.getValue(context); this.currentAnimationGroup.setValue(animationGroupToUse, context); diff --git a/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockFactory.ts b/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockFactory.ts index 13fd25bcaa48..8c37f366b018 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockFactory.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockFactory.ts @@ -96,6 +96,8 @@ export function blockFactory(blockName: FlowGraphBlockNames | string): () => Pro return async () => (await import("./Data/Math/flowGraphMathBlocks")).FlowGraphSaturateBlock; case FlowGraphBlockNames.MathInterpolation: return async () => (await import("./Data/Math/flowGraphMathBlocks")).FlowGraphMathInterpolationBlock; + case FlowGraphBlockNames.MathSlerp: + return async () => (await import("./Data/Math/flowGraphMathBlocks")).FlowGraphMathSlerpBlock; case FlowGraphBlockNames.Equality: return async () => (await import("./Data/Math/flowGraphMathBlocks")).FlowGraphEqualityBlock; case FlowGraphBlockNames.LessThan: diff --git a/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockNames.ts b/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockNames.ts index ddf2bfc96370..69c7801bb39a 100644 --- a/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockNames.ts +++ b/packages/dev/core/src/FlowGraph/Blocks/flowGraphBlockNames.ts @@ -44,6 +44,7 @@ export const enum FlowGraphBlockNames { Clamp = "FlowGraphClampBlock", Saturate = "FlowGraphSaturateBlock", MathInterpolation = "FlowGraphMathInterpolationBlock", + MathSlerp = "FlowGraphMathSlerpBlock", Equality = "FlowGraphEqualityBlock", LessThan = "FlowGraphLessThanBlock", LessThanOrEqual = "FlowGraphLessThanOrEqualBlock", diff --git a/packages/dev/core/src/FlowGraph/flowGraphPathConverterComponent.ts b/packages/dev/core/src/FlowGraph/flowGraphPathConverterComponent.ts index 3741b0dbd38b..4c88acf8604f 100644 --- a/packages/dev/core/src/FlowGraph/flowGraphPathConverterComponent.ts +++ b/packages/dev/core/src/FlowGraph/flowGraphPathConverterComponent.ts @@ -3,10 +3,28 @@ import { type FlowGraphBlock } from "./flowGraphBlock"; import { type FlowGraphContext } from "./flowGraphContext"; import { type FlowGraphDataConnection } from "./flowGraphDataConnection"; import { FlowGraphInteger } from "./CustomTypes/flowGraphInteger"; -import { RichTypeFlowGraphInteger } from "./flowGraphRichTypes"; +import { RichTypeAny } from "./flowGraphRichTypes"; import { type IObjectAccessor } from "./typeDefinitions"; -const PathHasTemplatesRegex = new RegExp(/\/\{(\w+)\}(?=\/|$)/g); +// KHR_interactivity JSON Pointer templates may use either bracket style: +// {name} → originally an integer template, repurposed by the "opaque reference" spec +// update for refs. Real-world assets mix both conventions, so the bracket +// style by itself is not enough to determine the input's type. +// [name] → integer template (post-ref-update spec). +// We therefore accept both and decide how to substitute at resolution time based on the +// runtime value supplied to the input socket (FlowGraphInteger / number → int substitution, +// string → ref substitution by extracting the matching JSON-Pointer segment). +const RefTemplateRegex = new RegExp(/\/\{(\w+)\}(?=\/|$)/g); +const IntTemplateRegex = new RegExp(/\/\[(\w+)\](?=\/|$)/g); + +interface IPathTemplateInfo { + /** Template variable name (without surrounding brackets). */ + name: string; + /** Bracket style used in the source path; preserved so we replace the right placeholder. */ + style: "curly" | "square"; + /** The connection that supplies the runtime value for substitution. */ + connection: FlowGraphDataConnection; +} /** * @experimental @@ -14,24 +32,40 @@ const PathHasTemplatesRegex = new RegExp(/\/\{(\w+)\}(?=\/|$)/g); */ export class FlowGraphPathConverterComponent { /** - * The templated inputs for the provided path. + * The templated inputs for the provided path. Values may be FlowGraphInteger, number, or + * string (an opaque reference encoded as a JSON Pointer). */ - public readonly templatedInputs: FlowGraphDataConnection[] = []; + public readonly templatedInputs: FlowGraphDataConnection[] = []; + + /** Per-template metadata (name + bracket style + input connection). */ + public readonly templateInfos: IPathTemplateInfo[] = []; + public constructor( public path: string, public ownerBlock: FlowGraphBlock ) { - let match = PathHasTemplatesRegex.exec(path); const templateSet = new Set(); - while (match) { - const [, matchGroup] = match; - if (templateSet.has(matchGroup)) { - throw new Error("Duplicate template variable detected."); + + const collect = (regex: RegExp, style: "curly" | "square") => { + let match = regex.exec(path); + while (match) { + const [, name] = match; + if (templateSet.has(name)) { + throw new Error("Duplicate template variable detected."); + } + templateSet.add(name); + // Use RichTypeAny so the same socket can receive either an integer (legacy / + // [name] style) or a string ref (post-ref-update {name} style); the value's + // runtime type drives the substitution behaviour in getAccessor. + const conn = ownerBlock.registerDataInput(name, RichTypeAny, undefined); + this.templatedInputs.push(conn); + this.templateInfos.push({ name, style, connection: conn }); + match = regex.exec(path); } - templateSet.add(matchGroup); - this.templatedInputs.push(ownerBlock.registerDataInput(matchGroup, RichTypeFlowGraphInteger, new FlowGraphInteger(0))); - match = PathHasTemplatesRegex.exec(path); - } + }; + + collect(RefTemplateRegex, "curly"); + collect(IntTemplateRegex, "square"); } /** @@ -43,13 +77,102 @@ export class FlowGraphPathConverterComponent { */ public getAccessor(pathConverter: IPathToObjectConverter, context: FlowGraphContext): IObjectInfo { let finalPath = this.path; - for (const templatedInput of this.templatedInputs) { - const valueToReplace = templatedInput.getValue(context).value; - if (typeof valueToReplace !== "number" || valueToReplace < 0) { - throw new Error("Invalid value for templated input."); - } - finalPath = finalPath.replace(`{${templatedInput.name}}`, valueToReplace.toString()); + for (const info of this.templateInfos) { + const raw = info.connection.getValue(context); + const placeholder = info.style === "curly" ? `{${info.name}}` : `[${info.name}]`; + const substitution = ResolveTemplateSubstitution(this.path, info.name, raw); + finalPath = finalPath.replace(placeholder, substitution); } return pathConverter.convert(finalPath); } } + +/** + * Decide what string to splice into a templated path for a given runtime value. + * + * - FlowGraphInteger / number → use the integer's decimal representation. + * - string → treat as a JSON Pointer to a glTF object and pull the segment whose position + * in the ref matches the position of `{name}` (or `[name]`) in the surrounding template. + * Falls back to the last non-empty segment, then to the raw ref string. + * @param template the original templated path (used to locate the placeholder position) + * @param name the name of the template parameter being resolved + * @param raw the runtime value supplied for the template parameter + * @returns the substring to splice into the templated path in place of the placeholder + */ +function ResolveTemplateSubstitution(template: string, name: string, raw: any): string { + if (raw instanceof FlowGraphInteger) { + AssertNonNegativeInt(raw.value, name); + return raw.value.toString(); + } + if (typeof raw === "number") { + AssertNonNegativeInt(raw, name); + return raw.toString(); + } + if (typeof raw === "string") { + if (raw === "") { + throw new Error(`Templated reference input "${name}" is null.`); + } + return ExtractRefSubstitution(template, name, raw); + } + // Babylon object refs (e.g. a Mesh delivered by `event/onSelect.selectedNode`): + // the glTF loader stamps `_internalMetadata.gltf.pointers` with one entry + // per JSON-Pointer the object can be addressed by, e.g. a single-primitive + // Mesh holds both `/nodes/` and `/meshes//primitives/`. We pick + // the pointer whose root segment matches the segment in the template that + // immediately precedes the placeholder, so a template like + // `/nodes/{nodeRef}/globalMatrix` resolves against the `/nodes/` ref + // even if `/meshes//primitives/` was added to the object first. + if (raw && typeof raw === "object") { + const pointer = ExtractGltfPointerFromObject(raw, template, name); + if (pointer) { + return ExtractRefSubstitution(template, name, pointer); + } + } + throw new Error(`Invalid value for templated input "${name}": got ${typeof raw}.`); +} + +function ExtractGltfPointerFromObject(obj: any, template: string, name: string): string | undefined { + const pointers = obj?._internalMetadata?.gltf?.pointers; + if (!Array.isArray(pointers) || pointers.length === 0) { + return undefined; + } + const stringPointers = pointers.filter((p: unknown): p is string => typeof p === "string"); + if (stringPointers.length === 0) { + return undefined; + } + // Find the segment in the template that precedes the placeholder, e.g. + // "nodes" for "/nodes/{nodeRef}/globalMatrix". + const placeholders = [`{${name}}`, `[${name}]`]; + const templateSegments = template.split("/"); + const placeholderIndex = templateSegments.findIndex((s) => placeholders.indexOf(s) >= 0); + const expectedRoot = placeholderIndex > 0 ? templateSegments[placeholderIndex - 1] : undefined; + if (expectedRoot) { + const match = stringPointers.find((p) => p.split("/")[1] === expectedRoot); + if (match) { + return match; + } + } + return stringPointers[0]; +} + +function AssertNonNegativeInt(value: number, name: string): void { + if (typeof value !== "number" || value < 0 || !Number.isFinite(value)) { + throw new Error(`Invalid value for templated input "${name}": ${value}.`); + } +} + +function ExtractRefSubstitution(template: string, name: string, refValue: string): string { + const templateSegments = template.split("/"); + const placeholders = [`{${name}}`, `[${name}]`]; + const placeholderIndex = templateSegments.findIndex((s) => placeholders.indexOf(s) >= 0); + const refSegments = refValue.split("/"); + if (placeholderIndex >= 0 && placeholderIndex < refSegments.length && refSegments[placeholderIndex] !== "") { + return refSegments[placeholderIndex]; + } + for (let i = refSegments.length - 1; i >= 0; i--) { + if (refSegments[i] !== "") { + return refSegments[i]; + } + } + return refValue; +} diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity.ts index 1fac13605384..83faa5242d7b 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity.ts @@ -12,6 +12,10 @@ import { addToBlockFactory } from "core/FlowGraph/Blocks/flowGraphBlockFactory"; import { Quaternion, Vector3 } from "core/Maths/math.vector"; import { type Scene } from "core/scene"; import { type IAnimation } from "../glTFLoaderInterfaces"; +import { CompositePathToObjectConverter, type IPathConverterPrefixEntry } from "./compositePathToObjectConverter"; +import { BabylonScenePathToObjectConverter, BABYLON_SCENE_OBJECT_MODEL_PREFIX, CreateDefaultBabylonSceneObjectModelTree } from "./babylonScenePathToObjectConverter"; +import { type IObjectAccessor } from "core/FlowGraph/typeDefinitions"; +import { type IPathToObjectConverter } from "core/ObjectModel/objectModelInterfaces"; const NAME = "KHR_interactivity"; @@ -39,7 +43,8 @@ export class KHR_interactivity implements IGLTFLoaderExtension { */ public enabled: boolean; - private _pathConverter?: GLTFPathToObjectConverter; + private _gltfPathConverter?: GLTFPathToObjectConverter; + private _pathConverter?: CompositePathToObjectConverter; /** * @internal @@ -47,13 +52,31 @@ export class KHR_interactivity implements IGLTFLoaderExtension { */ constructor(private _loader: GLTFLoader) { this.enabled = this._loader.isExtensionUsed(NAME); - this._pathConverter = GetPathToObjectConverter(this._loader.gltf); + this._gltfPathConverter = GetPathToObjectConverter(this._loader.gltf); + const scene = _loader.babylonScene; + if (this._gltfPathConverter) { + // Build a composite that handles both: + // - The Babylon-scene namespace (`/extensions/BABYLON_scene_objects/...`), + // used by ref values that point at scene objects not described by the + // source glTF (e.g. refs emitted by engine-side event blocks). + // - The standard glTF object model (everything else), via the existing + // glTF converter as a fallback. + const initialPrefixes: IPathConverterPrefixEntry[] = []; + if (scene) { + initialPrefixes.push({ + prefix: BABYLON_SCENE_OBJECT_MODEL_PREFIX, + converter: new BabylonScenePathToObjectConverter(scene, CreateDefaultBabylonSceneObjectModelTree()), + }); + } + this._pathConverter = new CompositePathToObjectConverter( + initialPrefixes, + this._gltfPathConverter as unknown as IPathToObjectConverter + ); + } // avoid starting animations automatically. _loader._skipStartAnimationStep = true; // Update object model with new pointers - - const scene = _loader.babylonScene; if (scene) { _AddInteractivityObjectModel(scene); } @@ -61,6 +84,7 @@ export class KHR_interactivity implements IGLTFLoaderExtension { public dispose() { (this._loader as any) = null; + delete this._gltfPathConverter; delete this._pathConverter; } diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/declarationMapper.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/declarationMapper.ts index 03936ce0c72f..ae09f5d8d3e4 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/declarationMapper.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/declarationMapper.ts @@ -59,6 +59,12 @@ interface IGLTFToFlowGraphMappingObject { defaultValue?: any; } +/** + * Description of how a KHR_interactivity declaration (op such as + * `pointer/get`, `event/onSelect`, `math/add`) maps to one or more + * FlowGraph blocks. Used by {@link InteractivityGraphToFlowGraphParser} + * to translate the source glTF graph into the serialized FlowGraph form. + */ export interface IGLTFToFlowGraphMapping { /** * The type of the FlowGraph block(s). @@ -408,7 +414,16 @@ const gltfToFlowGraphMapping: { [key: string]: IGLTFToFlowGraphMapping } = { "math/clamp": getSimpleInputMapping(FlowGraphBlockNames.Clamp, ["a", "b", "c"]), "math/saturate": getSimpleInputMapping(FlowGraphBlockNames.Saturate), "math/mix": getSimpleInputMapping(FlowGraphBlockNames.MathInterpolation, ["a", "b", "c"]), + // Quaternion spherical-linear interpolation. Inputs are two unit + // quaternions and an unclamped float coefficient. + "math/quatSlerp": getSimpleInputMapping(FlowGraphBlockNames.MathSlerp, ["a", "b", "c"]), "math/eq": getSimpleInputMapping(FlowGraphBlockNames.Equality, ["a", "b"]), + // Reference equality. The spec defines `ref/eq` as: true if both refs are + // null, true if both refer to the same object (regardless of whether it + // exists), false otherwise. FlowGraphEqualityBlock falls through to a + // strict `===` comparison for non-vector/matrix/numeric types, which + // already produces the spec-defined behaviour for Babylon object refs. + "ref/eq": getSimpleInputMapping(FlowGraphBlockNames.Equality, ["a", "b"]), "math/lt": getSimpleInputMapping(FlowGraphBlockNames.LessThan, ["a", "b"]), "math/le": getSimpleInputMapping(FlowGraphBlockNames.LessThanOrEqual, ["a", "b"]), "math/gt": getSimpleInputMapping(FlowGraphBlockNames.GreaterThan, ["a", "b"]), @@ -1104,10 +1119,22 @@ const gltfToFlowGraphMapping: { [key: string]: IGLTFToFlowGraphMapping } = { flows: { err: { name: "error" }, }, + values: { + // New spec renames this output to `lastDelay` (ref). Internally we still produce a + // FlowGraphInteger; the index is unique per delay so it acts as the opaque handle. + lastDelay: { name: "lastDelayIndex" }, + }, }, }, "flow/cancelDelay": { blocks: [FlowGraphBlockNames.CancelDelay], + inputs: { + values: { + // New spec renames this input to `delay` (ref). The underlying block reads an int + // from `delayIndex`; when a ref-string flows in we coerce it via the path converter. + delay: { name: "delayIndex" }, + }, + }, }, "variable/get": { blocks: [FlowGraphBlockNames.GetVariable], diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/interactivityGraphParser.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/interactivityGraphParser.ts index a02190499245..f141140d1b58 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/interactivityGraphParser.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity/interactivityGraphParser.ts @@ -8,9 +8,21 @@ import { type FlowGraphBlockNames } from "core/FlowGraph/Blocks/flowGraphBlockNa import { FlowGraphConnectionType } from "core/FlowGraph/flowGraphConnection"; import { FlowGraphTypes } from "core/FlowGraph/flowGraphRichTypes"; +/** + * Description of a KHR_interactivity custom event, as parsed from the + * glTF `events` array. Used by the importer to register the event with the + * FlowGraph send/receive event blocks. + */ // eslint-disable-next-line @typescript-eslint/naming-convention export interface InteractivityEvent { + /** Identifier of the event, used to match send and receive blocks. */ eventId: string; + /** + * Optional payload schema for the event. Each entry describes one + * value carried by the event: an `id` (the FlowGraph data socket name), + * a `type` (glTF interactivity type name) and an optional default + * `value`. `eventData` (the boolean) is currently unused. + */ eventData?: { eventData: boolean; id: string; @@ -20,7 +32,7 @@ export interface InteractivityEvent { } // eslint-disable-next-line @typescript-eslint/naming-convention export const gltfTypeToBabylonType: { - [key: string]: { length: number; flowGraphType: FlowGraphTypes; elementType: "number" | "boolean" }; + [key: string]: { length: number; flowGraphType: FlowGraphTypes; elementType: "number" | "boolean" | "string" }; } = { float: { length: 1, flowGraphType: FlowGraphTypes.Number, elementType: "number" }, bool: { length: 1, flowGraphType: FlowGraphTypes.Boolean, elementType: "boolean" }, @@ -31,14 +43,27 @@ export const gltfTypeToBabylonType: { float2x2: { length: 4, flowGraphType: FlowGraphTypes.Matrix2D, elementType: "number" }, float3x3: { length: 9, flowGraphType: FlowGraphTypes.Matrix3D, elementType: "number" }, int: { length: 1, flowGraphType: FlowGraphTypes.Integer, elementType: "number" }, + // KHR_interactivity opaque reference type. Represented as a JSON Pointer string + // (e.g. "/nodes/17/") that addresses a glTF object. The empty string is the + // canonical "null reference" sentinel used by the parser. + ref: { length: 1, flowGraphType: FlowGraphTypes.String, elementType: "string" }, }; +/** + * Parses a KHR_interactivity graph definition (the raw glTF JSON object) into + * the serialized FlowGraph form consumed by {@link ParseFlowGraphAsync}. + * + * The class walks the interactivity types, declarations, variables, events + * and nodes in order, applies any importer-side transforms (e.g. the + * relative-pointer-prefix bake) and emits an {@link ISerializedFlowGraph} + * via {@link serializeToFlowGraph}. + */ export class InteractivityGraphToFlowGraphParser { /** * Note - the graph should be rejected if the same type is defined twice. * We currently don't validate that. */ - private _types: { length: number; flowGraphType: FlowGraphTypes; elementType: "number" | "boolean" }[] = []; + private _types: { length: number; flowGraphType: FlowGraphTypes; elementType: "number" | "boolean" | "string" }[] = []; private _mappings: { flowGraphMapping: IGLTFToFlowGraphMapping; fullOperationName: string }[] = []; private _staticVariables: { type: FlowGraphTypes; value: any[] }[] = []; private _events: InteractivityEvent[] = []; @@ -132,6 +157,10 @@ export class InteractivityGraphToFlowGraphParser { case FlowGraphTypes.Number: value.push(NaN); break; + case FlowGraphTypes.String: + // Default for a `ref`-typed value is the null reference, encoded as the empty string. + value.push("" as any); + break; case FlowGraphTypes.Vector2: value.push(NaN, NaN); break; @@ -214,6 +243,10 @@ export class InteractivityGraphToFlowGraphParser { throw new Error(`Error validating interactivity node ${this._interactivityGraph.declarations?.[node.declaration].op} - ${validationResult.error}`); } } + // Bake any static ref-typed value sockets into pointer templates that + // were authored as "relative" (no leading slash). See + // _bakeRelativePointerPrefix for full rationale. + this._bakeRelativePointerPrefix(node, mapping.fullOperationName); const blocks: ISerializedFlowGraphBlock[] = []; // create block(s) for this node using the mapping for (const blockType of mapping.flowGraphMapping.blocks) { @@ -225,6 +258,90 @@ export class InteractivityGraphToFlowGraphParser { } } + /** + * KHR_interactivity test assets such as `Calculator.glb` author + * `pointer/get` and `pointer/set` nodes whose `pointer` configuration value + * is a *relative* JSON Pointer (no leading slash), e.g. + * `extensions/KHR_node_visibility/visible`, paired with a static ref-typed + * value socket like `nodeRef = "/nodes/22/"` that supplies the absolute + * prefix. The standard spec algorithm only substitutes `{name}`/`[name]` + * template parameters and would leave the relative path untouched, so the + * effective JSON Pointer ends up invalid and the `nodeRef` socket has + * nowhere to land on the FlowGraph block (causing + * "Could not find data input with name nodeRef" failures at parse time). + * + * To support this convention we splice a static ref value into the + * pointer template at parse time, dropping the now-baked socket from the + * node's `values` map so the connection wiring step ignores it. Only + * literal/static refs are baked here; refs that come from an upstream + * connection (`{ node, socket }` shape) are left untouched, since we have + * no way to know their value at parse time. Such cases would still + * surface as the same parse-time error and need a richer runtime + * substitution strategy. + * @param node The interactivity node to patch in place. + * @param fullOperationName The fully qualified op name (e.g. `pointer/get`). + */ + private _bakeRelativePointerPrefix(node: IKHRInteractivity_Node, fullOperationName: string): void { + if (!fullOperationName.startsWith("pointer/")) { + return; + } + const pointerCfg = node.configuration?.pointer; + const template = pointerCfg?.value?.[0]; + if (typeof template !== "string" || template.startsWith("/")) { + return; + } + if (!node.values) { + return; + } + for (const name of Object.keys(node.values)) { + const socket = node.values[name] as IKHRInteractivity_Variable & { node?: number; socket?: string }; + // Skip non-ref-typed inputs. By KHR_interactivity convention the ref + // socket of a `pointer/*` op is named after the ref it carries + // (``nodeRef``, ``materialRef``, ``meshRef``, etc.) and never + // ``value`` — `value` is the data being read or written by the + // pointer op, not the pointer prefix. Only consider sockets whose + // declared type is ``ref`` (a literal whose serialized type maps to + // FlowGraphTypes.String) or whose name follows the *Ref convention. + const literal = socket?.value?.[0]; + const declaredType = (socket as any)?.type; + const isLiteralRef = typeof literal === "string" && literal.startsWith("/"); + const isTypedRef = typeof declaredType === "number" && this._types[declaredType]?.flowGraphType === FlowGraphTypes.String; + const isNamedRef = name.endsWith("Ref"); + if (!isLiteralRef && !isTypedRef && !isNamedRef) { + continue; + } + // Static-literal case: the socket has a hardcoded JSON-Pointer ref + // value (e.g. ``"/nodes/22/"``). Splice the literal into the template + // and drop the socket entirely so the connection-wiring step ignores + // it. This is the Calculator.glb pattern. + if (isLiteralRef) { + const trimmedRef = (literal as string).replace(/\/+$/, ""); + (pointerCfg!.value as any[])[0] = trimmedRef + "/" + template; + delete (node.values as any)[name]; + break; + } + // Dynamic-ref case: the socket is connected to an upstream output + // (``{ node, socket }``). We can't bake the value at parse time, so + // instead we rewrite the template so the existing template-parameter + // substitution machinery substitutes the runtime ref. For a socket + // named ``nodeRef`` and a relative template like + // ``extensions/KHR_node_visibility/visible``, rewrite to + // ``/nodes/{nodeRef}/extensions/KHR_node_visibility/visible``. The + // FlowGraphPathConverterComponent will register a ``nodeRef`` data + // input on the block and the connection-wiring step will hook it + // up to the upstream output. At runtime the runtime substitution + // logic extracts the matching JSON-Pointer segment from the ref + // string delivered by that connection. + // This is the MagicBall.glb pattern. + if (typeof socket?.node === "number") { + (pointerCfg!.value as any[])[0] = `/nodes/{${name}}/${template}`; + // Leave the value socket entry as is — it is now the data input + // that the substitution machinery will consume. + break; + } + } + } + private _getEmptyBlock(className: string, type: string): ISerializedFlowGraphBlock { return { uniqueId: RandomGUID(), @@ -462,10 +579,22 @@ export class InteractivityGraphToFlowGraphParser { outputConnection.connectedPointIds.push(inputConnection.uniqueId); } + /** + * Returns the deterministic FlowGraph user-variable name used for the + * static variable at the given declaration index. + * @param index zero-based index into the interactivity graph's `variables` array. + * @returns the FlowGraph variable name (e.g. `staticVariable_3`). + */ public getVariableName(index: number) { return "staticVariable_" + index; } + /** + * Serializes the parsed interactivity graph into the {@link ISerializedFlowGraph} + * payload consumed by `ParseFlowGraphAsync`. Performs node-connection wiring + * and seeds the execution context with the graph's static variables. + * @returns the serialized FlowGraph for the parsed KHR_interactivity graph. + */ public serializeToFlowGraph(): ISerializedFlowGraph { const context: ISerializedFlowGraphContext = { uniqueId: RandomGUID(), diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_hoverability.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_hoverability.ts index 2663ef7e4db6..f43c87ddb035 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_hoverability.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_hoverability.ts @@ -37,6 +37,10 @@ addNewInteractivityFlowGraphMapping("event/onHoverIn", NAME, { outputs: { values: { hoverNodeIndex: { name: "index", toBlock: FlowGraphBlockNames.IndexOf }, + // `hoveredNode` is the new ref-typed output from the Opaque-Reference + // spec update — the picked Babylon mesh itself, available directly + // from FlowGraphPointerOverEventBlock.meshUnderPointer (no IndexOf). + hoveredNode: { name: "meshUnderPointer", toBlock: FlowGraphBlockNames.PointerOverEvent }, controllerIndex: { name: "pointerId" }, }, flows: { @@ -105,6 +109,8 @@ addNewInteractivityFlowGraphMapping("event/onHoverOut", NAME, { outputs: { values: { hoverNodeIndex: { name: "index", toBlock: FlowGraphBlockNames.IndexOf }, + // Ref-typed output: the mesh that the pointer just left. + hoveredNode: { name: "meshOutOfPointer", toBlock: FlowGraphBlockNames.PointerOutEvent }, controllerIndex: { name: "pointerId" }, }, flows: { diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_selectability.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_selectability.ts index 1fedb31a8527..b216b2e70edc 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_selectability.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_selectability.ts @@ -36,6 +36,10 @@ addNewInteractivityFlowGraphMapping("event/onSelect", NAME, { outputs: { values: { selectedNodeIndex: { name: "index", toBlock: FlowGraphBlockNames.IndexOf }, + // `selectedNode` is the new ref-typed output from the Opaque-Reference + // spec update. It's the picked Babylon mesh itself, available directly + // from FlowGraphMeshPickEventBlock.pickedMesh — no IndexOf lookup needed. + selectedNode: { name: "pickedMesh", toBlock: FlowGraphBlockNames.MeshPickEvent }, controllerIndex: { name: "pointerId" }, selectionPoint: { name: "pickedPoint" }, selectionRayOrigin: { name: "pickOrigin" }, diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_visibility.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_visibility.ts index 5ca98672418c..8f348dac4b0c 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_visibility.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/KHR_node_visibility.ts @@ -80,7 +80,19 @@ export class KHR_node_visibility implements IGLTFLoaderExtension { if (babylonTransformNode) { babylonTransformNode.inheritVisibility = true; if (node.extensions && node.extensions.KHR_node_visibility && node.extensions.KHR_node_visibility.visible === false) { - babylonTransformNode.isVisible = false; + // Apply ``visible: false`` to the same set of meshes the + // runtime ``pointer/set`` accessor writes to. The wrapping + // ``babylonTransformNode`` is often a non-rendering + // ``TransformNode``, so setting ``isVisible`` only there + // leaves the primitive child meshes visible. Mirror the + // accessor in objectModelMapping/KHR_node_visibility.ts so + // assets that author hidden defaults (e.g. MagicBall.glb's + // FortuneWords) start hidden as intended. + (babylonTransformNode as AbstractMesh).isVisible = false; + node._primitiveBabylonMeshes?.forEach((mesh) => { + mesh.inheritVisibility = true; + mesh.isVisible = false; + }); } } } diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/babylonScenePathToObjectConverter.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/babylonScenePathToObjectConverter.ts new file mode 100644 index 000000000000..f75b6e75b6c7 --- /dev/null +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/babylonScenePathToObjectConverter.ts @@ -0,0 +1,346 @@ +/* eslint-disable @typescript-eslint/naming-convention */ + +import { type Scene } from "core/scene"; +import { type TransformNode } from "core/Meshes/transformNode"; +import { type AbstractMesh } from "core/Meshes/abstractMesh"; +import { type Material } from "core/Materials/material"; +import { type Vector3, Quaternion, type Matrix } from "core/Maths/math.vector"; +import { type IObjectAccessor } from "core/FlowGraph/typeDefinitions"; +import { type IObjectInfo, type IPathToObjectConverter } from "core/ObjectModel/objectModelInterfaces"; + +/** + * Root of the JSON-Pointer namespace under which Babylon-scene objects are + * addressed by KHR_interactivity refs that did not originate from the source + * glTF asset (e.g. refs emitted by engine-specific event blocks). + * + * Trailing `/` is intentional: it lets path-prefix dispatchers like + * {@link CompositePathToObjectConverter} match cleanly. + */ +export const BABYLON_SCENE_OBJECT_MODEL_PREFIX = "/extensions/BABYLON_scene_objects/"; + +/** + * Shape of the Babylon-scene object model tree consumed by + * {@link BabylonScenePathToObjectConverter}. Mirrors `IGLTFObjectModelTree` + * but is rooted at scene-asset arrays (`transformNodes`, `meshes`, …) and + * keyed by Babylon `uniqueId`. Initially we only expose the property leaves + * needed to validate the seam end-to-end; future leaves can be added without + * any path-converter changes. + */ +export interface IBabylonSceneObjectModelTree { + /** + * + */ + transformNodes: IBabylonObjectCollection; + /** + * + */ + meshes: IBabylonObjectCollection; + /** + * + */ + materials: IBabylonObjectCollection; +} + +/** + * Generic per-collection node in the tree. `length` exposes a `.length` + * accessor (mirroring glTF). `__array__` is the per-instance leaf hit when a + * uniqueId index appears in the path. + */ +export interface IBabylonObjectCollection { + /** + * + */ + length: IObjectAccessor; + /** + * + */ + __array__: IBabylonObjectLeaves; +} + +/** Per-instance accessors. Add new properties here as they are needed. */ +export interface IBabylonObjectLeaves { + /** Marks this position as a `getTarget` boundary so the resolver can hand back the instance itself. */ + __target__?: boolean; + /** + * + */ + name?: IObjectAccessor; + /** + * + */ + translation?: IObjectAccessor; + /** + * + */ + rotation?: IObjectAccessor; + /** + * + */ + scale?: IObjectAccessor; + /** + * + */ + matrix?: IObjectAccessor; + /** + * + */ + globalMatrix?: IObjectAccessor; + /** + * + */ + visible?: IObjectAccessor; +} + +/** + * Type aliases used internally to keep the per-property accessor signatures + * narrow without forcing every leaf to be re-typed at the use site. + */ +type AnyAccessor = IObjectAccessor; + +/** + * Resolves JSON Pointer paths in the `/extensions/BABYLON_scene_objects/...` + * namespace to Babylon scene objects. + * + * The path layout is `/{root}/{collection}/{uniqueId}/{property}` where + * `{root}` is the literal `extensions/BABYLON_scene_objects` prefix and + * `{uniqueId}` is the Babylon `uniqueId` (stable per session) of the target + * instance. For example: + * + * - `/extensions/BABYLON_scene_objects/transformNodes/42/translation` + * - `/extensions/BABYLON_scene_objects/meshes/17/visible` + * + * Composite path dispatchers (see {@link CompositePathToObjectConverter}) + * route paths starting with the prefix here; everything else continues to be + * resolved by the standard glTF converter. + */ +export class BabylonScenePathToObjectConverter implements IPathToObjectConverter { + public constructor( + private _scene: Scene, + private _tree: IBabylonSceneObjectModelTree + ) {} + + /** + * @param path the full JSON Pointer (must start with the Babylon prefix) + * @returns an object-info container holding the resolved instance and accessor + */ + public convert(path: string): IObjectInfo { + if (!path.startsWith(BABYLON_SCENE_OBJECT_MODEL_PREFIX)) { + throw new Error(`BabylonScenePathToObjectConverter: path "${path}" does not start with the expected prefix "${BABYLON_SCENE_OBJECT_MODEL_PREFIX}".`); + } + + // Strip the namespace prefix and split. Ignore trailing empty segments + // so refs of the form "/extensions/BABYLON_scene_objects/transformNodes/42/" parse cleanly. + const tail = path.slice(BABYLON_SCENE_OBJECT_MODEL_PREFIX.length); + const parts = tail.split("/").filter((p) => p.length > 0); + if (parts.length === 0) { + throw new Error(`BabylonScenePathToObjectConverter: path "${path}" is missing a collection name.`); + } + + const collectionName = parts[0]; + const collection = (this._tree as unknown as Record | undefined>)[collectionName]; + if (!collection) { + throw new Error(`BabylonScenePathToObjectConverter: unknown collection "${collectionName}" in path "${path}".`); + } + + // Handle `.length` (no instance lookup). + if (parts.length === 2 && parts[1] === "length") { + const arr = this._getCollectionArray(collectionName); + return { object: arr, info: collection.length as AnyAccessor }; + } + + if (parts.length < 2) { + throw new Error(`BabylonScenePathToObjectConverter: path "${path}" is missing an instance id.`); + } + + const uniqueId = parseInt(parts[1], 10); + if (!Number.isFinite(uniqueId) || uniqueId < 0) { + throw new Error(`BabylonScenePathToObjectConverter: invalid uniqueId "${parts[1]}" in path "${path}".`); + } + + const instance = this._lookupInstanceByUniqueId(collectionName, uniqueId); + if (!instance) { + throw new Error(`BabylonScenePathToObjectConverter: no ${collectionName} instance found with uniqueId ${uniqueId} (path "${path}").`); + } + + // No property after the id → the ref itself is just a handle to the instance. + // The accessor's `get` and `getTarget` both return the instance. + if (parts.length === 2) { + return { + object: instance, + info: this._buildIdentityAccessor(instance), + }; + } + + // Walk the leaf descriptors for the requested property path. We keep this + // very simple right now: only one segment after the id is supported, which + // covers every property the initial leaves expose. Nested paths can be + // added later by extending the walker. + if (parts.length > 3) { + throw new Error(`BabylonScenePathToObjectConverter: nested property paths are not yet supported (path "${path}").`); + } + const propertyName = parts[2]; + const leaf = (collection.__array__ as Record | boolean | undefined>)[propertyName]; + if (!leaf || typeof leaf === "boolean") { + throw new Error(`BabylonScenePathToObjectConverter: property "${propertyName}" is not registered on ${collectionName} (path "${path}").`); + } + + return { + object: instance, + info: leaf as AnyAccessor, + }; + } + + private _getCollectionArray(collectionName: string): readonly any[] { + switch (collectionName) { + case "transformNodes": + return this._scene.transformNodes; + case "meshes": + return this._scene.meshes; + case "materials": + return this._scene.materials; + default: + return []; + } + } + + private _lookupInstanceByUniqueId(collectionName: string, uniqueId: number): unknown | undefined { + switch (collectionName) { + case "transformNodes": { + const direct = this._scene.transformNodes.find((n) => n.uniqueId === uniqueId); + if (direct) { + return direct; + } + // Meshes are also transform nodes; allow the same path to resolve them. + return this._scene.meshes.find((m) => m.uniqueId === uniqueId); + } + case "meshes": + return this._scene.meshes.find((m) => m.uniqueId === uniqueId); + case "materials": + return this._scene.materials.find((m) => m.uniqueId === uniqueId); + default: + return undefined; + } + } + + private _buildIdentityAccessor(instance: unknown): AnyAccessor { + return { + type: "object", + get: () => instance, + getTarget: () => instance, + isReadOnly: true, + }; + } +} + +/** + * Builds the default Babylon-scene object-model tree. + * + * We deliberately start with a minimal set of properties: the goal of this + * tree is to prove the seam (refs in the BABYLON namespace resolving through + * the same `FlowGraphJsonPointerParserBlock` that the glTF refs use) without + * committing to a complete property surface in this PR. Add new leaves here + * as concrete event-source operations need them. + * @returns a fresh Babylon-scene object-model tree with the default property surface. + */ +export function CreateDefaultBabylonSceneObjectModelTree(): IBabylonSceneObjectModelTree { + return { + transformNodes: { + length: { + type: "number", + get: (arr: TransformNode[]) => arr.length, + getTarget: (arr: TransformNode[]) => arr, + }, + __array__: { + __target__: true, + name: { + type: "string", + get: (n: TransformNode) => n.name, + set: (v: string, n: TransformNode) => { + n.name = v; + }, + getTarget: (n: TransformNode) => n, + }, + translation: { + type: "Vector3", + get: (n: TransformNode) => n.position, + set: (v: Vector3, n: TransformNode) => n.position.copyFrom(v), + getTarget: (n: TransformNode) => n, + }, + rotation: { + type: "Quaternion", + get: (n: TransformNode) => n.rotationQuaternion ?? Quaternion.RotationYawPitchRoll(n.rotation.y, n.rotation.x, n.rotation.z), + set: (v: Quaternion, n: TransformNode) => { + if (!n.rotationQuaternion) { + n.rotationQuaternion = v.clone(); + } else { + n.rotationQuaternion.copyFrom(v); + } + }, + getTarget: (n: TransformNode) => n, + }, + scale: { + type: "Vector3", + get: (n: TransformNode) => n.scaling, + set: (v: Vector3, n: TransformNode) => n.scaling.copyFrom(v), + getTarget: (n: TransformNode) => n, + }, + matrix: { + type: "Matrix", + get: (n: TransformNode) => n.computeWorldMatrix(false), + getTarget: (n: TransformNode) => n, + isReadOnly: true, + }, + globalMatrix: { + type: "Matrix", + get: (n: TransformNode) => n.computeWorldMatrix(true), + getTarget: (n: TransformNode) => n, + isReadOnly: true, + }, + }, + }, + meshes: { + length: { + type: "number", + get: (arr: AbstractMesh[]) => arr.length, + getTarget: (arr: AbstractMesh[]) => arr, + }, + __array__: { + __target__: true, + name: { + type: "string", + get: (m: AbstractMesh) => m.name, + set: (v: string, m: AbstractMesh) => { + m.name = v; + }, + getTarget: (m: AbstractMesh) => m, + }, + visible: { + type: "boolean", + get: (m: AbstractMesh) => m.isVisible, + set: (v: boolean, m: AbstractMesh) => { + m.isVisible = v; + }, + getTarget: (m: AbstractMesh) => m, + }, + }, + }, + materials: { + length: { + type: "number", + get: (arr: Material[]) => arr.length, + getTarget: (arr: Material[]) => arr, + }, + __array__: { + __target__: true, + name: { + type: "string", + get: (m: Material) => m.name, + set: (v: string, m: Material) => { + m.name = v; + }, + getTarget: (m: Material) => m, + }, + }, + }, + }; +} diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/compositePathToObjectConverter.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/compositePathToObjectConverter.ts new file mode 100644 index 000000000000..a21b33caf715 --- /dev/null +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/compositePathToObjectConverter.ts @@ -0,0 +1,65 @@ +import { type IObjectInfo, type IPathToObjectConverter } from "core/ObjectModel/objectModelInterfaces"; + +/** + * Entry in the composite converter's prefix table. The first entry whose + * `prefix` matches the start of the path wins. + */ +export interface IPathConverterPrefixEntry { + /** Path prefix to match, e.g. `"/extensions/BABYLON_scene_objects/"`. */ + prefix: string; + /** Converter to delegate to when the path starts with `prefix`. */ + converter: IPathToObjectConverter; +} + +/** + * Composite path-to-object converter that dispatches by path prefix. + * + * The KHR_interactivity object model lives at the top of the JSON tree + * (`/nodes/...`, `/materials/...`, `/extensions/...`) and is resolved by + * `GLTFPathToObjectConverter` (see `gltfPathToObjectConverter`). + * + * Babylon-specific extensions can register additional namespaces here + * (for example `/extensions/BABYLON_scene_objects/...` for refs that point + * at scene objects not described by the source glTF) without forcing every + * caller to know about them — `FlowGraphJsonPointerParserBlock` and the + * existing template substitution machinery treat any path uniformly. + * + * Prefix entries are tried in order; if none matches, the fallback converter + * is used. The fallback is typically the glTF converter, since standard + * KHR_interactivity pointer paths sit at the JSON root and have no shared + * prefix that would distinguish them from a missing namespace. + */ +export class CompositePathToObjectConverter implements IPathToObjectConverter { + /** + * @param _prefixes prefix-keyed converter table, tried in order + * @param _fallback converter used when no prefix entry matches + */ + public constructor( + private _prefixes: IPathConverterPrefixEntry[], + private _fallback: IPathToObjectConverter + ) {} + + /** + * Adds a new prefix entry at the front of the lookup list so it is tried + * before any entries registered earlier. Useful for late-registered + * loader extensions that want to override or augment a previously + * registered namespace. + * @param entry the entry to add + */ + public addPrefix(entry: IPathConverterPrefixEntry): void { + this._prefixes.unshift(entry); + } + + /** + * @param path the JSON Pointer path to resolve + * @returns an object accessor for the resolved property + */ + public convert(path: string): IObjectInfo { + for (const { prefix, converter } of this._prefixes) { + if (path.startsWith(prefix)) { + return converter.convert(path); + } + } + return this._fallback.convert(path); + } +} diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/gltfPathToObjectConverter.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/gltfPathToObjectConverter.ts index ecc3858d69f3..1f0eba93a1dd 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/gltfPathToObjectConverter.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/gltfPathToObjectConverter.ts @@ -14,6 +14,18 @@ export const OptionalPathExceptionsList: { // get the node as object when reading an extension regex: new RegExp(`^/nodes/\\d+/extensions/`), }, + { + // weights may be undefined on nodes without morph targets + regex: new RegExp(`^/nodes/\\d+/weights`), + }, + { + // weights may be undefined on meshes without morph targets + regex: new RegExp(`^/meshes/\\d+/weights`), + }, + { + // KHR_texture_transform may not be present on texture info objects + regex: new RegExp(`/extensions/KHR_texture_transform/`), + }, ]; /** @@ -61,6 +73,15 @@ export class GLTFPathToObjectConverter implements const parts = path.split("/"); parts.shift(); + // KHR_interactivity Opaque-Reference spec uses a trailing slash to mean + // "ref to the resource itself" (e.g. `/animations/0/` is a ref to the + // animation). Drop the trailing empty segment so we resolve to the + // accessor for the resource rather than descending into a non-existent + // empty-named child. + if (parts.length > 0 && parts[parts.length - 1] === "") { + parts.pop(); + } + //if the last part has ".length" in it, separate that as an extra part if (parts[parts.length - 1].includes(".length")) { const lastPart = parts[parts.length - 1]; @@ -73,13 +94,27 @@ export class GLTFPathToObjectConverter implements for (const part of parts) { const isLength = part === "length"; - if (isLength && !infoTree.__array__) { - throw new Error(`Path ${path} is invalid`); + if (isLength) { + // For .length, check if the current level has a 'length' accessor + if (infoTree.length) { + infoTree = infoTree.length; + } else if (infoTree.__array__) { + // Fallback: length of an array that doesn't have explicit length accessor + throw new Error(`Path ${path} is invalid - no length accessor`); + } else { + throw new Error(`Path ${path} is invalid`); + } + // Set target to the current object tree (the array itself) + // Only update target if objectTree is defined, otherwise keep the last valid target + if (objectTree !== undefined) { + target = objectTree; + } + continue; } if (infoTree.__ignoreObjectTree__) { ignoreObjectTree = true; } - if (infoTree.__array__ && !isLength) { + if (infoTree.__array__) { infoTree = infoTree.__array__; } else { infoTree = infoTree[part]; @@ -87,20 +122,43 @@ export class GLTFPathToObjectConverter implements throw new Error(`Path ${path} is invalid`); } } - if (!ignoreObjectTree) { + // __passThroughTarget__: skip objectTree traversal for this property. + // The accessor functions will receive the parent target (e.g., the INode) + // instead of the child property (e.g., node.weights which may be undefined). + if (infoTree.__passThroughTarget__) { + ignoreObjectTree = true; + } else if (!ignoreObjectTree) { if (objectTree === undefined) { // check if the path is in the exception list. If it is, break and return the last object that was found const exception = OptionalPathExceptionsList.find((e) => e.regex.test(path)); if (!exception) { throw new Error(`Path ${path} is invalid`); } - } else if (!isLength) { + } else { objectTree = objectTree?.[part]; } + } else { + // When ignoring object tree traversal and encountering a numeric array index, + // wrap the accessor functions to pass the index through as the second argument. + const numericIndex = parseInt(part); + if (!isNaN(numericIndex) && typeof infoTree.get === "function") { + const orig = infoTree; + infoTree = { ...orig }; + infoTree.get = (target: any) => orig.get(target, numericIndex); + if (typeof orig.set === "function") { + infoTree.set = (value: any, target: any) => orig.set(value, target, numericIndex); + } + if (typeof orig.getTarget === "function") { + infoTree.getTarget = (target: any) => orig.getTarget(target, numericIndex); + } + } } - if (infoTree.__target__ || isLength) { - target = objectTree; + if (infoTree.__target__) { + // Only update target if objectTree is defined, otherwise keep the last valid target + if (objectTree !== undefined) { + target = objectTree; + } } } diff --git a/packages/dev/loaders/src/glTF/2.0/Extensions/objectModelMapping.ts b/packages/dev/loaders/src/glTF/2.0/Extensions/objectModelMapping.ts index 1daa1dc04a81..1bdb9def754a 100644 --- a/packages/dev/loaders/src/glTF/2.0/Extensions/objectModelMapping.ts +++ b/packages/dev/loaders/src/glTF/2.0/Extensions/objectModelMapping.ts @@ -1,6 +1,8 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { type TransformNode } from "core/Meshes/transformNode"; +import { type AbstractMesh } from "core/Meshes/abstractMesh"; +import { type MorphTargetManager } from "core/Morph/morphTargetManager"; import { type IAnimation, type ICamera, @@ -9,7 +11,10 @@ import { type IEXTLightsArea_Light, type IMaterial, type IMesh, + type IMeshPrimitive, type INode, + type IScene, + type ISkin, } from "../glTFLoaderInterfaces"; import { type Vector3, Matrix, Quaternion, Vector2 } from "core/Maths/math.vector"; import { Constants } from "core/Engines/constants"; @@ -26,22 +31,43 @@ import { type AnimationGroup } from "core/Animations/animationGroup"; import { type Mesh } from "core/Meshes/mesh"; import { type RectAreaLight } from "core/Lights/rectAreaLight"; +/** + * Top-level shape of the glTF Object Model accessor tree. Each property + * describes a navigable section of the JSON-Pointer namespace (e.g. `/nodes`, + * `/materials`, `/scenes`) that KHR_interactivity, KHR_animation_pointer and + * other extensions consume via {@link GetMappingForKey}. + */ export interface IGLTFObjectModelTree { + /** Read-only accessor for the active scene index (`/scene`). */ + scene: { __target__: boolean } & IObjectAccessor; + /** Accessor tree for `/cameras`. */ cameras: IGLTFObjectModelTreeCamerasObject; + /** Accessor tree for `/nodes`. */ nodes: IGLTFObjectModelTreeNodesObject; + /** Accessor tree for `/materials`. */ materials: IGLTFObjectModelTreeMaterialsObject; + /** Accessor tree for `/extensions` (root-level glTF extensions). */ extensions: IGLTFObjectModelTreeExtensionsObject; + /** Accessor tree for `/animations`. */ animations: { length: IObjectAccessor; __array__: {}; }; - meshes: { - length: IObjectAccessor; - __array__: {}; - }; + /** Accessor tree for `/meshes`. */ + meshes: IGLTFObjectModelTreeMeshesObject; + /** Accessor tree for `/scenes`. */ + scenes: IGLTFObjectModelTreeScenesObject; + /** Accessor tree for `/skins`. */ + skins: IGLTFObjectModelTreeSkinsObject; } +/** + * Accessor tree describing the `/nodes` section of the glTF Object Model. + * Exposes per-node TRS, ref-typed parent/children/camera/mesh/skin links, + * morph-target weights and node-extension properties. + */ export interface IGLTFObjectModelTreeNodesObject { + /** Number of nodes in the array. */ length: IObjectAccessor; __array__: { __target__: boolean; @@ -50,9 +76,22 @@ export interface IGLTFObjectModelTreeNodesObject; matrix: IObjectAccessor; globalMatrix: IObjectAccessor; + camera: IObjectAccessor; + mesh: IObjectAccessor; + skin: IObjectAccessor; + parent: IObjectAccessor; + children: { + length: IObjectAccessor; + __array__: { __target__: boolean } & IObjectAccessor; + }; weights: { + /** When true, the path converter skips objectTree traversal for this property, keeping the parent target. */ + __passThroughTarget__?: boolean; length: IObjectAccessor; - __array__: { __target__: boolean } & IObjectAccessor; + // The per-element target alternates between TransformNode (when the index is + // out of range or no morph host is reachable) and MorphTarget (when it is), + // so we widen the BabylonTargetType to `any` here. + __array__: { __target__: boolean } & IObjectAccessor; } & IObjectAccessor; extensions: { EXT_lights_ies?: { @@ -66,7 +105,13 @@ export interface IGLTFObjectModelTreeNodesObject; __array__: { __target__: boolean; orthographic: { @@ -84,9 +129,17 @@ export interface IGLTFObjectModelTreeCamerasObject { }; } +/** + * Accessor tree describing the `/materials` section of the glTF Object Model. + * Covers core PBR properties as well as the family of KHR_materials_* extensions. + */ export interface IGLTFObjectModelTreeMaterialsObject { + /** Number of materials in the array. */ + length: IObjectAccessor; __array__: { __target__: boolean; + doubleSided: IObjectAccessor; + alphaCutoff: IObjectAccessor; pbrMetallicRoughness: { baseColorFactor: IObjectAccessor; metallicFactor: IObjectAccessor>; @@ -245,9 +298,70 @@ interface ITextureDefinition { scale: IObjectAccessor; } -export interface IGLTFObjectModelTreeMeshesObject {} +/** + * Accessor tree describing the `/meshes` section of the glTF Object Model. + * Exposes per-mesh primitives (and their material refs) and the mesh-level + * morph-target weights array. + */ +export interface IGLTFObjectModelTreeMeshesObject { + /** Number of meshes in the array. */ + length: IObjectAccessor; + __array__: { + __target__: boolean; + primitives: { + length: IObjectAccessor; + __array__: { + __target__: boolean; + material: IObjectAccessor; + }; + }; + weights: { + length: IObjectAccessor; + __array__: { __target__: boolean } & IObjectAccessor; + }; + }; +} + +/** + * Accessor tree describing the `/scenes` section of the glTF Object Model. + * Per-scene root-node refs are exposed under `nodes/{i}`. + */ +export interface IGLTFObjectModelTreeScenesObject { + /** Number of scenes in the array. */ + length: IObjectAccessor; + __array__: { + __target__: boolean; + nodes: { + length: IObjectAccessor; + __array__: { __target__: boolean } & IObjectAccessor; + }; + }; +} + +/** + * Accessor tree describing the `/skins` section of the glTF Object Model. + * Joint and skeleton properties are exposed as JSON-Pointer refs. + */ +export interface IGLTFObjectModelTreeSkinsObject { + /** Number of skins in the array. */ + length: IObjectAccessor; + __array__: { + __target__: boolean; + joints: { + length: IObjectAccessor; + __array__: { __target__: boolean } & IObjectAccessor; + }; + skeleton: IObjectAccessor; + }; +} +/** + * Accessor tree describing root-level glTF extensions exposed through the + * Object Model. Currently covers the punctual / area / IES / image-based + * light extension families. + */ export interface IGLTFObjectModelTreeExtensionsObject { + /** Accessor tree for `/extensions/KHR_lights_punctual`. */ KHR_lights_punctual: { lights: { length: IObjectAccessor; @@ -263,6 +377,7 @@ export interface IGLTFObjectModelTreeExtensionsObject { }; }; }; + /** Accessor tree for `/extensions/EXT_lights_area`. */ EXT_lights_area: { lights: { length: IObjectAccessor; @@ -277,11 +392,13 @@ export interface IGLTFObjectModelTreeExtensionsObject { }; }; }; + /** Accessor tree for `/extensions/EXT_lights_ies`. */ EXT_lights_ies: { lights: { length: IObjectAccessor; }; }; + /** Accessor tree for `/extensions/EXT_lights_image_based`. */ EXT_lights_image_based: { lights: { __array__: { @@ -325,24 +442,74 @@ const nodesTree: IGLTFObjectModelTreeNodesObject = { getPropertyName: [() => "scaling"], }, weights: { + // Skip glTF objectTree traversal — weights may be undefined on the glTF node + // but accessible via the Babylon MorphTargetManager on the INode's meshes + __passThroughTarget__: true, length: { type: "number", - get: (node: INode) => node._numMorphTargets, - getTarget: (node: INode) => node._babylonTransformNode, - getPropertyName: [() => "influence"], + get: (node: INode) => { + // KHR_interactivity treats /nodes//weights.length as always-valid. + // Resolve via Babylon scene graph (direct or descendant) and fall back + // to 0 when nothing is reachable so the pointer/get reports isValid=true. + const found = _findNodeMorphTargets(node); + return found ? found.mtm.numTargets : 0; + }, + getTarget: (node: INode) => node?._babylonTransformNode, + getPropertyName: [() => "length"], }, __array__: { __target__: true, type: "number", - get: (node: INode, index?: number) => (index !== undefined ? node._primitiveBabylonMeshes?.[0].morphTargetManager?.getTarget(index).influence : undefined), - // set: (value: number, node: INode, index?: number) => node._babylonTransformNode?.getMorphTargetManager()?.getTarget(index)?.setInfluence(value), - getTarget: (node: INode) => node._babylonTransformNode, + get: (node: INode, index?: number) => { + const found = _findNodeMorphTargets(node); + if (found && index !== undefined && index >= 0 && index < found.mtm.numTargets) { + return _roundFloat32Artifact(found.mtm.getTarget(index).influence); + } + // KHR_interactivity treats /nodes//weights/ as valid (returning the + // type's default of 0) when the node has its own mesh or has reachable + // morph targets, and as invalid only when no mesh is reachable at all. + if (node?.mesh !== undefined || found) { + return 0; + } + return undefined; + }, + set: (value: any, node: INode, index?: number) => { + const numValue = typeof value === "number" ? value : typeof value?.value === "number" ? value.value : value; + const found = _findNodeMorphTargets(node); + if (!found || index === undefined || index < 0 || index >= found.mtm.numTargets) { + return; + } + // Fan out to every mesh that shares this morph target manager so + // multi-primitive meshes stay in sync. + for (const mesh of found.meshes) { + const target = mesh.morphTargetManager?.getTarget(index); + if (target) { + target.influence = numValue; + } + } + }, + getTarget: (node: INode, index?: number) => { + const found = _findNodeMorphTargets(node); + if (found && index !== undefined && index >= 0 && index < found.mtm.numTargets) { + return found.mtm.getTarget(index); + } + return node?._babylonTransformNode; + }, getPropertyName: [() => "influence"], }, type: "number[]", - get: (node: INode, index?: number) => [0], // TODO: get the weights correctly - // set: (value: number, node: INode, index?: number) => node._babylonTransformNode?.getMorphTargetManager()?.getTarget(index)?.setInfluence(value), - getTarget: (node: INode) => node._babylonTransformNode, + get: (node: INode) => { + const found = _findNodeMorphTargets(node); + if (!found) { + return []; + } + const weights: number[] = []; + for (let i = 0; i < found.mtm.numTargets; i++) { + weights.push(_roundFloat32Artifact(found.mtm.getTarget(i).influence)); + } + return weights; + }, + getTarget: (node: INode) => node?._babylonTransformNode, getPropertyName: [() => "influence"], }, // readonly! @@ -378,6 +545,51 @@ const nodesTree: IGLTFObjectModelTreeNodesObject = { getTarget: (node: INode) => node._babylonTransformNode, isReadOnly: true, }, + camera: { + type: "string", + // Per KHR_interactivity Object Model: read-only ref pointing to the + // attached camera, encoded as a JSON Pointer string. Empty string + // when no camera is attached (the spec's null-ref convention). + get: (node: INode) => (node.camera !== undefined ? `/cameras/${node.camera}/` : ""), + getTarget: (node: INode) => node, + isReadOnly: true, + }, + mesh: { + type: "string", + get: (node: INode) => (node.mesh !== undefined ? `/meshes/${node.mesh}/` : ""), + getTarget: (node: INode) => node, + isReadOnly: true, + }, + skin: { + type: "string", + get: (node: INode) => (node.skin !== undefined ? `/skins/${node.skin}/` : ""), + getTarget: (node: INode) => node, + isReadOnly: true, + }, + parent: { + type: "string", + get: (node: INode) => (node.parent && node.parent.index !== undefined ? `/nodes/${node.parent.index}/` : ""), + getTarget: (node: INode) => node, + isReadOnly: true, + }, + children: { + length: { + type: "number", + get: (children: number[]) => children?.length ?? 0, + getTarget: (children: number[]) => children ?? [], + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + type: "string", + // The wrapping converter passes the indexed child value (an + // INode index) as `childIndex`; convert it to a JSON Pointer + // ref string so ref/eq comparisons work as authored. + get: (childIndex: any) => (typeof childIndex === "number" ? `/nodes/${childIndex}/` : ""), + getTarget: () => ({ __nodeIndex: true }), + isReadOnly: true, + }, + }, extensions: { EXT_lights_ies: { multiplier: { @@ -436,20 +648,74 @@ const animationsTree = { getTarget: (animations: IAnimation[]) => animations.map((animation) => animation._babylonAnimationGroup!), getPropertyName: [() => "length"], }, - __array__: {}, + __array__: { + // Indexed access to the animation. KHR_interactivity Opaque-Reference + // spec defines the trailing-slash form ``/animations//`` as a ref + // to the animation itself; we surface that here as a JSON-Pointer ref + // string so blocks like ``animation/start`` can consume it directly. + // Use the animation's own ``index`` property (populated by the loader's + // ArrayItem.Assign step) so the ref is resolved without needing a + // separate index payload from the path converter. + __target__: true, + type: "string", + get: (animation: IAnimation) => (animation && typeof animation.index === "number" ? `/animations/${animation.index}/` : ""), + getTarget: (animation: IAnimation) => animation._babylonAnimationGroup, + isReadOnly: true, + }, }; -const meshesTree = { +const meshesTree: IGLTFObjectModelTreeMeshesObject = { length: { type: "number", get: (meshes: IMesh[]) => meshes.length, getTarget: (meshes: IMesh[]) => meshes.map((mesh) => mesh.primitives[0]._instanceData?.babylonSourceMesh), getPropertyName: [() => "length"], }, - __array__: {}, + __array__: { + __target__: true, + primitives: { + length: { + type: "number", + get: (primitives: IMeshPrimitive[]) => primitives?.length ?? 0, + getTarget: (primitives: IMeshPrimitive[]) => primitives ?? [], + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + material: { + type: "string", + // Read-only ref to the assigned material, JSON Pointer encoded. + get: (primitive: IMeshPrimitive) => (primitive.material !== undefined ? `/materials/${primitive.material}/` : ""), + getTarget: (primitive: IMeshPrimitive) => primitive, + isReadOnly: true, + }, + }, + }, + weights: { + length: { + type: "number", + get: (weights: number[]) => weights?.length ?? 0, + getTarget: (weights: number[]) => weights ?? [], + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + type: "number", + get: (weightValue: any) => weightValue, + getTarget: () => ({ __weightValue: true }), + isReadOnly: true, + }, + }, + }, }; const camerasTree: IGLTFObjectModelTreeCamerasObject = { + length: { + type: "number", + get: (cameras: ICamera[]) => cameras.length, + getTarget: (cameras: ICamera[]) => cameras.map((camera) => camera._babylonCamera!), + getPropertyName: [() => "length"], + }, __array__: { __target__: true, orthographic: { @@ -548,8 +814,38 @@ const camerasTree: IGLTFObjectModelTreeCamerasObject = { }; const materialsTree: IGLTFObjectModelTreeMaterialsObject = { + length: { + type: "number", + get: (materials: IMaterial[]) => materials.length, + getTarget: (materials: IMaterial[]) => materials.map((material) => material._data?.[Constants.MATERIAL_TriangleFillMode]?.babylonMaterial as PBRMaterial), + getPropertyName: [() => "length"], + }, __array__: { __target__: true, + doubleSided: { + type: "boolean", + get: (material, index?, payload?) => !GetMaterial(material, index, payload)?.backFaceCulling, + set: (value: boolean, material, index?, payload?) => { + const mat = GetMaterial(material, index, payload); + if (mat) { + mat.backFaceCulling = !value; + } + }, + getTarget: (material, index?, payload?) => GetMaterial(material, index, payload), + getPropertyName: [() => "backFaceCulling"], + }, + alphaCutoff: { + type: "number", + get: (material, index?, payload?) => GetMaterial(material, index, payload)?.alphaCutOff, + set: (value: number, material, index?, payload?) => { + const mat = GetMaterial(material, index, payload); + if (mat) { + mat.alphaCutOff = value; + } + }, + getTarget: (material, index?, payload?) => GetMaterial(material, index, payload), + getPropertyName: [() => "alphaCutOff"], + }, emissiveFactor: { type: "Color3", get: (material, index?, payload?) => GetMaterial(material, index, payload).emissiveColor, @@ -660,9 +956,12 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, anisotropyRotation: { type: "number", - get: (material, index?, payload?) => GetMaterial(material, index, payload).anisotropy.angle, + get: (material, index?, payload?) => GetMaterial(material, index, payload)?.anisotropy?.angle, set: (value: number, material, index?, payload?) => { - GetMaterial(material, index, payload).anisotropy.angle = value; + const mat = GetMaterial(material, index, payload); + if (mat) { + mat.anisotropy.angle = value; + } }, getTarget: (material, index?, payload?) => GetMaterial(material, index, payload), getPropertyName: [() => "anisotropy.angle"], @@ -794,7 +1093,7 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, sheenRoughnessTexture: { extensions: { - KHR_texture_transform: GenerateTextureMap("sheen", "thicknessTexture"), + KHR_texture_transform: GenerateTextureMap("sheen", "textureRoughness"), }, }, }, @@ -834,7 +1133,10 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, transmissionTexture: { extensions: { - KHR_texture_transform: GenerateTextureMap("subSurface", "refractionIntensityTexture"), + KHR_texture_transform: GenerateTextureMap("subSurface", "refractionIntensityTexture", { + extensionKey: "KHR_materials_transmission", + texturePath: ["transmissionTexture"], + }), }, }, }, @@ -847,7 +1149,10 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, diffuseTransmissionTexture: { extensions: { - KHR_texture_transform: GenerateTextureMap("subSurface", "translucencyIntensityTexture"), + KHR_texture_transform: GenerateTextureMap("subSurface", "translucencyIntensityTexture", { + extensionKey: "KHR_materials_diffuse_transmission", + texturePath: ["diffuseTransmissionTexture"], + }), }, }, diffuseTransmissionColorFactor: { @@ -858,7 +1163,10 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, diffuseTransmissionColorTexture: { extensions: { - KHR_texture_transform: GenerateTextureMap("subSurface", "translucencyColorTexture"), + KHR_texture_transform: GenerateTextureMap("subSurface", "translucencyColorTexture", { + extensionKey: "KHR_materials_diffuse_transmission", + texturePath: ["diffuseTransmissionColorTexture"], + }), }, }, }, @@ -883,7 +1191,7 @@ const materialsTree: IGLTFObjectModelTreeMaterialsObject = { }, thicknessTexture: { extensions: { - KHR_texture_transform: GenerateTextureMap("subSurface", "thicknessTexture"), + KHR_texture_transform: GenerateTextureMap("subSurface", "thicknessTexture", { extensionKey: "KHR_materials_volume", texturePath: ["thicknessTexture"] }), }, }, }, @@ -1045,7 +1353,191 @@ function GetTexture(material: IMaterial, payload: any, textureType: keyof PBRMat function GetMaterial(material: IMaterial, _index?: number, payload?: any) { return material._data?.[payload?.fillMode ?? Constants.MATERIAL_TriangleFillMode]?.babylonMaterial as PBRMaterial; } -function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: string): ITextureDefinition { +function _getNodeMorphTargetManager(node: INode): any { + const tn = node?._babylonTransformNode; + if (!tn) { + return undefined; + } + // Single primitive: transform node IS the mesh with morphTargetManager + if ((tn as any).morphTargetManager) { + return (tn as any).morphTargetManager; + } + // Multiple primitives: check each primitive mesh and its source + const primMeshes = node._primitiveBabylonMeshes; + if (primMeshes) { + for (const mesh of primMeshes) { + if (mesh?.morphTargetManager) { + return mesh.morphTargetManager; + } + // Check source mesh for instanced meshes + if ((mesh as any)?.sourceMesh?.morphTargetManager) { + return (mesh as any).sourceMesh.morphTargetManager; + } + } + } + return undefined; +} + +/** + * Result of a morph-target lookup: the active manager plus every Babylon mesh + * that shares it, so writes (set) can fan out across all primitives. + */ +interface IMorphTargetLookup { + mtm: MorphTargetManager; + meshes: AbstractMesh[]; +} + +/** + * KHR_interactivity test assets routinely use a glTF hierarchy where the + * **parent** node has no `mesh` but a descendant does. The Khronos morph-weight + * tests query `/nodes//weights/*` and expect the result to come from + * the morph targets of the first descendant mesh. To support this we walk the + * Babylon-side scene graph below the queried INode looking for a Mesh that has + * a morphTargetManager. + * + * For multi-primitive meshes (one INode → several Babylon meshes parented to a + * wrapper TransformNode) we also collect the sibling primitives so a `set` + * touches every mesh that shares the manager. + * @param node the glTF node to start the lookup from + * @returns the active morph target manager and every Babylon mesh that shares + * it, or `undefined` when no morph target manager is reachable from the node. + */ +function _findNodeMorphTargets(node: INode): IMorphTargetLookup | undefined { + const tn = node?._babylonTransformNode; + if (!tn) { + return undefined; + } + // Direct: this node's own mesh has a morph target manager. + const directMtm: MorphTargetManager | undefined = _getNodeMorphTargetManager(node); + if (directMtm && node._primitiveBabylonMeshes && node._primitiveBabylonMeshes.length > 0) { + return { mtm: directMtm, meshes: node._primitiveBabylonMeshes as AbstractMesh[] }; + } + // Fallback: search descendants in the Babylon scene graph for the first + // mesh that has a morph target manager, then collect every sibling mesh + // that shares it (covers the multi-primitive case). + const descendants = tn.getDescendants(false); + for (const desc of descendants) { + const candidate = desc as AbstractMesh; + const mtm: MorphTargetManager | undefined = (candidate as any).morphTargetManager ?? (candidate as any).sourceMesh?.morphTargetManager; + if (!mtm) { + continue; + } + const meshes: AbstractMesh[] = []; + const parent = candidate.parent; + if (parent) { + for (const sib of parent.getChildMeshes(true)) { + const sibMtm: MorphTargetManager | undefined = (sib as any).morphTargetManager ?? (sib as any).sourceMesh?.morphTargetManager; + if (sibMtm === mtm) { + meshes.push(sib); + } + } + } + if (meshes.length === 0) { + meshes.push(candidate); + } + return { mtm, meshes }; + } + return undefined; +} + +/** + * Collapse float32-precision artifacts back to the closest "clean" double. + * + * glTF stores numbers as JSON, but tools usually serialize float32 morph weights + * with their full double-precision text — `0.1` becomes `0.10000000149011612`. + * KHR_interactivity tests then compare the read-back weight via strict `math/eq` + * against literals like `0.1`, which fails because the two doubles aren't bitwise + * equal. Rounding the value to 7 significant figures (the precision of a float32) + * recovers the original "clean" double for any value that survived a float32 + * round-trip while leaving genuinely high-precision doubles essentially intact. + * @param v the value to round + * @returns the rounded value, or the input unchanged if it is not finite + */ +function _roundFloat32Artifact(v: number): number { + if (!Number.isFinite(v)) { + return v; + } + return parseFloat(v.toPrecision(7)); +} +/** + * Coordinate where a Babylon `Texture` lives on a PBRMaterial vs where its + * KHR_texture_transform definition lives in the source glTF JSON. Used by + * {@link GenerateTextureMap} to provide a glTF-side fallback when the loader + * extension that owns the texture decided not to materialise it on the + * Babylon material (e.g. KHR_materials_volume early-returns when there is no + * `thicknessFactor`, leaving `subSurface.thicknessTexture` null even though + * the glTF JSON has the texture+transform fully defined). + * + * The fallback gives KHR_interactivity `pointer/get` and `pointer/set` a + * place to read/write the transform values so round-trip tests succeed even + * when the texture is not yet active in the renderer. If/when the loader + * later activates the texture, the seeded values can be picked up from the + * glTF JSON. + */ +interface IGltfTextureTransformPath { + /** + * The path of nested keys under `material.extensions.` to reach the + * texture-info object. For example `["thicknessTexture"]` for + * `KHR_materials_volume.thicknessTexture`. + */ + extensionKey: string; + texturePath: string[]; +} + +/** + * Read the KHR_texture_transform object stored in the source glTF JSON for a + * texture-info that lives under one of the material's extensions, creating + * empty parent objects on demand so callers can write through it. Returns + * `undefined` when the input shape is incompatible. + * @param material the source IMaterial owning the extension + * @param gltfPath the path describing where the texture-info lives + * @param createMissing when true, create missing parent objects so writes succeed + * @returns the glTF-side KHR_texture_transform object, or undefined + */ +function _gltfTextureTransform(material: IMaterial, gltfPath: IGltfTextureTransformPath, createMissing: boolean): any | undefined { + if (!material) { + return undefined; + } + if (createMissing && !material.extensions) { + (material as any).extensions = {}; + } + let cursor: any = material.extensions?.[gltfPath.extensionKey]; + if (!cursor) { + if (!createMissing) { + return undefined; + } + cursor = {}; + (material as any).extensions[gltfPath.extensionKey] = cursor; + } + for (const key of gltfPath.texturePath) { + let next = cursor[key]; + if (!next) { + if (!createMissing) { + return undefined; + } + next = {}; + cursor[key] = next; + } + cursor = next; + } + if (!cursor.extensions) { + if (!createMissing) { + return undefined; + } + cursor.extensions = {}; + } + let xform = cursor.extensions.KHR_texture_transform; + if (!xform) { + if (!createMissing) { + return undefined; + } + xform = {}; + cursor.extensions.KHR_texture_transform = xform; + } + return xform; +} + +function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: string, gltfPath?: IGltfTextureTransformPath): ITextureDefinition { return { offset: { componentsCount: 2, @@ -1053,12 +1545,29 @@ function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: st type: "Vector2", get: (material, _index?, payload?) => { const texture = GetTexture(material, payload, textureType, textureInObject); - return new Vector2(texture?.uOffset, texture?.vOffset); + if (texture) { + return new Vector2(texture.uOffset, texture.vOffset); + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, false); + const o = xform?.offset; + return new Vector2(o?.[0] ?? 0, o?.[1] ?? 0); + } + return new Vector2(0, 0); }, getTarget: GetMaterial, set: (value, material, _index?, payload?) => { const texture = GetTexture(material, payload, textureType, textureInObject); - ((texture.uOffset = value.x), (texture.vOffset = value.y)); + if (texture) { + texture.uOffset = value.x; + texture.vOffset = value.y; + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, true); + if (xform) { + xform.offset = [value.x, value.y]; + } + } }, getPropertyName: [ () => `${textureType}${textureInObject ? "." + textureInObject : ""}.uOffset`, @@ -1067,9 +1576,30 @@ function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: st }, rotation: { type: "number", - get: (material, _index?, payload?) => GetTexture(material, payload, textureType, textureInObject)?.wAng, + get: (material, _index?, payload?) => { + const texture = GetTexture(material, payload, textureType, textureInObject); + if (texture) { + return texture.wAng; + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, false); + return xform?.rotation ?? 0; + } + return 0; + }, getTarget: GetMaterial, - set: (value, material, _index?, payload?) => (GetTexture(material, payload, textureType, textureInObject).wAng = value), + set: (value, material, _index?, payload?) => { + const texture = GetTexture(material, payload, textureType, textureInObject); + if (texture) { + texture.wAng = value; + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, true); + if (xform) { + xform.rotation = value; + } + } + }, getPropertyName: [() => `${textureType}${textureInObject ? "." + textureInObject : ""}.wAng`], }, scale: { @@ -1077,12 +1607,29 @@ function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: st type: "Vector2", get: (material, _index?, payload?) => { const texture = GetTexture(material, payload, textureType, textureInObject); - return new Vector2(texture?.uScale, texture?.vScale); + if (texture) { + return new Vector2(texture.uScale, texture.vScale); + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, false); + const s = xform?.scale; + return new Vector2(s?.[0] ?? 1, s?.[1] ?? 1); + } + return new Vector2(1, 1); }, getTarget: GetMaterial, set: (value, material, index?, payload?) => { const texture = GetTexture(material, payload, textureType, textureInObject); - ((texture.uScale = value.x), (texture.vScale = value.y)); + if (texture) { + texture.uScale = value.x; + texture.vScale = value.y; + } + if (gltfPath) { + const xform = _gltfTextureTransform(material, gltfPath, true); + if (xform) { + xform.scale = [value.x, value.y]; + } + } }, getPropertyName: [ () => `${textureType}${textureInObject ? "." + textureInObject : ""}.uScale`, @@ -1092,13 +1639,91 @@ function GenerateTextureMap(textureType: keyof PBRMaterial, textureInObject?: st }; } +const scenesTree: IGLTFObjectModelTreeScenesObject = { + length: { + type: "number", + get: (scenes: IScene[]) => scenes.length, + getTarget: (scenes: IScene[]) => scenes, + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + nodes: { + length: { + type: "number", + get: (nodes: number[]) => nodes?.length ?? 0, + getTarget: (nodes: number[]) => nodes ?? [], + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + type: "string", + // Indexed scene root: the underlying value is the INode index; + // KHR_interactivity expects a ref-typed JSON Pointer string. + get: (nodeIndex: any) => (typeof nodeIndex === "number" ? `/nodes/${nodeIndex}/` : ""), + getTarget: () => ({ __nodeIndex: true }), + isReadOnly: true, + }, + }, + }, +}; + +const skinsTree: IGLTFObjectModelTreeSkinsObject = { + length: { + type: "number", + get: (skins: ISkin[]) => skins.length, + getTarget: (skins: ISkin[]) => skins.map((skin) => skin._data?.babylonSkeleton), + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + joints: { + length: { + type: "number", + get: (joints: number[]) => joints?.length ?? 0, + getTarget: (joints: number[]) => joints ?? [], + getPropertyName: [() => "length"], + }, + __array__: { + __target__: true, + type: "string", + // Indexed skin joint: returns a ref to the joint node. + get: (jointIndex: any) => (typeof jointIndex === "number" ? `/nodes/${jointIndex}/` : ""), + getTarget: () => ({ __nodeIndex: true }), + isReadOnly: true, + }, + }, + skeleton: { + type: "string", + // Skin's skeleton root: returns a ref to the root node, or empty + // (null ref) when no skeleton root is declared. + get: (skin: ISkin) => { + const skeleton = (skin as any).skeleton; + return typeof skeleton === "number" ? `/nodes/${skeleton}/` : ""; + }, + getTarget: (skin: ISkin) => skin, + isReadOnly: true, + }, + }, +}; + const objectModelMapping: IGLTFObjectModelTree = { + scene: { + __target__: true, + type: "number", + get: (sceneIndex: any) => sceneIndex ?? 0, + getTarget: () => ({ __gltfRoot: true }), + isReadOnly: true, + getPropertyName: [() => "scene"], + }, cameras: camerasTree, nodes: nodesTree, materials: materialsTree, extensions: extensionsTree, animations: animationsTree, meshes: meshesTree, + scenes: scenesTree, + skins: skinsTree, }; /** diff --git a/packages/public/glTF2Interface/babylon.glTF2Interface.d.ts b/packages/public/glTF2Interface/babylon.glTF2Interface.d.ts index 461d34bf1e16..11fa921d8b4f 100644 --- a/packages/public/glTF2Interface/babylon.glTF2Interface.d.ts +++ b/packages/public/glTF2Interface/babylon.glTF2Interface.d.ts @@ -1529,11 +1529,11 @@ declare namespace BABYLON.GLTF2 { */ type NodeIndex = number; /** - * Value types supported (in js it is either boolean or number) + * Value types supported (in js it is either boolean or number, or string for opaque references) */ - type ValueType = (boolean | number)[]; + type ValueType = (boolean | number | string)[]; - type ValueSignature = "bool" | "float" | "float2" | "float3" | "float4" | "float2x2" | "float3x3" | "float4x4" | "int" | "custom"; + type ValueSignature = "bool" | "float" | "float2" | "float3" | "float4" | "float2x2" | "float3x3" | "float4x4" | "int" | "ref" | "custom"; type ConfigurationValueType = (boolean | number | string)[]; diff --git a/packages/tools/sandbox/src/main.ts b/packages/tools/sandbox/src/main.ts index a5f4dcaf717d..4b8bf5f85fd2 100644 --- a/packages/tools/sandbox/src/main.ts +++ b/packages/tools/sandbox/src/main.ts @@ -8,9 +8,9 @@ * the Show args and dispatches a "babylonSandboxReady" event that we pick up * here to start the sandbox with the correct version info. */ -// Register GLTF/GLB loader (2.0 sub-loader assigns GLTFFileLoader._CreateGLTF2Loader). -// The sandbox loads .glb/.gltf files directly via SceneLoader. -import "loaders/glTF/2.0/glTFLoader"; +// Register GLTF/GLB loader and all glTF 2.0 extensions (side-effect imports +// register each KHR_/EXT_ extension via registerGLTFExtension). +import "loaders/glTF/2.0"; import { Sandbox } from "./sandbox"; const HostElement = document.getElementById("host-element") as HTMLElement; diff --git a/packages/tools/sandbox/src/sandbox.tsx b/packages/tools/sandbox/src/sandbox.tsx index f1dfe9ad97eb..620a4a05edba 100644 --- a/packages/tools/sandbox/src/sandbox.tsx +++ b/packages/tools/sandbox/src/sandbox.tsx @@ -16,6 +16,7 @@ import { Color3, Color4 } from "core/Maths/math"; import { FilesInputStore } from "core/Misc/filesInputStore"; import "./scss/main.scss"; +import "core/Loading/loadingScreen"; import fullScreenLogo from "./img/logo-fullscreen.svg"; import { type AbstractEngine } from "core/Engines/abstractEngine"; import { ImageProcessingConfiguration } from "core/Materials/imageProcessingConfiguration";