fix: Handle nested braces in expression parsing

packages/@n8n/codemirror-lang/package.json

@@ -38,6 +38,9 @@
     "format": "biome format --write src test",
     "format:check": "biome ci src test"
   },
+  "dependencies": {
+    "esprima-next": "5.8.4"
+  },
   "peerDependencies": {
     "@codemirror/language": "catalog:",
     "@lezer/highlight": "catalog:",

packages/@n8n/codemirror-lang/src/expressions/expressions.grammar

@@ -4,18 +4,10 @@ entity { Plaintext | Resolvable }
 
 @tokens {
   Plaintext { ![{] Plaintext? | "{" (@eof | ![{] Plaintext?) }
+}
 
-  OpenMarker[closedBy="CloseMarker"] { "{{" }
-
-  CloseMarker[openedBy="OpenMarker"] { "}}" }
-
-  Resolvable {
-    OpenMarker resolvableChar* CloseMarker
-  }
-
-  resolvableChar { unicodeChar | "}" ![}] | "\\}}" }
-
-	unicodeChar { $[\u0000-\u007C] | $[\u007E-\u{10FFFF}] }
+@external tokens tokens from "./tokens" {
+  Resolvable
 }
 
 @detectDelim

packages/@n8n/codemirror-lang/src/expressions/grammar.terms.ts

@@ -1,4 +1,4 @@
 // This file was generated by lezer-generator. You probably shouldn't edit it.
-export const Program = 1,
-	Plaintext = 2,
-	Resolvable = 3;
+export const Resolvable = 1,
+	Program = 2,
+	Plaintext = 3;

packages/@n8n/codemirror-lang/src/expressions/grammar.ts

@@ -1,17 +1,18 @@
 // This file was generated by lezer-generator. You probably shouldn't edit it.
 import { LRParser } from '@lezer/lr';
+import { tokens } from './tokens';
 export const parser = LRParser.deserialize({
 	version: 14,
-	states: "nQQOPOOOOOO'#Cb'#CbOOOO'#C`'#C`QQOPOOOOOO-E6^-E6^",
-	stateData: 'Y~OQPORPO~O',
+	states: "nQQOROOOOOQ'#Cb'#CbOOOQ'#C`'#C`QQOROOOOOQ-E6^-E6^",
+	stateData: 'Y~OPPORPO~O',
 	goto: 'bVPPPPWP^QRORSRTQOR',
-	nodeNames: '⚠ Program Plaintext Resolvable',
+	nodeNames: '⚠ Resolvable Program Plaintext',
 	maxTerm: 6,
 	skippedNodes: [0],
 	repeatNodeCount: 1,
 	tokenData:
-		"%o~RTO#ob#o#p!h#p;'Sb;'S;=`!]<%lOb~gTQ~O#ob#o#pv#p;'Sb;'S;=`!]<%lOb~yUO#ob#p;'Sb;'S;=`!]<%l~b~Ob~~!c~!`P;=`<%lb~!hOQ~~!kVO#ob#o#p#Q#p;'Sb;'S;=`!]<%l~b~Ob~~!c~#TVO#O#Q#O#P#j#P#q#Q#q#r%Q#r;'S#Q;'S;=`%i<%lO#Q~#mVO#O#Q#O#P#j#P#q#Q#q#r$S#r;'S#Q;'S;=`%i<%lO#Q~$VTO#q#Q#q#r$f#r;'S#Q;'S;=`%i<%lO#Q~$kVR~O#O#Q#O#P#j#P#q#Q#q#r%Q#r;'S#Q;'S;=`%i<%lO#Q~%TTO#q#Q#q#r%d#r;'S#Q;'S;=`%i<%lO#Q~%iOR~~%lP;=`<%l#Q",
-	tokenizers: [0],
-	topRules: { Program: [0, 1] },
+		"!h~RTO#ob#o#pv#p;'Sb;'S;=`!]<%lOb~gTR~O#ob#o#pv#p;'Sb;'S;=`!]<%lOb~yUO#ob#p;'Sb;'S;=`!]<%l~b~Ob~~!c~!`P;=`<%lb~!hOR~",
+	tokenizers: [0, tokens],
+	topRules: { Program: [0, 2] },
 	tokenPrec: 0,
 });

packages/@n8n/codemirror-lang/src/expressions/tokens.ts

@@ -0,0 +1,96 @@
+import { ExternalTokenizer } from '@lezer/lr';
+import { parseScript } from 'esprima-next';
+
+import { Resolvable } from './grammar.terms';
+
+const BACKSLASH = 92;
+const BRACE_LEFT = 123;
+const BRACE_RIGHT = 125;
+const EXPRESSION_STATEMENT = 'ExpressionStatement';
+const EMPTY_STATEMENT = 'EmptyStatement';
+
+const parseCandidateProgram = (source: string) => {
+	try {
+		return parseScript(source);
+	} catch {
+		return null;
+	}
+};
+
+const isSentinelExpressionProgram = (source: string) => {
+	const program = parseCandidateProgram(source);
+	const lastType = String(program?.body.at(-1)?.type);
+	return (
+		program !== null &&
+		program.body.length >= 2 &&
+		lastType === EXPRESSION_STATEMENT &&
+		program.body
+			.slice(0, -1)
+			.every(
+				(statement) =>
+					String(statement.type) === EMPTY_STATEMENT ||
+					String(statement.type) === EXPRESSION_STATEMENT,
+			)
+	);
+};
+
+const isParsableExpressionCandidate = (source: string, cache: Map<string, boolean>) => {
+	const cached = cache.get(source);
+	if (cached !== undefined) {
+		return cached;
+	}
+
+	const result =
+		isSentinelExpressionProgram(`${source};0`) || isSentinelExpressionProgram(`(${source});0`);
+
+	cache.set(source, result);
+	return result;
+};
+
+export const tokens = new ExternalTokenizer((input) => {
+	if (input.next !== BRACE_LEFT || input.peek(1) !== BRACE_LEFT) {
+		return;
+	}
+
+	input.advance(2);
+
+	let code = '';
+	let firstFallbackEnd: number | null = null;
+	const parseableExpressionCache = new Map<string, boolean>();
+
+	for (;;) {
+		const char = Number(input.next);
+
+		if (char < 0) {
+			if (firstFallbackEnd !== null) {
+				input.acceptTokenTo(Resolvable, firstFallbackEnd);
+			}
+			return;
+		}
+
+		const nextChar = Number(input.peek(1));
+
+		if (char === BACKSLASH && nextChar === BRACE_RIGHT && Number(input.peek(2)) === BRACE_RIGHT) {
+			code += '\\}}';
+			input.advance(3);
+			continue;
+		}
+
+		if (char === BRACE_RIGHT && nextChar === BRACE_RIGHT) {
+			const candidateEnd = input.pos + 2;
+			firstFallbackEnd ??= candidateEnd;
+
+			if (isParsableExpressionCandidate(code, parseableExpressionCache)) {
+				input.acceptTokenTo(Resolvable, candidateEnd);
+				return;
+			}
+
+			code += '}';
+			input.advance();
+			continue;
+		}
+
+		code += String.fromCharCode(char);
+		input.advance();
+	}
+});

packages/@n8n/codemirror-lang/test/expressions/cases.txt

@@ -158,6 +158,110 @@ Program(Resolvable)
 
 Program(Resolvable)
 
+# Resolvable containing object literal with nested braces
+
+{{ {values:{}} }}
+
+==>
+
+Program(Resolvable)
+
+# Resolvable containing bare object literal ending at the closing marker
+
+{{ {a:1}}}
+
+==>
+
+Program(Resolvable)
+
+# Resolvable containing nested object literals
+
+{{ {values:{ nested: {} }} }}
+
+==>
+
+Program(Resolvable)
+
+# Resolvable containing nested object literals with multiple internal closing pairs
+
+{{ {a:{c:{}}, b:{}} }}
+
+==>
+
+Program(Resolvable)
+
+# Resolvable containing regex literal followed by plaintext
+
+{{ /[/*]/.test(x) }} tail
+
+==>
+
+Program(Resolvable,Plaintext)
+
+# Resolvable containing string literal with left brace followed by another expression
+
+{{ "{foo" }} and {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
+# Resolvable containing regex literal with left brace followed by another expression
+
+{{ /{/.test(x) }} and {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
+# Resolvable containing comment with left brace followed by another expression
+
+{{ /* { */ 1 }} and {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
+# Resolvable containing string literal with closing braces followed by another expression
+
+{{ "{{foo}}" }} and {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
+# Resolvable containing bare object literal followed by plaintext
+
+{{ {a:1}}} tail
+
+==>
+
+Program(Resolvable,Plaintext)
+
+# Resolvable containing bare object literal followed by plaintext and another expression
+
+{{ {a:1}}} tail {{ $json.n }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
+# Resolvable containing bare object literal with classic function body followed by plaintext
+
+{{ {foo:function(){return 1}}}} tail
+
+==>
+
+Program(Resolvable,Plaintext)
+
+# Resolvable containing bare object literal with method body followed by plaintext and another expression
+
+{{ {foo(){return 1}}}} tail {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)
+
 # Resolvable containing double-brace-wrapped text with escaping
 
 {{ he {{ abc \}} llo }}
@@ -324,3 +428,11 @@ Program(Resolvable)
 ==>
 
 Program(Resolvable)
+
+# Line comments containing closing braces do not swallow following expressions
+{{ // }}
+1 }} and {{ 2 }}
+
+==>
+
+Program(Resolvable,Plaintext,Resolvable)