From c3b659de6573ef8dc56b5770f1d5d6f03ce0959a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:26:43 +0000 Subject: [PATCH 1/6] Initial plan From ab1643f2981794ac237bd5376928332ed1f4fcca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:32:37 +0000 Subject: [PATCH 2/6] Fix nested curly braces in template strings by tracking brace depth Co-authored-by: TwitchBronBron <2544493+TwitchBronBron@users.noreply.github.com> --- src/lexer/Lexer.spec.ts | 68 +++++++++++++++++++++++++++++++++++++++++ src/lexer/Lexer.ts | 25 +++++++++++++-- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 6dfaf2c3f..f0a5eb7a6 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -796,6 +796,74 @@ describe('lexer', () => { TokenKind.Eof ]); }); + + it('handles nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles deeply nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{a: {b: 1}}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, // a + TokenKind.Colon, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, // b + TokenKind.Colon, + TokenKind.IntegerLiteral, // 1 + TokenKind.RightCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles mixed expressions with nested braces', () => { + let tokens = Lexer.scan('thing = `${arr[0]} and ${{key: value}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.Identifier, // arr + TokenKind.LeftSquareBracket, + TokenKind.IntegerLiteral, // 0 + TokenKind.RightSquareBracket, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, // " and " + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, // key + TokenKind.Colon, + TokenKind.Identifier, // value + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); }); // string literals describe('double literals', () => { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 6f45df185..9d2f03d40 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -612,9 +612,28 @@ export class Lexer { this.advance(); this.advance(); this.addToken(TokenKind.TemplateStringExpressionBegin); - while (!this.isAtEnd() && !this.check('}')) { - this.start = this.current; - this.scanToken(); + let braceDepth = 0; + while (!this.isAtEnd()) { + if (this.check('}')) { + if (braceDepth === 0) { + // This is the closing brace for the template expression + break; + } else { + // This is a regular right brace inside the expression + braceDepth--; + this.start = this.current; + this.advance(); + this.addToken(TokenKind.RightCurlyBrace); + } + } else if (this.check('{')) { + // Track nested braces + braceDepth++; + this.start = this.current; + this.scanToken(); + } else { + this.start = this.current; + this.scanToken(); + } } if (this.check('}')) { this.advance(); From 3829c165b4294f535e9189d272f73d43b6c3ae1e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 18:19:23 +0000 Subject: [PATCH 3/6] Refactor nested brace handling to use state-based approach Co-authored-by: TwitchBronBron <2544493+TwitchBronBron@users.noreply.github.com> --- src/lexer/Lexer.ts | 82 +++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 9d2f03d40..a15d0e65f 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -67,6 +67,16 @@ export class Lexer { */ private leadingWhitespace = ''; + /** + * Tracks whether we're currently inside a template string expression + */ + private isInTemplateExpression = false; + + /** + * Tracks the depth of nested braces within a template string expression + */ + private templateExpressionBraceDepth = 0; + /** * A convenience function, equivalent to `new Lexer().scan(toScan)`, that converts a string * containing BrightScript code to an array of `Token` objects that will later be used to build @@ -299,6 +309,26 @@ export class Lexer { } else { this.addToken(TokenKind.Question); } + }, + '{': function (this: Lexer) { + if (this.isInTemplateExpression) { + this.templateExpressionBraceDepth++; + } + this.addToken(TokenKind.LeftCurlyBrace); + }, + '}': function (this: Lexer) { + if (this.isInTemplateExpression) { + if (this.templateExpressionBraceDepth > 0) { + this.templateExpressionBraceDepth--; + this.addToken(TokenKind.RightCurlyBrace); + } else { + // This is the closing brace for the template expression + this.isInTemplateExpression = false; + this.addToken(TokenKind.TemplateStringExpressionEnd); + } + } else { + this.addToken(TokenKind.RightCurlyBrace); + } } }; @@ -332,8 +362,6 @@ export class Lexer { ')': TokenKind.RightParen, '=': TokenKind.Equal, ',': TokenKind.Comma, - '{': TokenKind.LeftCurlyBrace, - '}': TokenKind.RightCurlyBrace, '[': TokenKind.LeftSquareBracket, ']': TokenKind.RightSquareBracket, '^': TokenKind.Caret, @@ -525,6 +553,14 @@ export class Lexer { * string is terminated by a newline or the end of input. */ private templateString() { + // Save current template expression state (for nested template strings) + const savedIsInTemplateExpression = this.isInTemplateExpression; + const savedTemplateExpressionBraceDepth = this.templateExpressionBraceDepth; + + // Reset template expression state for this template string + this.isInTemplateExpression = false; + this.templateExpressionBraceDepth = 0; + this.addToken(TokenKind.BackTick); while (!this.isAtEnd() && !this.check('`')) { //handle line/column tracking when capturing newlines @@ -612,38 +648,14 @@ export class Lexer { this.advance(); this.advance(); this.addToken(TokenKind.TemplateStringExpressionBegin); - let braceDepth = 0; - while (!this.isAtEnd()) { - if (this.check('}')) { - if (braceDepth === 0) { - // This is the closing brace for the template expression - break; - } else { - // This is a regular right brace inside the expression - braceDepth--; - this.start = this.current; - this.advance(); - this.addToken(TokenKind.RightCurlyBrace); - } - } else if (this.check('{')) { - // Track nested braces - braceDepth++; - this.start = this.current; - this.scanToken(); - } else { - this.start = this.current; - this.scanToken(); - } - } - if (this.check('}')) { - this.advance(); - this.addToken(TokenKind.TemplateStringExpressionEnd); - } else { - this.diagnostics.push({ - ...DiagnosticMessages.unexpectedConditionalCompilationString(), - range: this.rangeOf() - }); + // Enter template expression mode + this.isInTemplateExpression = true; + this.templateExpressionBraceDepth = 0; + + while (!this.isAtEnd() && this.isInTemplateExpression) { + this.start = this.current; + this.scanToken(); } this.start = this.current; @@ -660,6 +672,10 @@ export class Lexer { this.advance(); this.addToken(TokenKind.BackTick); } + + // Restore template expression state (for nested template strings) + this.isInTemplateExpression = savedIsInTemplateExpression; + this.templateExpressionBraceDepth = savedTemplateExpressionBraceDepth; } private templateQuasiString() { From 87c7e1408582c86934c2f51c18e2ae93fd1c9159 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 20:33:28 +0000 Subject: [PATCH 4/6] Implement stack-based approach for nested template expressions Co-authored-by: TwitchBronBron <2544493+TwitchBronBron@users.noreply.github.com> --- src/lexer/Lexer.spec.ts | 26 ++++++++++++ src/lexer/Lexer.ts | 88 +++++++++++++++++++++++------------------ 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index f0a5eb7a6..d16bb8577 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -864,6 +864,32 @@ describe('lexer', () => { TokenKind.Eof ]); }); + + it('handles nested template expressions', () => { + let tokens = Lexer.scan('print `one${`two${`three${`four`}`}`}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Print, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, // one + TokenKind.TemplateStringExpressionBegin, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, // two + TokenKind.TemplateStringExpressionBegin, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, // three + TokenKind.TemplateStringExpressionBegin, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, // four + TokenKind.BackTick, + TokenKind.TemplateStringExpressionEnd, + TokenKind.BackTick, + TokenKind.TemplateStringExpressionEnd, + TokenKind.BackTick, + TokenKind.TemplateStringExpressionEnd, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); }); // string literals describe('double literals', () => { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index a15d0e65f..526660e61 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -68,14 +68,36 @@ export class Lexer { private leadingWhitespace = ''; /** - * Tracks whether we're currently inside a template string expression + * Stack to track nested template string expression state. + * Each entry contains the brace depth for that template expression level. + * Empty stack means we're not in any template expression. */ - private isInTemplateExpression = false; + private templateExpressionStack: number[] = []; /** - * Tracks the depth of nested braces within a template string expression + * Returns true if we're currently inside any template string expression */ - private templateExpressionBraceDepth = 0; + private get isInTemplateExpression(): boolean { + return this.templateExpressionStack.length > 0; + } + + /** + * Returns the current template expression brace depth (0 if not in template expression) + */ + private get templateExpressionBraceDepth(): number { + return this.templateExpressionStack.length > 0 + ? this.templateExpressionStack[this.templateExpressionStack.length - 1] + : 0; + } + + /** + * Sets the current template expression brace depth + */ + private set templateExpressionBraceDepth(depth: number) { + if (this.templateExpressionStack.length > 0) { + this.templateExpressionStack[this.templateExpressionStack.length - 1] = depth; + } + } /** * A convenience function, equivalent to `new Lexer().scan(toScan)`, that converts a string @@ -157,6 +179,27 @@ export class Lexer { '"': Lexer.prototype.string, '\'': Lexer.prototype.comment, '`': Lexer.prototype.templateString, + '{': function (this: Lexer) { + if (this.isInTemplateExpression) { + this.templateExpressionBraceDepth++; + } + this.addToken(TokenKind.LeftCurlyBrace); + }, + '}': function (this: Lexer) { + if (this.isInTemplateExpression) { + if (this.templateExpressionBraceDepth > 0) { + this.templateExpressionBraceDepth--; + this.addToken(TokenKind.RightCurlyBrace); + } else { + // This is the closing brace for the template expression + // Pop the current template expression level from the stack + this.templateExpressionStack.pop(); + this.addToken(TokenKind.TemplateStringExpressionEnd); + } + } else { + this.addToken(TokenKind.RightCurlyBrace); + } + }, '.': function (this: Lexer) { // this might be a float/double literal, because decimals without a leading 0 // are allowed @@ -309,26 +352,6 @@ export class Lexer { } else { this.addToken(TokenKind.Question); } - }, - '{': function (this: Lexer) { - if (this.isInTemplateExpression) { - this.templateExpressionBraceDepth++; - } - this.addToken(TokenKind.LeftCurlyBrace); - }, - '}': function (this: Lexer) { - if (this.isInTemplateExpression) { - if (this.templateExpressionBraceDepth > 0) { - this.templateExpressionBraceDepth--; - this.addToken(TokenKind.RightCurlyBrace); - } else { - // This is the closing brace for the template expression - this.isInTemplateExpression = false; - this.addToken(TokenKind.TemplateStringExpressionEnd); - } - } else { - this.addToken(TokenKind.RightCurlyBrace); - } } }; @@ -553,14 +576,6 @@ export class Lexer { * string is terminated by a newline or the end of input. */ private templateString() { - // Save current template expression state (for nested template strings) - const savedIsInTemplateExpression = this.isInTemplateExpression; - const savedTemplateExpressionBraceDepth = this.templateExpressionBraceDepth; - - // Reset template expression state for this template string - this.isInTemplateExpression = false; - this.templateExpressionBraceDepth = 0; - this.addToken(TokenKind.BackTick); while (!this.isAtEnd() && !this.check('`')) { //handle line/column tracking when capturing newlines @@ -649,9 +664,8 @@ export class Lexer { this.advance(); this.addToken(TokenKind.TemplateStringExpressionBegin); - // Enter template expression mode - this.isInTemplateExpression = true; - this.templateExpressionBraceDepth = 0; + // Enter template expression mode by pushing a new level onto the stack + this.templateExpressionStack.push(0); while (!this.isAtEnd() && this.isInTemplateExpression) { this.start = this.current; @@ -672,10 +686,6 @@ export class Lexer { this.advance(); this.addToken(TokenKind.BackTick); } - - // Restore template expression state (for nested template strings) - this.isInTemplateExpression = savedIsInTemplateExpression; - this.templateExpressionBraceDepth = savedTemplateExpressionBraceDepth; } private templateQuasiString() { From a2412fc3811a509114fe5f5b94a5be559e41735f Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 4 Sep 2026 11:31:23 -0400 Subject: [PATCH 5/6] Rework nested template brace handling Replace the earlier lexer-state approach with a local brace-depth counter scoped to each `${...}`. Token scanning stays entirely in `scanToken`, and nested template strings recurse naturally, so no shared lexer state is needed to support arbitrary nesting depth. Co-Authored-By: Claude Opus 5 (1M context) --- src/lexer/Lexer.spec.ts | 234 ++++++++++++++++++++++++---------------- src/lexer/Lexer.ts | 106 ++++++++---------- 2 files changed, 183 insertions(+), 157 deletions(-) diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 54ac561a9..0d1a9985a 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -572,6 +572,146 @@ describe('lexer', () => { expect(tokens[1].text).to.deep.equal(`hello `); }); + it('handles nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles deeply nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{a: {b: 1}}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.IntegerLiteral, + TokenKind.RightCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles curly braces alongside other expressions', () => { + let tokens = Lexer.scan('thing = `${arr[0]} and ${{key: value}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.Identifier, + TokenKind.LeftSquareBracket, + TokenKind.IntegerLiteral, + TokenKind.RightSquareBracket, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.Identifier, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles deeply nested template strings', () => { + let tokens = Lexer.scan('print `one${`two${`three${`four`}`}`}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Print, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //one + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //two + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //three + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //four + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.Eof + ]); + }); + + it('handles nested curly braces inside nested template strings', () => { + let tokens = Lexer.scan('print `a${`b${{c: {}}}`}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Print, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //a + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //b + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.Eof + ]); + }); + + it('flags an unterminated template string expression', () => { + const { diagnostics } = Lexer.scan('thing = `${{}`'); + expect(diagnostics).to.be.lengthOf(1); + }); + it('real example, which is causing issues in the formatter', () => { let { tokens } = Lexer.scan(` function getItemXML(item) @@ -796,100 +936,6 @@ describe('lexer', () => { TokenKind.Eof ]); }); - - it('handles nested curly braces', () => { - let tokens = Lexer.scan('thing = `${{}}`').tokens; - expect(tokens.map(x => x.kind)).to.eql([ - TokenKind.Identifier, - TokenKind.Equal, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, - TokenKind.TemplateStringExpressionBegin, - TokenKind.LeftCurlyBrace, - TokenKind.RightCurlyBrace, - TokenKind.TemplateStringExpressionEnd, - TokenKind.TemplateStringQuasi, - TokenKind.BackTick, - TokenKind.Eof - ]); - }); - - it('handles deeply nested curly braces', () => { - let tokens = Lexer.scan('thing = `${{a: {b: 1}}}`').tokens; - expect(tokens.map(x => x.kind)).to.eql([ - TokenKind.Identifier, - TokenKind.Equal, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, - TokenKind.TemplateStringExpressionBegin, - TokenKind.LeftCurlyBrace, - TokenKind.Identifier, // a - TokenKind.Colon, - TokenKind.LeftCurlyBrace, - TokenKind.Identifier, // b - TokenKind.Colon, - TokenKind.IntegerLiteral, // 1 - TokenKind.RightCurlyBrace, - TokenKind.RightCurlyBrace, - TokenKind.TemplateStringExpressionEnd, - TokenKind.TemplateStringQuasi, - TokenKind.BackTick, - TokenKind.Eof - ]); - }); - - it('handles mixed expressions with nested braces', () => { - let tokens = Lexer.scan('thing = `${arr[0]} and ${{key: value}}`').tokens; - expect(tokens.map(x => x.kind)).to.eql([ - TokenKind.Identifier, - TokenKind.Equal, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, - TokenKind.TemplateStringExpressionBegin, - TokenKind.Identifier, // arr - TokenKind.LeftSquareBracket, - TokenKind.IntegerLiteral, // 0 - TokenKind.RightSquareBracket, - TokenKind.TemplateStringExpressionEnd, - TokenKind.TemplateStringQuasi, // " and " - TokenKind.TemplateStringExpressionBegin, - TokenKind.LeftCurlyBrace, - TokenKind.Identifier, // key - TokenKind.Colon, - TokenKind.Identifier, // value - TokenKind.RightCurlyBrace, - TokenKind.TemplateStringExpressionEnd, - TokenKind.TemplateStringQuasi, - TokenKind.BackTick, - TokenKind.Eof - ]); - }); - - it('handles nested template expressions', () => { - let tokens = Lexer.scan('print `one${`two${`three${`four`}`}`}`').tokens; - expect(tokens.map(x => x.kind)).to.eql([ - TokenKind.Print, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, // one - TokenKind.TemplateStringExpressionBegin, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, // two - TokenKind.TemplateStringExpressionBegin, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, // three - TokenKind.TemplateStringExpressionBegin, - TokenKind.BackTick, - TokenKind.TemplateStringQuasi, // four - TokenKind.BackTick, - TokenKind.TemplateStringExpressionEnd, - TokenKind.BackTick, - TokenKind.TemplateStringExpressionEnd, - TokenKind.BackTick, - TokenKind.TemplateStringExpressionEnd, - TokenKind.BackTick, - TokenKind.Eof - ]); - }); }); // string literals describe('double literals', () => { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 29c33703b..e7007f19c 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -67,38 +67,6 @@ export class Lexer { */ private leadingWhitespace = ''; - /** - * Stack to track nested template string expression state. - * Each entry contains the brace depth for that template expression level. - * Empty stack means we're not in any template expression. - */ - private templateExpressionStack: number[] = []; - - /** - * Returns true if we're currently inside any template string expression - */ - private get isInTemplateExpression(): boolean { - return this.templateExpressionStack.length > 0; - } - - /** - * Returns the current template expression brace depth (0 if not in template expression) - */ - private get templateExpressionBraceDepth(): number { - return this.templateExpressionStack.length > 0 - ? this.templateExpressionStack[this.templateExpressionStack.length - 1] - : 0; - } - - /** - * Sets the current template expression brace depth - */ - private set templateExpressionBraceDepth(depth: number) { - if (this.templateExpressionStack.length > 0) { - this.templateExpressionStack[this.templateExpressionStack.length - 1] = depth; - } - } - /** * A convenience function, equivalent to `new Lexer().scan(toScan)`, that converts a string * containing BrightScript code to an array of `Token` objects that will later be used to build @@ -179,27 +147,6 @@ export class Lexer { '"': Lexer.prototype.string, '\'': Lexer.prototype.comment, '`': Lexer.prototype.templateString, - '{': function (this: Lexer) { - if (this.isInTemplateExpression) { - this.templateExpressionBraceDepth++; - } - this.addToken(TokenKind.LeftCurlyBrace); - }, - '}': function (this: Lexer) { - if (this.isInTemplateExpression) { - if (this.templateExpressionBraceDepth > 0) { - this.templateExpressionBraceDepth--; - this.addToken(TokenKind.RightCurlyBrace); - } else { - // This is the closing brace for the template expression - // Pop the current template expression level from the stack - this.templateExpressionStack.pop(); - this.addToken(TokenKind.TemplateStringExpressionEnd); - } - } else { - this.addToken(TokenKind.RightCurlyBrace); - } - }, '.': function (this: Lexer) { // this might be a float/double literal, because decimals without a leading 0 // are allowed @@ -385,6 +332,8 @@ export class Lexer { ')': TokenKind.RightParen, '=': TokenKind.Equal, ',': TokenKind.Comma, + '{': TokenKind.LeftCurlyBrace, + '}': TokenKind.RightCurlyBrace, '[': TokenKind.LeftSquareBracket, ']': TokenKind.RightSquareBracket, '^': TokenKind.Caret, @@ -679,16 +628,7 @@ export class Lexer { this.advance(); this.advance(); this.addToken(TokenKind.TemplateStringExpressionBegin); - - // Enter template expression mode by pushing a new level onto the stack - this.templateExpressionStack.push(0); - - while (!this.isAtEnd() && this.isInTemplateExpression) { - this.start = this.current; - this.scanToken(); - } - - this.start = this.current; + this.templateStringExpression(); } else { this.advance(); } @@ -704,6 +644,46 @@ export class Lexer { } } + /** + * Scans the contents of a `${...}` template string expression, stopping just after the `}` that closes it. + * Assumes the `${` has already been consumed and its token emitted. + * + * All actual token scanning is delegated to `scanToken`, so this method's only job is deciding + * which `}` terminates the expression. A `}` that closes a brace opened _inside_ the expression + * (an associative array literal, for example) is not the terminator, so we track how many + * unclosed `{` we've scanned past and only stop at a `}` seen at depth zero. + * + * Nested template strings need no special handling here: `scanToken` routes a backtick back into + * `templateString`, which recurses into this method for its own expressions. Each level therefore + * gets its own `depth` local, so arbitrarily deep nesting works without any shared state. + */ + private templateStringExpression() { + let depth = 0; + while (!this.isAtEnd()) { + if (this.check('}')) { + if (depth === 0) { + //this is the `}` that closes the expression + this.advance(); + this.addToken(TokenKind.TemplateStringExpressionEnd); + this.start = this.current; + return; + } + depth--; + } else if (this.check('{')) { + depth++; + } + this.start = this.current; + this.scanToken(); + } + + //we hit the end of the file before finding the closing `}` + this.diagnostics.push({ + ...DiagnosticMessages.unexpectedConditionalCompilationString(), + range: this.rangeOf() + }); + this.start = this.current; + } + private templateQuasiString() { let value = this.source.slice(this.start, this.current); if (value !== '`') { // if this is an empty string straight after an expression, then we'll accidentally consume the backtick From 04531e754a9181ff3b8f2785bf87cbe36f950521 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 4 Sep 2026 12:49:12 -0400 Subject: [PATCH 6/6] Add template expression edge case tests Cover the cases where a `}` or `{` appears somewhere the depth counter must not see it: inside a string literal, inside a comment, and inside an inline function body. Also cover a multi-line AA literal and an empty expression. Co-Authored-By: Claude Opus 5 (1M context) --- src/lexer/Lexer.spec.ts | 100 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 0d1a9985a..6ee9dd940 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -707,6 +707,106 @@ describe('lexer', () => { ]); }); + it('ignores curly braces inside strings within an expression', () => { + //the `}` here belongs to the string, so it must not end the expression + let tokens = Lexer.scan('thing = `${{ "a}b": "{" }}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.StringLiteral, + TokenKind.Colon, + TokenKind.StringLiteral, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('ignores curly braces inside a comment within an expression', () => { + let tokens = Lexer.scan('thing = `${1 \'}\n}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.IntegerLiteral, + TokenKind.Comment, + TokenKind.Newline, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an associative array literal spanning multiple lines', () => { + let tokens = Lexer.scan('thing = `${{\n b: 1\n}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Newline, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.IntegerLiteral, + TokenKind.Newline, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an inline function containing braces in an expression', () => { + let tokens = Lexer.scan('thing = `${sub()\nprint {}\nend sub}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.Sub, + TokenKind.LeftParen, + TokenKind.RightParen, + TokenKind.Newline, + TokenKind.Print, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.Newline, + TokenKind.EndSub, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an empty expression', () => { + let tokens = Lexer.scan('thing = `${}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + it('flags an unterminated template string expression', () => { const { diagnostics } = Lexer.scan('thing = `${{}`'); expect(diagnostics).to.be.lengthOf(1);