Fix nested curly braces in template strings

src/lexer/Lexer.spec.ts

@@ -572,6 +572,246 @@ describe('lexer', () => {
             expect(tokens[1].text).to.deep.equal(`hello `);
         });
 
+        it('handles nested curly braces', () => {
+            let tokens = Lexer.scan('thing = `${{}}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('handles deeply nested curly braces', () => {
+            let tokens = Lexer.scan('thing = `${{a: {b: 1}}}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.Identifier,
+                TokenKind.Colon,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.Identifier,
+                TokenKind.Colon,
+                TokenKind.IntegerLiteral,
+                TokenKind.RightCurlyBrace,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('handles curly braces alongside other expressions', () => {
+            let tokens = Lexer.scan('thing = `${arr[0]} and ${{key: value}}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.Identifier,
+                TokenKind.LeftSquareBracket,
+                TokenKind.IntegerLiteral,
+                TokenKind.RightSquareBracket,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.Identifier,
+                TokenKind.Colon,
+                TokenKind.Identifier,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('handles deeply nested template strings', () => {
+            let tokens = Lexer.scan('print `one${`two${`three${`four`}`}`}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Print,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //one
+                TokenKind.TemplateStringExpressionBegin,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //two
+                TokenKind.TemplateStringExpressionBegin,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //three
+                TokenKind.TemplateStringExpressionBegin,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //four
+                TokenKind.BackTick,
+
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi, //empty
+                TokenKind.BackTick,
+
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi, //empty
+                TokenKind.BackTick,
+
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi, //empty
+                TokenKind.BackTick,
+
+                TokenKind.Eof
+            ]);
+        });
+
+        it('handles nested curly braces inside nested template strings', () => {
+            let tokens = Lexer.scan('print `a${`b${{c: {}}}`}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Print,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //a
+                TokenKind.TemplateStringExpressionBegin,
+
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi, //b
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.Identifier,
+                TokenKind.Colon,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.RightCurlyBrace,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi, //empty
+                TokenKind.BackTick,
+
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi, //empty
+                TokenKind.BackTick,
+
+                TokenKind.Eof
+            ]);
+        });
+
+        it('ignores curly braces inside strings within an expression', () => {
+            //the `}` here belongs to the string, so it must not end the expression
+            let tokens = Lexer.scan('thing = `${{ "a}b": "{" }}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.StringLiteral,
+                TokenKind.Colon,
+                TokenKind.StringLiteral,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('ignores curly braces inside a comment within an expression', () => {
+            let tokens = Lexer.scan('thing = `${1 \'}\n}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.IntegerLiteral,
+                TokenKind.Comment,
+                TokenKind.Newline,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('supports an associative array literal spanning multiple lines', () => {
+            let tokens = Lexer.scan('thing = `${{\n    b: 1\n}}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.Newline,
+                TokenKind.Identifier,
+                TokenKind.Colon,
+                TokenKind.IntegerLiteral,
+                TokenKind.Newline,
+                TokenKind.RightCurlyBrace,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('supports an inline function containing braces in an expression', () => {
+            let tokens = Lexer.scan('thing = `${sub()\nprint {}\nend sub}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.Sub,
+                TokenKind.LeftParen,
+                TokenKind.RightParen,
+                TokenKind.Newline,
+                TokenKind.Print,
+                TokenKind.LeftCurlyBrace,
+                TokenKind.RightCurlyBrace,
+                TokenKind.Newline,
+                TokenKind.EndSub,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('supports an empty expression', () => {
+            let tokens = Lexer.scan('thing = `${}`').tokens;
+            expect(tokens.map(x => x.kind)).to.eql([
+                TokenKind.Identifier,
+                TokenKind.Equal,
+                TokenKind.BackTick,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.TemplateStringExpressionBegin,
+                TokenKind.TemplateStringExpressionEnd,
+                TokenKind.TemplateStringQuasi,
+                TokenKind.BackTick,
+                TokenKind.Eof
+            ]);
+        });
+
+        it('flags an unterminated template string expression', () => {
+            const { diagnostics } = Lexer.scan('thing = `${{}`');
+            expect(diagnostics).to.be.lengthOf(1);
+        });
+
         it('real example, which is causing issues in the formatter', () => {
             let { tokens } = Lexer.scan(`
                 function getItemXML(item)

src/lexer/Lexer.ts

@@ -628,22 +628,7 @@ export class Lexer {
                 this.advance();
                 this.advance();
                 this.addToken(TokenKind.TemplateStringExpressionBegin);
-                while (!this.isAtEnd() && !this.check('}')) {
-                    this.start = this.current;
-                    this.scanToken();
-                }
-                if (this.check('}')) {
-                    this.advance();
-                    this.addToken(TokenKind.TemplateStringExpressionEnd);
-                } else {
-
-                    this.diagnostics.push({
-                        ...DiagnosticMessages.unexpectedConditionalCompilationString(),
-                        range: this.rangeOf()
-                    });
-                }
-
-                this.start = this.current;
+                this.templateStringExpression();
             } else {
                 this.advance();
             }
@@ -659,6 +644,46 @@ export class Lexer {
         }
     }
 
+    /**
+     * Scans the contents of a `${...}` template string expression, stopping just after the `}` that closes it.
+     * Assumes the `${` has already been consumed and its token emitted.
+     *
+     * All actual token scanning is delegated to `scanToken`, so this method's only job is deciding
+     * which `}` terminates the expression. A `}` that closes a brace opened _inside_ the expression
+     * (an associative array literal, for example) is not the terminator, so we track how many
+     * unclosed `{` we've scanned past and only stop at a `}` seen at depth zero.
+     *
+     * Nested template strings need no special handling here: `scanToken` routes a backtick back into
+     * `templateString`, which recurses into this method for its own expressions. Each level therefore
+     * gets its own `depth` local, so arbitrarily deep nesting works without any shared state.
+     */
+    private templateStringExpression() {
+        let depth = 0;
+        while (!this.isAtEnd()) {
+            if (this.check('}')) {
+                if (depth === 0) {
+                    //this is the `}` that closes the expression
+                    this.advance();
+                    this.addToken(TokenKind.TemplateStringExpressionEnd);
+                    this.start = this.current;
+                    return;
+                }
+                depth--;
+            } else if (this.check('{')) {
+                depth++;
+            }
+            this.start = this.current;
+            this.scanToken();
+        }
+
+        //we hit the end of the file before finding the closing `}`
+        this.diagnostics.push({
+            ...DiagnosticMessages.unexpectedConditionalCompilationString(),
+            range: this.rangeOf()
+        });
+        this.start = this.current;
+    }
+
     private templateQuasiString() {
         let value = this.source.slice(this.start, this.current);
         if (value !== '`') { // if this is an empty string straight after an expression, then we'll accidentally consume the backtick