diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index a37a1d930..1bc2129b2 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -572,6 +572,246 @@ describe('lexer', () => { expect(tokens[1].text).to.deep.equal(`hello `); }); + it('handles nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles deeply nested curly braces', () => { + let tokens = Lexer.scan('thing = `${{a: {b: 1}}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.IntegerLiteral, + TokenKind.RightCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles curly braces alongside other expressions', () => { + let tokens = Lexer.scan('thing = `${arr[0]} and ${{key: value}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.Identifier, + TokenKind.LeftSquareBracket, + TokenKind.IntegerLiteral, + TokenKind.RightSquareBracket, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.Identifier, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('handles deeply nested template strings', () => { + let tokens = Lexer.scan('print `one${`two${`three${`four`}`}`}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Print, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //one + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //two + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //three + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //four + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.Eof + ]); + }); + + it('handles nested curly braces inside nested template strings', () => { + let tokens = Lexer.scan('print `a${`b${{c: {}}}`}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Print, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //a + TokenKind.TemplateStringExpressionBegin, + + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, //b + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, //empty + TokenKind.BackTick, + + TokenKind.Eof + ]); + }); + + it('ignores curly braces inside strings within an expression', () => { + //the `}` here belongs to the string, so it must not end the expression + let tokens = Lexer.scan('thing = `${{ "a}b": "{" }}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.StringLiteral, + TokenKind.Colon, + TokenKind.StringLiteral, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('ignores curly braces inside a comment within an expression', () => { + let tokens = Lexer.scan('thing = `${1 \'}\n}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.IntegerLiteral, + TokenKind.Comment, + TokenKind.Newline, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an associative array literal spanning multiple lines', () => { + let tokens = Lexer.scan('thing = `${{\n b: 1\n}}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.LeftCurlyBrace, + TokenKind.Newline, + TokenKind.Identifier, + TokenKind.Colon, + TokenKind.IntegerLiteral, + TokenKind.Newline, + TokenKind.RightCurlyBrace, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an inline function containing braces in an expression', () => { + let tokens = Lexer.scan('thing = `${sub()\nprint {}\nend sub}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.Sub, + TokenKind.LeftParen, + TokenKind.RightParen, + TokenKind.Newline, + TokenKind.Print, + TokenKind.LeftCurlyBrace, + TokenKind.RightCurlyBrace, + TokenKind.Newline, + TokenKind.EndSub, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('supports an empty expression', () => { + let tokens = Lexer.scan('thing = `${}`').tokens; + expect(tokens.map(x => x.kind)).to.eql([ + TokenKind.Identifier, + TokenKind.Equal, + TokenKind.BackTick, + TokenKind.TemplateStringQuasi, + TokenKind.TemplateStringExpressionBegin, + TokenKind.TemplateStringExpressionEnd, + TokenKind.TemplateStringQuasi, + TokenKind.BackTick, + TokenKind.Eof + ]); + }); + + it('flags an unterminated template string expression', () => { + const { diagnostics } = Lexer.scan('thing = `${{}`'); + expect(diagnostics).to.be.lengthOf(1); + }); + it('real example, which is causing issues in the formatter', () => { let { tokens } = Lexer.scan(` function getItemXML(item) diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 05cc7d791..e7007f19c 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -628,22 +628,7 @@ export class Lexer { this.advance(); this.advance(); this.addToken(TokenKind.TemplateStringExpressionBegin); - while (!this.isAtEnd() && !this.check('}')) { - this.start = this.current; - this.scanToken(); - } - if (this.check('}')) { - this.advance(); - this.addToken(TokenKind.TemplateStringExpressionEnd); - } else { - - this.diagnostics.push({ - ...DiagnosticMessages.unexpectedConditionalCompilationString(), - range: this.rangeOf() - }); - } - - this.start = this.current; + this.templateStringExpression(); } else { this.advance(); } @@ -659,6 +644,46 @@ export class Lexer { } } + /** + * Scans the contents of a `${...}` template string expression, stopping just after the `}` that closes it. + * Assumes the `${` has already been consumed and its token emitted. + * + * All actual token scanning is delegated to `scanToken`, so this method's only job is deciding + * which `}` terminates the expression. A `}` that closes a brace opened _inside_ the expression + * (an associative array literal, for example) is not the terminator, so we track how many + * unclosed `{` we've scanned past and only stop at a `}` seen at depth zero. + * + * Nested template strings need no special handling here: `scanToken` routes a backtick back into + * `templateString`, which recurses into this method for its own expressions. Each level therefore + * gets its own `depth` local, so arbitrarily deep nesting works without any shared state. + */ + private templateStringExpression() { + let depth = 0; + while (!this.isAtEnd()) { + if (this.check('}')) { + if (depth === 0) { + //this is the `}` that closes the expression + this.advance(); + this.addToken(TokenKind.TemplateStringExpressionEnd); + this.start = this.current; + return; + } + depth--; + } else if (this.check('{')) { + depth++; + } + this.start = this.current; + this.scanToken(); + } + + //we hit the end of the file before finding the closing `}` + this.diagnostics.push({ + ...DiagnosticMessages.unexpectedConditionalCompilationString(), + range: this.rangeOf() + }); + this.start = this.current; + } + private templateQuasiString() { let value = this.source.slice(this.start, this.current); if (value !== '`') { // if this is an empty string straight after an expression, then we'll accidentally consume the backtick