# HG changeset patch # User mhaupt # Date 1446026045 -3600 # Node ID 2e284c36d51f3ee0c0f806cc0d4ff5d78734d8fa # Parent 4a0312f2894bcbe1fd20266c8fda8d983bd2fcf6 8134941: Implement ES6 template literal support Reviewed-by: attila, hannesw Contributed-by: andreas.woess@oracle.com diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/ir/RuntimeNode.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/ir/RuntimeNode.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/ir/RuntimeNode.java Wed Oct 28 10:54:05 2015 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -83,7 +83,9 @@ /** is undefined */ IS_UNDEFINED(TokenType.EQ_STRICT, Type.BOOLEAN, 2), /** is not undefined */ - IS_NOT_UNDEFINED(TokenType.NE_STRICT, Type.BOOLEAN, 2); + IS_NOT_UNDEFINED(TokenType.NE_STRICT, Type.BOOLEAN, 2), + /** Get template object from raw and cooked string arrays. */ + GET_TEMPLATE_OBJECT(TokenType.TEMPLATE, Type.SCRIPT_OBJECT, 2); /** token type */ private final TokenType tokenType; diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Wed Oct 28 10:54:05 2015 +0100 @@ -46,6 +46,10 @@ import static jdk.nashorn.internal.parser.TokenType.REGEX; import static jdk.nashorn.internal.parser.TokenType.RPAREN; import static jdk.nashorn.internal.parser.TokenType.STRING; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_HEAD; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_MIDDLE; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_TAIL; import static jdk.nashorn.internal.parser.TokenType.XML; import java.io.Serializable; @@ -96,6 +100,8 @@ private final boolean pauseOnFunctionBody; private boolean pauseOnNextLeftBrace; + private int templateExpressionOpenBraces; + private static final String SPACETAB = " \t"; // ASCII space and tab private static final String LFCR = "\n\r"; // line feed and carriage return (ctrl-m) @@ -392,13 +398,19 @@ } /** - * Test if char is a string delimiter, e.g. '\' or '"'. Also scans exec - * strings ('`') in scripting mode. + * Test if char is a string delimiter, e.g. '\' or '"'. * @param ch a char * @return true if string delimiter */ protected boolean isStringDelimiter(final char ch) { - return ch == '\'' || ch == '"' || (scripting && ch == '`'); + return ch == '\'' || ch == '"'; + } + + /** + * Test if char is a template literal delimiter ('`'). + */ + private static boolean isTemplateDelimiter(char ch) { + return ch == '`'; } /** @@ -943,6 +955,10 @@ sb.append(next); break; } + } else if (ch0 == '\r') { + // Convert CR-LF or CR to LF line terminator. + sb.append('\n'); + skip(ch1 == '\n' ? 2 : 1); } else { // Add regular character. sb.append(ch0); @@ -958,7 +974,7 @@ /** * Scan over a string literal. - * @param add true if we nare not just scanning but should actually modify the token stream + * @param add true if we are not just scanning but should actually modify the token stream */ protected void scanString(final boolean add) { // Type of string. @@ -1034,6 +1050,70 @@ } /** + * Scan over a template string literal. + */ + private void scanTemplate() { + assert ch0 == '`'; + TokenType type = TEMPLATE; + + // Skip over quote and record beginning of string content. + skip(1); + State stringState = saveState(); + + // Scan until close quote + while (!atEOF()) { + // Skip over escaped character. + if (ch0 == '`') { + skip(1); + // Record end of string. + stringState.setLimit(position - 1); + add(type == TEMPLATE ? type : TEMPLATE_TAIL, stringState.position, stringState.limit); + return; + } else if (ch0 == '$' && ch1 == '{') { + skip(2); + stringState.setLimit(position - 2); + add(type == TEMPLATE ? TEMPLATE_HEAD : type, stringState.position, stringState.limit); + + // scan to RBRACE + Lexer expressionLexer = new Lexer(this, saveState()); + expressionLexer.templateExpressionOpenBraces = 1; + expressionLexer.lexify(); + restoreState(expressionLexer.saveState()); + + // scan next middle or tail of the template literal + assert ch0 == '}'; + type = TEMPLATE_MIDDLE; + + // Skip over rbrace and record beginning of string content. + skip(1); + stringState = saveState(); + + continue; + } else if (ch0 == '\\') { + skip(1); + // EscapeSequence + if (!isEscapeCharacter(ch0)) { + error(Lexer.message("invalid.escape.char"), TEMPLATE, position, limit); + } + if (isEOL(ch0)) { + // LineContinuation + skipEOL(false); + continue; + } + } else if (isEOL(ch0)) { + // LineTerminatorSequence + skipEOL(false); + continue; + } + + // Skip literal character. + skip(1); + } + + error(Lexer.message("missing.close.quote"), TEMPLATE, position, limit); + } + + /** * Is the given character a valid escape char after "\" ? * * @param ch character to be checked @@ -1621,6 +1701,16 @@ // Scan and add a number. scanNumber(); } else if ((type = TokenLookup.lookupOperator(ch0, ch1, ch2, ch3)) != null) { + if (templateExpressionOpenBraces > 0) { + if (type == LBRACE) { + templateExpressionOpenBraces++; + } else if (type == RBRACE) { + if (--templateExpressionOpenBraces == 0) { + break; + } + } + } + // Get the number of characters in the token. final int typeLength = type.getLength(); // Skip that many characters. @@ -1644,6 +1734,12 @@ } else if (Character.isDigit(ch0)) { // Scan and add a number. scanNumber(); + } else if (isTemplateDelimiter(ch0) && es6) { + // Scan and add template in ES6 mode. + scanTemplate(); + } else if (isTemplateDelimiter(ch0) && scripting) { + // Scan and add an exec string ('`') in scripting mode. + scanString(true); } else { // Don't recognize this character. skip(1); @@ -1699,6 +1795,11 @@ return valueOfIdent(start, len); // String case REGEX: return valueOfPattern(start, len); // RegexToken::LexerToken + case TEMPLATE: + case TEMPLATE_HEAD: + case TEMPLATE_MIDDLE: + case TEMPLATE_TAIL: + return valueOfString(start, len, true); // String case XML: return valueOfXML(start, len); // XMLToken::LexerToken case DIRECTIVE_COMMENT: @@ -1711,6 +1812,45 @@ } /** + * Get the raw string value of a template literal string part. + * + * @param token template string token + * @return raw string + */ + public String valueOfRawString(final long token) { + final int start = Token.descPosition(token); + final int length = Token.descLength(token); + + // Save the current position. + final int savePosition = position; + // Calculate the end position. + final int end = start + length; + // Reset to beginning of string. + reset(start); + + // Buffer for recording characters. + final StringBuilder sb = new StringBuilder(length); + + // Scan until end of string. + while (position < end) { + if (ch0 == '\r') { + // Convert CR-LF or CR to LF line terminator. + sb.append('\n'); + skip(ch1 == '\n' ? 2 : 1); + } else { + // Add regular character. + sb.append(ch0); + skip(1); + } + } + + // Restore position. + reset(savePosition); + + return sb.toString(); + } + + /** * Get the correctly localized error message for a given message id format arguments * @param msgId message id * @param args format arguments diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Wed Oct 28 10:54:05 2015 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -51,6 +51,10 @@ import static jdk.nashorn.internal.parser.TokenType.RBRACKET; import static jdk.nashorn.internal.parser.TokenType.RPAREN; import static jdk.nashorn.internal.parser.TokenType.SEMICOLON; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_HEAD; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_MIDDLE; +import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_TAIL; import static jdk.nashorn.internal.parser.TokenType.TERNARY; import static jdk.nashorn.internal.parser.TokenType.WHILE; @@ -64,6 +68,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; + import jdk.internal.dynalink.support.NameCodec; import jdk.nashorn.internal.codegen.CompilerConstants; import jdk.nashorn.internal.codegen.Namespace; @@ -1926,10 +1931,10 @@ * Literal * ArrayLiteral * ObjectLiteral + * RegularExpressionLiteral + * TemplateLiteral * ( Expression ) * - * See 11.1 - * * Parse primary expression. * @return Expression node. */ @@ -1989,6 +1994,9 @@ expect(RPAREN); return expression; + case TEMPLATE: + case TEMPLATE_HEAD: + return templateLiteral(); default: // In this context some operator tokens mark the start of a literal. @@ -2387,6 +2395,8 @@ } /** + * Parse left hand side expression. + * * LeftHandSideExpression : * NewExpression * CallExpression @@ -2396,10 +2406,8 @@ * CallExpression Arguments * CallExpression [ Expression ] * CallExpression . IdentifierName + * CallExpression TemplateLiteral * - * See 11.2 - * - * Parse left hand side expression. * @return Expression node. */ private Expression leftHandSideExpression() { @@ -2426,7 +2434,7 @@ callToken = token; switch (type) { - case LPAREN: + case LPAREN: { // Get NEW or FUNCTION arguments. final List arguments = optimizeList(argumentList()); @@ -2434,8 +2442,8 @@ lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false); break; - - case LBRACKET: + } + case LBRACKET: { next(); // Get array index. @@ -2447,8 +2455,8 @@ lhs = new IndexNode(callToken, finish, lhs, rhs); break; - - case PERIOD: + } + case PERIOD: { next(); final IdentNode property = getIdentifierName(); @@ -2457,7 +2465,16 @@ lhs = new AccessNode(callToken, finish, lhs, property.getName()); break; - + } + case TEMPLATE: + case TEMPLATE_HEAD: { + // tagged template literal + final List arguments = templateLiteralArgumentList(); + + lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false); + + break; + } default: break loop; } @@ -2516,16 +2533,16 @@ } /** + * Parse member expression. + * * MemberExpression : * PrimaryExpression * FunctionExpression * MemberExpression [ Expression ] * MemberExpression . IdentifierName + * MemberExpression TemplateLiteral * new MemberExpression Arguments * - * See 11.2 - * - * Parse member expression. * @return Expression node. */ private Expression memberExpression() { @@ -2582,6 +2599,16 @@ break; } + case TEMPLATE: + case TEMPLATE_HEAD: { + // tagged template literal + final int callLine = line; + final List arguments = templateLiteralArgumentList(); + + lhs = new CallNode(callLine, callToken, finish, lhs, arguments, false); + + break; + } default: break loop; } @@ -3035,6 +3062,20 @@ final ParserState parserState = (ParserState)data.getEndParserState(); assert parserState != null; + if (k < stream.last() && start < parserState.position && parserState.position <= Token.descPosition(stream.get(stream.last()))) { + // RBRACE is already in the token stream, so fast forward to it + for (; k < stream.last(); k++) { + long nextToken = stream.get(k + 1); + if (Token.descPosition(nextToken) == parserState.position && Token.descType(nextToken) == RBRACE) { + token = stream.get(k); + type = Token.descType(token); + next(); + assert type == RBRACE && start == parserState.position; + return true; + } + } + } + stream.reset(); lexer = parserState.createLexer(source, lexer, stream, scripting && !env._no_syntax_extensions, env._es6); line = parserState.line; @@ -3425,6 +3466,79 @@ } } + /** + * Parse untagged template literal as string concatenation. + */ + private Expression templateLiteral() { + assert type == TEMPLATE || type == TEMPLATE_HEAD; + final boolean noSubstitutionTemplate = type == TEMPLATE; + long lastLiteralToken = token; + LiteralNode literal = getLiteral(); + if (noSubstitutionTemplate) { + return literal; + } + + Expression concat = literal; + TokenType lastLiteralType; + do { + Expression expression = expression(); + if (type != TEMPLATE_MIDDLE && type != TEMPLATE_TAIL) { + throw error(AbstractParser.message("unterminated.template.expression"), token); + } + concat = new BinaryNode(Token.recast(lastLiteralToken, TokenType.ADD), concat, expression); + lastLiteralType = type; + lastLiteralToken = token; + literal = getLiteral(); + concat = new BinaryNode(Token.recast(lastLiteralToken, TokenType.ADD), concat, literal); + } while (lastLiteralType == TEMPLATE_MIDDLE); + return concat; + } + + /** + * Parse tagged template literal as argument list. + * @return argument list for a tag function call (template object, ...substitutions) + */ + private List templateLiteralArgumentList() { + assert type == TEMPLATE || type == TEMPLATE_HEAD; + final ArrayList argumentList = new ArrayList<>(); + final ArrayList rawStrings = new ArrayList<>(); + final ArrayList cookedStrings = new ArrayList<>(); + argumentList.add(null); // filled at the end + + final long templateToken = token; + final boolean hasSubstitutions = type == TEMPLATE_HEAD; + addTemplateLiteralString(rawStrings, cookedStrings); + + if (hasSubstitutions) { + TokenType lastLiteralType; + do { + Expression expression = expression(); + if (type != TEMPLATE_MIDDLE && type != TEMPLATE_TAIL) { + throw error(AbstractParser.message("unterminated.template.expression"), token); + } + argumentList.add(expression); + + lastLiteralType = type; + addTemplateLiteralString(rawStrings, cookedStrings); + } while (lastLiteralType == TEMPLATE_MIDDLE); + } + + final LiteralNode rawStringArray = LiteralNode.newInstance(templateToken, finish, rawStrings); + final LiteralNode cookedStringArray = LiteralNode.newInstance(templateToken, finish, cookedStrings); + final RuntimeNode templateObject = new RuntimeNode(templateToken, finish, RuntimeNode.Request.GET_TEMPLATE_OBJECT, rawStringArray, cookedStringArray); + argumentList.set(0, templateObject); + return optimizeList(argumentList); + } + + private void addTemplateLiteralString(final ArrayList rawStrings, final ArrayList cookedStrings) { + final long stringToken = token; + final String rawString = lexer.valueOfRawString(stringToken); + final String cookedString = (String) getValue(); + next(); + rawStrings.add(LiteralNode.newInstance(stringToken, finish, rawString)); + cookedStrings.add(LiteralNode.newInstance(stringToken, finish, cookedString)); + } + @Override public String toString() { return "'JavaScript Parsing'"; diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Token.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Token.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Token.java Wed Oct 28 10:54:05 2015 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,11 +71,21 @@ public static long withDelimiter(final long token) { final TokenType tokenType = Token.descType(token); switch(tokenType) { - case STRING: case ESCSTRING: case EXECSTRING: { + case STRING: + case ESCSTRING: + case EXECSTRING: + case TEMPLATE: + case TEMPLATE_TAIL: { final int start = Token.descPosition(token) - 1; final int len = Token.descLength(token) + 2; return toDesc(tokenType, start, len); } + case TEMPLATE_HEAD: + case TEMPLATE_MIDDLE: { + final int start = Token.descPosition(token) - 1; + final int len = Token.descLength(token) + 3; + return toDesc(tokenType, start, len); + } default: { return token; } diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/TokenType.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/TokenType.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/TokenType.java Wed Oct 28 10:54:05 2015 +0100 @@ -183,6 +183,10 @@ XML (LITERAL, null), OBJECT (LITERAL, null), ARRAY (LITERAL, null), + TEMPLATE (LITERAL, null), + TEMPLATE_HEAD (LITERAL, null), + TEMPLATE_MIDDLE(LITERAL, null), + TEMPLATE_TAIL (LITERAL, null), COMMALEFT (IR, null), DECPOSTFIX (IR, null), diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/ScriptRuntime.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/ScriptRuntime.java Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/ScriptRuntime.java Wed Oct 28 10:54:05 2015 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,6 +45,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; + import jdk.internal.dynalink.beans.StaticClass; import jdk.nashorn.api.scripting.JSObject; import jdk.nashorn.api.scripting.ScriptObjectMirror; @@ -1050,4 +1051,20 @@ context.getLogger(ApplySpecialization.class).info("Overwrote special name '" + name +"' - invalidating switchpoint"); SwitchPoint.invalidateAll(new SwitchPoint[] { sp }); } + + /** + * ES6 12.2.9.3 Runtime Semantics: GetTemplateObject(templateLiteral). + * + * @param rawStrings array of template raw values + * @param cookedStrings array of template values + * @return template object + */ + public static ScriptObject GET_TEMPLATE_OBJECT(final Object rawStrings, final Object cookedStrings) { + final ScriptObject template = (ScriptObject)cookedStrings; + final ScriptObject rawObj = (ScriptObject)rawStrings; + assert rawObj.getArray().length() == template.getArray().length(); + template.addOwnProperty("raw", Property.NOT_WRITABLE | Property.NOT_ENUMERABLE | Property.NOT_CONFIGURABLE, rawObj.freeze()); + template.freeze(); + return template; + } } diff -r 4a0312f2894b -r 2e284c36d51f nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/resources/Messages.properties --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/resources/Messages.properties Wed Jul 05 20:56:54 2017 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/resources/Messages.properties Wed Oct 28 10:54:05 2015 +0100 @@ -60,6 +60,7 @@ parser.error.regex.syntax={0} parser.error.trailing.comma.in.json=Trailing comma is not allowed in JSON parser.error.missing.const.assignment=Missing assignment to constant "{0}" +parser.error.unterminated.template.expression=Expected } after expression in template literal # strict mode error messages parser.error.strict.no.with="with" statement cannot be used in strict mode