Implement handling of arrow functions in the parser
authormarja@chromium.org <marja@chromium.org@ce2b1a6d-e550-0410-aec6-3dcde31c8c00>
Mon, 14 Jul 2014 07:55:45 +0000 (07:55 +0000)
committermarja@chromium.org <marja@chromium.org@ce2b1a6d-e550-0410-aec6-3dcde31c8c00>
Mon, 14 Jul 2014 07:55:45 +0000 (07:55 +0000)
Arrow functions are parsed from ParseAssignmentExpression(). Handling the
parameter list is done by letting ParseConditionalExpression() parse a comma
separated list of identifiers, and it returns a tree of BinaryOperation nodes
with VariableProxy leaves, or a single VariableProxy if there is only one
parameter. When the arrow token "=>" is found, the VariableProxy nodes are
passed to ParseArrowFunctionLiteral(), which will then skip parsing the
paramaeter list. This avoids having to rewind when the arrow is found and
restart parsing the parameter list.

Note that the empty parameter list "()" is handled directly in
ParsePrimaryExpression(): after is has consumed the opening parenthesis,
if a closing parenthesis follows, then the only valid input is an arrow
function. In this case, ParsePrimaryExpression() directly calls
ParseArrowFunctionLiteral(), to avoid needing to return a sentinel value
to signal the empty parameter list. Because it will consume the body of
the arrow function, ParseAssignmentExpression() will not see the arrow
"=>" token as next, and return the already-parser expression.

The implementation is done in ParserBase, so it was needed to do some
additions to ParserBase, ParserTraits and PreParserTraits. Some of the
glue code can be removed later on when more more functionality is moved
to ParserBase.

Additionally, this adds a runtime flag "harmony_arrow_functions"
(disabled by default); enabling "harmony" will enable it as well.

BUG=v8:2700
LOG=N
R=marja@chromium.org

Review URL: https://codereview.chromium.org/383983002

Patch from Adrián Pérez de Castro <aperez@igalia.com>.

git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@22366 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/ast.h
src/flag-definitions.h
src/messages.js
src/parser.cc
src/parser.h
src/preparser.h
src/scanner.cc
src/scanner.h
src/token.h
test/cctest/test-parsing.cc

index 3308fba..84d828e 100644 (file)
--- a/src/ast.h
+++ b/src/ast.h
@@ -344,6 +344,11 @@ class Expression : public AstNode {
   Bounds bounds() const { return bounds_; }
   void set_bounds(Bounds bounds) { bounds_ = bounds; }
 
+  // Whether the expression is parenthesized
+  unsigned parenthesization_level() const { return parenthesization_level_; }
+  bool is_parenthesized() const { return parenthesization_level_ > 0; }
+  void increase_parenthesization_level() { ++parenthesization_level_; }
+
   // Type feedback information for assignments and properties.
   virtual bool IsMonomorphic() {
     UNREACHABLE();
@@ -370,6 +375,7 @@ class Expression : public AstNode {
       : AstNode(pos),
         zone_(zone),
         bounds_(Bounds::Unbounded(zone)),
+        parenthesization_level_(0),
         id_(GetNextId(zone)),
         test_id_(GetNextId(zone)) {}
   void set_to_boolean_types(byte types) { to_boolean_types_ = types; }
@@ -379,6 +385,7 @@ class Expression : public AstNode {
  private:
   Bounds bounds_;
   byte to_boolean_types_;
+  unsigned parenthesization_level_;
 
   const BailoutId id_;
   const TypeFeedbackId test_id_;
index b6f858d..3f43c7e 100644 (file)
@@ -165,6 +165,7 @@ DEFINE_BOOL(harmony_numeric_literals, false,
 DEFINE_BOOL(harmony_strings, false, "enable harmony string")
 DEFINE_BOOL(harmony_arrays, false, "enable harmony arrays")
 DEFINE_BOOL(harmony_maths, false, "enable harmony math functions")
+DEFINE_BOOL(harmony_arrow_functions, false, "enable harmony arrow functions")
 DEFINE_BOOL(harmony, false, "enable all harmony features (except typeof)")
 
 DEFINE_IMPLICATION(harmony, harmony_scoping)
@@ -176,6 +177,7 @@ DEFINE_IMPLICATION(harmony, harmony_iteration)
 DEFINE_IMPLICATION(harmony, harmony_numeric_literals)
 DEFINE_IMPLICATION(harmony, harmony_strings)
 DEFINE_IMPLICATION(harmony, harmony_arrays)
+DEFINE_IMPLICATION(harmony, harmony_arrow_functions)
 DEFINE_IMPLICATION(harmony_modules, harmony_scoping)
 DEFINE_IMPLICATION(harmony_collections, harmony_symbols)
 DEFINE_IMPLICATION(harmony_generators, harmony_symbols)
index 12cc197..4f786b5 100644 (file)
@@ -154,6 +154,7 @@ var kMessages = {
   strict_cannot_assign:          ["Cannot assign to read only '", "%0", "' in strict mode"],
   strict_poison_pill:            ["'caller', 'callee', and 'arguments' properties may not be accessed on strict mode functions or the arguments objects for calls to them"],
   strict_caller:                 ["Illegal access to a strict mode caller function."],
+  malformed_arrow_function_parameter_list: ["Malformed arrow function parameter list"],
   generator_poison_pill:         ["'caller' and 'arguments' properties may not be accessed on generator functions."],
   unprotected_let:               ["Illegal let declaration in unprotected statement context."],
   unprotected_const:             ["Illegal const declaration in unprotected statement context."],
index 3542b4c..2098f35 100644 (file)
@@ -718,6 +718,7 @@ Parser::Parser(CompilationInfo* info)
   set_allow_lazy(false);  // Must be explicitly enabled.
   set_allow_generators(FLAG_harmony_generators);
   set_allow_for_of(FLAG_harmony_iteration);
+  set_allow_arrow_functions(FLAG_harmony_arrow_functions);
   set_allow_harmony_numeric_literals(FLAG_harmony_numeric_literals);
   for (int feature = 0; feature < v8::Isolate::kUseCounterFeatureCount;
        ++feature) {
@@ -3271,6 +3272,68 @@ Handle<FixedArray> CompileTimeValue::GetElements(Handle<FixedArray> value) {
 }
 
 
+bool CheckAndDeclareArrowParameter(ParserTraits* traits, Expression* expression,
+                                   Scope* scope, int* num_params,
+                                   Scanner::Location* dupe_loc) {
+  // Case for empty parameter lists:
+  //   () => ...
+  if (expression == NULL) return true;
+
+  // Too many parentheses around expression:
+  //   (( ... )) => ...
+  if (expression->parenthesization_level() > 1) return false;
+
+  // Case for a single parameter:
+  //   (foo) => ...
+  //   foo => ...
+  if (expression->IsVariableProxy()) {
+    if (expression->AsVariableProxy()->is_this()) return false;
+
+    const AstRawString* raw_name = expression->AsVariableProxy()->raw_name();
+    if (traits->IsEvalOrArguments(raw_name) ||
+        traits->IsFutureStrictReserved(raw_name))
+      return false;
+
+    if (scope->IsDeclared(raw_name)) {
+      *dupe_loc = Scanner::Location(
+          expression->position(), expression->position() + raw_name->length());
+      return false;
+    }
+
+    scope->DeclareParameter(raw_name, VAR);
+    ++(*num_params);
+    return true;
+  }
+
+  // Case for more than one parameter:
+  //   (foo, bar [, ...]) => ...
+  if (expression->IsBinaryOperation()) {
+    BinaryOperation* binop = expression->AsBinaryOperation();
+    if (binop->op() != Token::COMMA || binop->left()->is_parenthesized() ||
+        binop->right()->is_parenthesized())
+      return false;
+
+    return CheckAndDeclareArrowParameter(traits, binop->left(), scope,
+                                         num_params, dupe_loc) &&
+           CheckAndDeclareArrowParameter(traits, binop->right(), scope,
+                                         num_params, dupe_loc);
+  }
+
+  // Any other kind of expression is not a valid parameter list.
+  return false;
+}
+
+
+int ParserTraits::DeclareArrowParametersFromExpression(
+    Expression* expression, Scope* scope, Scanner::Location* dupe_loc,
+    bool* ok) {
+  int num_params = 0;
+  *ok = CheckAndDeclareArrowParameter(this, expression, scope, &num_params,
+                                      dupe_loc);
+  return num_params;
+}
+
+
 FunctionLiteral* Parser::ParseFunctionLiteral(
     const AstRawString* function_name,
     Scanner::Location function_name_location,
@@ -3680,6 +3743,7 @@ PreParser::PreParseResult Parser::ParseLazyFunctionBodyWithPreParser(
     reusable_preparser_->set_allow_lazy(true);
     reusable_preparser_->set_allow_generators(allow_generators());
     reusable_preparser_->set_allow_for_of(allow_for_of());
+    reusable_preparser_->set_allow_arrow_functions(allow_arrow_functions());
     reusable_preparser_->set_allow_harmony_numeric_literals(
         allow_harmony_numeric_literals());
   }
index d0b67b7..d88ba96 100644 (file)
@@ -351,9 +351,13 @@ class ParserTraits {
 
     // Used by FunctionState and BlockState.
     typedef v8::internal::Scope Scope;
+    typedef v8::internal::Scope* ScopePtr;
     typedef Variable GeneratorVariable;
     typedef v8::internal::Zone Zone;
 
+    typedef v8::internal::AstProperties AstProperties;
+    typedef Vector<VariableProxy*> ParameterIdentifierVector;
+
     // Return types for traversing functions.
     typedef const AstRawString* Identifier;
     typedef v8::internal::Expression* Expression;
@@ -388,6 +392,7 @@ class ParserTraits {
 
   // Helper functions for recursive descent.
   bool IsEvalOrArguments(const AstRawString* identifier) const;
+  V8_INLINE bool IsFutureStrictReserved(const AstRawString* identifier) const;
 
   // Returns true if the expression is of type "this.foo".
   static bool IsThisProperty(Expression* expression);
@@ -501,14 +506,19 @@ class ParserTraits {
   static Expression* EmptyExpression() {
     return NULL;
   }
+  static Expression* EmptyArrowParamList() { return NULL; }
   static Literal* EmptyLiteral() {
     return NULL;
   }
+
   // Used in error return values.
   static ZoneList<Expression*>* NullExpressionList() {
     return NULL;
   }
 
+  // Non-NULL empty string.
+  V8_INLINE const AstRawString* EmptyIdentifierString();
+
   // Odd-ball literal creators.
   Literal* GetLiteralTheHole(int position,
                              AstNodeFactory<AstConstructionVisitor>* factory);
@@ -537,6 +547,13 @@ class ParserTraits {
   ZoneList<v8::internal::Statement*>* NewStatementList(int size, Zone* zone) {
     return new(zone) ZoneList<v8::internal::Statement*>(size, zone);
   }
+  V8_INLINE Scope* NewScope(Scope* parent_scope, ScopeType scope_type);
+
+  // Utility functions
+  int DeclareArrowParametersFromExpression(Expression* expression, Scope* scope,
+                                           Scanner::Location* dupe_loc,
+                                           bool* ok);
+  V8_INLINE AstValueFactory* ast_value_factory();
 
   // Temporary glue; these functions will move to ParserBase.
   Expression* ParseV8Intrinsic(bool* ok);
@@ -549,6 +566,14 @@ class ParserTraits {
       FunctionLiteral::FunctionType type,
       FunctionLiteral::ArityRestriction arity_restriction,
       bool* ok);
+  V8_INLINE void SkipLazyFunctionBody(const AstRawString* name,
+                                      int* materialized_literal_count,
+                                      int* expected_property_count, bool* ok);
+  V8_INLINE ZoneList<Statement*>* ParseEagerFunctionBody(
+      const AstRawString* name, int pos, Variable* fvar,
+      Token::Value fvar_init_op, bool is_generator, bool* ok);
+  V8_INLINE void CheckConflictingVarDeclarations(v8::internal::Scope* scope,
+                                                 bool* ok);
 
  private:
   Parser* parser_;
@@ -782,6 +807,50 @@ class Parser : public ParserBase<ParserTraits> {
 };
 
 
+bool ParserTraits::IsFutureStrictReserved(
+    const AstRawString* identifier) const {
+  return identifier->IsOneByteEqualTo("yield") ||
+         parser_->scanner()->IdentifierIsFutureStrictReserved(identifier);
+}
+
+
+Scope* ParserTraits::NewScope(Scope* parent_scope, ScopeType scope_type) {
+  return parser_->NewScope(parent_scope, scope_type);
+}
+
+
+const AstRawString* ParserTraits::EmptyIdentifierString() {
+  return parser_->ast_value_factory_->empty_string();
+}
+
+
+void ParserTraits::SkipLazyFunctionBody(const AstRawString* function_name,
+                                        int* materialized_literal_count,
+                                        int* expected_property_count,
+                                        bool* ok) {
+  return parser_->SkipLazyFunctionBody(
+      function_name, materialized_literal_count, expected_property_count, ok);
+}
+
+
+ZoneList<Statement*>* ParserTraits::ParseEagerFunctionBody(
+    const AstRawString* name, int pos, Variable* fvar,
+    Token::Value fvar_init_op, bool is_generator, bool* ok) {
+  return parser_->ParseEagerFunctionBody(name, pos, fvar, fvar_init_op,
+                                         is_generator, ok);
+}
+
+void ParserTraits::CheckConflictingVarDeclarations(v8::internal::Scope* scope,
+                                                   bool* ok) {
+  parser_->CheckConflictingVarDeclarations(scope, ok);
+}
+
+
+AstValueFactory* ParserTraits::ast_value_factory() {
+  return parser_->ast_value_factory_;
+}
+
+
 // Support for handling complex values (array and object literals) that
 // can be fully handled at compile time.
 class CompileTimeValue: public AllStatic {
index 902d20a..15104ae 100644 (file)
@@ -62,11 +62,10 @@ class ParserBase : public Traits {
   // Shorten type names defined by Traits.
   typedef typename Traits::Type::Expression ExpressionT;
   typedef typename Traits::Type::Identifier IdentifierT;
+  typedef typename Traits::Type::FunctionLiteral FunctionLiteralT;
 
-  ParserBase(Scanner* scanner, uintptr_t stack_limit,
-             v8::Extension* extension,
-             ParserRecorder* log,
-             typename Traits::Type::Zone* zone,
+  ParserBase(Scanner* scanner, uintptr_t stack_limit, v8::Extension* extension,
+             ParserRecorder* log, typename Traits::Type::Zone* zone,
              typename Traits::Type::Parser this_object)
       : Traits(this_object),
         parenthesized_function_(false),
@@ -83,7 +82,8 @@ class ParserBase : public Traits {
         allow_natives_syntax_(false),
         allow_generators_(false),
         allow_for_of_(false),
-        zone_(zone) { }
+        allow_arrow_functions_(false),
+        zone_(zone) {}
 
   // Getters that indicate whether certain syntactical constructs are
   // allowed to be parsed by this instance of the parser.
@@ -91,6 +91,7 @@ class ParserBase : public Traits {
   bool allow_natives_syntax() const { return allow_natives_syntax_; }
   bool allow_generators() const { return allow_generators_; }
   bool allow_for_of() const { return allow_for_of_; }
+  bool allow_arrow_functions() const { return allow_arrow_functions_; }
   bool allow_modules() const { return scanner()->HarmonyModules(); }
   bool allow_harmony_scoping() const { return scanner()->HarmonyScoping(); }
   bool allow_harmony_numeric_literals() const {
@@ -103,6 +104,7 @@ class ParserBase : public Traits {
   void set_allow_natives_syntax(bool allow) { allow_natives_syntax_ = allow; }
   void set_allow_generators(bool allow) { allow_generators_ = allow; }
   void set_allow_for_of(bool allow) { allow_for_of_ = allow; }
+  void set_allow_arrow_functions(bool allow) { allow_arrow_functions_ = allow; }
   void set_allow_modules(bool allow) { scanner()->SetHarmonyModules(allow); }
   void set_allow_harmony_scoping(bool allow) {
     scanner()->SetHarmonyScoping(allow);
@@ -152,6 +154,11 @@ class ParserBase : public Traits {
         typename Traits::Type::Scope* scope,
         typename Traits::Type::Zone* zone = NULL,
         AstValueFactory* ast_value_factory = NULL);
+    FunctionState(FunctionState** function_state_stack,
+                  typename Traits::Type::Scope** scope_stack,
+                  typename Traits::Type::Scope** scope,
+                  typename Traits::Type::Zone* zone = NULL,
+                  AstValueFactory* ast_value_factory = NULL);
     ~FunctionState();
 
     int NextMaterializedLiteralIndex() {
@@ -442,6 +449,15 @@ class ParserBase : public Traits {
   ExpressionT ParseMemberExpression(bool* ok);
   ExpressionT ParseMemberExpressionContinuation(ExpressionT expression,
                                                 bool* ok);
+  ExpressionT ParseArrowFunctionLiteral(int start_pos, ExpressionT params_ast,
+                                        bool* ok);
+  ExpressionT ParseArrowFunctionLiteralBody(
+      FunctionState* function_state, typename Traits::Type::ScopePtr scope,
+      int num_parameters, const Scanner::Location& eval_args_error_loc,
+      const Scanner::Location& dupe_error_loc,
+      const Scanner::Location& reserved_loc,
+      FunctionLiteral::IsParenthesizedFlag parenthesized, int start_pos,
+      bool* ok);
 
   // Checks if the expression is a valid reference expression (e.g., on the
   // left-hand side of assignments). Although ruled out by ECMA as early errors,
@@ -526,6 +542,7 @@ class ParserBase : public Traits {
   bool allow_natives_syntax_;
   bool allow_generators_;
   bool allow_for_of_;
+  bool allow_arrow_functions_;
 
   typename Traits::Type::Zone* zone_;  // Only used by Parser.
 };
@@ -555,15 +572,23 @@ class PreParserIdentifier {
   static PreParserIdentifier Yield() {
     return PreParserIdentifier(kYieldIdentifier);
   }
-  bool IsEval() { return type_ == kEvalIdentifier; }
-  bool IsArguments() { return type_ == kArgumentsIdentifier; }
-  bool IsEvalOrArguments() { return type_ >= kEvalIdentifier; }
-  bool IsYield() { return type_ == kYieldIdentifier; }
-  bool IsFutureReserved() { return type_ == kFutureReservedIdentifier; }
-  bool IsFutureStrictReserved() {
+  bool IsEval() const { return type_ == kEvalIdentifier; }
+  bool IsArguments() const { return type_ == kArgumentsIdentifier; }
+  bool IsEvalOrArguments() const { return type_ >= kEvalIdentifier; }
+  bool IsYield() const { return type_ == kYieldIdentifier; }
+  bool IsFutureReserved() const { return type_ == kFutureReservedIdentifier; }
+  bool IsFutureStrictReserved() const {
     return type_ == kFutureStrictReservedIdentifier;
   }
-  bool IsValidStrictVariable() { return type_ == kUnknownIdentifier; }
+  bool IsValidStrictVariable() const { return type_ == kUnknownIdentifier; }
+
+  // Allow identifier->name()[->length()] to work. The preparser
+  // does not need the actual positions/lengths of the identifiers.
+  const PreParserIdentifier* operator->() const { return this; }
+  const PreParserIdentifier raw_name() const { return *this; }
+
+  int position() const { return 0; }
+  int length() const { return 0; }
 
  private:
   enum Type {
@@ -579,6 +604,7 @@ class PreParserIdentifier {
   Type type_;
 
   friend class PreParserExpression;
+  friend class PreParserScope;
 };
 
 
@@ -594,10 +620,26 @@ class PreParserExpression {
   }
 
   static PreParserExpression FromIdentifier(PreParserIdentifier id) {
-    return PreParserExpression(kIdentifierFlag |
+    return PreParserExpression(kTypeIdentifier |
                                (id.type_ << kIdentifierShift));
   }
 
+  static PreParserExpression BinaryOperation(PreParserExpression left,
+                                             Token::Value op,
+                                             PreParserExpression right) {
+    int code = ((op == Token::COMMA) && !left.is_parenthesized() &&
+                !right.is_parenthesized())
+                   ? left.ArrowParamListBit() & right.ArrowParamListBit()
+                   : 0;
+    return PreParserExpression(kTypeBinaryOperation | code);
+  }
+
+  static PreParserExpression EmptyArrowParamList() {
+    // Any expression for which IsValidArrowParamList() returns true
+    // will work here.
+    return FromIdentifier(PreParserIdentifier::Default());
+  }
+
   static PreParserExpression StringLiteral() {
     return PreParserExpression(kUnknownStringLiteral);
   }
@@ -622,18 +664,20 @@ class PreParserExpression {
     return PreParserExpression(kCallExpression);
   }
 
-  bool IsIdentifier() { return (code_ & kIdentifierFlag) != 0; }
+  bool IsIdentifier() const { return (code_ & kTypeMask) == kTypeIdentifier; }
 
-  PreParserIdentifier AsIdentifier() {
+  PreParserIdentifier AsIdentifier() const {
     ASSERT(IsIdentifier());
     return PreParserIdentifier(
         static_cast<PreParserIdentifier::Type>(code_ >> kIdentifierShift));
   }
 
-  bool IsStringLiteral() { return (code_ & kStringLiteralFlag) != 0; }
+  bool IsStringLiteral() const {
+    return (code_ & kTypeMask) == kTypeStringLiteral;
+  }
 
   bool IsUseStrictLiteral() {
-    return (code_ & kStringLiteralMask) == kUseStrictString;
+    return (code_ & kUseStrictString) == kUseStrictString;
   }
 
   bool IsThis() { return code_ == kThisExpression; }
@@ -650,12 +694,30 @@ class PreParserExpression {
     return IsIdentifier() || IsProperty();
   }
 
+  bool IsValidArrowParamList() const {
+    return (ArrowParamListBit() & kBinaryOperationArrowParamList) != 0 &&
+           (code_ & kMultiParenthesizedExpression) == 0;
+  }
+
   // At the moment PreParser doesn't track these expression types.
   bool IsFunctionLiteral() const { return false; }
   bool IsCallNew() const { return false; }
 
   PreParserExpression AsFunctionLiteral() { return *this; }
 
+  bool IsBinaryOperation() const {
+    return (code_ & kTypeMask) == kTypeBinaryOperation;
+  }
+
+  bool is_parenthesized() const {
+    return (code_ & kParenthesizedExpression) != 0;
+  }
+
+  void increase_parenthesization_level() {
+    code_ |= is_parenthesized() ? kMultiParenthesizedExpression
+                                : kParenthesizedExpression;
+  }
+
   // Dummy implementation for making expression->somefunc() work in both Parser
   // and PreParser.
   PreParserExpression* operator->() { return this; }
@@ -664,33 +726,69 @@ class PreParserExpression {
   void set_index(int index) {}  // For YieldExpressions
   void set_parenthesized() {}
 
+  int position() const { return RelocInfo::kNoPosition; }
+  void set_function_token_position(int position) {}
+  void set_ast_properties(int* ast_properties) {}
+  void set_dont_optimize_reason(BailoutReason dont_optimize_reason) {}
+
+  bool operator==(const PreParserExpression& other) const {
+    return code_ == other.code_;
+  }
+  bool operator!=(const PreParserExpression& other) const {
+    return code_ != other.code_;
+  }
+
  private:
-  // Least significant 2 bits are used as flags. Bits 0 and 1 represent
-  // identifiers or strings literals, and are mutually exclusive, but can both
-  // be absent. If the expression is an identifier or a string literal, the
-  // other bits describe the type (see PreParserIdentifier::Type and string
-  // literal constants below).
+  // Least significant 2 bits are used as expression type. The third least
+  // significant bit tracks whether an expression is parenthesized. If the
+  // expression is an identifier or a string literal, the other bits
+  // describe the type/ (see PreParserIdentifier::Type and string literal
+  // constants below). For binary operations, the other bits are flags
+  // which further describe the contents of the expression.
   enum {
     kUnknownExpression = 0,
-    // Identifiers
-    kIdentifierFlag = 1,  // Used to detect labels.
-    kIdentifierShift = 3,
+    kTypeMask = 1 | 2,
+    kParenthesizedExpression = (1 << 2),
+    kMultiParenthesizedExpression = (1 << 3),
 
-    kStringLiteralFlag = 2,  // Used to detect directive prologue.
-    kUnknownStringLiteral = kStringLiteralFlag,
-    kUseStrictString = kStringLiteralFlag | 8,
+    // Identifiers
+    kTypeIdentifier = 1,  // Used to detect labels.
+    kIdentifierShift = 5,
+    kTypeStringLiteral = 2,  // Used to detect directive prologue.
+    kUnknownStringLiteral = kTypeStringLiteral,
+    kUseStrictString = kTypeStringLiteral | 32,
     kStringLiteralMask = kUseStrictString,
 
+    // Binary operations. Those are needed to detect certain keywords and
+    // duplicated identifier in parameter lists for arrow functions, because
+    // they are initially parsed as comma-separated expressions.
+    kTypeBinaryOperation = 3,
+    kBinaryOperationArrowParamList = (1 << 4),
+
     // Below here applies if neither identifier nor string literal. Reserve the
     // 2 least significant bits for flags.
-    kThisExpression = 1 << 2,
-    kThisPropertyExpression = 2 << 2,
-    kPropertyExpression = 3 << 2,
-    kCallExpression = 4 << 2
+    kThisExpression = (1 << 4),
+    kThisPropertyExpression = (2 << 4),
+    kPropertyExpression = (3 << 4),
+    kCallExpression = (4 << 4)
   };
 
   explicit PreParserExpression(int expression_code) : code_(expression_code) {}
 
+  V8_INLINE int ArrowParamListBit() const {
+    if (IsBinaryOperation()) return code_ & kBinaryOperationArrowParamList;
+    if (IsIdentifier()) {
+      const PreParserIdentifier ident = AsIdentifier();
+      // A valid identifier can be an arrow function parameter list
+      // except for eval, arguments, yield, and reserved keywords.
+      if (ident.IsEval() || ident.IsArguments() || ident.IsYield() ||
+          ident.IsFutureStrictReserved())
+        return 0;
+      return kBinaryOperationArrowParamList;
+    }
+    return 0;
+  }
+
   int code_;
 };
 
@@ -772,7 +870,8 @@ class PreParserStatementList {
 
 class PreParserScope {
  public:
-  explicit PreParserScope(PreParserScope* outer_scope, ScopeType scope_type)
+  explicit PreParserScope(PreParserScope* outer_scope, ScopeType scope_type,
+                          void* = NULL)
       : scope_type_(scope_type) {
     strict_mode_ = outer_scope ? outer_scope->strict_mode() : SLOPPY;
   }
@@ -781,6 +880,19 @@ class PreParserScope {
   StrictMode strict_mode() const { return strict_mode_; }
   void SetStrictMode(StrictMode strict_mode) { strict_mode_ = strict_mode; }
 
+  // When PreParser is in use, lazy compilation is already being done,
+  // things cannot get lazier than that.
+  bool AllowsLazyCompilation() const { return false; }
+
+  void set_start_position(int position) {}
+  void set_end_position(int position) {}
+
+  bool IsDeclared(const PreParserIdentifier& identifier) const { return false; }
+  void DeclareParameter(const PreParserIdentifier& identifier, VariableMode) {}
+
+  // Allow scope->Foo() to work.
+  PreParserScope* operator->() { return this; }
+
  private:
   ScopeType scope_type_;
   StrictMode strict_mode_;
@@ -844,7 +956,7 @@ class PreParserFactory {
   PreParserExpression NewBinaryOperation(Token::Value op,
                                          PreParserExpression left,
                                          PreParserExpression right, int pos) {
-    return PreParserExpression::Default();
+    return PreParserExpression::BinaryOperation(left, op, right);
   }
   PreParserExpression NewCompareOperation(Token::Value op,
                                           PreParserExpression left,
@@ -885,6 +997,31 @@ class PreParserFactory {
                                  int pos) {
     return PreParserExpression::Default();
   }
+  PreParserStatement NewReturnStatement(PreParserExpression expression,
+                                        int pos) {
+    return PreParserStatement::Default();
+  }
+  PreParserExpression NewFunctionLiteral(
+      PreParserIdentifier name, AstValueFactory* ast_value_factory,
+      const PreParserScope& scope, PreParserStatementList body,
+      int materialized_literal_count, int expected_property_count,
+      int handler_count, int parameter_count,
+      FunctionLiteral::ParameterFlag has_duplicate_parameters,
+      FunctionLiteral::FunctionType function_type,
+      FunctionLiteral::IsFunctionFlag is_function,
+      FunctionLiteral::IsParenthesizedFlag is_parenthesized,
+      FunctionLiteral::IsGeneratorFlag is_generator, int position) {
+    return PreParserExpression::Default();
+  }
+
+  // Return the object itself as AstVisitor and implement the needed
+  // dummy method right in this class.
+  PreParserFactory* visitor() { return this; }
+  BailoutReason dont_optimize_reason() { return kNoReason; }
+  int* ast_properties() {
+    static int dummy = 42;
+    return &dummy;
+  }
 };
 
 
@@ -899,11 +1036,16 @@ class PreParserTraits {
 
     // Used by FunctionState and BlockState.
     typedef PreParserScope Scope;
+    typedef PreParserScope ScopePtr;
+
     // PreParser doesn't need to store generator variables.
     typedef void GeneratorVariable;
     // No interaction with Zones.
     typedef void Zone;
 
+    typedef int AstProperties;
+    typedef Vector<PreParserIdentifier> ParameterIdentifierVector;
+
     // Return types for traversing functions.
     typedef PreParserIdentifier Identifier;
     typedef PreParserExpression Expression;
@@ -946,6 +1088,10 @@ class PreParserTraits {
     return expression.AsIdentifier();
   }
 
+  static bool IsFutureStrictReserved(PreParserIdentifier identifier) {
+    return identifier.IsYield() || identifier.IsFutureStrictReserved();
+  }
+
   static bool IsBoilerplateProperty(PreParserExpression property) {
     // PreParser doesn't count boilerplate properties.
     return false;
@@ -1009,6 +1155,9 @@ class PreParserTraits {
       const char* type, Handle<Object> arg, int pos) {
     return PreParserExpression::Default();
   }
+  PreParserScope NewScope(PreParserScope* outer_scope, ScopeType scope_type) {
+    return PreParserScope(outer_scope, scope_type);
+  }
 
   // Reporting errors.
   void ReportMessageAt(Scanner::Location location,
@@ -1025,9 +1174,15 @@ class PreParserTraits {
   static PreParserIdentifier EmptyIdentifier() {
     return PreParserIdentifier::Default();
   }
+  static PreParserIdentifier EmptyIdentifierString() {
+    return PreParserIdentifier::Default();
+  }
   static PreParserExpression EmptyExpression() {
     return PreParserExpression::Default();
   }
+  static PreParserExpression EmptyArrowParamList() {
+    return PreParserExpression::EmptyArrowParamList();
+  }
   static PreParserExpression EmptyLiteral() {
     return PreParserExpression::Default();
   }
@@ -1081,6 +1236,31 @@ class PreParserTraits {
     return PreParserExpressionList();
   }
 
+  V8_INLINE void SkipLazyFunctionBody(PreParserIdentifier function_name,
+                                      int* materialized_literal_count,
+                                      int* expected_property_count, bool* ok) {
+    UNREACHABLE();
+  }
+
+  V8_INLINE PreParserStatementList
+      ParseEagerFunctionBody(PreParserIdentifier function_name, int pos,
+                             Variable* fvar, Token::Value fvar_init_op,
+                             bool is_generator, bool* ok);
+
+  // Utility functions
+  int DeclareArrowParametersFromExpression(PreParserExpression expression,
+                                           PreParserScope* scope,
+                                           Scanner::Location* dupe_loc,
+                                           bool* ok) {
+    // TODO(aperez): Detect duplicated identifiers in paramlists.
+    *ok = expression.IsValidArrowParamList();
+    return 0;
+  }
+
+  static AstValueFactory* ast_value_factory() { return NULL; }
+
+  void CheckConflictingVarDeclarations(PreParserScope scope, bool* ok) {}
+
   // Temporary glue; these functions will move to ParserBase.
   PreParserExpression ParseV8Intrinsic(bool* ok);
   PreParserExpression ParseFunctionLiteral(
@@ -1131,7 +1311,7 @@ class PreParser : public ParserBase<PreParserTraits> {
   // during parsing.
   PreParseResult PreParseProgram() {
     PreParserScope scope(scope_, GLOBAL_SCOPE);
-    FunctionState top_scope(&function_state_, &scope_, &scope, NULL);
+    FunctionState top_scope(&function_state_, &scope_, &scope);
     bool ok = true;
     int start_position = scanner()->peek_location().beg_pos;
     ParseSourceElements(Token::EOS, &ok);
@@ -1213,6 +1393,14 @@ class PreParser : public ParserBase<PreParserTraits> {
   Expression ParseObjectLiteral(bool* ok);
   Expression ParseV8Intrinsic(bool* ok);
 
+  V8_INLINE void SkipLazyFunctionBody(PreParserIdentifier function_name,
+                                      int* materialized_literal_count,
+                                      int* expected_property_count, bool* ok);
+  V8_INLINE PreParserStatementList
+      ParseEagerFunctionBody(PreParserIdentifier function_name, int pos,
+                             Variable* fvar, Token::Value fvar_init_op,
+                             bool is_generator, bool* ok);
+
   Expression ParseFunctionLiteral(
       Identifier name,
       Scanner::Location function_name_location,
@@ -1227,6 +1415,28 @@ class PreParser : public ParserBase<PreParserTraits> {
   bool CheckInOrOf(bool accept_OF);
 };
 
+
+PreParserStatementList PreParser::ParseEagerFunctionBody(
+    PreParserIdentifier function_name, int pos, Variable* fvar,
+    Token::Value fvar_init_op, bool is_generator, bool* ok) {
+  ParsingModeScope parsing_mode(this, PARSE_EAGERLY);
+
+  ParseSourceElements(Token::RBRACE, ok);
+  if (!*ok) return PreParserStatementList();
+
+  Expect(Token::RBRACE, ok);
+  return PreParserStatementList();
+}
+
+
+PreParserStatementList PreParserTraits::ParseEagerFunctionBody(
+    PreParserIdentifier function_name, int pos, Variable* fvar,
+    Token::Value fvar_init_op, bool is_generator, bool* ok) {
+  return pre_parser_->ParseEagerFunctionBody(function_name, pos, fvar,
+                                             fvar_init_op, is_generator, ok);
+}
+
+
 template<class Traits>
 ParserBase<Traits>::FunctionState::FunctionState(
     FunctionState** function_state_stack,
@@ -1252,7 +1462,32 @@ ParserBase<Traits>::FunctionState::FunctionState(
 }
 
 
-template<class Traits>
+template <class Traits>
+ParserBase<Traits>::FunctionState::FunctionState(
+    FunctionState** function_state_stack,
+    typename Traits::Type::Scope** scope_stack,
+    typename Traits::Type::Scope** scope,
+    typename Traits::Type::Zone* extra_param,
+    AstValueFactory* ast_value_factory)
+    : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize),
+      next_handler_index_(0),
+      expected_property_count_(0),
+      is_generator_(false),
+      generator_object_variable_(NULL),
+      function_state_stack_(function_state_stack),
+      outer_function_state_(*function_state_stack),
+      scope_stack_(scope_stack),
+      outer_scope_(*scope_stack),
+      saved_ast_node_id_(0),
+      extra_param_(extra_param),
+      factory_(extra_param, ast_value_factory) {
+  *scope_stack_ = *scope;
+  *function_state_stack = this;
+  Traits::SetUpFunctionState(this, extra_param);
+}
+
+
+template <class Traits>
 ParserBase<Traits>::FunctionState::~FunctionState() {
   *scope_stack_ = outer_scope_;
   *function_state_stack_ = outer_function_state_;
@@ -1469,11 +1704,20 @@ ParserBase<Traits>::ParsePrimaryExpression(bool* ok) {
 
     case Token::LPAREN:
       Consume(Token::LPAREN);
-      // Heuristically try to detect immediately called functions before
-      // seeing the call parentheses.
-      parenthesized_function_ = (peek() == Token::FUNCTION);
-      result = this->ParseExpression(true, CHECK_OK);
-      Expect(Token::RPAREN, CHECK_OK);
+      if (allow_arrow_functions() && peek() == Token::RPAREN) {
+        // Arrow functions are the only expression type constructions
+        // for which an empty parameter list "()" is valid input.
+        Consume(Token::RPAREN);
+        return this->ParseArrowFunctionLiteral(pos, this->EmptyArrowParamList(),
+                                               CHECK_OK);
+      } else {
+        // Heuristically try to detect immediately called functions before
+        // seeing the call parentheses.
+        parenthesized_function_ = (peek() == Token::FUNCTION);
+        result = this->ParseExpression(true, CHECK_OK);
+        result->increase_parenthesization_level();
+        Expect(Token::RPAREN, CHECK_OK);
+      }
       break;
 
     case Token::MOD:
@@ -1745,6 +1989,7 @@ typename ParserBase<Traits>::ExpressionT
 ParserBase<Traits>::ParseAssignmentExpression(bool accept_IN, bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   ArrowFunction
   //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
@@ -1758,6 +2003,10 @@ ParserBase<Traits>::ParseAssignmentExpression(bool accept_IN, bool* ok) {
   ExpressionT expression =
       this->ParseConditionalExpression(accept_IN, CHECK_OK);
 
+  if (allow_arrow_functions() && peek() == Token::ARROW)
+    return this->ParseArrowFunctionLiteral(lhs_location.beg_pos, expression,
+                                           CHECK_OK);
+
   if (!Token::IsAssignmentOp(peek())) {
     if (fni_ != NULL) fni_->Leave();
     // Parsed conditional expression only (no assignment).
@@ -2184,6 +2433,105 @@ ParserBase<Traits>::ParseMemberExpressionContinuation(ExpressionT expression,
 }
 
 
+template <class Traits>
+typename ParserBase<Traits>::ExpressionT ParserBase<
+    Traits>::ParseArrowFunctionLiteral(int start_pos, ExpressionT params_ast,
+                                       bool* ok) {
+  // TODO(aperez): Change this to use ARROW_SCOPE
+  typename Traits::Type::ScopePtr scope =
+      this->NewScope(scope_, FUNCTION_SCOPE);
+
+  FunctionState function_state(&function_state_, &scope_, &scope, zone(),
+                               this->ast_value_factory());
+  Scanner::Location dupe_error_loc = Scanner::Location::invalid();
+  int num_params = Traits::DeclareArrowParametersFromExpression(
+      params_ast, scope_, &dupe_error_loc, ok);
+  if (!*ok) {
+    ReportMessageAt(Scanner::Location(start_pos, scanner()->location().beg_pos),
+                    "malformed_arrow_function_parameter_list");
+    return this->EmptyExpression();
+  }
+
+  if (num_params > Code::kMaxArguments) {
+    ReportMessageAt(Scanner::Location(params_ast->position(), position()),
+                    "too_many_parameters");
+    *ok = false;
+    return this->EmptyExpression();
+  }
+
+  ExpressionT expression = ParseArrowFunctionLiteralBody(
+      &function_state, scope, num_params, Scanner::Location::invalid(),
+      dupe_error_loc, Scanner::Location::invalid(),
+      FunctionLiteral::kNotParenthesized, start_pos, CHECK_OK);
+  return expression;
+}
+
+
+template <class Traits>
+typename ParserBase<Traits>::ExpressionT
+ParserBase<Traits>::ParseArrowFunctionLiteralBody(
+    FunctionState* function_state, typename Traits::Type::ScopePtr scope,
+    int num_parameters, const Scanner::Location& eval_args_error_loc,
+    const Scanner::Location& dupe_error_loc,
+    const Scanner::Location& reserved_loc,
+    FunctionLiteral::IsParenthesizedFlag parenthesized, int start_pos,
+    bool* ok) {
+  int materialized_literal_count = -1;
+  int expected_property_count = -1;
+
+  Expect(Token::ARROW, CHECK_OK);
+
+  if (peek() == Token::LBRACE) {
+    // Multiple statemente body
+    Consume(Token::LBRACE);
+    bool is_lazily_parsed =
+        (mode() == PARSE_LAZILY && scope_->AllowsLazyCompilation());
+    if (is_lazily_parsed) {
+      this->SkipLazyFunctionBody(this->EmptyIdentifier(),
+                                 &materialized_literal_count,
+                                 &expected_property_count, CHECK_OK);
+    } else {
+      this->ParseEagerFunctionBody(this->EmptyIdentifier(),
+                                   RelocInfo::kNoPosition, NULL,
+                                   Token::INIT_VAR, false,  // Not a generator.
+                                   CHECK_OK);
+    }
+  } else {
+    // Single-expression body
+    ParseAssignmentExpression(true, CHECK_OK);
+  }
+
+  scope->set_start_position(start_pos);
+  scope->set_end_position(scanner()->location().end_pos);
+
+  // Arrow function *parameter lists* are always checked as in strict mode.
+  this->CheckStrictFunctionNameAndParameters(
+      this->EmptyIdentifier(), false, Scanner::Location::invalid(),
+      Scanner::Location::invalid(), dupe_error_loc,
+      Scanner::Location::invalid(), CHECK_OK);
+
+  // Validate strict mode.
+  if (strict_mode() == STRICT) {
+    CheckOctalLiteral(start_pos, scanner()->location().end_pos, CHECK_OK);
+  }
+
+  if (allow_harmony_scoping() && strict_mode() == STRICT)
+    this->CheckConflictingVarDeclarations(scope, CHECK_OK);
+
+  // TODO(aperez): Generate a proper FunctionLiteral instead of
+  // returning a dummy value.
+  FunctionLiteralT function_literal = factory()->NewFunctionLiteral(
+      this->EmptyIdentifierString(), this->ast_value_factory(), scope,
+      this->NewStatementList(0, zone()), 0, 0, 0, num_parameters,
+      FunctionLiteral::kNoDuplicateParameters,
+      FunctionLiteral::ANONYMOUS_EXPRESSION, FunctionLiteral::kIsFunction,
+      FunctionLiteral::kNotParenthesized, FunctionLiteral::kNotGenerator,
+      start_pos);
+  function_literal->set_function_token_position(start_pos);
+  return function_literal;
+}
+
+
 template <typename Traits>
 typename ParserBase<Traits>::ExpressionT
 ParserBase<Traits>::CheckAndRewriteReferenceExpression(
index 3c68201..e0858c7 100644 (file)
@@ -466,10 +466,12 @@ void Scanner::Scan() {
         break;
 
       case '=':
-        // = == ===
+        // = == === =>
         Advance();
         if (c0_ == '=') {
           token = Select('=', Token::EQ_STRICT, Token::EQ);
+        } else if (c0_ == '>') {
+          token = Select(Token::ARROW);
         } else {
           token = Token::ASSIGN;
         }
@@ -1006,6 +1008,16 @@ static Token::Value KeywordOrIdentifierToken(const uint8_t* input,
 }
 
 
+bool Scanner::IdentifierIsFutureStrictReserved(
+    const AstRawString* string) const {
+  // Keywords are always 1-byte strings.
+  return string->is_one_byte() &&
+         Token::FUTURE_STRICT_RESERVED_WORD ==
+             KeywordOrIdentifierToken(string->raw_data(), string->length(),
+                                      harmony_scoping_, harmony_modules_);
+}
+
+
 Token::Value Scanner::ScanIdentifierOrKeyword() {
   ASSERT(unicode_cache_->IsIdentifierStart(c0_));
   LiteralScope literal(this);
index d71b994..3f57a19 100644 (file)
@@ -458,6 +458,8 @@ class Scanner {
     return &source_mapping_url_;
   }
 
+  bool IdentifierIsFutureStrictReserved(const AstRawString* string) const;
+
  private:
   // The current and look-ahead token.
   struct TokenDesc {
index 6156036..2e7d5d9 100644 (file)
@@ -25,138 +25,139 @@ namespace internal {
 
 #define IGNORE_TOKEN(name, string, precedence)
 
-#define TOKEN_LIST(T, K)                                                \
-  /* End of source indicator. */                                        \
-  T(EOS, "EOS", 0)                                                      \
-                                                                        \
-  /* Punctuators (ECMA-262, section 7.7, page 15). */                   \
-  T(LPAREN, "(", 0)                                                     \
-  T(RPAREN, ")", 0)                                                     \
-  T(LBRACK, "[", 0)                                                     \
-  T(RBRACK, "]", 0)                                                     \
-  T(LBRACE, "{", 0)                                                     \
-  T(RBRACE, "}", 0)                                                     \
-  T(COLON, ":", 0)                                                      \
-  T(SEMICOLON, ";", 0)                                                  \
-  T(PERIOD, ".", 0)                                                     \
-  T(CONDITIONAL, "?", 3)                                                \
-  T(INC, "++", 0)                                                       \
-  T(DEC, "--", 0)                                                       \
-                                                                        \
-  /* Assignment operators. */                                           \
-  /* IsAssignmentOp() and Assignment::is_compound() relies on */        \
-  /* this block of enum values being contiguous and sorted in the */    \
-  /* same order! */                                                     \
-  T(INIT_VAR, "=init_var", 2)  /* AST-use only. */                      \
-  T(INIT_LET, "=init_let", 2)  /* AST-use only. */                      \
-  T(INIT_CONST, "=init_const", 2)  /* AST-use only. */                  \
-  T(INIT_CONST_LEGACY, "=init_const_legacy", 2)  /* AST-use only. */    \
-  T(ASSIGN, "=", 2)                                                     \
-  T(ASSIGN_BIT_OR, "|=", 2)                                             \
-  T(ASSIGN_BIT_XOR, "^=", 2)                                            \
-  T(ASSIGN_BIT_AND, "&=", 2)                                            \
-  T(ASSIGN_SHL, "<<=", 2)                                               \
-  T(ASSIGN_SAR, ">>=", 2)                                               \
-  T(ASSIGN_SHR, ">>>=", 2)                                              \
-  T(ASSIGN_ADD, "+=", 2)                                                \
-  T(ASSIGN_SUB, "-=", 2)                                                \
-  T(ASSIGN_MUL, "*=", 2)                                                \
-  T(ASSIGN_DIV, "/=", 2)                                                \
-  T(ASSIGN_MOD, "%=", 2)                                                \
-                                                                        \
-  /* Binary operators sorted by precedence. */                          \
-  /* IsBinaryOp() relies on this block of enum values */                \
-  /* being contiguous and sorted in the same order! */                  \
-  T(COMMA, ",", 1)                                                      \
-  T(OR, "||", 4)                                                        \
-  T(AND, "&&", 5)                                                       \
-  T(BIT_OR, "|", 6)                                                     \
-  T(BIT_XOR, "^", 7)                                                    \
-  T(BIT_AND, "&", 8)                                                    \
-  T(SHL, "<<", 11)                                                      \
-  T(SAR, ">>", 11)                                                      \
-  T(SHR, ">>>", 11)                                                     \
-  T(ROR, "rotate right", 11)   /* only used by Crankshaft */            \
-  T(ADD, "+", 12)                                                       \
-  T(SUB, "-", 12)                                                       \
-  T(MUL, "*", 13)                                                       \
-  T(DIV, "/", 13)                                                       \
-  T(MOD, "%", 13)                                                       \
-                                                                        \
-  /* Compare operators sorted by precedence. */                         \
-  /* IsCompareOp() relies on this block of enum values */               \
-  /* being contiguous and sorted in the same order! */                  \
-  T(EQ, "==", 9)                                                        \
-  T(NE, "!=", 9)                                                        \
-  T(EQ_STRICT, "===", 9)                                                \
-  T(NE_STRICT, "!==", 9)                                                \
-  T(LT, "<", 10)                                                        \
-  T(GT, ">", 10)                                                        \
-  T(LTE, "<=", 10)                                                      \
-  T(GTE, ">=", 10)                                                      \
-  K(INSTANCEOF, "instanceof", 10)                                       \
-  K(IN, "in", 10)                                                       \
-                                                                        \
-  /* Unary operators. */                                                \
-  /* IsUnaryOp() relies on this block of enum values */                 \
-  /* being contiguous and sorted in the same order! */                  \
-  T(NOT, "!", 0)                                                        \
-  T(BIT_NOT, "~", 0)                                                    \
-  K(DELETE, "delete", 0)                                                \
-  K(TYPEOF, "typeof", 0)                                                \
-  K(VOID, "void", 0)                                                    \
-                                                                        \
-  /* Keywords (ECMA-262, section 7.5.2, page 13). */                    \
-  K(BREAK, "break", 0)                                                  \
-  K(CASE, "case", 0)                                                    \
-  K(CATCH, "catch", 0)                                                  \
-  K(CONTINUE, "continue", 0)                                            \
-  K(DEBUGGER, "debugger", 0)                                            \
-  K(DEFAULT, "default", 0)                                              \
-  /* DELETE */                                                          \
-  K(DO, "do", 0)                                                        \
-  K(ELSE, "else", 0)                                                    \
-  K(FINALLY, "finally", 0)                                              \
-  K(FOR, "for", 0)                                                      \
-  K(FUNCTION, "function", 0)                                            \
-  K(IF, "if", 0)                                                        \
-  /* IN */                                                              \
-  /* INSTANCEOF */                                                      \
-  K(NEW, "new", 0)                                                      \
-  K(RETURN, "return", 0)                                                \
-  K(SWITCH, "switch", 0)                                                \
-  K(THIS, "this", 0)                                                    \
-  K(THROW, "throw", 0)                                                  \
-  K(TRY, "try", 0)                                                      \
-  /* TYPEOF */                                                          \
-  K(VAR, "var", 0)                                                      \
-  /* VOID */                                                            \
-  K(WHILE, "while", 0)                                                  \
-  K(WITH, "with", 0)                                                    \
-                                                                        \
-  /* Literals (ECMA-262, section 7.8, page 16). */                      \
-  K(NULL_LITERAL, "null", 0)                                            \
-  K(TRUE_LITERAL, "true", 0)                                            \
-  K(FALSE_LITERAL, "false", 0)                                          \
-  T(NUMBER, NULL, 0)                                                    \
-  T(STRING, NULL, 0)                                                    \
-                                                                        \
-  /* Identifiers (not keywords or future reserved words). */            \
-  T(IDENTIFIER, NULL, 0)                                                \
-                                                                        \
-  /* Future reserved words (ECMA-262, section 7.6.1.2). */              \
-  T(FUTURE_RESERVED_WORD, NULL, 0)                                      \
-  T(FUTURE_STRICT_RESERVED_WORD, NULL, 0)                               \
-  K(CONST, "const", 0)                                                  \
-  K(EXPORT, "export", 0)                                                \
-  K(IMPORT, "import", 0)                                                \
-  K(LET, "let", 0)                                                      \
-  K(YIELD, "yield", 0)                                                  \
-                                                                        \
-  /* Illegal token - not able to scan. */                               \
-  T(ILLEGAL, "ILLEGAL", 0)                                              \
-                                                                        \
-  /* Scanner-internal use only. */                                      \
+#define TOKEN_LIST(T, K)                                             \
+  /* End of source indicator. */                                     \
+  T(EOS, "EOS", 0)                                                   \
+                                                                     \
+  /* Punctuators (ECMA-262, section 7.7, page 15). */                \
+  T(LPAREN, "(", 0)                                                  \
+  T(RPAREN, ")", 0)                                                  \
+  T(LBRACK, "[", 0)                                                  \
+  T(RBRACK, "]", 0)                                                  \
+  T(LBRACE, "{", 0)                                                  \
+  T(RBRACE, "}", 0)                                                  \
+  T(COLON, ":", 0)                                                   \
+  T(SEMICOLON, ";", 0)                                               \
+  T(PERIOD, ".", 0)                                                  \
+  T(CONDITIONAL, "?", 3)                                             \
+  T(INC, "++", 0)                                                    \
+  T(DEC, "--", 0)                                                    \
+  T(ARROW, "=>", 0)                                                  \
+                                                                     \
+  /* Assignment operators. */                                        \
+  /* IsAssignmentOp() and Assignment::is_compound() relies on */     \
+  /* this block of enum values being contiguous and sorted in the */ \
+  /* same order! */                                                  \
+  T(INIT_VAR, "=init_var", 2)                   /* AST-use only. */  \
+  T(INIT_LET, "=init_let", 2)                   /* AST-use only. */  \
+  T(INIT_CONST, "=init_const", 2)               /* AST-use only. */  \
+  T(INIT_CONST_LEGACY, "=init_const_legacy", 2) /* AST-use only. */  \
+  T(ASSIGN, "=", 2)                                                  \
+  T(ASSIGN_BIT_OR, "|=", 2)                                          \
+  T(ASSIGN_BIT_XOR, "^=", 2)                                         \
+  T(ASSIGN_BIT_AND, "&=", 2)                                         \
+  T(ASSIGN_SHL, "<<=", 2)                                            \
+  T(ASSIGN_SAR, ">>=", 2)                                            \
+  T(ASSIGN_SHR, ">>>=", 2)                                           \
+  T(ASSIGN_ADD, "+=", 2)                                             \
+  T(ASSIGN_SUB, "-=", 2)                                             \
+  T(ASSIGN_MUL, "*=", 2)                                             \
+  T(ASSIGN_DIV, "/=", 2)                                             \
+  T(ASSIGN_MOD, "%=", 2)                                             \
+                                                                     \
+  /* Binary operators sorted by precedence. */                       \
+  /* IsBinaryOp() relies on this block of enum values */             \
+  /* being contiguous and sorted in the same order! */               \
+  T(COMMA, ",", 1)                                                   \
+  T(OR, "||", 4)                                                     \
+  T(AND, "&&", 5)                                                    \
+  T(BIT_OR, "|", 6)                                                  \
+  T(BIT_XOR, "^", 7)                                                 \
+  T(BIT_AND, "&", 8)                                                 \
+  T(SHL, "<<", 11)                                                   \
+  T(SAR, ">>", 11)                                                   \
+  T(SHR, ">>>", 11)                                                  \
+  T(ROR, "rotate right", 11) /* only used by Crankshaft */           \
+  T(ADD, "+", 12)                                                    \
+  T(SUB, "-", 12)                                                    \
+  T(MUL, "*", 13)                                                    \
+  T(DIV, "/", 13)                                                    \
+  T(MOD, "%", 13)                                                    \
+                                                                     \
+  /* Compare operators sorted by precedence. */                      \
+  /* IsCompareOp() relies on this block of enum values */            \
+  /* being contiguous and sorted in the same order! */               \
+  T(EQ, "==", 9)                                                     \
+  T(NE, "!=", 9)                                                     \
+  T(EQ_STRICT, "===", 9)                                             \
+  T(NE_STRICT, "!==", 9)                                             \
+  T(LT, "<", 10)                                                     \
+  T(GT, ">", 10)                                                     \
+  T(LTE, "<=", 10)                                                   \
+  T(GTE, ">=", 10)                                                   \
+  K(INSTANCEOF, "instanceof", 10)                                    \
+  K(IN, "in", 10)                                                    \
+                                                                     \
+  /* Unary operators. */                                             \
+  /* IsUnaryOp() relies on this block of enum values */              \
+  /* being contiguous and sorted in the same order! */               \
+  T(NOT, "!", 0)                                                     \
+  T(BIT_NOT, "~", 0)                                                 \
+  K(DELETE, "delete", 0)                                             \
+  K(TYPEOF, "typeof", 0)                                             \
+  K(VOID, "void", 0)                                                 \
+                                                                     \
+  /* Keywords (ECMA-262, section 7.5.2, page 13). */                 \
+  K(BREAK, "break", 0)                                               \
+  K(CASE, "case", 0)                                                 \
+  K(CATCH, "catch", 0)                                               \
+  K(CONTINUE, "continue", 0)                                         \
+  K(DEBUGGER, "debugger", 0)                                         \
+  K(DEFAULT, "default", 0)                                           \
+  /* DELETE */                                                       \
+  K(DO, "do", 0)                                                     \
+  K(ELSE, "else", 0)                                                 \
+  K(FINALLY, "finally", 0)                                           \
+  K(FOR, "for", 0)                                                   \
+  K(FUNCTION, "function", 0)                                         \
+  K(IF, "if", 0)                                                     \
+  /* IN */                                                           \
+  /* INSTANCEOF */                                                   \
+  K(NEW, "new", 0)                                                   \
+  K(RETURN, "return", 0)                                             \
+  K(SWITCH, "switch", 0)                                             \
+  K(THIS, "this", 0)                                                 \
+  K(THROW, "throw", 0)                                               \
+  K(TRY, "try", 0)                                                   \
+  /* TYPEOF */                                                       \
+  K(VAR, "var", 0)                                                   \
+  /* VOID */                                                         \
+  K(WHILE, "while", 0)                                               \
+  K(WITH, "with", 0)                                                 \
+                                                                     \
+  /* Literals (ECMA-262, section 7.8, page 16). */                   \
+  K(NULL_LITERAL, "null", 0)                                         \
+  K(TRUE_LITERAL, "true", 0)                                         \
+  K(FALSE_LITERAL, "false", 0)                                       \
+  T(NUMBER, NULL, 0)                                                 \
+  T(STRING, NULL, 0)                                                 \
+                                                                     \
+  /* Identifiers (not keywords or future reserved words). */         \
+  T(IDENTIFIER, NULL, 0)                                             \
+                                                                     \
+  /* Future reserved words (ECMA-262, section 7.6.1.2). */           \
+  T(FUTURE_RESERVED_WORD, NULL, 0)                                   \
+  T(FUTURE_STRICT_RESERVED_WORD, NULL, 0)                            \
+  K(CONST, "const", 0)                                               \
+  K(EXPORT, "export", 0)                                             \
+  K(IMPORT, "import", 0)                                             \
+  K(LET, "let", 0)                                                   \
+  K(YIELD, "yield", 0)                                               \
+                                                                     \
+  /* Illegal token - not able to scan. */                            \
+  T(ILLEGAL, "ILLEGAL", 0)                                           \
+                                                                     \
+  /* Scanner-internal use only. */                                   \
   T(WHITESPACE, NULL, 0)
 
 
index 8a1e429..e21eabd 100644 (file)
@@ -286,6 +286,7 @@ TEST(StandAlonePreParser) {
       "function foo(x, y) { return x + y; }",
       "%ArgleBargle(glop);",
       "var x = new new Function('this.x = 42');",
+      "var f = (x, y) => x + y;",
       NULL
   };
 
@@ -302,6 +303,7 @@ TEST(StandAlonePreParser) {
     i::PreParser preparser(&scanner, &log, stack_limit);
     preparser.set_allow_lazy(true);
     preparser.set_allow_natives_syntax(true);
+    preparser.set_allow_arrow_functions(true);
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
     CHECK(!log.HasError());
@@ -1185,7 +1187,8 @@ enum ParserFlag {
   kAllowModules,
   kAllowGenerators,
   kAllowForOf,
-  kAllowHarmonyNumericLiterals
+  kAllowHarmonyNumericLiterals,
+  kAllowArrowFunctions
 };
 
 
@@ -1206,6 +1209,7 @@ void SetParserFlags(i::ParserBase<Traits>* parser,
   parser->set_allow_for_of(flags.Contains(kAllowForOf));
   parser->set_allow_harmony_numeric_literals(
       flags.Contains(kAllowHarmonyNumericLiterals));
+  parser->set_allow_arrow_functions(flags.Contains(kAllowArrowFunctions));
 }
 
 
@@ -1412,7 +1416,7 @@ TEST(ParserSync) {
 
   static const ParserFlag flags1[] = {
     kAllowLazy, kAllowHarmonyScoping, kAllowModules, kAllowGenerators,
-    kAllowForOf
+    kAllowForOf, kAllowArrowFunctions
   };
   for (int i = 0; context_data[i][0] != NULL; ++i) {
     for (int j = 0; statement_data[j] != NULL; ++j) {
@@ -1489,7 +1493,7 @@ void RunParserSyncTest(const char* context_data[][2],
 
   static const ParserFlag default_flags[] = {
     kAllowLazy, kAllowHarmonyScoping, kAllowModules, kAllowGenerators,
-    kAllowForOf, kAllowNativesSyntax
+    kAllowForOf, kAllowNativesSyntax, kAllowArrowFunctions
   };
   ParserFlag* generated_flags = NULL;
   if (flags == NULL) {
@@ -1566,6 +1570,10 @@ TEST(ErrorsEvalAndArguments) {
     "function foo(arguments) { }",
     "function foo(bar, eval) { }",
     "function foo(bar, arguments) { }",
+    "(eval) => { }",
+    "(arguments) => { }",
+    "(foo, eval) => { }",
+    "(foo, arguments) => { }",
     "eval = 1;",
     "arguments = 1;",
     "var foo = eval = 1;",
@@ -1622,6 +1630,7 @@ TEST(NoErrorsEvalAndArgumentsStrict) {
   const char* context_data[][2] = {
     { "\"use strict\";", "" },
     { "function test_func() { \"use strict\";", "}" },
+    { "() => { \"use strict\"; ", "}" },
     { NULL, NULL }
   };
 
@@ -1637,7 +1646,9 @@ TEST(NoErrorsEvalAndArgumentsStrict) {
     NULL
   };
 
-  RunParserSyncTest(context_data, statement_data, kSuccess);
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kSuccess, NULL, 0,
+                    always_flags, ARRAY_SIZE(always_flags));
 }
 
 
@@ -1649,6 +1660,7 @@ TEST(ErrorsFutureStrictReservedWords) {
   const char* context_data[][2] = {
     { "\"use strict\";", "" },
     { "function test_func() {\"use strict\"; ", "}"},
+    { "() => { \"use strict\"; ", "}" },
     { NULL, NULL }
   };
 
@@ -1667,7 +1679,9 @@ TEST(ErrorsFutureStrictReservedWords) {
     NULL
   };
 
-  RunParserSyncTest(context_data, statement_data, kError);
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kError, NULL, 0, always_flags,
+                    ARRAY_SIZE(always_flags));
 }
 
 
@@ -1675,6 +1689,7 @@ TEST(NoErrorsFutureStrictReservedWords) {
   const char* context_data[][2] = {
     { "", "" },
     { "function test_func() {", "}"},
+    { "() => {", "}" },
     { NULL, NULL }
   };
 
@@ -1693,7 +1708,9 @@ TEST(NoErrorsFutureStrictReservedWords) {
     NULL
   };
 
-  RunParserSyncTest(context_data, statement_data, kSuccess);
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kSuccess, NULL, 0,
+                    always_flags, ARRAY_SIZE(always_flags));
 }
 
 
@@ -1706,6 +1723,8 @@ TEST(ErrorsReservedWords) {
     { "\"use strict\";", "" },
     { "var eval; function test_func() {", "}"},
     { "var eval; function test_func() {\"use strict\"; ", "}"},
+    { "var eval; () => {", "}"},
+    { "var eval; () => {\"use strict\"; ", "}"},
     { NULL, NULL }
   };
 
@@ -1716,6 +1735,8 @@ TEST(ErrorsReservedWords) {
     "function super() { }",
     "function foo(super) { }",
     "function foo(bar, super) { }",
+    "(super) => { }",
+    "(bar, super) => { }",
     "super = 1;",
     "var foo = super = 1;",
     "++super;",
@@ -1794,7 +1815,9 @@ TEST(NoErrorsYieldSloppyAllModes) {
     NULL
   };
 
-  RunParserSyncTest(context_data, statement_data, kSuccess);
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kSuccess, NULL, 0,
+                    always_flags, ARRAY_SIZE(always_flags));
 }
 
 
@@ -1849,6 +1872,7 @@ TEST(ErrorsYieldStrict) {
     { "\"use strict\"; function * gen() { function not_gen() {", "} }" },
     { "\"use strict\"; (function not_gen() {", "})" },
     { "\"use strict\"; (function * gen() { (function not_gen() {", "}) })" },
+    { "() => {\"use strict\"; ", "}" },
     { NULL, NULL }
   };
 
@@ -2059,6 +2083,7 @@ TEST(ErrorsIllegalWordsAsLabelsSloppy) {
   const char* context_data[][2] = {
     { "", ""},
     { "function test_func() {", "}" },
+    { "() => {", "}" },
     { NULL, NULL }
   };
 
@@ -2076,6 +2101,7 @@ TEST(ErrorsIllegalWordsAsLabelsStrict) {
   const char* context_data[][2] = {
     { "\"use strict\";", "" },
     { "function test_func() {\"use strict\"; ", "}"},
+    { "() => {\"use strict\"; ", "}" },
     { NULL, NULL }
   };
 
@@ -2095,8 +2121,10 @@ TEST(NoErrorsIllegalWordsAsLabels) {
   const char* context_data[][2] = {
     { "", ""},
     { "function test_func() {", "}" },
+    { "() => {", "}" },
     { "\"use strict\";", "" },
     { "\"use strict\"; function test_func() {", "}" },
+    { "\"use strict\"; () => {", "}" },
     { NULL, NULL }
   };
 
@@ -2107,7 +2135,9 @@ TEST(NoErrorsIllegalWordsAsLabels) {
     NULL
   };
 
-  RunParserSyncTest(context_data, statement_data, kSuccess);
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kSuccess, NULL, 0,
+                    always_flags, ARRAY_SIZE(always_flags));
 }
 
 
@@ -2116,6 +2146,7 @@ TEST(ErrorsParenthesizedLabels) {
   const char* context_data[][2] = {
     { "", ""},
     { "function test_func() {", "}" },
+    { "() => {", "}" },
     { NULL, NULL }
   };
 
@@ -2956,3 +2987,160 @@ TEST(UseAsmUseCount) {
              "function bar() { \"use asm\"; var baz = 1; }");
   CHECK_EQ(2, use_counts[v8::Isolate::kUseAsm]);
 }
+
+
+TEST(ErrorsArrowFunctions) {
+  // Tests that parser and preparser generate the same kind of errors
+  // on invalid arrow function syntax.
+  const char* context_data[][2] = {
+    {"", ";"},
+    {"v = ", ";"},
+    {"bar ? (", ") : baz;"},
+    {"bar ? baz : (", ");"},
+    {"bar[", "];"},
+    {"bar, ", ";"},
+    {"", ", bar;"},
+    {NULL, NULL}
+  };
+
+  const char* statement_data[] = {
+    "=> 0",
+    "=>",
+    "() =>",
+    "=> {}",
+    ") => {}",
+    ", => {}",
+    "(,) => {}",
+    "return => {}",
+    "() => {'value': 42}",
+
+    // Check that the early return introduced in ParsePrimaryExpression
+    // does not accept stray closing parentheses.
+    ")",
+    ") => 0",
+    "foo[()]",
+    "()",
+
+    // Parameter lists with extra parens should be recognized as errors.
+    "(()) => 0",
+    "((x)) => 0",
+    "((x, y)) => 0",
+    "(x, (y)) => 0",
+    "((x, y, z)) => 0",
+    "(x, (y, z)) => 0",
+    "((x, y), z) => 0",
+
+    // Parameter lists are always validated as strict, so those are errors.
+    "eval => {}",
+    "arguments => {}",
+    "yield => {}",
+    "interface => {}",
+    "(eval) => {}",
+    "(arguments) => {}",
+    "(yield) => {}",
+    "(interface) => {}",
+    "(eval, bar) => {}",
+    "(bar, eval) => {}",
+    "(bar, arguments) => {}",
+    "(bar, yield) => {}",
+    "(bar, interface) => {}",
+    // TODO(aperez): Detecting duplicates does not work in PreParser.
+    // "(bar, bar) => {}",
+
+    // The parameter list is parsed as an expression, but only
+    // a comma-separated list of identifier is valid.
+    "32 => {}",
+    "(32) => {}",
+    "(a, 32) => {}",
+    "if => {}",
+    "(if) => {}",
+    "(a, if) => {}",
+    "a + b => {}",
+    "(a + b) => {}",
+    "(a + b, c) => {}",
+    "(a, b - c) => {}",
+    "\"a\" => {}",
+    "(\"a\") => {}",
+    "(\"a\", b) => {}",
+    "(a, \"b\") => {}",
+    "-a => {}",
+    "(-a) => {}",
+    "(-a, b) => {}",
+    "(a, -b) => {}",
+    "{} => {}",
+    "({}) => {}",
+    "(a, {}) => {}",
+    "({}, a) => {}",
+    "a++ => {}",
+    "(a++) => {}",
+    "(a++, b) => {}",
+    "(a, b++) => {}",
+    "[] => {}",
+    "([]) => {}",
+    "(a, []) => {}",
+    "([], a) => {}",
+    "(a = b) => {}",
+    "(a = b, c) => {}",
+    "(a, b = c) => {}",
+    "(foo ? bar : baz) => {}",
+    "(a, foo ? bar : baz) => {}",
+    "(foo ? bar : baz, a) => {}",
+    NULL
+  };
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
+
+
+TEST(NoErrorsArrowFunctions) {
+  // Tests that parser and preparser accept valid arrow functions syntax.
+  const char* context_data[][2] = {
+    {"", ";"},
+    {"bar ? (", ") : baz;"},
+    {"bar ? baz : (", ");"},
+    {"bar, ", ";"},
+    {"", ", bar;"},
+    {NULL, NULL}
+  };
+
+  const char* statement_data[] = {
+    "() => {}",
+    "() => { return 42 }",
+    "x => { return x; }",
+    "(x) => { return x; }",
+    "(x, y) => { return x + y; }",
+    "(x, y, z) => { return x + y + z; }",
+    "(x, y) => { x.a = y; }",
+    "() => 42",
+    "x => x",
+    "x => x * x",
+    "(x) => x",
+    "(x) => x * x",
+    "(x, y) => x + y",
+    "(x, y, z) => x, y, z",
+    "(x, y) => x.a = y",
+    "() => ({'value': 42})",
+    "x => y => x + y",
+    "(x, y) => (u, v) => x*u + y*v",
+    "(x, y) => z => z * (x + y)",
+    "x => (y, z) => z * (x + y)",
+
+    // Those are comma-separated expressions, with arrow functions as items.
+    // They stress the code for validating arrow function parameter lists.
+    "a, b => 0",
+    "a, b, (c, d) => 0",
+    "(a, b, (c, d) => 0)",
+    "(a, b) => 0, (c, d) => 1",
+    "(a, b => {}, a => a + 1)",
+    "((a, b) => {}, (a => a + 1))",
+    "(a, (a, (b, c) => 0))",
+
+    // Arrow has more precedence, this is the same as: foo ? bar : (baz = {})
+    "foo ? bar : baz => {}",
+    NULL
+  };
+
+  static const ParserFlag always_flags[] = {kAllowArrowFunctions};
+  RunParserSyncTest(context_data, statement_data, kSuccess, NULL, 0,
+                    always_flags, ARRAY_SIZE(always_flags));
+}