From 2816f1968020b93be8190f8611b27e010da12174 Mon Sep 17 00:00:00 2001 From: "mstarzinger@chromium.org" Date: Tue, 2 Apr 2013 17:34:59 +0000 Subject: [PATCH] Add parser support for generators. This patchset begins by adding support for "yield", which is unlike other tokens in JS. In a generator, whether strict or classic, it is a syntactic keyword. In classic mode it is an identifier. In strict mode it is reserved. This patch adds YIELD as a token to the scanner, and adapts the preparser and parser appropriately. It also parses "function*", indicating that a function is actually a generator, for both eagerly and lazily parsed functions. Currently "yield" just compiles as "return". BUG=v8:2355 TEST=mjsunit/harmony/generators-parsing Review URL: https://codereview.chromium.org/12646003 Patch from Andy Wingo . git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@14116 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/ast.cc | 1 + src/ast.h | 53 +++++++++++++-- src/compiler.cc | 1 + src/flag-definitions.h | 2 + src/full-codegen.cc | 28 ++++++++ src/hydrogen.cc | 10 +++ src/objects-inl.h | 1 + src/objects.h | 4 ++ src/parser.cc | 94 ++++++++++++++++++++++----- src/parser.h | 10 +++ src/preparser.cc | 60 ++++++++++++++--- src/preparser.h | 33 +++++++--- src/prettyprinter.cc | 11 ++++ src/scanner.cc | 2 +- src/scanner.h | 3 +- src/token.h | 1 + test/mjsunit/harmony/generators-parsing.js | 100 +++++++++++++++++++++++++++++ 17 files changed, 374 insertions(+), 40 deletions(-) create mode 100644 test/mjsunit/harmony/generators-parsing.js diff --git a/src/ast.cc b/src/ast.cc index 1ed0176..7d7a5b2 100644 --- a/src/ast.cc +++ b/src/ast.cc @@ -1062,6 +1062,7 @@ DONT_OPTIMIZE_NODE(ModuleVariable) DONT_OPTIMIZE_NODE(ModulePath) DONT_OPTIMIZE_NODE(ModuleUrl) DONT_OPTIMIZE_NODE(ModuleStatement) +DONT_OPTIMIZE_NODE(Yield) DONT_OPTIMIZE_NODE(WithStatement) DONT_OPTIMIZE_NODE(TryCatchStatement) DONT_OPTIMIZE_NODE(TryFinallyStatement) diff --git a/src/ast.h b/src/ast.h index 725b468..948343b 100644 --- a/src/ast.h +++ b/src/ast.h @@ -102,6 +102,7 @@ namespace internal { V(ObjectLiteral) \ V(ArrayLiteral) \ V(Assignment) \ + V(Yield) \ V(Throw) \ V(Property) \ V(Call) \ @@ -1953,6 +1954,31 @@ class Assignment: public Expression { }; +class Yield: public Expression { + public: + DECLARE_NODE_TYPE(Yield) + + Expression* expression() const { return expression_; } + bool is_delegating_yield() const { return is_delegating_yield_; } + virtual int position() const { return pos_; } + + protected: + Yield(Isolate* isolate, + Expression* expression, + bool is_delegating_yield, + int pos) + : Expression(isolate), + expression_(expression), + is_delegating_yield_(is_delegating_yield), + pos_(pos) { } + + private: + Expression* expression_; + bool is_delegating_yield_; + int pos_; +}; + + class Throw: public Expression { public: DECLARE_NODE_TYPE(Throw) @@ -1993,6 +2019,11 @@ class FunctionLiteral: public Expression { kNotParenthesized }; + enum IsGeneratorFlag { + kIsGenerator, + kNotGenerator + }; + DECLARE_NODE_TYPE(FunctionLiteral) Handle name() const { return name_; } @@ -2053,6 +2084,10 @@ class FunctionLiteral: public Expression { bitfield_ = IsParenthesized::update(bitfield_, kIsParenthesized); } + bool is_generator() { + return IsGenerator::decode(bitfield_) == kIsGenerator; + } + int ast_node_count() { return ast_properties_.node_count(); } AstProperties::Flags* flags() { return ast_properties_.flags(); } void set_ast_properties(AstProperties* ast_properties) { @@ -2073,7 +2108,8 @@ class FunctionLiteral: public Expression { Type type, ParameterFlag has_duplicate_parameters, IsFunctionFlag is_function, - IsParenthesizedFlag is_parenthesized) + IsParenthesizedFlag is_parenthesized, + IsGeneratorFlag is_generator) : Expression(isolate), name_(name), scope_(scope), @@ -2093,7 +2129,8 @@ class FunctionLiteral: public Expression { Pretenure::encode(false) | HasDuplicateParameters::encode(has_duplicate_parameters) | IsFunction::encode(is_function) | - IsParenthesized::encode(is_parenthesized); + IsParenthesized::encode(is_parenthesized) | + IsGenerator::encode(is_generator); } private: @@ -2118,6 +2155,7 @@ class FunctionLiteral: public Expression { class HasDuplicateParameters: public BitField {}; class IsFunction: public BitField {}; class IsParenthesized: public BitField {}; + class IsGenerator: public BitField {}; }; @@ -2916,6 +2954,12 @@ class AstNodeFactory BASE_EMBEDDED { VISIT_AND_RETURN(Assignment, assign) } + Yield* NewYield(Expression* expression, bool is_delegating_yield, int pos) { + Yield* yield = + new(zone_) Yield(isolate_, expression, is_delegating_yield, pos); + VISIT_AND_RETURN(Yield, yield) + } + Throw* NewThrow(Expression* exception, int pos) { Throw* t = new(zone_) Throw(isolate_, exception, pos); VISIT_AND_RETURN(Throw, t) @@ -2934,13 +2978,14 @@ class AstNodeFactory BASE_EMBEDDED { FunctionLiteral::ParameterFlag has_duplicate_parameters, FunctionLiteral::Type type, FunctionLiteral::IsFunctionFlag is_function, - FunctionLiteral::IsParenthesizedFlag is_parenthesized) { + FunctionLiteral::IsParenthesizedFlag is_parenthesized, + FunctionLiteral::IsGeneratorFlag is_generator) { FunctionLiteral* lit = new(zone_) FunctionLiteral( isolate_, name, scope, body, materialized_literal_count, expected_property_count, handler_count, has_only_simple_this_property_assignments, this_property_assignments, parameter_count, type, has_duplicate_parameters, is_function, - is_parenthesized); + is_parenthesized, is_generator); // Top-level literal doesn't count for the AST's properties. if (is_function == FunctionLiteral::kIsFunction) { visitor_.VisitFunctionLiteral(lit); diff --git a/src/compiler.cc b/src/compiler.cc index 2c4dae5..6f9b901 100644 --- a/src/compiler.cc +++ b/src/compiler.cc @@ -1121,6 +1121,7 @@ void Compiler::SetFunctionInfo(Handle function_info, function_info->set_dont_optimize(lit->flags()->Contains(kDontOptimize)); function_info->set_dont_inline(lit->flags()->Contains(kDontInline)); function_info->set_dont_cache(lit->flags()->Contains(kDontCache)); + function_info->set_is_generator(lit->is_generator()); } diff --git a/src/flag-definitions.h b/src/flag-definitions.h index db36183..ea72168 100644 --- a/src/flag-definitions.h +++ b/src/flag-definitions.h @@ -150,6 +150,7 @@ DEFINE_bool(harmony_observation, false, "enable harmony object observation (implies harmony collections") DEFINE_bool(harmony_typed_arrays, false, "enable harmony typed arrays") +DEFINE_bool(harmony_generators, false, "enable harmony generators") DEFINE_bool(harmony, false, "enable all harmony features (except typeof)") DEFINE_implication(harmony, harmony_scoping) DEFINE_implication(harmony, harmony_modules) @@ -157,6 +158,7 @@ DEFINE_implication(harmony, harmony_symbols) DEFINE_implication(harmony, harmony_proxies) DEFINE_implication(harmony, harmony_collections) DEFINE_implication(harmony, harmony_observation) +DEFINE_implication(harmony, harmony_generators) DEFINE_implication(harmony_modules, harmony_scoping) DEFINE_implication(harmony_observation, harmony_collections) DEFINE_implication(harmony, harmony_typed_arrays) diff --git a/src/full-codegen.cc b/src/full-codegen.cc index cb6f228..1c6a0b9 100644 --- a/src/full-codegen.cc +++ b/src/full-codegen.cc @@ -232,6 +232,12 @@ void BreakableStatementChecker::VisitAssignment(Assignment* expr) { } +void BreakableStatementChecker::VisitYield(Yield* expr) { + // Yield is breakable if the expression is. + Visit(expr->expression()); +} + + void BreakableStatementChecker::VisitThrow(Throw* expr) { // Throw is breakable if the expression is. Visit(expr->exception()); @@ -1538,6 +1544,28 @@ void FullCodeGenerator::VisitSharedFunctionInfoLiteral( } +void FullCodeGenerator::VisitYield(Yield* expr) { + if (expr->is_delegating_yield()) + UNIMPLEMENTED(); + + Comment cmnt(masm_, "[ Yield"); + VisitForAccumulatorValue(expr->expression()); + // TODO(wingo): Assert that the operand stack depth is 0, at least while + // general yield expressions are unimplemented. + + // TODO(wingo): What follows is as in VisitReturnStatement. Replace it with a + // call to a builtin that will resume the generator. + NestedStatement* current = nesting_stack_; + int stack_depth = 0; + int context_length = 0; + while (current != NULL) { + current = current->Exit(&stack_depth, &context_length); + } + __ Drop(stack_depth); + EmitReturnSequence(); +} + + void FullCodeGenerator::VisitThrow(Throw* expr) { Comment cmnt(masm_, "[ Throw"); VisitForStackValue(expr->exception()); diff --git a/src/hydrogen.cc b/src/hydrogen.cc index c2305d7..20973f0 100644 --- a/src/hydrogen.cc +++ b/src/hydrogen.cc @@ -4122,6 +4122,10 @@ void HOptimizedGraphBuilder::VisitExpressions( bool HOptimizedGraphBuilder::BuildGraph() { + if (info()->function()->is_generator()) { + Bailout("function is a generator"); + return false; + } Scope* scope = info()->scope(); if (scope->HasIllegalRedeclaration()) { Bailout("function with illegal redeclaration"); @@ -7010,6 +7014,12 @@ void HOptimizedGraphBuilder::VisitAssignment(Assignment* expr) { } +void HOptimizedGraphBuilder::VisitYield(Yield* expr) { + // Generators are not optimized, so we should never get here. + UNREACHABLE(); +} + + void HOptimizedGraphBuilder::VisitThrow(Throw* expr) { ASSERT(!HasStackOverflow()); ASSERT(current_block() != NULL); diff --git a/src/objects-inl.h b/src/objects-inl.h index 61e9101..8d6f307 100644 --- a/src/objects-inl.h +++ b/src/objects-inl.h @@ -4519,6 +4519,7 @@ BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_optimize, kDontOptimize) BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_inline, kDontInline) BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_cache, kDontCache) +BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, is_generator, kIsGenerator) void SharedFunctionInfo::BeforeVisitingPointers() { if (IsInobjectSlackTrackingInProgress()) DetachInitialMap(); diff --git a/src/objects.h b/src/objects.h index bd3a284..f8afdbc 100644 --- a/src/objects.h +++ b/src/objects.h @@ -5948,6 +5948,9 @@ class SharedFunctionInfo: public HeapObject { // Indicates that code for this function cannot be cached. DECL_BOOLEAN_ACCESSORS(dont_cache) + // Indicates that this function is a generator. + DECL_BOOLEAN_ACCESSORS(is_generator) + // Indicates whether or not the code in the shared function support // deoptimization. inline bool has_deoptimization_support(); @@ -6174,6 +6177,7 @@ class SharedFunctionInfo: public HeapObject { kDontOptimize, kDontInline, kDontCache, + kIsGenerator, kCompilerHintsCount // Pseudo entry }; diff --git a/src/parser.cc b/src/parser.cc index cdc0adb..e468cb9 100644 --- a/src/parser.cc +++ b/src/parser.cc @@ -486,10 +486,12 @@ class Parser::BlockState BASE_EMBEDDED { Parser::FunctionState::FunctionState(Parser* parser, Scope* scope, + bool is_generator, Isolate* isolate) : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize), next_handler_index_(0), expected_property_count_(0), + is_generator_(is_generator), only_simple_this_property_assignments_(false), this_property_assignments_(isolate->factory()->empty_fixed_array()), parser_(parser), @@ -642,7 +644,10 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info, } ParsingModeScope parsing_mode(this, mode); - FunctionState function_state(this, scope, isolate()); // Enters 'scope'. + bool is_generator = false; + // Enters 'scope'. + FunctionState function_state(this, scope, is_generator, isolate()); + top_scope_->SetLanguageMode(info->language_mode()); ZoneList* body = new(zone()) ZoneList(16, zone()); bool ok = true; @@ -680,7 +685,8 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info, FunctionLiteral::kNoDuplicateParameters, FunctionLiteral::ANONYMOUS_EXPRESSION, FunctionLiteral::kGlobalOrEval, - FunctionLiteral::kNotParenthesized); + FunctionLiteral::kNotParenthesized, + FunctionLiteral::kNotGenerator); result->set_ast_properties(factory()->visitor()->ast_properties()); } else if (stack_overflow_) { isolate()->StackOverflow(); @@ -754,7 +760,8 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source, scope = Scope::DeserializeScopeChain(info()->closure()->context(), scope, zone()); } - FunctionState function_state(this, scope, isolate()); + bool is_generator = false; // Top scope is not a generator. + FunctionState function_state(this, scope, is_generator, isolate()); ASSERT(scope->language_mode() != STRICT_MODE || !info()->is_classic_mode()); ASSERT(scope->language_mode() != EXTENDED_MODE || info()->is_extended_mode()); @@ -768,6 +775,7 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source, bool ok = true; result = ParseFunctionLiteral(name, false, // Strict mode name already checked. + shared_info->is_generator(), RelocInfo::kNoPosition, type, &ok); @@ -1132,6 +1140,7 @@ Statement* Parser::ParseModuleElement(ZoneStringList* labels, // ModuleDeclaration // ImportDeclaration // ExportDeclaration + // GeneratorDeclaration switch (peek()) { case Token::FUNCTION: @@ -1430,6 +1439,7 @@ Statement* Parser::ParseExportDeclaration(bool* ok) { // 'export' Identifier (',' Identifier)* ';' // 'export' VariableDeclaration // 'export' FunctionDeclaration + // 'export' GeneratorDeclaration // 'export' ModuleDeclaration // // TODO(ES6): implement structuring ExportSpecifiers @@ -1509,6 +1519,7 @@ Statement* Parser::ParseBlockElement(ZoneStringList* labels, // BlockElement (aka SourceElement): // LetDeclaration // ConstDeclaration + // GeneratorDeclaration switch (peek()) { case Token::FUNCTION: @@ -1628,6 +1639,10 @@ Statement* Parser::ParseStatement(ZoneStringList* labels, bool* ok) { // FunctionDeclaration // Common language extension is to allow function declaration in place // of any statement. This language extension is disabled in strict mode. + // + // In Harmony mode, this case also handles the extension: + // Statement: + // GeneratorDeclaration if (!top_scope_->is_classic_mode()) { ReportMessageAt(scanner().peek_location(), "strict_function", Vector::empty()); @@ -1890,13 +1905,18 @@ Statement* Parser::ParseNativeDeclaration(bool* ok) { Statement* Parser::ParseFunctionDeclaration(ZoneStringList* names, bool* ok) { // FunctionDeclaration :: // 'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}' + // GeneratorDeclaration :: + // 'function' '*' Identifier '(' FormalParameterListopt ')' + // '{' FunctionBody '}' Expect(Token::FUNCTION, CHECK_OK); int function_token_position = scanner().location().beg_pos; + bool is_generator = FLAG_harmony_generators && Check(Token::MUL); bool is_strict_reserved = false; Handle name = ParseIdentifierOrStrictReservedWord( &is_strict_reserved, CHECK_OK); FunctionLiteral* fun = ParseFunctionLiteral(name, is_strict_reserved, + is_generator, function_token_position, FunctionLiteral::DECLARATION, CHECK_OK); @@ -3004,8 +3024,13 @@ Expression* Parser::ParseExpression(bool accept_IN, bool* ok) { Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) { // AssignmentExpression :: // ConditionalExpression + // YieldExpression // LeftHandSideExpression AssignmentOperator AssignmentExpression + if (peek() == Token::YIELD && is_generator()) { + return ParseYieldExpression(ok); + } + if (fni_ != NULL) fni_->Enter(); Expression* expression = ParseConditionalExpression(accept_IN, CHECK_OK); @@ -3074,6 +3099,17 @@ Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) { } +Expression* Parser::ParseYieldExpression(bool* ok) { + // YieldExpression :: + // 'yield' '*'? AssignmentExpression + int position = scanner().peek_location().beg_pos; + Expect(Token::YIELD, CHECK_OK); + bool is_yield_star = Check(Token::MUL); + Expression* expression = ParseAssignmentExpression(false, CHECK_OK); + return factory()->NewYield(expression, is_yield_star, position); +} + + // Precedence = 3 Expression* Parser::ParseConditionalExpression(bool accept_IN, bool* ok) { // ConditionalExpression :: @@ -3450,6 +3486,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack, if (peek() == Token::FUNCTION) { Expect(Token::FUNCTION, CHECK_OK); int function_token_position = scanner().location().beg_pos; + bool is_generator = FLAG_harmony_generators && Check(Token::MUL); Handle name; bool is_strict_reserved_name = false; if (peek_any_identifier()) { @@ -3461,6 +3498,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack, : FunctionLiteral::NAMED_EXPRESSION; result = ParseFunctionLiteral(name, is_strict_reserved_name, + is_generator, function_token_position, type, CHECK_OK); @@ -3604,6 +3642,7 @@ Expression* Parser::ParsePrimaryExpression(bool* ok) { break; case Token::IDENTIFIER: + case Token::YIELD: case Token::FUTURE_STRICT_RESERVED_WORD: { Handle name = ParseIdentifier(CHECK_OK); if (fni_ != NULL) fni_->PushVariableName(name); @@ -4009,6 +4048,7 @@ ObjectLiteral::Property* Parser::ParseObjectLiteralGetSet(bool is_getter, FunctionLiteral* value = ParseFunctionLiteral(name, false, // reserved words are allowed here + false, // not a generator RelocInfo::kNoPosition, FunctionLiteral::ANONYMOUS_EXPRESSION, CHECK_OK); @@ -4310,6 +4350,7 @@ class SingletonLogger : public ParserRecorder { FunctionLiteral* Parser::ParseFunctionLiteral(Handle function_name, bool name_is_strict_reserved, + bool is_generator, int function_token_position, FunctionLiteral::Type type, bool* ok) { @@ -4344,9 +4385,12 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle function_name, FunctionLiteral::IsParenthesizedFlag parenthesized = parenthesized_function_ ? FunctionLiteral::kIsParenthesized : FunctionLiteral::kNotParenthesized; + FunctionLiteral::IsGeneratorFlag generator = is_generator + ? FunctionLiteral::kIsGenerator + : FunctionLiteral::kNotGenerator; AstProperties ast_properties; // Parse function body. - { FunctionState function_state(this, scope, isolate()); + { FunctionState function_state(this, scope, is_generator, isolate()); top_scope_->SetScopeName(function_name); // FormalParameterList :: @@ -4584,7 +4628,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle function_name, duplicate_parameters, type, FunctionLiteral::kIsFunction, - parenthesized); + parenthesized, + generator); function_literal->set_function_token_position(function_token_position); function_literal->set_ast_properties(&ast_properties); @@ -4606,10 +4651,12 @@ preparser::PreParser::PreParseResult Parser::LazyParseFunctionLiteral( stack_limit, do_allow_lazy, allow_natives_syntax_, - allow_modules_); + allow_modules_, + FLAG_harmony_generators); } preparser::PreParser::PreParseResult result = reusable_preparser_->PreParseLazyFunction(top_scope_->language_mode(), + is_generator(), logger); return result; } @@ -4672,7 +4719,8 @@ bool Parser::peek_any_identifier() { Token::Value next = peek(); return next == Token::IDENTIFIER || next == Token::FUTURE_RESERVED_WORD || - next == Token::FUTURE_STRICT_RESERVED_WORD; + next == Token::FUTURE_STRICT_RESERVED_WORD || + next == Token::YIELD; } @@ -4744,13 +4792,17 @@ Literal* Parser::GetLiteralTheHole() { // Parses an identifier that is valid for the current scope, in particular it // fails on strict mode future reserved keywords in a strict scope. Handle Parser::ParseIdentifier(bool* ok) { - if (!top_scope_->is_classic_mode()) { - Expect(Token::IDENTIFIER, ok); - } else if (!Check(Token::IDENTIFIER)) { - Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok); + Token::Value next = Next(); + if (next == Token::IDENTIFIER || + (top_scope_->is_classic_mode() && + (next == Token::FUTURE_STRICT_RESERVED_WORD || + (next == Token::YIELD && !is_generator())))) { + return GetSymbol(ok); + } else { + ReportUnexpectedToken(next); + *ok = false; + return Handle(); } - if (!*ok) return Handle(); - return GetSymbol(ok); } @@ -4758,12 +4810,17 @@ Handle Parser::ParseIdentifier(bool* ok) { // whether it is strict mode future reserved. Handle Parser::ParseIdentifierOrStrictReservedWord( bool* is_strict_reserved, bool* ok) { - *is_strict_reserved = false; - if (!Check(Token::IDENTIFIER)) { - Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok); + Token::Value next = Next(); + if (next == Token::IDENTIFIER) { + *is_strict_reserved = false; + } else if (next == Token::FUTURE_STRICT_RESERVED_WORD || + (next == Token::YIELD && !is_generator())) { *is_strict_reserved = true; + } else { + ReportUnexpectedToken(next); + *ok = false; + return Handle(); } - if (!*ok) return Handle(); return GetSymbol(ok); } @@ -5875,6 +5932,9 @@ ScriptDataImpl* ParserApi::PreParse(Utf16CharacterStream* source, if (FLAG_lazy && (extension == NULL)) { flags |= kAllowLazy; } + if (FLAG_harmony_generators) { + flags |= kAllowGenerators; + } CompleteParserRecorder recorder; return DoPreParse(source, flags, &recorder); } diff --git a/src/parser.h b/src/parser.h index 6dcf7f1..fc4aba2 100644 --- a/src/parser.h +++ b/src/parser.h @@ -474,6 +474,7 @@ class Parser { public: FunctionState(Parser* parser, Scope* scope, + bool is_generator, Isolate* isolate); ~FunctionState(); @@ -504,6 +505,8 @@ class Parser { void AddProperty() { expected_property_count_++; } int expected_property_count() { return expected_property_count_; } + bool is_generator() const { return is_generator_; } + AstNodeFactory* factory() { return &factory_; } private: @@ -518,6 +521,9 @@ class Parser { // Properties count estimation. int expected_property_count_; + // Indicates that this function is a generator. + bool is_generator_; + // Keeps track of assignments to properties of this. Used for // optimizing constructors. bool only_simple_this_property_assignments_; @@ -631,6 +637,7 @@ class Parser { Expression* ParseExpression(bool accept_IN, bool* ok); Expression* ParseAssignmentExpression(bool accept_IN, bool* ok); + Expression* ParseYieldExpression(bool* ok); Expression* ParseConditionalExpression(bool accept_IN, bool* ok); Expression* ParseBinaryExpression(int prec, bool accept_IN, bool* ok); Expression* ParseUnaryExpression(bool* ok); @@ -674,6 +681,7 @@ class Parser { ZoneList* ParseArguments(bool* ok); FunctionLiteral* ParseFunctionLiteral(Handle var_name, bool name_is_reserved, + bool is_generator, int function_token_position, FunctionLiteral::Type type, bool* ok); @@ -703,6 +711,8 @@ class Parser { return scanner().Next(); } + bool is_generator() const { return current_function_state_->is_generator(); } + bool peek_any_identifier(); INLINE(void Consume(Token::Value token)); diff --git a/src/preparser.cc b/src/preparser.cc index c461d8a..c61a08d 100644 --- a/src/preparser.cc +++ b/src/preparser.cc @@ -53,12 +53,13 @@ int isfinite(double value); namespace preparser { PreParser::PreParseResult PreParser::PreParseLazyFunction( - i::LanguageMode mode, i::ParserRecorder* log) { + i::LanguageMode mode, bool is_generator, i::ParserRecorder* log) { log_ = log; // Lazy functions always have trivial outer scopes (no with/catch scopes). Scope top_scope(&scope_, kTopLevelScope); set_language_mode(mode); Scope function_scope(&scope_, kFunctionScope); + function_scope.set_is_generator(is_generator); ASSERT_EQ(i::Token::LBRACE, scanner_->current_token()); bool ok = true; int start_position = scanner_->peek_location().beg_pos; @@ -154,6 +155,7 @@ PreParser::Statement PreParser::ParseSourceElement(bool* ok) { // SourceElement: // LetDeclaration // ConstDeclaration + // GeneratorDeclaration switch (peek()) { case i::Token::FUNCTION: @@ -294,19 +296,23 @@ PreParser::Statement PreParser::ParseStatement(bool* ok) { PreParser::Statement PreParser::ParseFunctionDeclaration(bool* ok) { // FunctionDeclaration :: // 'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}' + // GeneratorDeclaration :: + // 'function' '*' Identifier '(' FormalParameterListopt ')' + // '{' FunctionBody '}' Expect(i::Token::FUNCTION, CHECK_OK); + bool is_generator = allow_generators_ && Check(i::Token::MUL); Identifier identifier = ParseIdentifier(CHECK_OK); i::Scanner::Location location = scanner_->location(); - Expression function_value = ParseFunctionLiteral(CHECK_OK); + Expression function_value = ParseFunctionLiteral(is_generator, CHECK_OK); if (function_value.IsStrictFunction() && !identifier.IsValidStrictVariable()) { // Strict mode violation, using either reserved word or eval/arguments // as name of strict function. const char* type = "strict_function_name"; - if (identifier.IsFutureStrictReserved()) { + if (identifier.IsFutureStrictReserved() || identifier.IsYield()) { type = "strict_reserved_word"; } ReportMessageAt(location, type, NULL); @@ -475,7 +481,9 @@ PreParser::Statement PreParser::ParseExpressionOrLabelledStatement(bool* ok) { Expression expr = ParseExpression(true, CHECK_OK); if (expr.IsRawIdentifier()) { ASSERT(!expr.AsIdentifier().IsFutureReserved()); - ASSERT(is_classic_mode() || !expr.AsIdentifier().IsFutureStrictReserved()); + ASSERT(is_classic_mode() || + (!expr.AsIdentifier().IsFutureStrictReserved() && + !expr.AsIdentifier().IsYield())); if (peek() == i::Token::COLON) { Consume(i::Token::COLON); return ParseStatement(ok); @@ -810,8 +818,13 @@ PreParser::Expression PreParser::ParseAssignmentExpression(bool accept_IN, bool* ok) { // AssignmentExpression :: // ConditionalExpression + // YieldExpression // LeftHandSideExpression AssignmentOperator AssignmentExpression + if (scope_->is_generator() && peek() == i::Token::YIELD) { + return ParseYieldExpression(ok); + } + i::Scanner::Location before = scanner_->peek_location(); Expression expression = ParseConditionalExpression(accept_IN, CHECK_OK); @@ -842,6 +855,19 @@ PreParser::Expression PreParser::ParseAssignmentExpression(bool accept_IN, // Precedence = 3 +PreParser::Expression PreParser::ParseYieldExpression(bool* ok) { + // YieldExpression :: + // 'yield' '*'? AssignmentExpression + Consume(i::Token::YIELD); + Check(i::Token::MUL); + + ParseAssignmentExpression(false, CHECK_OK); + + return Expression::Default(); +} + + +// Precedence = 3 PreParser::Expression PreParser::ParseConditionalExpression(bool accept_IN, bool* ok) { // ConditionalExpression :: @@ -1034,11 +1060,13 @@ PreParser::Expression PreParser::ParseMemberWithNewPrefixesExpression( Expression result = Expression::Default(); if (peek() == i::Token::FUNCTION) { Consume(i::Token::FUNCTION); + + bool is_generator = allow_generators_ && Check(i::Token::MUL); Identifier identifier = Identifier::Default(); if (peek_any_identifier()) { identifier = ParseIdentifier(CHECK_OK); } - result = ParseFunctionLiteral(CHECK_OK); + result = ParseFunctionLiteral(is_generator, CHECK_OK); if (result.IsStrictFunction() && !identifier.IsValidStrictVariable()) { StrictModeIdentifierViolation(scanner_->location(), "strict_function_name", @@ -1112,6 +1140,7 @@ PreParser::Expression PreParser::ParsePrimaryExpression(bool* ok) { case i::Token::FUTURE_RESERVED_WORD: case i::Token::FUTURE_STRICT_RESERVED_WORD: + case i::Token::YIELD: case i::Token::IDENTIFIER: { Identifier id = ParseIdentifier(CHECK_OK); result = Expression::FromIdentifier(id); @@ -1257,7 +1286,7 @@ PreParser::Expression PreParser::ParseObjectLiteral(bool* ok) { } PropertyType type = is_getter ? kGetterProperty : kSetterProperty; CheckDuplicate(&duplicate_finder, name, type, CHECK_OK); - ParseFunctionLiteral(CHECK_OK); + ParseFunctionLiteral(false, CHECK_OK); if (peek() != i::Token::RBRACE) { Expect(i::Token::COMMA, CHECK_OK); } @@ -1344,7 +1373,8 @@ PreParser::Arguments PreParser::ParseArguments(bool* ok) { } -PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) { +PreParser::Expression PreParser::ParseFunctionLiteral(bool is_generator, + bool* ok) { // Function :: // '(' FormalParameterList? ')' '{' FunctionBody '}' @@ -1352,6 +1382,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) { ScopeType outer_scope_type = scope_->type(); bool inside_with = scope_->IsInsideWith(); Scope function_scope(&scope_, kFunctionScope); + function_scope.set_is_generator(is_generator); // FormalParameterList :: // '(' (Identifier)*[','] ')' Expect(i::Token::LPAREN, CHECK_OK); @@ -1497,6 +1528,8 @@ PreParser::Identifier PreParser::GetIdentifierSymbol() { } else if (scanner_->current_token() == i::Token::FUTURE_STRICT_RESERVED_WORD) { return Identifier::FutureStrictReserved(); + } else if (scanner_->current_token() == i::Token::YIELD) { + return Identifier::Yield(); } if (scanner_->is_literal_ascii()) { // Detect strict-mode poison words. @@ -1523,6 +1556,14 @@ PreParser::Identifier PreParser::ParseIdentifier(bool* ok) { *ok = false; return GetIdentifierSymbol(); } + case i::Token::YIELD: + if (scope_->is_generator()) { + // 'yield' in a generator is only valid as part of a YieldExpression. + ReportMessageAt(scanner_->location(), "unexpected_token", "yield"); + *ok = false; + return Identifier::Yield(); + } + // FALLTHROUGH case i::Token::FUTURE_STRICT_RESERVED_WORD: if (!is_classic_mode()) { i::Scanner::Location location = scanner_->location(); @@ -1580,7 +1621,7 @@ void PreParser::StrictModeIdentifierViolation(i::Scanner::Location location, const char* type = eval_args_type; if (identifier.IsFutureReserved()) { type = "reserved_word"; - } else if (identifier.IsFutureStrictReserved()) { + } else if (identifier.IsFutureStrictReserved() || identifier.IsYield()) { type = "strict_reserved_word"; } if (!is_classic_mode()) { @@ -1634,7 +1675,8 @@ bool PreParser::peek_any_identifier() { i::Token::Value next = peek(); return next == i::Token::IDENTIFIER || next == i::Token::FUTURE_RESERVED_WORD || - next == i::Token::FUTURE_STRICT_RESERVED_WORD; + next == i::Token::FUTURE_STRICT_RESERVED_WORD || + next == i::Token::YIELD; } diff --git a/src/preparser.h b/src/preparser.h index ad52d74..35eaec2 100644 --- a/src/preparser.h +++ b/src/preparser.h @@ -117,7 +117,8 @@ class PreParser { uintptr_t stack_limit, bool allow_lazy, bool allow_natives_syntax, - bool allow_modules) + bool allow_modules, + bool allow_generators) : scanner_(scanner), log_(log), scope_(NULL), @@ -128,6 +129,7 @@ class PreParser { allow_lazy_(allow_lazy), allow_modules_(allow_modules), allow_natives_syntax_(allow_natives_syntax), + allow_generators_(allow_generators), parenthesized_function_(false), harmony_scoping_(scanner->HarmonyScoping()) { } @@ -144,19 +146,22 @@ class PreParser { bool allow_lazy = (flags & i::kAllowLazy) != 0; bool allow_natives_syntax = (flags & i::kAllowNativesSyntax) != 0; bool allow_modules = (flags & i::kAllowModules) != 0; + bool allow_generators = (flags & i::kAllowGenerators) != 0; return PreParser(scanner, log, stack_limit, allow_lazy, - allow_natives_syntax, allow_modules).PreParse(); + allow_natives_syntax, allow_modules, + allow_generators).PreParse(); } // Parses a single function literal, from the opening parentheses before // parameters to the closing brace after the body. // Returns a FunctionEntry describing the body of the function in enough // detail that it can be lazily compiled. - // The scanner is expected to have matched the "function" keyword and - // parameters, and have consumed the initial '{'. + // The scanner is expected to have matched the "function" or "function*" + // keyword and parameters, and have consumed the initial '{'. // At return, unless an error occurred, the scanner is positioned before the // the final '}'. PreParseResult PreParseLazyFunction(i::LanguageMode mode, + bool is_generator, i::ParserRecorder* log); private: @@ -240,9 +245,13 @@ class PreParser { static Identifier FutureStrictReserved() { return Identifier(kFutureStrictReservedIdentifier); } + static Identifier Yield() { + return Identifier(kYieldIdentifier); + } bool IsEval() { return type_ == kEvalIdentifier; } bool IsArguments() { return type_ == kArgumentsIdentifier; } bool IsEvalOrArguments() { return type_ >= kEvalIdentifier; } + bool IsYield() { return type_ == kYieldIdentifier; } bool IsFutureReserved() { return type_ == kFutureReservedIdentifier; } bool IsFutureStrictReserved() { return type_ == kFutureStrictReservedIdentifier; @@ -254,6 +263,7 @@ class PreParser { kUnknownIdentifier, kFutureReservedIdentifier, kFutureStrictReservedIdentifier, + kYieldIdentifier, kEvalIdentifier, kArgumentsIdentifier }; @@ -347,7 +357,7 @@ class PreParser { // Identifiers and string literals can be parenthesized. // They no longer work as labels or directive prologues, // but are still recognized in other contexts. - return Expression(code_ | kParentesizedExpressionFlag); + return Expression(code_ | kParenthesizedExpressionFlag); } // For other types of expressions, it's not important to remember // the parentheses. @@ -373,7 +383,8 @@ class PreParser { kUseStrictString = kStringLiteralFlag | 8, kStringLiteralMask = kUseStrictString, - kParentesizedExpressionFlag = 4, // Only if identifier or string literal. + // Only if identifier or string literal. + kParenthesizedExpressionFlag = 4, // Below here applies if neither identifier nor string literal. kThisExpression = 4, @@ -451,7 +462,8 @@ class PreParser { expected_properties_(0), with_nesting_count_(0), language_mode_( - (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE) { + (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE), + is_generator_(false) { *variable = this; } ~Scope() { *variable_ = prev_; } @@ -461,6 +473,8 @@ class PreParser { int expected_properties() { return expected_properties_; } int materialized_literal_count() { return materialized_literal_count_; } bool IsInsideWith() { return with_nesting_count_ != 0; } + bool is_generator() { return is_generator_; } + void set_is_generator(bool is_generator) { is_generator_ = is_generator; } bool is_classic_mode() { return language_mode_ == i::CLASSIC_MODE; } @@ -492,6 +506,7 @@ class PreParser { int expected_properties_; int with_nesting_count_; i::LanguageMode language_mode_; + bool is_generator_; }; // Preparse the program. Only called in PreParseProgram after creating @@ -557,6 +572,7 @@ class PreParser { Expression ParseExpression(bool accept_IN, bool* ok); Expression ParseAssignmentExpression(bool accept_IN, bool* ok); + Expression ParseYieldExpression(bool* ok); Expression ParseConditionalExpression(bool accept_IN, bool* ok); Expression ParseBinaryExpression(int prec, bool accept_IN, bool* ok); Expression ParseUnaryExpression(bool* ok); @@ -572,7 +588,7 @@ class PreParser { Expression ParseV8Intrinsic(bool* ok); Arguments ParseArguments(bool* ok); - Expression ParseFunctionLiteral(bool* ok); + Expression ParseFunctionLiteral(bool is_generator, bool* ok); void ParseLazyFunctionLiteralBody(bool* ok); Identifier ParseIdentifier(bool* ok); @@ -664,6 +680,7 @@ class PreParser { bool allow_lazy_; bool allow_modules_; bool allow_natives_syntax_; + bool allow_generators_; bool parenthesized_function_; bool harmony_scoping_; }; diff --git a/src/prettyprinter.cc b/src/prettyprinter.cc index df6183a..50a71ce 100644 --- a/src/prettyprinter.cc +++ b/src/prettyprinter.cc @@ -353,6 +353,12 @@ void PrettyPrinter::VisitAssignment(Assignment* node) { } +void PrettyPrinter::VisitYield(Yield* node) { + Print("yield "); + Visit(node->expression()); +} + + void PrettyPrinter::VisitThrow(Throw* node) { Print("throw "); Visit(node->exception()); @@ -1059,6 +1065,11 @@ void AstPrinter::VisitAssignment(Assignment* node) { } +void AstPrinter::VisitYield(Yield* node) { + PrintIndentedVisit("YIELD", node->expression()); +} + + void AstPrinter::VisitThrow(Throw* node) { PrintIndentedVisit("THROW", node->exception()); } diff --git a/src/scanner.cc b/src/scanner.cc index bd2db58..ef2dc2c 100755 --- a/src/scanner.cc +++ b/src/scanner.cc @@ -877,7 +877,7 @@ uc32 Scanner::ScanIdentifierUnicodeEscape() { KEYWORD("while", Token::WHILE) \ KEYWORD("with", Token::WITH) \ KEYWORD_GROUP('y') \ - KEYWORD("yield", Token::FUTURE_STRICT_RESERVED_WORD) + KEYWORD("yield", Token::YIELD) static Token::Value KeywordOrIdentifierToken(const char* input, diff --git a/src/scanner.h b/src/scanner.h index a454750..dd1bfb8 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -53,7 +53,8 @@ enum ParsingFlags { kLanguageModeMask = 0x03, kAllowLazy = 0x04, kAllowNativesSyntax = 0x08, - kAllowModules = 0x10 + kAllowModules = 0x10, + kAllowGenerators = 0x20 }; STATIC_ASSERT((kLanguageModeMask & CLASSIC_MODE) == CLASSIC_MODE); diff --git a/src/token.h b/src/token.h index 4078a15..04d7f76 100644 --- a/src/token.h +++ b/src/token.h @@ -174,6 +174,7 @@ namespace internal { K(EXPORT, "export", 0) \ K(IMPORT, "import", 0) \ K(LET, "let", 0) \ + K(YIELD, "yield", 0) \ \ /* Illegal token - not able to scan. */ \ T(ILLEGAL, "ILLEGAL", 0) \ diff --git a/test/mjsunit/harmony/generators-parsing.js b/test/mjsunit/harmony/generators-parsing.js new file mode 100644 index 0000000..0e5494d --- /dev/null +++ b/test/mjsunit/harmony/generators-parsing.js @@ -0,0 +1,100 @@ +// Copyright 2013 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Flags: --harmony-generators + +// Test basic generator syntax. + +// Yield statements. +function* g() { yield 3; yield 4; } + +// Yield expressions. +function* g() { (yield 3) + (yield 4); } + +// You can have a generator in strict mode. +function* g() { "use strict"; yield 3; yield 4; } + +// Generator expression. +(function* () { yield 3; }); + +// Named generator expression. +(function* g() { yield 3; }); + +// A generator without a yield is specified as causing an early error. This +// behavior is currently unimplemented. See +// https://bugs.ecmascript.org/show_bug.cgi?id=1283. +function* g() { } + +// A YieldExpression in the RHS of a YieldExpression is currently specified as +// causing an early error. This behavior is currently unimplemented. See +// https://bugs.ecmascript.org/show_bug.cgi?id=1283. +function* g() { yield yield 1; } +function* g() { yield 3 + (yield 4); } + +// Generator definitions with a name of "yield" are not specifically ruled out +// by the spec, as the `yield' name is outside the generator itself. However, +// in strict-mode, "yield" is an invalid identifier. +function* yield() { (yield 3) + (yield 4); } +assertThrows("function* yield() { \"use strict\"; (yield 3) + (yield 4); }", + SyntaxError); + +// In classic mode, yield is a normal identifier, outside of generators. +function yield(yield) { yield: yield (yield + yield (0)); } + +// Yield is always valid as a key in an object literal. +({ yield: 1 }); +function* g() { yield ({ yield: 1 }) } +function* g() { yield ({ get yield() { return 1; }}) } + +// Checks that yield is a valid label in classic mode, but not valid in a strict +// mode or in generators. +function f() { yield: 1 } +assertThrows("function f() { \"use strict\"; yield: 1 }", SyntaxError) +assertThrows("function f*() { yield: 1 }", SyntaxError) + +// Yield is only a keyword in the body of the generator, not in nested +// functions. +function* g() { function f() { yield (yield + yield (0)); } } + +// Yield needs a RHS. +assertThrows("function* g() { yield; }", SyntaxError); + +// Yield in a generator is not an identifier. +assertThrows("function* g() { yield = 10; }", SyntaxError); + +// Yield binds very loosely, so this parses as "yield (3 + yield 4)", which is +// invalid. +assertThrows("function* g() { yield 3 + yield 4; }", SyntaxError); + +// Yield is still a future-reserved-word in strict mode +assertThrows("function f() { \"use strict\"; var yield = 13; }", SyntaxError); + +// The name of the NFE is let-bound in G, so is invalid. +assertThrows("function* g() { yield (function yield() {}); }", SyntaxError); + +// In generators, yield is invalid as a formal argument name. +assertThrows("function* g(yield) { yield (10); }", SyntaxError); -- 2.7.4