bool Check(Token::Value token);
void ExpectSemicolon(bool* ok);
+ Handle<String> GetSymbol(bool* ok);
+
// Get odd-ball literals.
Literal* GetLiteralUndefined();
Literal* GetLiteralTheHole();
virtual Scope* NewScope(Scope* parent, Scope::Type type, bool inside_with);
- virtual Handle<String> LookupSymbol(const char* string, int length) {
+ virtual Handle<String> LookupSymbol(int index, Vector<const char> string) {
return Handle<String>();
}
// Records the occurrence of a function.
virtual FunctionEntry LogFunction(int start) { return FunctionEntry(); }
+ virtual void LogSymbol(int start, Vector<const char> symbol) {}
// Return the current position in the function entry log.
- virtual int position() { return 0; }
+ virtual int function_position() { return 0; }
+ virtual int symbol_position() { return 0; }
+ virtual int symbol_ids() { return 0; }
virtual void LogError() { }
};
class AstBuildingParserFactory : public ParserFactory {
public:
- AstBuildingParserFactory() : ParserFactory(false) { }
+ explicit AstBuildingParserFactory(int expected_symbols)
+ : ParserFactory(false), symbol_cache_(expected_symbols) { }
virtual Scope* NewScope(Scope* parent, Scope::Type type, bool inside_with);
- virtual Handle<String> LookupSymbol(const char* string, int length) {
- return Factory::LookupSymbol(Vector<const char>(string, length));
+ virtual Handle<String> LookupSymbol(int symbol_id,
+ Vector<const char> string) {
+ // If there is no preparse data, we have no simpler way to identify similar
+ // symbols.
+ if (symbol_id < 0) return Factory::LookupSymbol(string);
+ return LookupCachedSymbol(symbol_id, string);
+ }
+
+ Handle<String> LookupCachedSymbol(int symbol_id,
+ Vector<const char> string) {
+ // Make sure the cache is large enough to hold the symbol identifier.
+ if (symbol_cache_.length() <= symbol_id) {
+ // Increase length to index + 1.
+ symbol_cache_.AddBlock(Handle<String>::null(),
+ symbol_id + 1 - symbol_cache_.length());
+ }
+ Handle<String> result = symbol_cache_.at(symbol_id);
+ if (result.is_null()) {
+ result = Factory::LookupSymbol(string);
+ symbol_cache_.at(symbol_id) = result;
+ return result;
+ }
+ Counters::total_preparse_symbols_skipped.Increment();
+ return result;
}
virtual Handle<String> EmptySymbol() {
}
virtual Statement* EmptyStatement();
+ private:
+ List<Handle<String> > symbol_cache_;
};
public:
ParserRecorder();
virtual FunctionEntry LogFunction(int start);
+ virtual void LogSymbol(int start, Vector<const char> literal) {
+ int hash = vector_hash(literal);
+ HashMap::Entry* entry = symbol_table_.Lookup(&literal, hash, true);
+ int id = static_cast<int>(reinterpret_cast<intptr_t>(entry->value));
+ if (id == 0) {
+ // Put (symbol_id_ + 1) into entry and increment it.
+ symbol_id_++;
+ entry->value = reinterpret_cast<void*>(symbol_id_);
+ Vector<Vector<const char> > symbol = symbol_entries_.AddBlock(1, literal);
+ entry->key = &symbol[0];
+ } else {
+ // Log a reuse of an earlier seen symbol.
+ symbol_store_.Add(start);
+ symbol_store_.Add(id - 1);
+ }
+ }
virtual void LogError() { }
virtual void LogMessage(Scanner::Location loc,
const char* message,
Vector<const char*> args);
Vector<unsigned> ExtractData() {
- int total_size = ScriptDataImpl::kHeaderSize + store_.size();
+ int function_size = function_store_.size();
+ int symbol_size = symbol_store_.size();
+ int total_size = ScriptDataImpl::kHeaderSize + function_size + symbol_size;
Vector<unsigned> data = Vector<unsigned>::New(total_size);
+ preamble_[ScriptDataImpl::kFunctionsSizeOffset] = function_size;
+ preamble_[ScriptDataImpl::kSymbolCountOffset] = symbol_id_;
memcpy(data.start(), preamble_, sizeof(preamble_));
- if (ScriptDataImpl::kHeaderSize < total_size) {
- store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize, total_size));
+ int symbol_start = ScriptDataImpl::kHeaderSize + function_size;
+ if (function_size > 0) {
+ function_store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize,
+ symbol_start));
+ }
+ if (symbol_size > 0) {
+ symbol_store_.WriteTo(data.SubVector(symbol_start, total_size));
}
return data;
}
- virtual int position() { return store_.size(); }
+
+ virtual int function_position() { return function_store_.size(); }
+ virtual int symbol_position() { return symbol_store_.size(); }
+ virtual int symbol_ids() { return symbol_id_; }
private:
- Collector<unsigned> store_;
+ Collector<unsigned> function_store_;
+ Collector<unsigned> symbol_store_;
+ Collector<Vector<const char> > symbol_entries_;
+ HashMap symbol_table_;
+ int symbol_id_;
+
+ static int vector_hash(Vector<const char> string) {
+ int hash = 0;
+ for (int i = 0; i < string.length(); i++) {
+ int c = string[i];
+ hash += c;
+ hash += (hash << 10);
+ hash ^= (hash >> 6);
+ }
+ return hash;
+ }
+
+ static bool vector_compare(void* a, void* b) {
+ Vector<const char>* string1 = reinterpret_cast<Vector<const char>* >(a);
+ Vector<const char>* string2 = reinterpret_cast<Vector<const char>* >(b);
+ int length = string1->length();
+ if (string2->length() != length) return false;
+ return memcmp(string1->start(), string2->start(), length) == 0;
+ }
+
unsigned preamble_[ScriptDataImpl::kHeaderSize];
#ifdef DEBUG
int prev_start;
void ScriptDataImpl::SkipFunctionEntry(int start) {
- ASSERT(index_ + FunctionEntry::kSize <= store_.length());
- ASSERT(static_cast<int>(store_[index_]) == start);
- index_ += FunctionEntry::kSize;
+ ASSERT(function_index_ + FunctionEntry::kSize <= store_.length());
+ ASSERT(static_cast<int>(store_[function_index_]) == start);
+ function_index_ += FunctionEntry::kSize;
}
FunctionEntry ScriptDataImpl::GetFunctionEntry(int start) {
// The current pre-data entry must be a FunctionEntry with the given
// start position.
- if ((index_ + FunctionEntry::kSize <= store_.length())
- && (static_cast<int>(store_[index_]) == start)) {
- int index = index_;
- index_ += FunctionEntry::kSize;
+ if ((function_index_ + FunctionEntry::kSize <= store_.length())
+ && (static_cast<int>(store_[function_index_]) == start)) {
+ int index = function_index_;
+ function_index_ += FunctionEntry::kSize;
return FunctionEntry(store_.SubVector(index,
index + FunctionEntry::kSize));
}
}
-bool ScriptDataImpl::SanityCheck() {
- if (store_.length() < static_cast<int>(ScriptDataImpl::kHeaderSize)) {
- return false;
+int ScriptDataImpl::GetSymbolIdentifier(int start) {
+ int next = symbol_index_ + 2;
+ if (next <= store_.length()
+ && static_cast<int>(store_[symbol_index_]) == start) {
+ symbol_index_ = next;
+ return store_[next - 1];
}
+ return symbol_id_++;
+}
+
+
+
+bool ScriptDataImpl::SanityCheck() {
+ // Check that the header data is valid and doesn't specify
+ // point to positions outside the store.
+ if (store_.length() < ScriptDataImpl::kHeaderSize) return false;
if (magic() != ScriptDataImpl::kMagicNumber) return false;
if (version() != ScriptDataImpl::kCurrentVersion) return false;
+ if (has_error()) {
+ // Extra sane sanity check for error message encoding.
+ if (store_.length() <= kHeaderSize + kMessageTextPos) return false;
+ if (Read(kMessageStartPos) > Read(kMessageEndPos)) return false;
+ unsigned arg_count = Read(kMessageArgCountPos);
+ int pos = kMessageTextPos;
+ for (unsigned int i = 0; i <= arg_count; i++) {
+ if (store_.length() <= kHeaderSize + pos) return false;
+ int length = static_cast<int>(Read(pos));
+ if (length < 0) return false;
+ pos += 1 + length;
+ }
+ if (store_.length() < kHeaderSize + pos) return false;
+ return true;
+ }
+ // Check that the space allocated for function entries is sane.
+ int functions_size =
+ static_cast<int>(store_[ScriptDataImpl::kFunctionsSizeOffset]);
+ if (functions_size < 0) return false;
+ if (functions_size % FunctionEntry::kSize != 0) return false;
+ // Check that the count of symbols is non-negative.
+ int symbol_count =
+ static_cast<int>(store_[ScriptDataImpl::kSymbolCountOffset]);
+ if (symbol_count < 0) return false;
+ // Check that the total size has room both function entries.
+ int minimum_size =
+ ScriptDataImpl::kHeaderSize + functions_size;
+ if (store_.length() < minimum_size) return false;
return true;
}
ParserRecorder::ParserRecorder()
- : store_(0) {
+ : function_store_(0),
+ symbol_store_(0),
+ symbol_entries_(0),
+ symbol_table_(vector_compare),
+ symbol_id_(0) {
#ifdef DEBUG
prev_start = -1;
#endif
preamble_[ScriptDataImpl::kMagicOffset] = ScriptDataImpl::kMagicNumber;
preamble_[ScriptDataImpl::kVersionOffset] = ScriptDataImpl::kCurrentVersion;
preamble_[ScriptDataImpl::kHasErrorOffset] = false;
+ preamble_[ScriptDataImpl::kFunctionsSizeOffset] = 0;
+ preamble_[ScriptDataImpl::kSymbolCountOffset] = 0;
preamble_[ScriptDataImpl::kSizeOffset] = 0;
- ASSERT_EQ(4, ScriptDataImpl::kHeaderSize);
+ ASSERT_EQ(6, ScriptDataImpl::kHeaderSize);
}
void ParserRecorder::WriteString(Vector<const char> str) {
- store_.Add(str.length());
+ function_store_.Add(str.length());
for (int i = 0; i < str.length(); i++) {
- store_.Add(str[i]);
+ function_store_.Add(str[i]);
}
}
Vector<const char*> args) {
if (has_error()) return;
preamble_[ScriptDataImpl::kHasErrorOffset] = true;
- store_.Reset();
- store_.Add(loc.beg_pos);
- store_.Add(loc.end_pos);
- store_.Add(args.length());
+ function_store_.Reset();
+ STATIC_ASSERT(ScriptDataImpl::kMessageStartPos == 0);
+ function_store_.Add(loc.beg_pos);
+ STATIC_ASSERT(ScriptDataImpl::kMessageEndPos == 1);
+ function_store_.Add(loc.end_pos);
+ STATIC_ASSERT(ScriptDataImpl::kMessageArgCountPos == 2);
+ function_store_.Add(args.length());
+ STATIC_ASSERT(ScriptDataImpl::kMessageTextPos == 3);
WriteString(CStrVector(message));
for (int i = 0; i < args.length(); i++) {
WriteString(CStrVector(args[i]));
Scanner::Location ScriptDataImpl::MessageLocation() {
- int beg_pos = Read(0);
- int end_pos = Read(1);
+ int beg_pos = Read(kMessageStartPos);
+ int end_pos = Read(kMessageEndPos);
return Scanner::Location(beg_pos, end_pos);
}
const char* ScriptDataImpl::BuildMessage() {
- unsigned* start = ReadAddress(3);
+ unsigned* start = ReadAddress(kMessageTextPos);
return ReadString(start, NULL);
}
Vector<const char*> ScriptDataImpl::BuildArgs() {
- int arg_count = Read(2);
+ int arg_count = Read(kMessageArgCountPos);
const char** array = NewArray<const char*>(arg_count);
- int pos = ScriptDataImpl::kHeaderSize + Read(3);
+ // Position after the string starting at position 3.
+ int pos = kMessageTextPos + 1 + Read(kMessageTextPos);
for (int i = 0; i < arg_count; i++) {
int count = 0;
array[i] = ReadString(ReadAddress(pos), &count);
return &store_[ScriptDataImpl::kHeaderSize + position];
}
-void ScriptDataImpl::FindStart(int position) {
- // Only search forwards, and linearly for now.
- while ((index_ < store_.length())
- && (static_cast<int>(store_[index_])) < position) {
- index_ += FunctionEntry::kSize;
- }
-}
-
FunctionEntry ParserRecorder::LogFunction(int start) {
#ifdef DEBUG
prev_start = start;
#endif
if (has_error()) return FunctionEntry();
- FunctionEntry result(store_.AddBlock(FunctionEntry::kSize, 0));
+ FunctionEntry result(function_store_.AddBlock(FunctionEntry::kSize, 0));
result.set_start_pos(start);
return result;
}
public:
AstBuildingParser(Handle<Script> script, bool allow_natives_syntax,
v8::Extension* extension, ScriptDataImpl* pre_data)
- : Parser(script, allow_natives_syntax, extension, PARSE,
- factory(), log(), pre_data) { }
+ : Parser(script,
+ allow_natives_syntax,
+ extension,
+ PARSE,
+ factory(),
+ log(),
+ pre_data),
+ factory_(pre_data ? pre_data->symbol_count() : 16) { }
virtual void ReportMessageAt(Scanner::Location loc, const char* message,
Vector<const char*> args);
virtual VariableProxy* Declare(Handle<String> name, Variable::Mode mode,
}
+Handle<String> Parser::GetSymbol(bool* ok) {
+ if (pre_data() != NULL) {
+ int symbol_id =
+ pre_data()->GetSymbolIdentifier(scanner_.location().beg_pos);
+ if (symbol_id < 0) {
+ ReportInvalidPreparseData(Factory::empty_symbol(), ok);
+ return Handle<String>::null();
+ }
+ return factory()->LookupSymbol(symbol_id, scanner_.literal());
+ }
+ log()->LogSymbol(scanner_.location().beg_pos, scanner_.literal());
+ return factory()->LookupSymbol(-1, scanner_.literal());
+}
+
+
void AstBuildingParser::ReportMessageAt(Scanner::Location source_location,
const char* type,
Vector<const char*> args) {
case Token::STRING: {
Consume(Token::STRING);
- Handle<String> symbol =
- factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ Handle<String> symbol = GetSymbol(CHECK_OK);
result = NEW(Literal(symbol));
if (fni_ != NULL) fni_->PushLiteralName(symbol);
break;
Token::Value next = Next();
// TODO(820): Allow NUMBER and STRING as well (and handle array indices).
if (next == Token::IDENTIFIER || Token::IsKeyword(next)) {
- Handle<String> name =
- factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ Handle<String> name = GetSymbol(CHECK_OK);
FunctionLiteral* value =
ParseFunctionLiteral(name,
RelocInfo::kNoPosition,
}
case Token::STRING: {
Consume(Token::STRING);
- Handle<String> string =
- factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ Handle<String> string = GetSymbol(CHECK_OK);
if (fni_ != NULL) fni_->PushLiteralName(string);
uint32_t index;
if (!string.is_null() && string->AsArrayIndex(&index)) {
default:
if (Token::IsKeyword(next)) {
Consume(next);
- Handle<String> string =
- factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ Handle<String> string = GetSymbol(CHECK_OK);
key = NEW(Literal(string));
} else {
// Unexpected token.
}
Counters::total_preparse_skipped.Increment(end_pos - function_block_pos);
scanner_.SeekForward(end_pos);
- pre_data()->Skip(entry.predata_skip());
+ pre_data()->Skip(entry.predata_function_skip(),
+ entry.predata_symbol_skip(),
+ entry.symbol_id_skip());
materialized_literal_count = entry.literal_count();
expected_property_count = entry.property_count();
only_simple_this_property_assignments = false;
pre_data()->SkipFunctionEntry(function_block_pos);
}
FunctionEntry entry = log()->LogFunction(function_block_pos);
- int predata_position_before = log()->position();
+ int predata_function_position_before = log()->function_position();
+ int predata_symbol_position_before = log()->symbol_position();
+ int symbol_ids_before = log()->symbol_ids();
ParseSourceElements(&body, Token::RBRACE, CHECK_OK);
materialized_literal_count = temp_scope.materialized_literal_count();
expected_property_count = temp_scope.expected_property_count();
entry.set_end_pos(end_pos);
entry.set_literal_count(materialized_literal_count);
entry.set_property_count(expected_property_count);
- entry.set_predata_skip(log()->position() - predata_position_before);
+ entry.set_predata_function_skip(
+ log()->function_position() - predata_function_position_before);
+ entry.set_predata_symbol_skip(
+ log()->symbol_position() - predata_symbol_position_before);
+ entry.set_symbol_id_skip(
+ log()->symbol_ids() - symbol_ids_before);
}
}
Handle<String> Parser::ParseIdentifier(bool* ok) {
Expect(Token::IDENTIFIER, ok);
if (!*ok) return Handle<String>();
- return factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ return GetSymbol(ok);
}
*ok = false;
return Handle<String>();
}
- return factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ return GetSymbol(ok);
}
*is_get = strcmp(token, "get") == 0;
*is_set = !*is_get && strcmp(token, "set") == 0;
}
- return factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ return GetSymbol(ok);
}
if (peek() != Token::RBRACE) {
do {
Expect(Token::STRING, CHECK_OK);
- Handle<String> key = factory()->LookupSymbol(scanner_.literal_string(),
- scanner_.literal_length());
+ Handle<String> key = GetSymbol(CHECK_OK);
Expect(Token::COLON, CHECK_OK);
Expression* value = ParseJsonValue(CHECK_OK);
Literal* key_literal;
ScriptDataImpl::~ScriptDataImpl() {
- store_.Dispose();
+ if (owns_store_) store_.Dispose();
}
backing_[kPropertyCountOffset] = value;
}
- int predata_skip() { return backing_[kPredataSkipOffset]; }
- void set_predata_skip(int value) {
- backing_[kPredataSkipOffset] = value;
+ int predata_function_skip() { return backing_[kPredataFunctionSkipOffset]; }
+ void set_predata_function_skip(int value) {
+ backing_[kPredataFunctionSkipOffset] = value;
}
+ int predata_symbol_skip() { return backing_[kPredataSymbolSkipOffset]; }
+ void set_predata_symbol_skip(int value) {
+ backing_[kPredataSymbolSkipOffset] = value;
+ }
+
+ int symbol_id_skip() { return backing_[kSymbolIdSkipOffset]; }
+ void set_symbol_id_skip(int value) {
+ backing_[kSymbolIdSkipOffset] = value;
+ }
+
+
bool is_valid() { return backing_.length() > 0; }
- static const int kSize = 5;
+ static const int kSize = 7;
private:
Vector<unsigned> backing_;
static const int kEndPosOffset = 1;
static const int kLiteralCountOffset = 2;
static const int kPropertyCountOffset = 3;
- static const int kPredataSkipOffset = 4;
+ static const int kPredataFunctionSkipOffset = 4;
+ static const int kPredataSymbolSkipOffset = 5;
+ static const int kSymbolIdSkipOffset = 6;
};
public:
explicit ScriptDataImpl(Vector<unsigned> store)
: store_(store),
- index_(kHeaderSize) { }
+ function_index_(kHeaderSize),
+ symbol_id_(0),
+ owns_store_(true) {
+ Initialize();
+ }
+
+ void Initialize() {
+ if (store_.length() >= kHeaderSize) {
+ // Otherwise we won't satisfy the SanityCheck.
+ symbol_index_ = kHeaderSize + store_[kFunctionsSizeOffset];
+ }
+ }
+
+ // Create an empty ScriptDataImpl that is guaranteed to not satisfy
+ // a SanityCheck.
+ ScriptDataImpl() : store_(Vector<unsigned>()), owns_store_(false) { }
+
virtual ~ScriptDataImpl();
virtual int Length();
virtual const char* Data();
virtual bool HasError();
+
FunctionEntry GetFunctionEntry(int start);
+ int GetSymbolIdentifier(int start);
void SkipFunctionEntry(int start);
bool SanityCheck();
const char* BuildMessage();
Vector<const char*> BuildArgs();
+ int symbol_count() {
+ return (store_.length() > kHeaderSize) ? store_[kSymbolCountOffset] : 0;
+ }
+ // The following functions should only be called if SanityCheck has
+ // returned true.
bool has_error() { return store_[kHasErrorOffset]; }
unsigned magic() { return store_[kMagicOffset]; }
unsigned version() { return store_[kVersionOffset]; }
+
// Skip forward in the preparser data by the given number
// of unsigned ints.
- virtual void Skip(int entries) {
- ASSERT(entries >= 0);
- ASSERT(entries <= store_.length() - index_);
- index_ += entries;
+ virtual void Skip(int function_entries, int symbol_entries, int symbol_ids) {
+ ASSERT(function_entries >= 0);
+ ASSERT(function_entries
+ <= (static_cast<int>(store_[kFunctionsSizeOffset])
+ - (function_index_ - kHeaderSize)));
+ function_index_ += function_entries;
+ symbol_index_ += symbol_entries;
+ symbol_id_ += symbol_ids;
}
static const unsigned kMagicNumber = 0xBadDead;
- static const unsigned kCurrentVersion = 1;
+ static const unsigned kCurrentVersion = 2;
static const int kMagicOffset = 0;
static const int kVersionOffset = 1;
static const int kHasErrorOffset = 2;
- static const int kSizeOffset = 3;
- static const int kHeaderSize = 4;
+ static const int kFunctionsSizeOffset = 3;
+ static const int kSymbolCountOffset = 4;
+ static const int kSizeOffset = 5;
+ static const int kHeaderSize = 6;
+
+ static const int kMessageStartPos = 0;
+ static const int kMessageEndPos = 1;
+ static const int kMessageArgCountPos = 2;
+ static const int kMessageTextPos = 3;
private:
Vector<unsigned> store_;
- int index_;
+ int function_index_;
+ int symbol_index_;
+ int symbol_id_;
+ bool owns_store_;
unsigned Read(int position);
unsigned* ReadAddress(int position);
- void FindStart(int position);
+ ScriptDataImpl(const char* backing_store, int length)
+ : store_(reinterpret_cast<unsigned*>(const_cast<char*>(backing_store)),
+ length / sizeof(unsigned)),
+ function_index_(kHeaderSize),
+ symbol_id_(0),
+ owns_store_(false) {
+ ASSERT_EQ(0, reinterpret_cast<intptr_t>(backing_store) % sizeof(unsigned));
+ Initialize();
+ }
+
// Read strings written by ParserRecorder::WriteString.
static const char* ReadString(unsigned* start, int* chars);
+
+ friend class ScriptData;
};