From: vogelheim@chromium.org Date: Thu, 5 Jun 2014 13:06:21 +0000 (+0000) Subject: Support external startup data in V8. X-Git-Tag: upstream/4.7.83~8795 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=ba9f391bc066268dfc520b7ddde95b8700bd9740;p=platform%2Fupstream%2Fv8.git Support external startup data in V8. [Retry of crrev.com/293993021, which caused problems with 'ninja all' in Chromium. First patch set if a clean apply of crrev.com/293993021. Subsequent sets are the actual fix for that issue.] If the embedder chooses, the 'natives' (library sources) and the precompiled startup blob can be written to files during the build process and handed over to V8 at startup. The main purpose would be to reduce the size of the compiled binary for space constrained platforms. The build-time option is off by default. Nothing should change if it's not enabled. BUG= R=jochen@chromium.org Review URL: https://codereview.chromium.org/315033002 git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@21696 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- diff --git a/BUILD.gn b/BUILD.gn index 570a1a7..d219df2 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -16,6 +16,7 @@ v8_object_print = false v8_postmortem_support = false v8_use_default_platform = true v8_use_snapshot = true +v8_use_external_startup_data = false v8_enable_extra_checks = is_debug v8_target_arch = cpu_arch v8_random_seed = "314159265" @@ -109,6 +110,11 @@ config("features") { "ENABLE_HANDLE_ZAPPING", ] } + if (v8_use_external_startup_data == true) { + defines += [ + "V8_USE_EXTERNAL_STARTUP_DATA", + ] + } } config("toolchain") { @@ -315,6 +321,7 @@ source_set("v8_nosnapshot") { "$target_gen_dir/experimental-libraries.cc", "$target_gen_dir/trig-table.cc", "src/snapshot-empty.cc", + "src/snapshot-common.cc", ] configs -= [ "//build/config/compiler:chromium_code" ] @@ -643,7 +650,8 @@ source_set("v8_base") { "src/serialize.h", "src/small-pointer-list.h", "src/smart-pointers.h", - "src/snapshot-common.cc", + "src/snapshot-source-sink.cc", + "src/snapshot-source-sink.h", "src/snapshot.h", "src/spaces-inl.h", "src/spaces.cc", diff --git a/build/features.gypi b/build/features.gypi index e8f5b2f..3b0af91 100644 --- a/build/features.gypi +++ b/build/features.gypi @@ -59,6 +59,10 @@ # Use the v8 provided v8::Platform implementation. 'v8_use_default_platform%': 1, + + # Use external files for startup data blobs: + # the JS builtins sources and the start snapshot. + 'v8_use_external_startup_data%': 0, }, 'target_defaults': { 'conditions': [ @@ -87,9 +91,10 @@ 'defines': ['V8_USE_DEFAULT_PLATFORM',], }], ['v8_compress_startup_data=="bz2"', { - 'defines': [ - 'COMPRESS_STARTUP_DATA_BZ2', - ], + 'defines': ['COMPRESS_STARTUP_DATA_BZ2',], + }], + ['v8_use_external_startup_data==1', { + 'defines': ['V8_USE_EXTERNAL_STARTUP_DATA',], }], ], # conditions 'configurations': { diff --git a/include/v8.h b/include/v8.h index 25cb84e..a394cca 100644 --- a/include/v8.h +++ b/include/v8.h @@ -4643,6 +4643,24 @@ class V8_EXPORT V8 { static void SetDecompressedStartupData(StartupData* decompressed_data); /** + * Hand startup data to V8, in case the embedder has chosen to build + * V8 with external startup data. + * + * Note: + * - By default the startup data is linked into the V8 library, in which + * case this function is not meaningful. + * - If this needs to be called, it needs to be called before V8 + * tries to make use of its built-ins. + * - To avoid unnecessary copies of data, V8 will point directly into the + * given data blob, so pretty please keep it around until V8 exit. + * - Compression of the startup blob might be useful, but needs to + * handled entirely on the embedders' side. + * - The call will abort if the data is invalid. + */ + static void SetNativesDataBlob(StartupData* startup_blob); + static void SetSnapshotDataBlob(StartupData* startup_blob); + + /** * Adds a message listener. * * The same message listener can be added more than once and in that diff --git a/src/api.cc b/src/api.cc index a3b1b1d..ab13647 100644 --- a/src/api.cc +++ b/src/api.cc @@ -28,9 +28,7 @@ #include "src/icu_util.h" #include "src/json-parser.h" #include "src/messages.h" -#ifdef COMPRESS_STARTUP_DATA_BZ2 #include "src/natives.h" -#endif #include "src/parser.h" #include "src/platform.h" #include "src/platform/time.h" @@ -353,6 +351,24 @@ void V8::SetDecompressedStartupData(StartupData* decompressed_data) { } +void V8::SetNativesDataBlob(StartupData* natives_blob) { +#ifdef V8_USE_EXTERNAL_STARTUP_DATA + i::SetNativesFromFile(natives_blob); +#else + CHECK(false); +#endif +} + + +void V8::SetSnapshotDataBlob(StartupData* snapshot_blob) { +#ifdef V8_USE_EXTERNAL_STARTUP_DATA + i::SetSnapshotFromFile(snapshot_blob); +#else + CHECK(false); +#endif +} + + void V8::SetFatalErrorHandler(FatalErrorCallback that) { i::Isolate* isolate = i::Isolate::UncheckedCurrent(); isolate->set_exception_behavior(that); diff --git a/src/d8.cc b/src/d8.cc index 1ecfc27..4ed4f76 100644 --- a/src/d8.cc +++ b/src/d8.cc @@ -1320,6 +1320,15 @@ bool Shell::SetOptions(int argc, char* argv[]) { return false; } #endif // V8_SHARED +#ifdef V8_USE_EXTERNAL_STARTUP_DATA + else if (strncmp(argv[i], "--natives_blob=", 15) == 0) { + options.natives_blob = argv[i] + 15; + argv[i] = NULL; + } else if (strncmp(argv[i], "--snapshot_blob=", 16) == 0) { + options.snapshot_blob = argv[i] + 16; + argv[i] = NULL; + } +#endif // V8_USE_EXTERNAL_STARTUP_DATA } v8::V8::SetFlagsFromCommandLine(&argc, argv, true); @@ -1477,9 +1486,65 @@ class MockArrayBufferAllocator : public v8::ArrayBuffer::Allocator { }; +#ifdef V8_USE_EXTERNAL_STARTUP_DATA +class StartupDataHandler { + public: + StartupDataHandler(const char* natives_blob, + const char* snapshot_blob) { + Load(natives_blob, &natives_, v8::V8::SetNativesDataBlob); + Load(snapshot_blob, &snapshot_, v8::V8::SetSnapshotDataBlob); + } + + ~StartupDataHandler() { + delete[] natives_.data; + delete[] snapshot_.data; + } + + private: + void Load(const char* blob_file, + v8::StartupData* startup_data, + void (*setter_fn)(v8::StartupData*)) { + startup_data->data = NULL; + startup_data->compressed_size = 0; + startup_data->raw_size = 0; + + if (!blob_file) + return; + + FILE* file = fopen(blob_file, "rb"); + if (!file) + return; + + fseek(file, 0, SEEK_END); + startup_data->raw_size = ftell(file); + rewind(file); + + startup_data->data = new char[startup_data->raw_size]; + startup_data->compressed_size = fread( + const_cast(startup_data->data), 1, startup_data->raw_size, + file); + fclose(file); + + if (startup_data->raw_size == startup_data->compressed_size) + (*setter_fn)(startup_data); + } + + v8::StartupData natives_; + v8::StartupData snapshot_; + + // Disallow copy & assign. + StartupDataHandler(const StartupDataHandler& other); + void operator=(const StartupDataHandler& other); +}; +#endif // V8_USE_EXTERNAL_STARTUP_DATA + + int Shell::Main(int argc, char* argv[]) { if (!SetOptions(argc, argv)) return 1; v8::V8::InitializeICU(options.icu_data_file); +#ifdef V8_USE_EXTERNAL_STARTUP_DATA + StartupDataHandler startup_data(options.natives_blob, options.snapshot_blob); +#endif SetFlagsFromString("--trace-hydrogen-file=hydrogen.cfg"); SetFlagsFromString("--redirect-code-traces-to=code.asm"); ShellArrayBufferAllocator array_buffer_allocator; diff --git a/src/d8.h b/src/d8.h index bd70c8e..4244a5f 100644 --- a/src/d8.h +++ b/src/d8.h @@ -208,7 +208,9 @@ class ShellOptions { mock_arraybuffer_allocator(false), num_isolates(1), isolate_sources(NULL), - icu_data_file(NULL) { } + icu_data_file(NULL), + natives_blob(NULL), + snapshot_blob(NULL) { } ~ShellOptions() { delete[] isolate_sources; @@ -232,6 +234,8 @@ class ShellOptions { int num_isolates; SourceGroup* isolate_sources; const char* icu_data_file; + const char* natives_blob; + const char* snapshot_blob; }; #ifdef V8_SHARED diff --git a/src/flag-definitions.h b/src/flag-definitions.h index 5bd5ca7..e451567 100644 --- a/src/flag-definitions.h +++ b/src/flag-definitions.h @@ -634,8 +634,8 @@ DEFINE_string(raw_file, NULL, "A file to write the raw snapshot bytes to. " "(mksnapshot only)") DEFINE_string(raw_context_file, NULL, "A file to write the raw context " "snapshot bytes to. (mksnapshot only)") -DEFINE_bool(omit, false, "Omit raw snapshot bytes in generated code. " - "(mksnapshot only)") +DEFINE_string(startup_blob, NULL, "Write V8 startup blob file. " + "(mksnapshot only)") // code-stubs-hydrogen.cc DEFINE_bool(profile_hydrogen_code_stub_compilation, false, diff --git a/src/heap.cc b/src/heap.cc index 9e28d01..2271ae8 100644 --- a/src/heap.cc +++ b/src/heap.cc @@ -4964,7 +4964,7 @@ bool Heap::ConfigureHeap(int max_semi_space_size, max_semi_space_size_ = Page::kPageSize; } - if (Snapshot::IsEnabled()) { + if (Snapshot::HaveASnapshotToStartFrom()) { // If we are using a snapshot we always reserve the default amount // of memory for each semispace because code in the snapshot has // write-barrier code that relies on the size and alignment of new diff --git a/src/mksnapshot.cc b/src/mksnapshot.cc index d4262c4..91baeba 100644 --- a/src/mksnapshot.cc +++ b/src/mksnapshot.cc @@ -48,33 +48,81 @@ class SnapshotWriter { : fp_(GetFileDescriptorOrDie(snapshot_file)) , raw_file_(NULL) , raw_context_file_(NULL) - , compressor_(NULL) - , omit_(false) { + , startup_blob_file_(NULL) + , compressor_(NULL) { } ~SnapshotWriter() { fclose(fp_); if (raw_file_) fclose(raw_file_); if (raw_context_file_) fclose(raw_context_file_); + if (startup_blob_file_) fclose(startup_blob_file_); } void SetCompressor(Compressor* compressor) { compressor_ = compressor; } - void SetOmit(bool omit) { - omit_ = omit; - } - void SetRawFiles(const char* raw_file, const char* raw_context_file) { raw_file_ = GetFileDescriptorOrDie(raw_file); raw_context_file_ = GetFileDescriptorOrDie(raw_context_file); } + void SetStartupBlobFile(const char* startup_blob_file) { + if (startup_blob_file != NULL) + startup_blob_file_ = GetFileDescriptorOrDie(startup_blob_file); + } + void WriteSnapshot(const i::List& snapshot_data, const i::Serializer& serializer, const i::List& context_snapshot_data, const i::Serializer& context_serializer) const { + WriteSnapshotFile(snapshot_data, serializer, + context_snapshot_data, context_serializer); + MaybeWriteStartupBlob(snapshot_data, serializer, + context_snapshot_data, context_serializer); + } + + private: + void MaybeWriteStartupBlob(const i::List& snapshot_data, + const i::Serializer& serializer, + const i::List& context_snapshot_data, + const i::Serializer& context_serializer) const { + if (!startup_blob_file_) + return; + + i::List startup_blob; + ListSnapshotSink sink(&startup_blob); + + int spaces[] = { + i::NEW_SPACE, i::OLD_POINTER_SPACE, i::OLD_DATA_SPACE, i::CODE_SPACE, + i::MAP_SPACE, i::CELL_SPACE, i::PROPERTY_CELL_SPACE + }; + + i::byte* snapshot_bytes = reinterpret_cast(snapshot_data.begin()); + sink.PutBlob(snapshot_bytes, snapshot_data.length(), "snapshot"); + for (size_t i = 0; i < ARRAY_SIZE(spaces); ++i) + sink.PutInt(serializer.CurrentAllocationAddress(spaces[i]), "spaces"); + + i::byte* context_bytes = + reinterpret_cast(context_snapshot_data.begin()); + sink.PutBlob(context_bytes, context_snapshot_data.length(), "context"); + for (size_t i = 0; i < ARRAY_SIZE(spaces); ++i) + sink.PutInt(context_serializer.CurrentAllocationAddress(spaces[i]), + "spaces"); + + size_t written = fwrite(startup_blob.begin(), 1, startup_blob.length(), + startup_blob_file_); + if (written != (size_t)startup_blob.length()) { + i::PrintF("Writing snapshot file failed.. Aborting.\n"); + exit(1); + } + } + + void WriteSnapshotFile(const i::List& snapshot_data, + const i::Serializer& serializer, + const i::List& context_snapshot_data, + const i::Serializer& context_serializer) const { WriteFilePrefix(); WriteData("", snapshot_data, raw_file_); WriteData("context_", context_snapshot_data, raw_context_file_); @@ -83,7 +131,6 @@ class SnapshotWriter { WriteFileSuffix(); } - private: void WriteFilePrefix() const { fprintf(fp_, "// Autogenerated snapshot file. Do not edit.\n\n"); fprintf(fp_, "#include \"src/v8.h\"\n"); @@ -137,13 +184,12 @@ class SnapshotWriter { const i::List& source_data, const i::List* data_to_be_written) const { fprintf(fp_, "const byte Snapshot::%sdata_[] = {\n", prefix); - if (!omit_) - WriteSnapshotData(data_to_be_written); + WriteSnapshotData(data_to_be_written); fprintf(fp_, "};\n"); fprintf(fp_, "const int Snapshot::%ssize_ = %d;\n", prefix, data_to_be_written->length()); - if (data_to_be_written == &source_data && !omit_) { + if (data_to_be_written == &source_data) { fprintf(fp_, "const byte* Snapshot::%sraw_data_ = Snapshot::%sdata_;\n", prefix, prefix); fprintf(fp_, "const int Snapshot::%sraw_size_ = Snapshot::%ssize_;\n", @@ -196,8 +242,8 @@ class SnapshotWriter { FILE* fp_; FILE* raw_file_; FILE* raw_context_file_; + FILE* startup_blob_file_; Compressor* compressor_; - bool omit_; }; @@ -381,9 +427,10 @@ int main(int argc, char** argv) { { SnapshotWriter writer(argv[1]); - writer.SetOmit(i::FLAG_omit); if (i::FLAG_raw_file && i::FLAG_raw_context_file) writer.SetRawFiles(i::FLAG_raw_file, i::FLAG_raw_context_file); + if (i::FLAG_startup_blob) + writer.SetStartupBlobFile(i::FLAG_startup_blob); #ifdef COMPRESS_STARTUP_DATA_BZ2 BZip2Compressor bzip2; writer.SetCompressor(&bzip2); diff --git a/src/natives-external.cc b/src/natives-external.cc new file mode 100644 index 0000000..6d455ef --- /dev/null +++ b/src/natives-external.cc @@ -0,0 +1,190 @@ +// Copyright 2014 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/natives.h" + +#include "src/checks.h" +#include "src/list.h" +#include "src/list-inl.h" +#include "src/snapshot-source-sink.h" +#include "src/vector.h" + +namespace v8 { +namespace internal { + + +/** + * NativesStore stores the 'native' (builtin) JS libraries. + * + * NativesStore needs to be initialized before using V8, usually by the + * embedder calling v8::SetNativesDataBlob, which calls SetNativesFromFile + * below. + */ +class NativesStore { + public: + ~NativesStore() {} + + int GetBuiltinsCount() { return native_names_.length(); } + int GetDebuggerCount() { return debugger_count_; } + Vector GetScriptName(int index) { return native_names_[index]; } + Vector GetRawScriptSource(int index) { + return native_source_[index]; + } + + int GetIndex(const char* name) { + for (int i = 0; i < native_names_.length(); ++i) { + if (strcmp(name, native_names_[i].start()) == 0) { + return i; + } + } + ASSERT(false); + return -1; + } + + int GetRawScriptsSize() { + ASSERT(false); // Used for compression. Doesn't really make sense here. + return 0; + } + + Vector GetScriptsSource() { + ASSERT(false); // Used for compression. Doesn't really make sense here. + return Vector(); + } + + static NativesStore* MakeFromScriptsSource(SnapshotByteSource* source) { + NativesStore* store = new NativesStore; + + // We expect the libraries in the following format: + // int: # of debugger sources. + // 2N blobs: N pairs of source name + actual source. + // then, repeat for non-debugger sources. + int debugger_count = source->GetInt(); + for (int i = 0; i < debugger_count; ++i) + store->ReadNameAndContentPair(source); + int library_count = source->GetInt(); + for (int i = 0; i < library_count; ++i) + store->ReadNameAndContentPair(source); + + store->debugger_count_ = debugger_count; + return store; + } + + private: + NativesStore() : debugger_count_(0) {} + + bool ReadNameAndContentPair(SnapshotByteSource* bytes) { + const byte* name; + int name_length; + const byte* source; + int source_length; + bool success = bytes->GetBlob(&name, &name_length) && + bytes->GetBlob(&source, &source_length); + if (success) { + Vector name_vector( + reinterpret_cast(name), name_length); + Vector source_vector( + reinterpret_cast(source), source_length); + native_names_.Add(name_vector); + native_source_.Add(source_vector); + } + return success; + } + + List > native_names_; + List > native_source_; + int debugger_count_; + + DISALLOW_COPY_AND_ASSIGN(NativesStore); +}; + + +template +class NativesHolder { + public: + static NativesStore* get() { + ASSERT(holder_); + return holder_; + } + static void set(NativesStore* store) { + ASSERT(store); + holder_ = store; + } + + private: + static NativesStore* holder_; +}; + +template +NativesStore* NativesHolder::holder_ = NULL; + + +/** + * Read the Natives (library sources) blob, as generated by js2c + the build + * system. + */ +void SetNativesFromFile(StartupData* natives_blob) { + ASSERT(natives_blob); + ASSERT(natives_blob->data); + ASSERT(natives_blob->raw_size > 0); + + SnapshotByteSource bytes( + reinterpret_cast(natives_blob->data), + natives_blob->raw_size); + NativesHolder::set(NativesStore::MakeFromScriptsSource(&bytes)); + NativesHolder::set(NativesStore::MakeFromScriptsSource(&bytes)); + ASSERT(!bytes.HasMore()); +} + + +// Implement NativesCollection bsaed on NativesHolder + NativesStore. +// +// (The callers expect a purely static interface, since this is how the +// natives are usually compiled in. Since we implement them based on +// runtime content, we have to implement this indirection to offer +// a static interface.) +template +int NativesCollection::GetBuiltinsCount() { + return NativesHolder::get()->GetBuiltinsCount(); +} + +template +int NativesCollection::GetDebuggerCount() { + return NativesHolder::get()->GetDebuggerCount(); +} + +template +int NativesCollection::GetIndex(const char* name) { + return NativesHolder::get()->GetIndex(name); +} + +template +int NativesCollection::GetRawScriptsSize() { + return NativesHolder::get()->GetRawScriptsSize(); +} + +template +Vector NativesCollection::GetRawScriptSource(int index) { + return NativesHolder::get()->GetRawScriptSource(index); +} + +template +Vector NativesCollection::GetScriptName(int index) { + return NativesHolder::get()->GetScriptName(index); +} + +template +Vector NativesCollection::GetScriptsSource() { + return NativesHolder::get()->GetScriptsSource(); +} + + +// The compiler can't 'see' all uses of the static methods and hence +// my chose to elide them. This we'll explicitly instantiate these. +template class NativesCollection; +template class NativesCollection; +template class NativesCollection; +template class NativesCollection; + +} // namespace v8::internal +} // namespace v8 diff --git a/src/natives.h b/src/natives.h index 2f930dc..6ddedf0 100644 --- a/src/natives.h +++ b/src/natives.h @@ -5,6 +5,10 @@ #ifndef V8_NATIVES_H_ #define V8_NATIVES_H_ +#include "src/vector.h" + +namespace v8 { class StartupData; } // Forward declaration. + namespace v8 { namespace internal { @@ -39,6 +43,11 @@ class NativesCollection { typedef NativesCollection Natives; typedef NativesCollection ExperimentalNatives; +#ifdef V8_USE_EXTERNAL_STARTUP_DATA +// Used for reading the natives at runtime. Implementation in natives-empty.cc +void SetNativesFromFile(StartupData* natives_blob); +#endif + } } // namespace v8::internal #endif // V8_NATIVES_H_ diff --git a/src/serialize.cc b/src/serialize.cc index 4e5699c..118961b 100644 --- a/src/serialize.cc +++ b/src/serialize.cc @@ -16,6 +16,7 @@ #include "src/runtime.h" #include "src/serialize.h" #include "src/snapshot.h" +#include "src/snapshot-source-sink.h" #include "src/stub-cache.h" #include "src/v8threads.h" @@ -1203,19 +1204,6 @@ void Deserializer::ReadChunk(Object** current, } -void SnapshotByteSink::PutInt(uintptr_t integer, const char* description) { - ASSERT(integer < 1 << 22); - integer <<= 2; - int bytes = 1; - if (integer > 0xff) bytes = 2; - if (integer > 0xffff) bytes = 3; - integer |= bytes; - Put(static_cast(integer & 0xff), "IntPart1"); - if (bytes > 1) Put(static_cast((integer >> 8) & 0xff), "IntPart2"); - if (bytes > 2) Put(static_cast((integer >> 16) & 0xff), "IntPart3"); -} - - Serializer::Serializer(Isolate* isolate, SnapshotByteSink* sink) : isolate_(isolate), sink_(sink), @@ -1831,12 +1819,4 @@ void Serializer::InitializeCodeAddressMap() { } -bool SnapshotByteSource::AtEOF() { - if (0u + length_ - position_ > 2 * sizeof(uint32_t)) return false; - for (int x = position_; x < length_; x++) { - if (data_[x] != SerializerDeserializer::nop()) return false; - } - return true; -} - } } // namespace v8::internal diff --git a/src/serialize.h b/src/serialize.h index 6398e1e..d58e833 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -6,6 +6,9 @@ #define V8_SERIALIZE_H_ #include "src/hashmap.h" +#include "src/isolate.h" +#include "src/snapshot-source-sink.h" +#include "src/heap-profiler.h" namespace v8 { namespace internal { @@ -132,49 +135,6 @@ class ExternalReferenceDecoder { }; -class SnapshotByteSource { - public: - SnapshotByteSource(const byte* array, int length) - : data_(array), length_(length), position_(0) { } - - bool HasMore() { return position_ < length_; } - - int Get() { - ASSERT(position_ < length_); - return data_[position_++]; - } - - int32_t GetUnalignedInt() { -#if defined(V8_HOST_CAN_READ_UNALIGNED) && __BYTE_ORDER == __LITTLE_ENDIAN - int32_t answer; - ASSERT(position_ + sizeof(answer) <= length_ + 0u); - answer = *reinterpret_cast(data_ + position_); -#else - int32_t answer = data_[position_]; - answer |= data_[position_ + 1] << 8; - answer |= data_[position_ + 2] << 16; - answer |= data_[position_ + 3] << 24; -#endif - return answer; - } - - void Advance(int by) { position_ += by; } - - inline void CopyRaw(byte* to, int number_of_bytes); - - inline int GetInt(); - - bool AtEOF(); - - int position() { return position_; } - - private: - const byte* data_; - int length_; - int position_; -}; - - // The Serializer/Deserializer class is a common superclass for Serializer and // Deserializer which is used to store common constants and methods used by // both. @@ -268,26 +228,6 @@ class SerializerDeserializer: public ObjectVisitor { }; -int SnapshotByteSource::GetInt() { - // This way of variable-length encoding integers does not suffer from branch - // mispredictions. - uint32_t answer = GetUnalignedInt(); - int bytes = answer & 3; - Advance(bytes); - uint32_t mask = 0xffffffffu; - mask >>= 32 - (bytes << 3); - answer &= mask; - answer >>= 2; - return answer; -} - - -void SnapshotByteSource::CopyRaw(byte* to, int number_of_bytes) { - MemCopy(to, data_ + position_, number_of_bytes); - position_ += number_of_bytes; -} - - // A Deserializer reads a snapshot and reconstructs the Object graph it defines. class Deserializer: public SerializerDeserializer { public: @@ -368,18 +308,6 @@ class Deserializer: public SerializerDeserializer { }; -class SnapshotByteSink { - public: - virtual ~SnapshotByteSink() { } - virtual void Put(int byte, const char* description) = 0; - virtual void PutSection(int byte, const char* description) { - Put(byte, description); - } - void PutInt(uintptr_t integer, const char* description); - virtual int Position() = 0; -}; - - // Mapping objects to their location after deserialization. // This is used during building, but not at runtime by V8. class SerializationAddressMapper { diff --git a/src/snapshot-common.cc b/src/snapshot-common.cc index 33416f2..0e1fad5 100644 --- a/src/snapshot-common.cc +++ b/src/snapshot-common.cc @@ -15,43 +15,6 @@ namespace v8 { namespace internal { -static void ReserveSpaceForSnapshot(Deserializer* deserializer, - const char* file_name) { - int file_name_length = StrLength(file_name) + 10; - Vector name = Vector::New(file_name_length + 1); - OS::SNPrintF(name, "%s.size", file_name); - FILE* fp = OS::FOpen(name.start(), "r"); - CHECK_NE(NULL, fp); - int new_size, pointer_size, data_size, code_size, map_size, cell_size, - property_cell_size; -#ifdef _MSC_VER - // Avoid warning about unsafe fscanf from MSVC. - // Please note that this is only fine if %c and %s are not being used. -#define fscanf fscanf_s -#endif - CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size)); - CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size)); - CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size)); - CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size)); - CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size)); - CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size)); - CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size)); -#ifdef _MSC_VER -#undef fscanf -#endif - fclose(fp); - deserializer->set_reservation(NEW_SPACE, new_size); - deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size); - deserializer->set_reservation(OLD_DATA_SPACE, data_size); - deserializer->set_reservation(CODE_SPACE, code_size); - deserializer->set_reservation(MAP_SPACE, map_size); - deserializer->set_reservation(CELL_SPACE, cell_size); - deserializer->set_reservation(PROPERTY_CELL_SPACE, - property_cell_size); - name.Dispose(); -} - - void Snapshot::ReserveSpaceForLinkedInSnapshot(Deserializer* deserializer) { deserializer->set_reservation(NEW_SPACE, new_space_used_); deserializer->set_reservation(OLD_POINTER_SPACE, pointer_space_used_); @@ -64,21 +27,8 @@ void Snapshot::ReserveSpaceForLinkedInSnapshot(Deserializer* deserializer) { } -bool Snapshot::Initialize(const char* snapshot_file) { - if (snapshot_file) { - int len; - byte* str = ReadBytes(snapshot_file, &len); - if (!str) return false; - bool success; - { - SnapshotByteSource source(str, len); - Deserializer deserializer(&source); - ReserveSpaceForSnapshot(&deserializer, snapshot_file); - success = V8::Initialize(&deserializer); - } - DeleteArray(str); - return success; - } else if (size_ > 0) { +bool Snapshot::Initialize() { + if (size_ > 0) { ElapsedTimer timer; if (FLAG_profile_deserialization) { timer.Start(); @@ -123,4 +73,15 @@ Handle Snapshot::NewContextFromSnapshot(Isolate* isolate) { return Handle(Context::cast(root)); } + +#ifdef V8_USE_EXTERNAL_STARTUP_DATA +// Dummy implementations of Set*FromFile(..) APIs. +// +// These are meant for use with snapshot-external.cc. Should this file +// be compiled with those options we just supply these dummy implementations +// below. This happens when compiling the mksnapshot utility. +void SetNativesFromFile(StartupData* data) { CHECK(false); } +void SetSnapshotFromFile(StartupData* data) { CHECK(false); } +#endif // V8_USE_EXTERNAL_STARTUP_DATA + } } // namespace v8::internal diff --git a/src/snapshot-external.cc b/src/snapshot-external.cc new file mode 100644 index 0000000..5bfadc2 --- /dev/null +++ b/src/snapshot-external.cc @@ -0,0 +1,140 @@ +// Copyright 2006-2008 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Used for building with external snapshots. + +#include "src/snapshot.h" + +#include "src/v8.h" // for V8::Initialize +#include "src/serialize.h" +#include "src/snapshot-source-sink.h" + +namespace v8 { +namespace internal { + + +struct SnapshotImpl { + public: + const byte* data; + int size; + int new_space_used; + int pointer_space_used; + int data_space_used; + int code_space_used; + int map_space_used; + int cell_space_used; + int property_cell_space_used; + + const byte* context_data; + int context_size; + int context_new_space_used; + int context_pointer_space_used; + int context_data_space_used; + int context_code_space_used; + int context_map_space_used; + int context_cell_space_used; + int context_property_cell_space_used; +}; + + +static SnapshotImpl* snapshot_impl_ = NULL; + + +bool Snapshot::HaveASnapshotToStartFrom() { + return snapshot_impl_ != NULL; +} + + +bool Snapshot::Initialize() { + if (!HaveASnapshotToStartFrom()) + return false; + + ElapsedTimer timer; + if (FLAG_profile_deserialization) { + timer.Start(); + } + SnapshotByteSource source(snapshot_impl_->data, snapshot_impl_->size); + Deserializer deserializer(&source); + deserializer.set_reservation(NEW_SPACE, snapshot_impl_->new_space_used); + deserializer.set_reservation(OLD_POINTER_SPACE, + snapshot_impl_->pointer_space_used); + deserializer.set_reservation(OLD_DATA_SPACE, + snapshot_impl_->data_space_used); + deserializer.set_reservation(CODE_SPACE, snapshot_impl_->code_space_used); + deserializer.set_reservation(MAP_SPACE, snapshot_impl_->map_space_used); + deserializer.set_reservation(CELL_SPACE, snapshot_impl_->cell_space_used); + deserializer.set_reservation(PROPERTY_CELL_SPACE, + snapshot_impl_->property_cell_space_used); + bool success = V8::Initialize(&deserializer); + if (FLAG_profile_deserialization) { + double ms = timer.Elapsed().InMillisecondsF(); + PrintF("[Snapshot loading and deserialization took %0.3f ms]\n", ms); + } + return success; +} + + +Handle Snapshot::NewContextFromSnapshot(Isolate* isolate) { + if (!HaveASnapshotToStartFrom()) + return Handle(); + + SnapshotByteSource source(snapshot_impl_->context_data, + snapshot_impl_->context_size); + Deserializer deserializer(&source); + deserializer.set_reservation(NEW_SPACE, + snapshot_impl_->context_new_space_used); + deserializer.set_reservation(OLD_POINTER_SPACE, + snapshot_impl_->context_pointer_space_used); + deserializer.set_reservation(OLD_DATA_SPACE, + snapshot_impl_->context_data_space_used); + deserializer.set_reservation(CODE_SPACE, + snapshot_impl_->context_code_space_used); + deserializer.set_reservation(MAP_SPACE, + snapshot_impl_->context_map_space_used); + deserializer.set_reservation(CELL_SPACE, + snapshot_impl_->context_cell_space_used); + deserializer.set_reservation(PROPERTY_CELL_SPACE, + snapshot_impl_-> + context_property_cell_space_used); + Object* root; + deserializer.DeserializePartial(isolate, &root); + CHECK(root->IsContext()); + return Handle(Context::cast(root)); +} + + +void SetSnapshotFromFile(StartupData* snapshot_blob) { + ASSERT(snapshot_blob); + ASSERT(snapshot_blob->data); + ASSERT(snapshot_blob->raw_size > 0); + ASSERT(!snapshot_impl_); + + snapshot_impl_ = new SnapshotImpl; + SnapshotByteSource source(reinterpret_cast(snapshot_blob->data), + snapshot_blob->raw_size); + + bool success = source.GetBlob(&snapshot_impl_->data, + &snapshot_impl_->size); + snapshot_impl_->new_space_used = source.GetInt(); + snapshot_impl_->pointer_space_used = source.GetInt(); + snapshot_impl_->data_space_used = source.GetInt(); + snapshot_impl_->code_space_used = source.GetInt(); + snapshot_impl_->map_space_used = source.GetInt(); + snapshot_impl_->cell_space_used = source.GetInt(); + snapshot_impl_->property_cell_space_used = source.GetInt(); + + success &= source.GetBlob(&snapshot_impl_->context_data, + &snapshot_impl_->context_size); + snapshot_impl_->context_new_space_used = source.GetInt(); + snapshot_impl_->context_pointer_space_used = source.GetInt(); + snapshot_impl_->context_data_space_used = source.GetInt(); + snapshot_impl_->context_code_space_used = source.GetInt(); + snapshot_impl_->context_map_space_used = source.GetInt(); + snapshot_impl_->context_cell_space_used = source.GetInt(); + snapshot_impl_->context_property_cell_space_used = source.GetInt(); + + ASSERT(success); +} + +} } // namespace v8::internal diff --git a/src/snapshot-source-sink.cc b/src/snapshot-source-sink.cc new file mode 100644 index 0000000..9e6e604 --- /dev/null +++ b/src/snapshot-source-sink.cc @@ -0,0 +1,95 @@ +// Copyright 2014 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + + +#include "src/snapshot-source-sink.h" +#include "src/checks.h" + +#include "src/handles-inl.h" +#include "src/serialize.h" // for SerializerDeserializer::nop() in AtEOF() + + +namespace v8 { +namespace internal { + + +SnapshotByteSource::SnapshotByteSource(const byte* array, int length) + : data_(array), length_(length), position_(0) { +} + + +SnapshotByteSource::~SnapshotByteSource() { } + + +int32_t SnapshotByteSource::GetUnalignedInt() { + ASSERT(position_ < length_); // Require at least one byte left. +#if defined(V8_HOST_CAN_READ_UNALIGNED) && __BYTE_ORDER == __LITTLE_ENDIAN + int32_t answer = *reinterpret_cast(data_ + position_); +#else + int32_t answer = data_[position_]; + answer |= data_[position_ + 1] << 8; + answer |= data_[position_ + 2] << 16; + answer |= data_[position_ + 3] << 24; +#endif + return answer; +} + + +void SnapshotByteSource::CopyRaw(byte* to, int number_of_bytes) { + MemCopy(to, data_ + position_, number_of_bytes); + position_ += number_of_bytes; +} + + +void SnapshotByteSink::PutInt(uintptr_t integer, const char* description) { + ASSERT(integer < 1 << 22); + integer <<= 2; + int bytes = 1; + if (integer > 0xff) bytes = 2; + if (integer > 0xffff) bytes = 3; + integer |= bytes; + Put(static_cast(integer & 0xff), "IntPart1"); + if (bytes > 1) Put(static_cast((integer >> 8) & 0xff), "IntPart2"); + if (bytes > 2) Put(static_cast((integer >> 16) & 0xff), "IntPart3"); +} + +void SnapshotByteSink::PutRaw(byte* data, int number_of_bytes, + const char* description) { + for (int i = 0; i < number_of_bytes; ++i) { + Put(data[i], description); + } +} + +void SnapshotByteSink::PutBlob(byte* data, int number_of_bytes, + const char* description) { + PutInt(number_of_bytes, description); + PutRaw(data, number_of_bytes, description); +} + + +bool SnapshotByteSource::AtEOF() { + if (0u + length_ - position_ > 2 * sizeof(uint32_t)) return false; + for (int x = position_; x < length_; x++) { + if (data_[x] != SerializerDeserializer::nop()) return false; + } + return true; +} + + +bool SnapshotByteSource::GetBlob(const byte** data, int* number_of_bytes) { + int size = GetInt(); + *number_of_bytes = size; + + if (position_ + size < length_) { + *data = &data_[position_]; + Advance(size); + return true; + } else { + Advance(length_ - position_); // proceed until end. + return false; + } +} + +} // namespace v8::internal +} // namespace v8 diff --git a/src/snapshot-source-sink.h b/src/snapshot-source-sink.h new file mode 100644 index 0000000..0911e7c --- /dev/null +++ b/src/snapshot-source-sink.h @@ -0,0 +1,88 @@ +// Copyright 2012 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_SNAPSHOT_SOURCE_SINK_H_ +#define V8_SNAPSHOT_SOURCE_SINK_H_ + +#include "src/checks.h" +#include "src/utils.h" + +namespace v8 { +namespace internal { + + +/** + * Source to read snapshot and builtins files from. + * + * Note: Memory ownership remains with callee. + */ +class SnapshotByteSource V8_FINAL { + public: + SnapshotByteSource(const byte* array, int length); + ~SnapshotByteSource(); + + bool HasMore() { return position_ < length_; } + + int Get() { + ASSERT(position_ < length_); + return data_[position_++]; + } + + int32_t GetUnalignedInt(); + + void Advance(int by) { position_ += by; } + + void CopyRaw(byte* to, int number_of_bytes); + + inline int GetInt() { + // This way of variable-length encoding integers does not suffer from branch + // mispredictions. + uint32_t answer = GetUnalignedInt(); + int bytes = answer & 3; + Advance(bytes); + uint32_t mask = 0xffffffffu; + mask >>= 32 - (bytes << 3); + answer &= mask; + answer >>= 2; + return answer; + } + + bool GetBlob(const byte** data, int* number_of_bytes); + + bool AtEOF(); + + int position() { return position_; } + + private: + const byte* data_; + int length_; + int position_; + + DISALLOW_COPY_AND_ASSIGN(SnapshotByteSource); +}; + + +/** + * Sink to write snapshot files to. + * + * Subclasses must implement actual storage or i/o. + */ +class SnapshotByteSink { + public: + virtual ~SnapshotByteSink() { } + virtual void Put(int byte, const char* description) = 0; + virtual void PutSection(int byte, const char* description) { + Put(byte, description); + } + void PutInt(uintptr_t integer, const char* description); + void PutRaw(byte* data, int number_of_bytes, const char* description); + void PutBlob(byte* data, int number_of_bytes, const char* description); + virtual int Position() = 0; +}; + + +} // namespace v8::internal +} // namespace v8 + +#endif // V8_SNAPSHOT_SOURCE_SINK_H_ diff --git a/src/snapshot.h b/src/snapshot.h index 17191f0..b785cf5 100644 --- a/src/snapshot.h +++ b/src/snapshot.h @@ -12,23 +12,16 @@ namespace internal { class Snapshot { public: - // Initialize the VM from the given snapshot file. If snapshot_file is - // NULL, use the internal snapshot instead. Returns false if no snapshot + // Initialize the VM from the internal snapshot. Returns false if no snapshot // could be found. - static bool Initialize(const char* snapshot_file = NULL); + static bool Initialize(); static bool HaveASnapshotToStartFrom(); // Create a new context using the internal partial snapshot. static Handle NewContextFromSnapshot(Isolate* isolate); - // Returns whether or not the snapshot is enabled. - static bool IsEnabled() { return size_ != 0; } - - // Write snapshot to the given file. Returns true if snapshot was written - // successfully. - static bool WriteToFile(const char* snapshot_file); - + // These methods support COMPRESS_STARTUP_DATA_BZ2. static const byte* data() { return data_; } static int size() { return size_; } static int raw_size() { return raw_size_; } @@ -72,6 +65,10 @@ class Snapshot { DISALLOW_IMPLICIT_CONSTRUCTORS(Snapshot); }; +#ifdef V8_USE_EXTERNAL_STARTUP_DATA +void SetSnapshotFromFile(StartupData* snapshot_blob); +#endif + } } // namespace v8::internal #endif // V8_SNAPSHOT_H_ diff --git a/test/cctest/test-serialize.cc b/test/cctest/test-serialize.cc index 4c9d3fe..2dcf1c6 100644 --- a/test/cctest/test-serialize.cc +++ b/test/cctest/test-serialize.cc @@ -283,8 +283,60 @@ TEST(SerializeTwice) { //---------------------------------------------------------------------------- // Tests that the heap can be deserialized. + +static void ReserveSpaceForSnapshot(Deserializer* deserializer, + const char* file_name) { + int file_name_length = StrLength(file_name) + 10; + Vector name = Vector::New(file_name_length + 1); + OS::SNPrintF(name, "%s.size", file_name); + FILE* fp = OS::FOpen(name.start(), "r"); + name.Dispose(); + int new_size, pointer_size, data_size, code_size, map_size, cell_size, + property_cell_size; +#ifdef _MSC_VER + // Avoid warning about unsafe fscanf from MSVC. + // Please note that this is only fine if %c and %s are not being used. +#define fscanf fscanf_s +#endif + CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size)); + CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size)); + CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size)); + CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size)); + CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size)); + CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size)); + CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size)); +#ifdef _MSC_VER +#undef fscanf +#endif + fclose(fp); + deserializer->set_reservation(NEW_SPACE, new_size); + deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size); + deserializer->set_reservation(OLD_DATA_SPACE, data_size); + deserializer->set_reservation(CODE_SPACE, code_size); + deserializer->set_reservation(MAP_SPACE, map_size); + deserializer->set_reservation(CELL_SPACE, cell_size); + deserializer->set_reservation(PROPERTY_CELL_SPACE, property_cell_size); +} + + +bool InitializeFromFile(const char* snapshot_file) { + int len; + byte* str = ReadBytes(snapshot_file, &len); + if (!str) return false; + bool success; + { + SnapshotByteSource source(str, len); + Deserializer deserializer(&source); + ReserveSpaceForSnapshot(&deserializer, snapshot_file); + success = V8::Initialize(&deserializer); + } + DeleteArray(str); + return success; +} + + static void Deserialize() { - CHECK(Snapshot::Initialize(FLAG_testing_serialization_file)); + CHECK(InitializeFromFile(FLAG_testing_serialization_file)); } @@ -443,48 +495,13 @@ TEST(PartialSerialization) { } -static void ReserveSpaceForSnapshot(Deserializer* deserializer, - const char* file_name) { - int file_name_length = StrLength(file_name) + 10; - Vector name = Vector::New(file_name_length + 1); - OS::SNPrintF(name, "%s.size", file_name); - FILE* fp = OS::FOpen(name.start(), "r"); - name.Dispose(); - int new_size, pointer_size, data_size, code_size, map_size, cell_size, - property_cell_size; -#ifdef _MSC_VER - // Avoid warning about unsafe fscanf from MSVC. - // Please note that this is only fine if %c and %s are not being used. -#define fscanf fscanf_s -#endif - CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size)); - CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size)); - CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size)); - CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size)); - CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size)); - CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size)); - CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size)); -#ifdef _MSC_VER -#undef fscanf -#endif - fclose(fp); - deserializer->set_reservation(NEW_SPACE, new_size); - deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size); - deserializer->set_reservation(OLD_DATA_SPACE, data_size); - deserializer->set_reservation(CODE_SPACE, code_size); - deserializer->set_reservation(MAP_SPACE, map_size); - deserializer->set_reservation(CELL_SPACE, cell_size); - deserializer->set_reservation(PROPERTY_CELL_SPACE, property_cell_size); -} - - DEPENDENT_TEST(PartialDeserialization, PartialSerialization) { - if (!Snapshot::IsEnabled()) { + if (!Snapshot::HaveASnapshotToStartFrom()) { int file_name_length = StrLength(FLAG_testing_serialization_file) + 10; Vector startup_name = Vector::New(file_name_length + 1); OS::SNPrintF(startup_name, "%s.startup", FLAG_testing_serialization_file); - CHECK(Snapshot::Initialize(startup_name.start())); + CHECK(InitializeFromFile(startup_name.start())); startup_name.Dispose(); const char* file_name = FLAG_testing_serialization_file; @@ -596,7 +613,7 @@ DEPENDENT_TEST(ContextDeserialization, ContextSerialization) { Vector startup_name = Vector::New(file_name_length + 1); OS::SNPrintF(startup_name, "%s.startup", FLAG_testing_serialization_file); - CHECK(Snapshot::Initialize(startup_name.start())); + CHECK(InitializeFromFile(startup_name.start())); startup_name.Dispose(); const char* file_name = FLAG_testing_serialization_file; diff --git a/tools/concatenate-files.py b/tools/concatenate-files.py new file mode 100644 index 0000000..86bdf56 --- /dev/null +++ b/tools/concatenate-files.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright 2014 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This utility concatenates several files into one. On Unix-like systems +# it is equivalent to: +# cat file1 file2 file3 ...files... > target +# +# The reason for writing a seperate utility is that 'cat' is not available +# on all supported build platforms, but Python is, and hence this provides +# us with an easy and uniform way of doing this on all platforms. + +import optparse + + +def Concatenate(filenames): + """Concatenate files. + + Args: + files: Array of file names. + The last name is the target; all earlier ones are sources. + + Returns: + True, if the operation was successful. + """ + if len(filenames) < 2: + print "An error occured generating %s:\nNothing to do." % filenames[-1] + return False + + try: + with open(filenames[-1], "wb") as target: + for filename in filenames[:-1]: + with open(filename, "rb") as current: + target.write(current.read()) + return True + except IOError as e: + print "An error occured when writing %s:\n%s" % (filenames[-1], e) + return False + + +def main(): + parser = optparse.OptionParser() + parser.set_usage("""Concatenate several files into one. + Equivalent to: cat file1 ... > target.""") + (options, args) = parser.parse_args() + exit(0 if Concatenate(args) else 1) + + +if __name__ == "__main__": + main() diff --git a/tools/gyp/v8.gyp b/tools/gyp/v8.gyp index 096c8bd..8368d81 100644 --- a/tools/gyp/v8.gyp +++ b/tools/gyp/v8.gyp @@ -42,18 +42,22 @@ }, { 'toolsets': ['target'], }], - ['v8_use_snapshot=="true"', { + + ['v8_use_snapshot=="true" and v8_use_external_startup_data==0', { # The dependency on v8_base should come from a transitive # dependency however the Android toolchain requires libv8_base.a # to appear before libv8_snapshot.a so it's listed explicitly. 'dependencies': ['v8_base', 'v8_snapshot'], - }, - { + }], + ['v8_use_snapshot!="true" and v8_use_external_startup_data==0', { # The dependency on v8_base should come from a transitive # dependency however the Android toolchain requires libv8_base.a # to appear before libv8_snapshot.a so it's listed explicitly. 'dependencies': ['v8_base', 'v8_nosnapshot'], }], + ['v8_use_external_startup_data==1', { + 'dependencies': ['v8_base', 'v8_external_snapshot'], + }], ['component=="shared_library"', { 'type': '<(component)', 'sources': [ @@ -148,6 +152,7 @@ '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc', '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc', '<(INTERMEDIATE_DIR)/snapshot.cc', + '../../src/snapshot-common.cc', ], 'actions': [ { @@ -172,7 +177,7 @@ 'action': [ '<@(_inputs)', '<@(mksnapshot_flags)', - '<@(_outputs)' + '<@(INTERMEDIATE_DIR)/snapshot.cc' ], }, ], @@ -190,6 +195,7 @@ '<(SHARED_INTERMEDIATE_DIR)/libraries.cc', '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc', '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc', + '../../src/snapshot-common.cc', '../../src/snapshot-empty.cc', ], 'conditions': [ @@ -208,6 +214,80 @@ }], ] }, + { + 'target_name': 'v8_external_snapshot', + 'type': 'static_library', + 'conditions': [ + ['want_separate_host_toolset==1', { + 'toolsets': ['host', 'target'], + 'dependencies': [ + 'mksnapshot#host', + 'js2c#host', + 'generate_trig_table#host', + 'natives_blob#host', + ]}, { + 'toolsets': ['target'], + 'dependencies': [ + 'mksnapshot', + 'js2c', + 'generate_trig_table', + 'natives_blob', + ], + }], + ['component=="shared_library"', { + 'defines': [ + 'V8_SHARED', + 'BUILDING_V8_SHARED', + ], + 'direct_dependent_settings': { + 'defines': [ + 'V8_SHARED', + 'USING_V8_SHARED', + ], + }, + }], + ], + 'dependencies': [ + 'v8_base', + ], + 'include_dirs+': [ + '../..', + ], + 'sources': [ + '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc', + '../../src/natives-external.cc', + '../../src/snapshot-external.cc', + ], + 'actions': [ + { + 'action_name': 'run_mksnapshot (external)', + 'inputs': [ + '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)mksnapshot<(EXECUTABLE_SUFFIX)', + ], + 'outputs': [ + '<(INTERMEDIATE_DIR)/snapshot.cc', + '<(PRODUCT_DIR)/snapshot_blob.bin', + ], + 'variables': { + 'mksnapshot_flags': [ + '--log-snapshot-positions', + '--logfile', '<(INTERMEDIATE_DIR)/snapshot.log', + ], + 'conditions': [ + ['v8_random_seed!=0', { + 'mksnapshot_flags': ['--random-seed', '<(v8_random_seed)'], + }], + ], + }, + 'action': [ + '<@(_inputs)', + '<@(mksnapshot_flags)', + '<@(INTERMEDIATE_DIR)/snapshot.cc', + '--startup_blob', '<(PRODUCT_DIR)/snapshot_blob.bin', + ], + }, + ], + }, { 'target_name': 'generate_trig_table', 'type': 'none', 'conditions': [ @@ -541,8 +621,9 @@ '../../src/serialize.h', '../../src/small-pointer-list.h', '../../src/smart-pointers.h', - '../../src/snapshot-common.cc', '../../src/snapshot.h', + '../../src/snapshot-source-sink.cc', + '../../src/snapshot-source-sink.h', '../../src/spaces-inl.h', '../../src/spaces.cc', '../../src/spaces.h', @@ -1092,6 +1173,32 @@ ], }, { + 'target_name': 'natives_blob', + 'type': 'none', + 'conditions': [ + [ 'v8_use_external_startup_data==1', { + 'dependencies': ['js2c'], + 'actions': [{ + 'action_name': 'concatenate_natives_blob', + 'inputs': [ + '../../tools/concatenate-files.py', + '<(SHARED_INTERMEDIATE_DIR)/libraries.bin', + '<(SHARED_INTERMEDIATE_DIR)/libraries-experimental.bin', + ], + 'outputs': [ + '<(PRODUCT_DIR)/natives_blob.bin', + ], + 'action': ['python', '<@(_inputs)', '<@(_outputs)'], + }], + }], + ['want_separate_host_toolset==1', { + 'toolsets': ['host'], + }, { + 'toolsets': ['target'], + }], + ] + }, + { 'target_name': 'js2c', 'type': 'none', 'conditions': [ @@ -1147,6 +1254,8 @@ '../../src/harmony-array.js', '../../src/harmony-math.js' ], + 'libraries_bin_file': '<(SHARED_INTERMEDIATE_DIR)/libraries.bin', + 'libraries_experimental_bin_file': '<(SHARED_INTERMEDIATE_DIR)/libraries-experimental.bin', }, 'actions': [ { @@ -1162,12 +1271,20 @@ 'action': [ 'python', '../../tools/js2c.py', - '<@(_outputs)', + '<(SHARED_INTERMEDIATE_DIR)/libraries.cc', 'CORE', '<(v8_compress_startup_data)', '<@(library_files)', '<@(i18n_library_files)', ], + 'conditions': [ + [ 'v8_use_external_startup_data==1', { + 'outputs': ['<@(libraries_bin_file)'], + 'action': [ + '--startup_blob', '<@(libraries_bin_file)', + ], + }], + ], }, { 'action_name': 'js2c_experimental', @@ -1181,11 +1298,19 @@ 'action': [ 'python', '../../tools/js2c.py', - '<@(_outputs)', + '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc', 'EXPERIMENTAL', '<(v8_compress_startup_data)', '<@(experimental_library_files)' ], + 'conditions': [ + [ 'v8_use_external_startup_data==1', { + 'outputs': ['<@(libraries_experimental_bin_file)'], + 'action': [ + '--startup_blob', '<@(libraries_experimental_bin_file)' + ], + }], + ], }, ], }, diff --git a/tools/js2c.py b/tools/js2c.py index bda4656..3e56667 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -397,7 +397,7 @@ def PrepareSources(source_files): return result -def BuildMetadata(sources, source_bytes, native_type, omit): +def BuildMetadata(sources, source_bytes, native_type): """Build the meta data required to generate a libaries file. Args: @@ -405,7 +405,6 @@ def BuildMetadata(sources, source_bytes, native_type, omit): source_bytes: A list of source bytes. (The concatenation of all sources; might be compressed.) native_type: The parameter for the NativesCollection template. - omit: bool, whether we should omit the sources in the output. Returns: A dictionary for use with HEADER_TEMPLATE. @@ -438,7 +437,7 @@ def BuildMetadata(sources, source_bytes, native_type, omit): assert offset == len(raw_sources) # If we have the raw sources we can declare them accordingly. - have_raw_sources = source_bytes == raw_sources and not omit + have_raw_sources = source_bytes == raw_sources raw_sources_declaration = (RAW_SOURCES_DECLARATION if have_raw_sources else RAW_SOURCES_COMPRESSION_DECLARATION) @@ -446,7 +445,6 @@ def BuildMetadata(sources, source_bytes, native_type, omit): "builtin_count": len(sources.modules), "debugger_count": sum(sources.is_debugger_id), "sources_declaration": SOURCES_DECLARATION % ToCArray(source_bytes), - "sources_data": ToCArray(source_bytes) if not omit else "", "raw_sources_declaration": raw_sources_declaration, "raw_total_length": sum(map(len, sources.modules)), "total_length": total_length, @@ -477,10 +475,51 @@ def CompressMaybe(sources, compression_type): raise Error("Unknown compression type %s." % compression_type) -def JS2C(source, target, native_type, compression_type, raw_file, omit): +def PutInt(blob_file, value): + assert(value >= 0 and value < (1 << 20)) + size = 1 if (value < 1 << 6) else (2 if (value < 1 << 14) else 3) + value_with_length = (value << 2) | size + + byte_sequence = bytearray() + for i in xrange(size): + byte_sequence.append(value_with_length & 255) + value_with_length >>= 8; + blob_file.write(byte_sequence) + + +def PutStr(blob_file, value): + PutInt(blob_file, len(value)); + blob_file.write(value); + + +def WriteStartupBlob(sources, startup_blob): + """Write a startup blob, as expected by V8 Initialize ... + TODO(vogelheim): Add proper method name. + + Args: + sources: A Sources instance with the prepared sources. + startup_blob_file: Name of file to write the blob to. + """ + output = open(startup_blob, "wb") + + debug_sources = sum(sources.is_debugger_id); + PutInt(output, debug_sources) + for i in xrange(debug_sources): + PutStr(output, sources.names[i]); + PutStr(output, sources.modules[i]); + + PutInt(output, len(sources.names) - debug_sources) + for i in xrange(debug_sources, len(sources.names)): + PutStr(output, sources.names[i]); + PutStr(output, sources.modules[i]); + + output.close() + + +def JS2C(source, target, native_type, compression_type, raw_file, startup_blob): sources = PrepareSources(source) sources_bytes = CompressMaybe(sources, compression_type) - metadata = BuildMetadata(sources, sources_bytes, native_type, omit) + metadata = BuildMetadata(sources, sources_bytes, native_type) # Optionally emit raw file. if raw_file: @@ -488,6 +527,9 @@ def JS2C(source, target, native_type, compression_type, raw_file, omit): output.write(sources_bytes) output.close() + if startup_blob: + WriteStartupBlob(sources, startup_blob); + # Emit resulting source file. output = open(target, "w") output.write(HEADER_TEMPLATE % metadata) @@ -497,9 +539,9 @@ def JS2C(source, target, native_type, compression_type, raw_file, omit): def main(): parser = optparse.OptionParser() parser.add_option("--raw", action="store", - help="file to write the processed sources array to.") - parser.add_option("--omit", dest="omit", action="store_true", - help="Omit the raw sources from the generated code.") + help="file to write the processed sources array to.") + parser.add_option("--startup_blob", action="store", + help="file to write the startup blob to.") parser.set_usage("""js2c out.cc type compression sources.js ... out.cc: C code to be generated. type: type parameter for NativesCollection template. @@ -507,7 +549,7 @@ def main(): sources.js: JS internal sources or macros.py.""") (options, args) = parser.parse_args() - JS2C(args[3:], args[0], args[1], args[2], options.raw, options.omit) + JS2C(args[3:], args[0], args[1], args[2], options.raw, options.startup_blob) if __name__ == "__main__":