From ec58c05f4ee7245e6f778a1fde1d2a1f8da5d6e9 Mon Sep 17 00:00:00 2001 From: "mikhail.naganov@gmail.com" Date: Fri, 29 Apr 2011 12:08:33 +0000 Subject: [PATCH] Add support for startup data (snapshot) compression. This is for mobile platforms where application footprint size is important. To avoid including compression libraries into V8, we assume that the host machine have them (true for Linux), and rely on embedder to provide decompressed data. Currently, only snapshot data can be comressed. It is also possible to compress libraries sources, but it is more involved and will be addressed in another CL. BUG=none TEST=none Review URL: http://codereview.chromium.org/6901090 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@7724 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- SConstruct | 23 +++++- include/v8.h | 32 +++++++++ samples/process.cc | 4 ++ samples/shell.cc | 36 ++++++++++ src/api.cc | 55 ++++++++++++++ src/d8.h | 3 + src/list-inl.h | 12 +++- src/list.h | 5 +- src/mksnapshot.cc | 158 +++++++++++++++++++++++++++++------------ src/snapshot-common.cc | 5 +- src/snapshot-empty.cc | 4 ++ src/snapshot.h | 18 +++++ tools/gyp/v8.gyp | 25 ++++++- 13 files changed, 325 insertions(+), 55 deletions(-) diff --git a/SConstruct b/SConstruct index e7a16ad1a..c731276ae 100644 --- a/SConstruct +++ b/SConstruct @@ -213,6 +213,9 @@ LIBRARY_FLAGS = { }, 'gdbjit:on': { 'CPPDEFINES': ['ENABLE_GDB_JIT_INTERFACE'] + }, + 'compress_startup_data:bz2': { + 'CPPDEFINES': ['COMPRESS_STARTUP_DATA_BZ2'] } }, 'msvc': { @@ -355,6 +358,11 @@ MKSNAPSHOT_EXTRA_FLAGS = { 'os:win32': { 'LIBS': ['winmm', 'ws2_32'], }, + 'compress_startup_data:bz2': { + 'os:linux': { + 'LIBS': ['bz2'] + } + }, }, 'msvc': { 'all': { @@ -512,6 +520,12 @@ SAMPLE_FLAGS = { 'CCFLAGS': ['-g', '-O0'], 'CPPDEFINES': ['DEBUG'] }, + 'compress_startup_data:bz2': { + 'CPPDEFINES': ['COMPRESS_STARTUP_DATA_BZ2'], + 'os:linux': { + 'LIBS': ['bz2'] + } + }, }, 'msvc': { 'all': { @@ -975,7 +989,12 @@ SIMPLE_OPTIONS = { 'values': ['mips32r2', 'mips32r1'], 'default': 'mips32r2', 'help': 'mips variant' - } + }, + 'compress_startup_data': { + 'values': ['off', 'bz2'], + 'default': 'off', + 'help': 'compress startup data (snapshot) [Linux only]' + }, } ALL_OPTIONS = dict(PLATFORM_OPTIONS, **SIMPLE_OPTIONS) @@ -1098,6 +1117,8 @@ def VerifyOptions(env): print env['arch'] print env['simulator'] Abort("Option unalignedaccesses only supported for the ARM architecture.") + if env['os'] != 'linux' and env['compress_startup_data'] != 'off': + Abort("Startup data compression is only available on Linux") for (name, option) in ALL_OPTIONS.iteritems(): if (not name in env): message = ("A value for option %s must be specified (%s)." % diff --git a/include/v8.h b/include/v8.h index 4ab1ec2b4..b621f0110 100644 --- a/include/v8.h +++ b/include/v8.h @@ -2641,6 +2641,18 @@ class V8EXPORT Isolate { }; +class StartupData { + public: + enum CompressionAlgorithm { + kUncompressed, + kBZip2 + }; + + const char* data; + int compressed_size; + int raw_size; +}; + /** * Container class for static utility functions. */ @@ -2669,6 +2681,26 @@ class V8EXPORT V8 { */ static bool IsDead(); + /** + * The following 4 functions are to be used when V8 is built with + * the 'compress_startup_data' flag enabled. In this case, the + * embedder must decompress startup data prior to initializing V8. + * + * This is how interaction with V8 should look like: + * int compressed_data_count = v8::V8::GetCompressedStartupDataCount(); + * v8::StartupData* compressed_data = + * new v8::StartupData[compressed_data_count]; + * v8::V8::GetCompressedStartupData(compressed_data); + * ... decompress data (compressed_data can be updated in-place) ... + * v8::V8::SetDecompressedStartupData(compressed_data); + * ... now V8 can be initialized + * ... make sure the decompressed data stays valid until V8 shutdown + */ + static StartupData::CompressionAlgorithm GetCompressedStartupDataAlgorithm(); + static int GetCompressedStartupDataCount(); + static void GetCompressedStartupData(StartupData* compressed_data); + static void SetDecompressedStartupData(StartupData* decompressed_data); + /** * Adds a message listener. * diff --git a/samples/process.cc b/samples/process.cc index 6be4ea542..4a873b765 100644 --- a/samples/process.cc +++ b/samples/process.cc @@ -30,6 +30,10 @@ #include #include +#ifdef COMPRESS_STARTUP_DATA_BZ2 +#error Using compressed startup data is not supported for this sample +#endif + using namespace std; using namespace v8; diff --git a/samples/shell.cc b/samples/shell.cc index 51053aa0e..3640fa013 100644 --- a/samples/shell.cc +++ b/samples/shell.cc @@ -28,6 +28,9 @@ #include #include #include +#ifdef COMPRESS_STARTUP_DATA_BZ2 +#include +#endif #include #include #include @@ -299,6 +302,31 @@ int main(int argc, char* argv[]) { } } +#ifdef COMPRESS_STARTUP_DATA_BZ2 + ASSERT_EQ(v8::StartupData::kBZip2, + v8::V8::GetCompressedStartupDataAlgorithm()); + int compressed_data_count = v8::V8::GetCompressedStartupDataCount(); + v8::StartupData* compressed_data = new v8::StartupData[compressed_data_count]; + v8::V8::GetCompressedStartupData(compressed_data); + for (int i = 0; i < compressed_data_count; ++i) { + char* decompressed = new char[compressed_data[i].raw_size]; + unsigned int decompressed_size = compressed_data[i].raw_size; + int result = + BZ2_bzBuffToBuffDecompress(decompressed, + &decompressed_size, + const_cast(compressed_data[i].data), + compressed_data[i].compressed_size, + 0, 1); + if (result != BZ_OK) { + fprintf(stderr, "bzip error code: %d\n", result); + exit(1); + } + compressed_data[i].data = decompressed; + compressed_data[i].raw_size = decompressed_size; + } + v8::V8::SetDecompressedStartupData(compressed_data); +#endif // COMPRESS_STARTUP_DATA_BZ2 + v8::V8::SetFlagsFromCommandLine(&argc, argv, true); int result = 0; if (FLAG_stress_opt || FLAG_stress_deopt) { @@ -319,6 +347,14 @@ int main(int argc, char* argv[]) { result = RunMain(argc, argv); } v8::V8::Dispose(); + +#ifdef COMPRESS_STARTUP_DATA_BZ2 + for (int i = 0; i < compressed_data_count; ++i) { + delete[] compressed_data[i].data; + } + delete[] compressed_data; +#endif // COMPRESS_STARTUP_DATA_BZ2 + return result; } diff --git a/src/api.cc b/src/api.cc index e68335300..84c0be46c 100644 --- a/src/api.cc +++ b/src/api.cc @@ -311,6 +311,61 @@ static inline i::Isolate* EnterIsolateIfNeeded() { } +StartupData::CompressionAlgorithm V8::GetCompressedStartupDataAlgorithm() { +#ifdef COMPRESS_STARTUP_DATA_BZ2 + return StartupData::kBZip2; +#else + return StartupData::kUncompressed; +#endif +} + + +enum CompressedStartupDataItems { + kSnapshot = 0, + kSnapshotContext, + kCompressedStartupDataCount +}; + +int V8::GetCompressedStartupDataCount() { +#ifdef COMPRESS_STARTUP_DATA_BZ2 + return kCompressedStartupDataCount; +#else + return 0; +#endif +} + + +void V8::GetCompressedStartupData(StartupData* compressed_data) { +#ifdef COMPRESS_STARTUP_DATA_BZ2 + compressed_data[kSnapshot].data = + reinterpret_cast(i::Snapshot::data()); + compressed_data[kSnapshot].compressed_size = i::Snapshot::size(); + compressed_data[kSnapshot].raw_size = i::Snapshot::raw_size(); + + compressed_data[kSnapshotContext].data = + reinterpret_cast(i::Snapshot::context_data()); + compressed_data[kSnapshotContext].compressed_size = + i::Snapshot::context_size(); + compressed_data[kSnapshotContext].raw_size = i::Snapshot::context_raw_size(); +#endif +} + + +void V8::SetDecompressedStartupData(StartupData* decompressed_data) { +#ifdef COMPRESS_STARTUP_DATA_BZ2 + ASSERT_EQ(i::Snapshot::raw_size(), decompressed_data[kSnapshot].raw_size); + i::Snapshot::set_raw_data( + reinterpret_cast(decompressed_data[kSnapshot].data)); + + ASSERT_EQ(i::Snapshot::context_raw_size(), + decompressed_data[kSnapshotContext].raw_size); + i::Snapshot::set_context_raw_data( + reinterpret_cast( + decompressed_data[kSnapshotContext].data)); +#endif +} + + void V8::SetFatalErrorHandler(FatalErrorCallback that) { i::Isolate* isolate = EnterIsolateIfNeeded(); isolate->set_exception_behavior(that); diff --git a/src/d8.h b/src/d8.h index de1fe0de7..6c973ad79 100644 --- a/src/d8.h +++ b/src/d8.h @@ -31,6 +31,9 @@ #include "v8.h" #include "hashmap.h" +#ifdef COMPRESS_STARTUP_DATA_BZ2 +#error Using compressed startup data is not supported for D8 +#endif namespace v8 { diff --git a/src/list-inl.h b/src/list-inl.h index eeaea65f8..8ef7514f4 100644 --- a/src/list-inl.h +++ b/src/list-inl.h @@ -46,10 +46,16 @@ void List::Add(const T& element) { template void List::AddAll(const List& other) { - int result_length = length_ + other.length_; + AddAll(other.ToVector()); +} + + +template +void List::AddAll(const Vector& other) { + int result_length = length_ + other.length(); if (capacity_ < result_length) Resize(result_length); - for (int i = 0; i < other.length_; i++) { - data_[length_ + i] = other.data_[i]; + for (int i = 0; i < other.length(); i++) { + data_[length_ + i] = other.at(i); } length_ = result_length; } diff --git a/src/list.h b/src/list.h index 9a2e69897..ccb095c49 100644 --- a/src/list.h +++ b/src/list.h @@ -80,7 +80,7 @@ class List { INLINE(int length() const) { return length_; } INLINE(int capacity() const) { return capacity_; } - Vector ToVector() { return Vector(data_, length_); } + Vector ToVector() const { return Vector(data_, length_); } Vector ToConstVector() { return Vector(data_, length_); } @@ -91,6 +91,9 @@ class List { // Add all the elements from the argument list to this list. void AddAll(const List& other); + // Add all the elements from the vector to this list. + void AddAll(const Vector& other); + // Inserts the element at the specific index. void InsertAt(int index, const T& element); diff --git a/src/mksnapshot.cc b/src/mksnapshot.cc index 6ecbc8c55..13b3f8595 100644 --- a/src/mksnapshot.cc +++ b/src/mksnapshot.cc @@ -25,6 +25,9 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifdef COMPRESS_STARTUP_DATA_BZ2 +#include +#endif #include #include #include @@ -95,11 +98,53 @@ typedef std::map::iterator CounterMapIterator; static CounterMap counter_table_; -class CppByteSink : public i::SnapshotByteSink { +class Compressor { public: - explicit CppByteSink(const char* snapshot_file) - : bytes_written_(0), - partial_sink_(this) { + virtual ~Compressor() {} + virtual bool Compress(i::Vector input) = 0; + virtual i::Vector* output() = 0; +}; + + +class PartialSnapshotSink : public i::SnapshotByteSink { + public: + PartialSnapshotSink() : data_(), raw_size_(-1) { } + virtual ~PartialSnapshotSink() { data_.Free(); } + virtual void Put(int byte, const char* description) { + data_.Add(byte); + } + virtual int Position() { return data_.length(); } + void Print(FILE* fp) { + int length = Position(); + for (int j = 0; j < length; j++) { + if ((j & 0x1f) == 0x1f) { + fprintf(fp, "\n"); + } + if (j != 0) { + fprintf(fp, ","); + } + fprintf(fp, "%d", at(j)); + } + } + char at(int i) { return data_[i]; } + bool Compress(Compressor* compressor) { + ASSERT_EQ(-1, raw_size_); + raw_size_ = data_.length(); + if (!compressor->Compress(data_.ToVector())) return false; + data_.Clear(); + data_.AddAll(*compressor->output()); + return true; + } + int raw_size() { return raw_size_; } + private: + i::List data_; + int raw_size_; +}; + + +class CppByteSink : public PartialSnapshotSink { + public: + explicit CppByteSink(const char* snapshot_file) { fp_ = i::OS::FOpen(snapshot_file, "wb"); if (fp_ == NULL) { i::PrintF("Unable to write to snapshot file \"%s\"\n", snapshot_file); @@ -114,7 +159,18 @@ class CppByteSink : public i::SnapshotByteSink { } virtual ~CppByteSink() { - fprintf(fp_, "const int Snapshot::size_ = %d;\n\n", bytes_written_); + fprintf(fp_, "const int Snapshot::size_ = %d;\n", Position()); +#ifdef COMPRESS_STARTUP_DATA_BZ2 + fprintf(fp_, "const byte* Snapshot::raw_data_ = NULL;\n"); + fprintf(fp_, + "const int Snapshot::raw_size_ = %d;\n\n", + raw_size()); +#else + fprintf(fp_, + "const byte* Snapshot::raw_data_ = Snapshot::data_;\n"); + fprintf(fp_, + "const int Snapshot::raw_size_ = Snapshot::size_;\n\n"); +#endif fprintf(fp_, "} } // namespace v8::internal\n"); fclose(fp_); } @@ -127,7 +183,6 @@ class CppByteSink : public i::SnapshotByteSink { int map_space_used, int cell_space_used, int large_space_used) { - fprintf(fp_, "};\n\n"); fprintf(fp_, "const int Snapshot::new_space_used_ = %d;\n", new_space_used); fprintf(fp_, "const int Snapshot::pointer_space_used_ = %d;\n", @@ -151,57 +206,60 @@ class CppByteSink : public i::SnapshotByteSink { int length = partial_sink_.Position(); fprintf(fp_, "};\n\n"); fprintf(fp_, "const int Snapshot::context_size_ = %d;\n", length); + fprintf(fp_, + "const int Snapshot::context_raw_size_ = %d;\n", + partial_sink_.raw_size()); fprintf(fp_, "const byte Snapshot::context_data_[] = {\n"); - for (int j = 0; j < length; j++) { - if ((j & 0x1f) == 0x1f) { - fprintf(fp_, "\n"); - } - char byte = partial_sink_.at(j); - if (j != 0) { - fprintf(fp_, ","); - } - fprintf(fp_, "%d", byte); - } + partial_sink_.Print(fp_); + fprintf(fp_, "};\n\n"); +#ifdef COMPRESS_STARTUP_DATA_BZ2 + fprintf(fp_, "const byte* Snapshot::context_raw_data_ = NULL;\n"); +#else + fprintf(fp_, "const byte* Snapshot::context_raw_data_ =" + " Snapshot::context_data_;\n"); +#endif } - virtual void Put(int byte, const char* description) { - if (bytes_written_ != 0) { - fprintf(fp_, ","); - } - fprintf(fp_, "%d", byte); - bytes_written_++; - if ((bytes_written_ & 0x1f) == 0) { - fprintf(fp_, "\n"); - } + void WriteSnapshot() { + Print(fp_); } - virtual int Position() { - return bytes_written_; - } + PartialSnapshotSink* partial_sink() { return &partial_sink_; } + + private: + FILE* fp_; + PartialSnapshotSink partial_sink_; +}; - i::SnapshotByteSink* partial_sink() { return &partial_sink_; } - class PartialSnapshotSink : public i::SnapshotByteSink { - public: - explicit PartialSnapshotSink(CppByteSink* parent) - : parent_(parent), - data_() { } - virtual ~PartialSnapshotSink() { data_.Free(); } - virtual void Put(int byte, const char* description) { - data_.Add(byte); +#ifdef COMPRESS_STARTUP_DATA_BZ2 +class BZip2Compressor : public Compressor { + public: + BZip2Compressor() : output_(NULL) {} + virtual ~BZip2Compressor() { + delete output_; + } + virtual bool Compress(i::Vector input) { + delete output_; + output_ = new i::ScopedVector((input.length() * 101) / 100 + 1000); + unsigned int output_length_ = output_->length(); + int result = BZ2_bzBuffToBuffCompress(output_->start(), &output_length_, + input.start(), input.length(), + 9, 1, 0); + if (result == BZ_OK) { + output_->Truncate(output_length_); + return true; + } else { + fprintf(stderr, "bzlib error code: %d\n", result); + return false; } - virtual int Position() { return data_.length(); } - char at(int i) { return data_[i]; } - private: - CppByteSink* parent_; - i::List data_; - }; + } + virtual i::Vector* output() { return output_; } private: - FILE* fp_; - int bytes_written_; - PartialSnapshotSink partial_sink_; + i::ScopedVector* output_; }; +#endif int main(int argc, char** argv) { @@ -242,6 +300,14 @@ int main(int argc, char** argv) { ser.SerializeWeakReferences(); +#ifdef COMPRESS_STARTUP_DATA_BZ2 + BZip2Compressor compressor; + if (!sink.Compress(&compressor)) + return 1; + if (!sink.partial_sink()->Compress(&compressor)) + return 1; +#endif + sink.WriteSnapshot(); sink.WritePartialSnapshot(); sink.WriteSpaceUsed( diff --git a/src/snapshot-common.cc b/src/snapshot-common.cc index 7f8289580..ef89a5ef7 100644 --- a/src/snapshot-common.cc +++ b/src/snapshot-common.cc @@ -53,7 +53,7 @@ bool Snapshot::Initialize(const char* snapshot_file) { DeleteArray(str); return true; } else if (size_ > 0) { - Deserialize(data_, size_); + Deserialize(raw_data_, raw_size_); return true; } return false; @@ -71,7 +71,8 @@ Handle Snapshot::NewContextFromSnapshot() { map_space_used_, cell_space_used_, large_space_used_); - SnapshotByteSource source(context_data_, context_size_); + SnapshotByteSource source(context_raw_data_, + context_raw_size_); Deserializer deserializer(&source); Object* root; deserializer.DeserializePartial(&root); diff --git a/src/snapshot-empty.cc b/src/snapshot-empty.cc index cb26eb8c5..0b35720cc 100644 --- a/src/snapshot-empty.cc +++ b/src/snapshot-empty.cc @@ -35,9 +35,13 @@ namespace v8 { namespace internal { const byte Snapshot::data_[] = { 0 }; +const byte* Snapshot::raw_data_ = NULL; const int Snapshot::size_ = 0; +const int Snapshot::raw_size_ = 0; const byte Snapshot::context_data_[] = { 0 }; +const byte* Snapshot::context_raw_data_ = NULL; const int Snapshot::context_size_ = 0; +const int Snapshot::context_raw_size_ = 0; const int Snapshot::new_space_used_ = 0; const int Snapshot::pointer_space_used_ = 0; diff --git a/src/snapshot.h b/src/snapshot.h index bedd186e5..9e53af278 100644 --- a/src/snapshot.h +++ b/src/snapshot.h @@ -50,9 +50,25 @@ STATIC_CLASS Snapshot { // successfully. static bool WriteToFile(const char* snapshot_file); + static const byte* data() { return data_; } + static int size() { return size_; } + static int raw_size() { return raw_size_; } + static void set_raw_data(const byte* raw_data) { + raw_data_ = raw_data; + } + static const byte* context_data() { return context_data_; } + static int context_size() { return context_size_; } + static int context_raw_size() { return context_raw_size_; } + static void set_context_raw_data( + const byte* context_raw_data) { + context_raw_data_ = context_raw_data; + } + private: static const byte data_[]; + static const byte* raw_data_; static const byte context_data_[]; + static const byte* context_raw_data_; static const int new_space_used_; static const int pointer_space_used_; static const int data_space_used_; @@ -61,7 +77,9 @@ STATIC_CLASS Snapshot { static const int cell_space_used_; static const int large_space_used_; static const int size_; + static const int raw_size_; static const int context_size_; + static const int context_raw_size_; static bool Deserialize(const byte* content, int len); diff --git a/tools/gyp/v8.gyp b/tools/gyp/v8.gyp index a11d19a8b..77f13faa1 100644 --- a/tools/gyp/v8.gyp +++ b/tools/gyp/v8.gyp @@ -30,6 +30,7 @@ 'use_system_v8%': 0, 'msvs_use_common_release': 0, 'gcc_version%': 'unknown', + 'v8_compress_startup_data%': 'false', 'v8_target_arch%': '<(target_arch)', 'v8_use_snapshot%': 'true', 'v8_use_liveobjectlist%': 'false', @@ -76,6 +77,11 @@ 'LIVEOBJECTLIST', ], }], + ['v8_compress_startup_data=="bz2"', { + 'defines': [ + 'COMPRESS_STARTUP_DATA_BZ2', + ], + }], ], 'configurations': { 'Debug': { @@ -651,7 +657,14 @@ 'libraries': [ # Needed for clock_gettime() used by src/platform-linux.cc. '-lrt', - ]}, + ], + 'conditions': [ + ['v8_compress_startup_data=="bz2"', { + 'libraries': [ + '-lbz2', + ]}], + ], + }, 'sources': [ '../../src/platform-linux.cc', '../../src/platform-posix.cc' @@ -785,7 +798,11 @@ ['v8_target_arch=="arm" and host_arch=="x64" and _toolset=="host"', { 'cflags': ['-m32'], 'ldflags': ['-m32'], - }] + }], + ['v8_compress_startup_data=="bz2"', { + 'libraries': [ + '-lbz2', + ]}], ] }, { @@ -802,6 +819,10 @@ # This could be gotten by not setting chromium_code, if that's OK. 'defines': ['_CRT_SECURE_NO_WARNINGS'], }], + ['v8_compress_startup_data=="bz2"', { + 'libraries': [ + '-lbz2', + ]}], ], }, ], -- 2.34.1