From 51651fe338fe8002c9917d7ff19b47f6fb2af24c Mon Sep 17 00:00:00 2001 From: Ryan Hyun Choi Date: Tue, 9 Nov 2021 12:26:37 +0900 Subject: [PATCH] LWNode_Release_211109_610b330 Change-Id: Ibb82b2b15c428c92919eb45c5851c85347cd50c8 Signed-off-by: Ryan Choi --- .../escargotshim/include/lwnode/lwnode-loader.h | 69 ++++-- .../code/escargotshim/src/lwnode/lwnode-loader.cc | 260 +++++++++++---------- src/node_native_module_lwnode-inl.h | 154 ++++-------- 3 files changed, 232 insertions(+), 251 deletions(-) diff --git a/lwnode/code/escargotshim/include/lwnode/lwnode-loader.h b/lwnode/code/escargotshim/include/lwnode/lwnode-loader.h index e867590..38bd26d 100644 --- a/lwnode/code/escargotshim/include/lwnode/lwnode-loader.h +++ b/lwnode/code/escargotshim/include/lwnode/lwnode-loader.h @@ -17,6 +17,8 @@ #pragma once #include +#include +#include #include namespace Escargot { @@ -26,73 +28,90 @@ class ExecutionStateRef; namespace LWNode { -enum Encoding { UNKNOWN, ONE_BYTE, ONE_BYTE_LATIN1, TWO_BYTE }; +enum class Encoding : uint8_t { kUnknown, kAscii, kLatin1, kUtf16 }; struct FileData { void* buffer{nullptr}; long int size{0}; - Encoding encoding{ONE_BYTE}; + Encoding encoding{Encoding::kUnknown}; + std::string filePath; - FileData(void* buffer_, long int size_, Encoding encoding_) { + FileData(void* buffer_, + long int size_, + Encoding encoding_, + const std::string& filePath_) { buffer = buffer_; size = size_; encoding = encoding_; + filePath = filePath_; } FileData() = default; }; -class Loader { +class SourceReaderInterface { public: - using U8String = std::basic_string>; - - static FileData readFile(std::string filename, const Encoding fileEncoding); + virtual FileData read(std::string filename, const Encoding encodingHint) = 0; +}; - static void tryConvertUTF8ToLatin1(U8String& latin1String, - Encoding& encoding, - const uint8_t* buffer, - const size_t bufferSize, - const Encoding encodingHint); +class SourceReader : public SourceReaderInterface { + public: + static SourceReader* getInstance(); + FileData read(std::string filename, const Encoding encodingHint) override; - // should return string buffer - typedef void* (*LoadCallback)(void* callbackData); - // should free memoryPtr - typedef void (*UnloadCallback)(void* memoryPtr, void* callbackData); + private: + SourceReader() = default; +}; +class Loader { + public: class ReloadableSourceData { public: void* preloadedData{nullptr}; const char* path() { return path_; } + Encoding encoding() { return encoding_; } + SourceReaderInterface* sourceReader() { return sourceReader_; } + size_t preloadedDataLength() { return preloadedDataLength_; } size_t stringLength() { return isOneByteString() ? preloadedDataLength_ : preloadedDataLength_ / 2; } bool isOneByteString() { - return (encoding_ == ONE_BYTE) || (encoding_ == ONE_BYTE_LATIN1); + return (encoding_ == Encoding::kAscii) || + (encoding_ == Encoding::kLatin1); } - Encoding encoding() { return encoding_; } - static ReloadableSourceData* create(std::string sourcePath, - void* preloadedData, - size_t preloadedDataLength, - Encoding encoding); + static ReloadableSourceData* create(const FileData fileData, + SourceReaderInterface* sourceReader); private: char* path_{nullptr}; size_t preloadedDataLength_{0}; - Encoding encoding_{UNKNOWN}; + Encoding encoding_{Encoding::kUnknown}; ReloadableSourceData() = default; + SourceReaderInterface* sourceReader_{nullptr}; }; + // should return string buffer + typedef void* (*LoadCallback)(void* callbackData); + // should free memoryPtr + typedef void (*UnloadCallback)(void* memoryPtr, void* callbackData); + + static FileData createFileDataForReloadableString( + std::string filename, + std::unique_ptr> bufferHolder, + size_t bufferSize, + const Encoding encodingHint); + static Escargot::ValueRef* CreateReloadableSourceFromFile( Escargot::ExecutionStateRef* state, std::string fileName); static v8::MaybeLocal NewReloadableString( v8::Isolate* isolate, ReloadableSourceData* data, - LoadCallback loadCallback, - UnloadCallback unloadCallback); + LoadCallback loadCallback = nullptr, + UnloadCallback unloadCallback = nullptr); }; bool convertUTF8ToUTF16le(char** buffer, diff --git a/lwnode/code/escargotshim/src/lwnode/lwnode-loader.cc b/lwnode/code/escargotshim/src/lwnode/lwnode-loader.cc index bd19e69..760bf34 100644 --- a/lwnode/code/escargotshim/src/lwnode/lwnode-loader.cc +++ b/lwnode/code/escargotshim/src/lwnode/lwnode-loader.cc @@ -85,80 +85,66 @@ class FileScope { std::FILE* file_{nullptr}; }; -void Loader::tryConvertUTF8ToLatin1(U8String& latin1String, - Encoding& encoding, - const uint8_t* buffer, - const size_t bufferSize, - const Encoding encodingHint) { +static void tryConvertUTF8ToLatin1( + std::basic_string>& latin1String, + Encoding& encoding, + const uint8_t* buffer, + const size_t bufferSize, + const Encoding encodingHint) { bool isOneByteString = true; - if (encodingHint == UNKNOWN || encodingHint == ONE_BYTE_LATIN1) { + if (encodingHint == Encoding::kUnknown || encodingHint == Encoding::kLatin1) { if (UTF8Sequence::convertUTF8ToLatin1( latin1String, buffer, buffer + bufferSize) == false) { isOneByteString = false; } - } else if (encodingHint == TWO_BYTE) { + } else if (encodingHint == Encoding::kUtf16) { isOneByteString = false; } - encoding = UNKNOWN; + encoding = Encoding::kUnknown; if (isOneByteString) { if (latin1String.length() == bufferSize) { - encoding = ONE_BYTE; + encoding = Encoding::kAscii; } else { - encoding = ONE_BYTE_LATIN1; + encoding = Encoding::kLatin1; } } else { - encoding = TWO_BYTE; + encoding = Encoding::kUtf16; } } -FileData Loader::readFile(std::string filename, const Encoding encodingHint) { - FileScope fileScope(filename.c_str(), "rb"); - - std::FILE* file = fileScope.file(); - - if (!file) { - return FileData(); - } - - std::fseek(file, 0, SEEK_END); - long pos = std::ftell(file); - std::rewind(file); - - LWNODE_CHECK(pos >= 0); - - size_t bufferSize = (size_t)pos; - uint8_t* buffer = (uint8_t*)allocateStringBuffer(bufferSize + 1); - buffer[bufferSize] = '\0'; - - std::unique_ptr> bufferHolder( - buffer, freeStringBuffer); - - if (std::fread(buffer, sizeof(uint8_t), bufferSize, file) == 0) { - return FileData(); - } - - Loader::U8String latin1String; - Encoding encoding = UNKNOWN; - - if (encodingHint == ONE_BYTE) { - encoding = ONE_BYTE; - } else if (encodingHint == TWO_BYTE) { - encoding = TWO_BYTE; - } else if (encodingHint == ONE_BYTE_LATIN1) { - Loader::tryConvertUTF8ToLatin1( - latin1String, encoding, buffer, bufferSize, encodingHint); +FileData Loader::createFileDataForReloadableString( + std::string filename, + std::unique_ptr> bufferHolder, + size_t bufferSize, + const Encoding encodingHint) { + std::basic_string> latin1String; + Encoding encoding = Encoding::kUnknown; + + if (encodingHint == Encoding::kAscii) { + encoding = Encoding::kAscii; + } else if (encodingHint == Encoding::kUtf16) { + encoding = Encoding::kUtf16; + } else if (encodingHint == Encoding::kLatin1) { + tryConvertUTF8ToLatin1(latin1String, + encoding, + (uint8_t*)bufferHolder.get(), + bufferSize, + encodingHint); } else { - LWNODE_CHECK(encodingHint == UNKNOWN); - Loader::tryConvertUTF8ToLatin1( - latin1String, encoding, buffer, bufferSize, encodingHint); + LWNODE_CHECK(encodingHint == Encoding::kUnknown); + tryConvertUTF8ToLatin1(latin1String, + encoding, + (uint8_t*)bufferHolder.get(), + bufferSize, + encodingHint); } - if (encoding == TWO_BYTE) { + if (encoding == Encoding::kUtf16) { // Treat non-latin1 as UTF-8 and encode it as UTF-16 Little Endian. - if (encodingHint == UNKNOWN) { + if (encodingHint == Encoding::kUnknown) { LWNODE_LOG_INFO("%s contains characters outside of the Latin1 range.", filename.c_str()); } @@ -177,8 +163,8 @@ FileData Loader::readFile(std::string filename, const Encoding encodingHint) { bufferHolder.reset(newStringBuffer); bufferSize = newStringBufferSize; } else { - if (encoding == ONE_BYTE_LATIN1) { - if (encodingHint == UNKNOWN) { + if (encoding == Encoding::kLatin1) { + if (encodingHint == Encoding::kUnknown) { LWNODE_LOG_INFO("%s contains Latin1 characters.", filename.c_str()); } @@ -190,24 +176,59 @@ FileData Loader::readFile(std::string filename, const Encoding encodingHint) { } } - return FileData(bufferHolder.release(), bufferSize, encoding); + return FileData(bufferHolder.release(), bufferSize, encoding, filename); +} + +SourceReader* SourceReader::getInstance() { + static SourceReader s_singleton; + return &s_singleton; +} + +FileData SourceReader::read(std::string filename, const Encoding encodingHint) { + FileScope fileScope(filename.c_str(), "rb"); + + std::FILE* file = fileScope.file(); + + if (!file) { + return FileData(); + } + + std::fseek(file, 0, SEEK_END); + long pos = std::ftell(file); + std::rewind(file); + + LWNODE_CHECK(pos >= 0); + + size_t bufferSize = (size_t)pos; + uint8_t* buffer = (uint8_t*)allocateStringBuffer(bufferSize + 1); + buffer[bufferSize] = '\0'; + + std::unique_ptr> bufferHolder( + buffer, freeStringBuffer); + + if (std::fread(buffer, sizeof(uint8_t), bufferSize, file) == 0) { + return FileData(); + } + + return Loader::createFileDataForReloadableString( + filename, std::move(bufferHolder), bufferSize, encodingHint); } Loader::ReloadableSourceData* Loader::ReloadableSourceData::create( - std::string sourcePath, - void* preloadedData, - size_t preloadedDataLength, - Encoding encoding) { + const FileData fileData, SourceReaderInterface* sourceReader) { // NOTE: data and data->path should be managed by gc auto data = new (Memory::gcMalloc(sizeof(ReloadableSourceData))) ReloadableSourceData(); + + auto& sourcePath = fileData.filePath; data->path_ = (char*)Memory::gcMallocAtomic(sourcePath.size() + 1); std::copy(sourcePath.begin(), sourcePath.end(), data->path_); data->path_[sourcePath.size()] = '\0'; - data->preloadedData = preloadedData; - data->preloadedDataLength_ = preloadedDataLength; - data->encoding_ = encoding; + data->preloadedData = fileData.buffer; + data->preloadedDataLength_ = fileData.size; + data->encoding_ = fileData.encoding; + data->sourceReader_ = sourceReader; return data; } @@ -223,65 +244,21 @@ ValueRef* Loader::CreateReloadableSourceFromFile(ExecutionStateRef* state, auto lwContext = ContextWrap::fromEscargot(state->context()); auto isolate = lwContext->GetIsolate()->toV8(); - FileData dest = Loader::readFile(fileName, UNKNOWN); - - if (dest.buffer) { - auto data = Loader::ReloadableSourceData::create( - fileName, dest.buffer, dest.size, dest.encoding); - - HandleScope handleScope(isolate); - - v8::Local source = - Loader::NewReloadableString( - isolate, - data, - // Load-ReloadableSource - [](void* userData) -> void* { - auto data = - reinterpret_cast(userData); - - LWNODE_LOG_INFO(" * Load: %d (%d) %p %s (+%.2f kB)", - ++s_stat.loaded, - s_stat.reloaded, - data->preloadedData, - data->path(), - (float)data->preloadedDataLength() / 1024); - - if (data->preloadedData) { - auto buffer = data->preloadedData; - data->preloadedData = nullptr; - return buffer; // move memory ownership to js engine - } - s_stat.reloaded++; - - FileData dest = Loader::readFile(data->path(), data->encoding()); - LWNODE_CHECK_NOT_NULL(dest.buffer); - return dest.buffer; - }, - // Unload-ReloadableSource - [](void* preloadedData, void* userData) -> void { - auto data = - reinterpret_cast(userData); - - LWNODE_LOG_INFO("* Unload: %d (%d) %p %s (-%.2f kB)", - --s_stat.loaded, - s_stat.reloaded, - preloadedData, - data->path(), - (float)data->preloadedDataLength() / 1024); - - if (data->preloadedData) { - freeStringBuffer(data->preloadedData); - data->preloadedData = nullptr; - } - freeStringBuffer(preloadedData); - }) - .ToLocalChecked(); - - return CVAL(*source)->value()->asString(); + auto sourceReader = SourceReader::getInstance(); + FileData fileData = sourceReader->read(fileName, Encoding::kUnknown); + + if (fileData.buffer == nullptr) { + return ValueRef::createUndefined(); } - return ValueRef::createUndefined(); + HandleScope handleScope(isolate); + + v8::Local source = + NewReloadableString(isolate, + ReloadableSourceData::create(fileData, sourceReader)) + .ToLocalChecked(); + + return CVAL(*source)->value()->asString(); } MaybeLocal Loader::NewReloadableString(Isolate* isolate, @@ -295,6 +272,50 @@ MaybeLocal Loader::NewReloadableString(Isolate* isolate, } else if (data->stringLength() > v8::String::kMaxLength) { result = MaybeLocal(); } else { + if (loadCallback == nullptr && unloadCallback == nullptr) { + // set default load/unload callbacks + loadCallback = [](void* userData) -> void* { + auto data = reinterpret_cast(userData); + + LWNODE_LOG_INFO(" Load: %d (%d) %p %s (+%.2f kB)", + ++s_stat.loaded, + s_stat.reloaded, + data->preloadedData, + data->path(), + (float)data->preloadedDataLength() / 1024); + + if (data->preloadedData) { + auto buffer = data->preloadedData; + data->preloadedData = nullptr; + return buffer; // move memory ownership to js engine + } + s_stat.reloaded++; + + FileData fileData = + data->sourceReader()->read(data->path(), data->encoding()); + + LWNODE_CHECK_NOT_NULL(fileData.buffer); + return fileData.buffer; + }; + + unloadCallback = [](void* preloadedData, void* userData) -> void { + auto data = reinterpret_cast(userData); + + LWNODE_LOG_INFO(" Unload: %d (%d) %p %s (-%.2f kB)", + --s_stat.loaded, + s_stat.reloaded, + preloadedData, + data->path(), + (float)data->preloadedDataLength() / 1024); + + if (data->preloadedData) { + freeStringBuffer(data->preloadedData); + data->preloadedData = nullptr; + } + freeStringBuffer(preloadedData); + }; + } + Escargot::StringRef* reloadableString = Escargot::StringRef::createReloadableString( IsolateWrap::fromV8(isolate)->vmInstance(), @@ -303,6 +324,7 @@ MaybeLocal Loader::NewReloadableString(Isolate* isolate, data, // data should be gc-managed. loadCallback, unloadCallback); + result = v8::Utils::NewLocal(isolate, reloadableString); } diff --git a/src/node_native_module_lwnode-inl.h b/src/node_native_module_lwnode-inl.h index f8d31f2..7efb9dc 100644 --- a/src/node_native_module_lwnode-inl.h +++ b/src/node_native_module_lwnode-inl.h @@ -66,7 +66,7 @@ static ArchiveFileScope s_archiveFileScope; static std::map s_unzFileInfoDictionary; std::string getSelfProcPath() { - char path[PATH_MAX]; + char path[PATH_MAX + 1]; ssize_t length = readlink("/proc/self/exe", path, PATH_MAX); if (length < 0) { ERROR_AND_ABORT("readlink fails"); @@ -92,10 +92,10 @@ bool readCurrentFileFromArchive(const unzFile file, freeStringBuffer(buffer); buffer = nullptr; result = false; + } else { + (*buffer)[*fileSize] = '\0'; } - (*buffer)[*fileSize] = '\0'; - unzCloseCurrentFile(file); return result; } @@ -172,7 +172,7 @@ bool readFileFromArchive(const std::string& archiveFilename, FileData readFileFromArchive(std::string filename, const Encoding encodingHint) { - CHECK(encodingHint != UNKNOWN); + CHECK(encodingHint != Encoding::kUnknown); size_t bufferSize = 0; char* buffer = nullptr; @@ -193,57 +193,8 @@ FileData readFileFromArchive(std::string filename, std::unique_ptr> bufferHolder( buffer, freeStringBuffer); - Loader::U8String latin1String; - Encoding encoding = UNKNOWN; - - if (encodingHint == ONE_BYTE) { - encoding = ONE_BYTE; - } else if (encodingHint == TWO_BYTE) { - encoding = TWO_BYTE; - } else if (encodingHint == ONE_BYTE_LATIN1) { - Loader::tryConvertUTF8ToLatin1( - latin1String, encoding, (uint8_t*)buffer, bufferSize, encodingHint); - } else { - CHECK(encodingHint == UNKNOWN); - Loader::tryConvertUTF8ToLatin1( - latin1String, encoding, (uint8_t*)buffer, bufferSize, encodingHint); - } - - if (encoding == TWO_BYTE) { - // Treat non-latin1 as UTF-8 and encode it as UTF-16 Little Endian. - if (encodingHint == UNKNOWN) { - LWNODE_LOG_INFO("%s contains characters outside of the Latin1 range.", - filename.c_str()); - } - - char* newStringBuffer = nullptr; - size_t newStringBufferSize = 0; - - bool isConverted = convertUTF8ToUTF16le(&newStringBuffer, - &newStringBufferSize, - (const char*)bufferHolder.get(), - bufferSize); - if (isConverted == false) { - return FileData(); - } - - bufferHolder.reset(newStringBuffer); - bufferSize = newStringBufferSize; - } else { - if (encoding == ONE_BYTE_LATIN1) { - if (encodingHint == UNKNOWN) { - LWNODE_LOG_INFO("%s contains Latin1 characters.", filename.c_str()); - } - - bufferSize = latin1String.length(); - bufferHolder.reset(allocateStringBuffer(bufferSize + 1)); - ((uint8_t*)bufferHolder.get())[bufferSize] = '\0'; - - memcpy(bufferHolder.get(), latin1String.data(), bufferSize); - } - } - - return FileData(bufferHolder.release(), bufferSize, encoding); + return Loader::createFileDataForReloadableString( + filename, std::move(bufferHolder), bufferSize, encodingHint); } bool NativeModuleLoader::IsOneByte(const char* id) { @@ -266,71 +217,60 @@ static std::string getFileNameOnArchive(const char* id) { return filename; } -struct Stat { - int loaded{0}; - int reloaded{0}; +class SourceReaderOnArchive : public SourceReaderInterface { + public: + static SourceReaderOnArchive* getInstance() { + static SourceReaderOnArchive s_singleton; + return &s_singleton; + } + + FileData read(std::string filename, const Encoding encodingHint) { + CHECK(encodingHint != Encoding::kUnknown); + + size_t bufferSize = 0; + char* buffer = nullptr; + + static std::string s_externalBuiltinsPath; + + if (s_externalBuiltinsPath.empty()) { + std::string executablePath = getSelfProcPath(); + executablePath = executablePath.substr(0, executablePath.rfind('/') + 1); + s_externalBuiltinsPath = + executablePath + LWNODE_EXTERNAL_BUILTINS_FILENAME; + } + + if (readFileFromArchive( + s_externalBuiltinsPath, filename, &buffer, &bufferSize) == false) { + return FileData(); + } + + std::unique_ptr> bufferHolder( + buffer, freeStringBuffer); + + return Loader::createFileDataForReloadableString( + filename, std::move(bufferHolder), bufferSize, encodingHint); + } + + private: + SourceReaderOnArchive() = default; }; -static Stat s_stat; MaybeLocal NativeModuleLoader::LoadExternalBuiltinSource( Isolate* isolate, const char* id) { std::string filename = getFileNameOnArchive(id); - FileData fileData = - readFileFromArchive(filename, (IsOneByte(id) ? ONE_BYTE : TWO_BYTE)); + auto sourceReader = SourceReaderOnArchive::getInstance(); + + FileData fileData = sourceReader->read( + filename, (IsOneByte(id) ? Encoding::kAscii : Encoding::kUtf16)); if (fileData.buffer == nullptr) { ERROR_AND_ABORT("Failed to open builtins"); return MaybeLocal(); } - auto data = Loader::ReloadableSourceData::create( - filename, fileData.buffer, fileData.size, fileData.encoding); - return Loader::NewReloadableString( - isolate, - data, - // Load-ReloadableSource - [](void* userData) -> void* { - auto data = reinterpret_cast(userData); - - LWNODE_LOG_INFO(" Load: %d (%d) %p %s (+%.2f kB)", - ++s_stat.loaded, - s_stat.reloaded, - data->preloadedData, - data->path(), - (float)data->preloadedDataLength() / 1024); - - if (data->preloadedData) { - auto buffer = data->preloadedData; - data->preloadedData = nullptr; - return buffer; // move memory ownership to js engine - } - - s_stat.reloaded++; - - FileData fileData = readFileFromArchive(data->path(), data->encoding()); - - CHECK_NOT_NULL(fileData.buffer); - return fileData.buffer; - }, - // Unload-ReloadableSource - [](void* preloadedData, void* userData) -> void { - auto data = reinterpret_cast(userData); - - LWNODE_LOG_INFO("Unload: %d (%d) %p %s (-%.2f kB)", - --s_stat.loaded, - s_stat.reloaded, - preloadedData, - data->path(), - (float)data->preloadedDataLength() / 1024); - - if (data->preloadedData) { - freeStringBuffer(data->preloadedData); - data->preloadedData = nullptr; - } - freeStringBuffer(preloadedData); - }); + isolate, Loader::ReloadableSourceData::create(fileData, sourceReader)); } } // namespace native_module -- 2.7.4