From b42623ff9d6b1baf9fda11b16a8d2f512720d424 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 8 Jul 2021 10:42:44 +0300 Subject: [PATCH] port base64 encoding from 3.4 --- modules/core/src/persistence.cpp | 2393 +++++++++++----------- modules/core/src/persistence.hpp | 18 + modules/core/src/persistence_base64_encoding.cpp | 370 ++++ modules/core/src/persistence_base64_encoding.hpp | 127 ++ modules/core/src/persistence_impl.hpp | 231 +++ modules/core/src/persistence_json.cpp | 69 +- modules/core/src/persistence_xml.cpp | 30 +- modules/core/src/persistence_yml.cpp | 28 +- modules/core/test/test_io.cpp | 29 +- modules/python/test/test_filestorage_io.py | 93 + 10 files changed, 2156 insertions(+), 1232 deletions(-) create mode 100644 modules/core/src/persistence_base64_encoding.cpp create mode 100644 modules/core/src/persistence_base64_encoding.hpp create mode 100644 modules/core/src/persistence_impl.hpp diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 3232836..291931b 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -4,6 +4,8 @@ #include "precomp.hpp" #include "persistence.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" #include #include @@ -153,7 +155,7 @@ static int symbolToType(char c) return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); return static_cast(pos - symbols); } @@ -192,7 +194,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) } if( count <= 0 ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); fmt_pairs[i] = count; } @@ -208,7 +210,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) { i += 2; if( i >= max_len ) - CV_Error( CV_StsBadArg, "Too long data type specification" ); + CV_Error( cv::Error::StsBadArg, "Too long data type specification" ); } fmt_pairs[i] = 0; } @@ -275,7 +277,7 @@ int decodeSimpleFormat( const char* dt ) fmt_pair_count = decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); if( fmt_pair_count != 1 || fmt_pairs[0] >= CV_CN_MAX) - CV_Error( CV_StsError, "Too complex format for the matrix" ); + CV_Error( cv::Error::StsError, "Too complex format for the matrix" ); elem_type = CV_MAKETYPE( fmt_pairs[1], fmt_pairs[0] ); @@ -345,1450 +347,1483 @@ static inline void writeReal(uchar* p, double fval) #endif } -class FileStorage::Impl : public FileStorage_API -{ -public: - void init() - { - flags = 0; - buffer.clear(); - bufofs = 0; - state = UNDEFINED; - is_opened = false; - dummy_eof = false; - write_mode = false; - mem_mode = false; - space = 0; - wrap_margin = 71; - fmt = 0; - file = 0; - gzfile = 0; - empty_stream = true; - strbufv.clear(); - strbuf = 0; - strbufsize = strbufpos = 0; - roots.clear(); - - fs_data.clear(); - fs_data_ptrs.clear(); - fs_data_blksz.clear(); - freeSpaceOfs = 0; - - str_hash.clear(); - str_hash_data.clear(); - str_hash_data.resize(1); - str_hash_data[0] = '\0'; - - filename.clear(); - lineno = 0; - } - - Impl(FileStorage* _fs) - { - fs_ext = _fs; - init(); - } - - virtual ~Impl() - { - release(); - } - void release(String* out=0) - { - if( is_opened ) - { - if(out) - out->clear(); - if( write_mode ) - { - while( write_stack.size() > 1 ) - { - endWriteStruct(); - } - flush(); - if( fmt == FileStorage::FORMAT_XML ) - puts( "\n" ); - else if ( fmt == FileStorage::FORMAT_JSON ) - puts( "}\n" ); - } - if( mem_mode && out ) - { - *out = cv::String(outbuf.begin(), outbuf.end()); +void FileStorage::Impl::init() { + flags = 0; + buffer.clear(); + bufofs = 0; + state = UNDEFINED; + is_using_base64 = false; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + is_write_struct_delayed = false; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + base64_writer = nullptr; + is_opened = false; + dummy_eof = false; + write_mode = false; + mem_mode = false; + space = 0; + wrap_margin = 71; + fmt = 0; + file = 0; + gzfile = 0; + empty_stream = true; + + strbufv.clear(); + strbuf = 0; + strbufsize = strbufpos = 0; + roots.clear(); + + fs_data.clear(); + fs_data_ptrs.clear(); + fs_data_blksz.clear(); + freeSpaceOfs = 0; + + str_hash.clear(); + str_hash_data.clear(); + str_hash_data.resize(1); + str_hash_data[0] = '\0'; + + filename.clear(); + lineno = 0; +} + +FileStorage::Impl::Impl(FileStorage *_fs) { + fs_ext = _fs; + init(); +} + +FileStorage::Impl::~Impl() { + release(); +} + +void FileStorage::Impl::release(String *out) { + if (is_opened) { + if (out) + out->clear(); + if (write_mode) { + while (write_stack.size() > 1) { + endWriteStruct(); } + flush(); + if (fmt == FileStorage::FORMAT_XML) + puts("\n"); + else if (fmt == FileStorage::FORMAT_JSON) + puts("}\n"); + } + if (mem_mode && out) { + *out = cv::String(outbuf.begin(), outbuf.end()); } - closeFile(); - init(); } + closeFile(); + init(); +} - void analyze_file_name( const std::string& file_name, std::vector& params ) - { - params.clear(); - static const char not_file_name = '\n'; - static const char parameter_begin = '?'; - static const char parameter_separator = '&'; +void FileStorage::Impl::analyze_file_name(const std::string &file_name, std::vector ¶ms) { + params.clear(); + static const char not_file_name = '\n'; + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; - if( file_name.find(not_file_name, (size_t)0) != std::string::npos ) - return; + if (file_name.find(not_file_name, (size_t) 0) != std::string::npos) + return; - size_t beg = file_name.find_last_of(parameter_begin); - params.push_back(file_name.substr((size_t)0, beg)); + size_t beg = file_name.find_last_of(parameter_begin); + params.push_back(file_name.substr((size_t) 0, beg)); - if( beg != std::string::npos ) - { - size_t end = file_name.size(); - beg++; - for( size_t param_beg = beg, param_end = beg; - param_end < end; - param_beg = param_end + 1 ) - { - param_end = file_name.find_first_of( parameter_separator, param_beg ); - if( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end ) - { - params.push_back( file_name.substr( param_beg, param_end - param_beg ) ); - } + if (beg != std::string::npos) { + size_t end = file_name.size(); + beg++; + for (size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1) { + param_end = file_name.find_first_of(parameter_separator, param_beg); + if ((param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end) { + params.push_back(file_name.substr(param_beg, param_end - param_beg)); } } } +} - bool open( const char* filename_or_buf, int _flags, const char* encoding ) - { - _flags &= ~FileStorage::BASE64; - - bool ok = true; - release(); +bool FileStorage::Impl::open(const char *filename_or_buf, int _flags, const char *encoding) { + bool ok = true; + release(); - bool append = (_flags & 3) == FileStorage::APPEND; - mem_mode = (_flags & FileStorage::MEMORY) != 0; + bool append = (_flags & 3) == FileStorage::APPEND; + mem_mode = (_flags & FileStorage::MEMORY) != 0; - write_mode = (_flags & 3) != 0; + write_mode = (_flags & 3) != 0; + bool write_base64 = (write_mode || append) && (_flags & FileStorage::BASE64) != 0; - bool isGZ = false; - size_t fnamelen = 0; + bool isGZ = false; + size_t fnamelen = 0; - std::vector params; - //if ( !mem_mode ) - { - analyze_file_name( filename_or_buf, params ); - if( !params.empty() ) - filename = params[0]; + std::vector params; + //if ( !mem_mode ) + { + analyze_file_name(filename_or_buf, params); + if (!params.empty()) + filename = params[0]; - /*if( !write_base64 && params.size() >= 2 && - std::find(params.begin()+1, params.end(), std::string("base64")) != params.end()) - write_base64 = (write_mode || append);*/ - } + if (!write_base64 && params.size() >= 2 && + std::find(params.begin() + 1, params.end(), std::string("base64")) != params.end()) + write_base64 = (write_mode || append); + } - if( filename.size() == 0 && !mem_mode && !write_mode ) - CV_Error( CV_StsNullPtr, "NULL or empty filename" ); + if (filename.size() == 0 && !mem_mode && !write_mode) + CV_Error(cv::Error::StsNullPtr, "NULL or empty filename"); - if( mem_mode && append ) - CV_Error( CV_StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible" ); + if (mem_mode && append) + CV_Error(cv::Error::StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible"); - flags = _flags; + flags = _flags; - if( !mem_mode ) - { - char* dot_pos = strrchr((char*)filename.c_str(), '.'); - char compression = '\0'; + if (!mem_mode) { + char *dot_pos = strrchr((char *) filename.c_str(), '.'); + char compression = '\0'; - if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && - (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) - { - if( append ) - { - CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); - } - isGZ = true; - compression = dot_pos[3]; - if( compression ) - dot_pos[3] = '\0', fnamelen--; + if (dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && + (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0'))) { + if (append) { + CV_Error(cv::Error::StsNotImplemented, "Appending data to compressed file is not implemented"); } + isGZ = true; + compression = dot_pos[3]; + if (compression) + dot_pos[3] = '\0', fnamelen--; + } - if( !isGZ ) - { - file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t" ); - if( !file ) - return false; - } - else - { + if (!isGZ) { + file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t"); + if (!file) + return false; + } else { #if USE_ZLIB - char mode[] = { write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; - gzfile = gzopen(filename.c_str(), mode); - if( !gzfile ) - return false; + char mode[] = {write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0'}; + gzfile = gzopen(filename.c_str(), mode); + if (!gzfile) + return false; #else - CV_Error(CV_StsNotImplemented, "There is no compressed file storage support in this configuration"); + CV_Error(cv::Error::StsNotImplemented, "There is no compressed file storage support in this configuration"); #endif - } } + } - roots.clear(); - fs_data.clear(); - wrap_margin = 71; - fmt = FileStorage::FORMAT_AUTO; + roots.clear(); + fs_data.clear(); + wrap_margin = 71; + fmt = FileStorage::FORMAT_AUTO; - if( write_mode ) - { - fmt = flags & FileStorage::FORMAT_MASK; + if (write_mode) { + fmt = flags & FileStorage::FORMAT_MASK; - if( mem_mode ) - outbuf.clear(); + if (mem_mode) + outbuf.clear(); - if( fmt == FileStorage::FORMAT_AUTO && !filename.empty() ) - { - const char* dot_pos = NULL; - const char* dot_pos2 = NULL; - // like strrchr() implementation, but save two last positions simultaneously - for (const char* pos = &filename[0]; pos[0] != 0; pos++) - { - if( pos[0] == '.' ) - { - dot_pos2 = dot_pos; - dot_pos = pos; - } - } - if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) - { - dot_pos = dot_pos2; + if (fmt == FileStorage::FORMAT_AUTO && !filename.empty()) { + const char *dot_pos = NULL; + const char *dot_pos2 = NULL; + // like strrchr() implementation, but save two last positions simultaneously + for (const char *pos = &filename[0]; pos[0] != 0; pos++) { + if (pos[0] == '.') { + dot_pos2 = dot_pos; + dot_pos = pos; } - fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0 ) - ? FileStorage::FORMAT_XML - : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) - ? FileStorage::FORMAT_JSON - : FileStorage::FORMAT_YAML; - } - else if( fmt == FileStorage::FORMAT_AUTO ) - { - fmt = FileStorage::FORMAT_XML; } - - // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") - // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) - int buf_size = CV_FS_MAX_LEN*(fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - - if (append) - { - fseek( file, 0, SEEK_END ); - if (ftell(file) == 0) - append = false; + if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) { + dot_pos = dot_pos2; } + fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0) + ? FileStorage::FORMAT_XML + : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) + ? FileStorage::FORMAT_JSON + : FileStorage::FORMAT_YAML; + } else if (fmt == FileStorage::FORMAT_AUTO) { + fmt = FileStorage::FORMAT_XML; + } - write_stack.clear(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); - buffer.reserve(buf_size + 1024); - buffer.resize(buf_size); - bufofs = 0; + // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") + // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) + int buf_size = CV_FS_MAX_LEN * (fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - if( fmt == FileStorage::FORMAT_XML ) - { - size_t file_size = file ? (size_t)ftell(file) : (size_t)0; - if( !append || file_size == 0 ) - { - if( encoding && *encoding != '\0' ) - { - if( fs::strcasecmp(encoding, "UTF-16" ) == 0 ) - { - release(); - CV_Error( CV_StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); - } + if (append) { + fseek(file, 0, SEEK_END); + if (ftell(file) == 0) + append = false; + } - CV_Assert( strlen(encoding) < 1000 ); - char buf[1100]; - sprintf(buf, "\n", encoding); - puts( buf ); + write_stack.clear(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); + buffer.reserve(buf_size + 1024); + buffer.resize(buf_size); + bufofs = 0; + is_using_base64 = write_base64; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + + if (fmt == FileStorage::FORMAT_XML) { + size_t file_size = file ? (size_t) ftell(file) : (size_t) 0; + if (!append || file_size == 0) { + if (encoding && *encoding != '\0') { + if (fs::strcasecmp(encoding, "UTF-16") == 0) { + release(); + CV_Error(cv::Error::StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); } - else - puts( "\n" ); - puts( "\n" ); - } - else - { - int xml_buf_size = 1 << 10; - char substr[] = ""; - int last_occurrence = -1; - xml_buf_size = MIN(xml_buf_size, int(file_size)); - fseek( file, -xml_buf_size, SEEK_END ); - // find the last occurrence of - for(;;) - { - int line_offset = (int)ftell( file ); - const char* ptr0 = this->gets(xml_buf_size); - const char* ptr = NULL; - if( !ptr0 ) + + CV_Assert(strlen(encoding) < 1000); + char buf[1100]; + sprintf(buf, "\n", encoding); + puts(buf); + } else + puts("\n"); + puts("\n"); + } else { + int xml_buf_size = 1 << 10; + char substr[] = ""; + int last_occurrence = -1; + xml_buf_size = MIN(xml_buf_size, int(file_size)); + fseek(file, -xml_buf_size, SEEK_END); + // find the last occurrence of + for (;;) { + int line_offset = (int) ftell(file); + const char *ptr0 = this->gets(xml_buf_size); + const char *ptr = NULL; + if (!ptr0) + break; + ptr = ptr0; + for (;;) { + ptr = strstr(ptr, substr); + if (!ptr) break; - ptr = ptr0; - for(;;) - { - ptr = strstr( ptr, substr ); - if( !ptr ) - break; - last_occurrence = line_offset + (int)(ptr - ptr0); - ptr += strlen(substr); - } + last_occurrence = line_offset + (int) (ptr - ptr0); + ptr += strlen(substr); } - if( last_occurrence < 0 ) - { - release(); - CV_Error( CV_StsError, "Could not find in the end of file.\n" ); - } - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, last_occurrence, SEEK_SET ); - // replace the last "" with " ", which has the same length - puts( " " ); - fseek( file, 0, SEEK_END ); - puts( "\n" ); } - - emitter = createXMLEmitter(this); + if (last_occurrence < 0) { + release(); + CV_Error(cv::Error::StsError, "Could not find in the end of file.\n"); + } + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, last_occurrence, SEEK_SET); + // replace the last "" with " ", which has the same length + puts(" "); + fseek(file, 0, SEEK_END); + puts("\n"); } - else if( fmt == FileStorage::FORMAT_YAML ) - { - if( !append) - puts( "%YAML:1.0\n---\n" ); - else - puts( "...\n---\n" ); - emitter = createYAMLEmitter(this); - } + emitter = createXMLEmitter(this); + } else if (fmt == FileStorage::FORMAT_YAML) { + if (!append) + puts("%YAML:1.0\n---\n"); else - { - CV_Assert( fmt == FileStorage::FORMAT_JSON ); - if( !append ) - puts( "{\n" ); - else - { - bool valid = false; - long roffset = 0; - for ( ; - fseek( file, roffset, SEEK_END ) == 0; - roffset -= 1 ) - { - const char end_mark = '}'; - if ( fgetc( file ) == end_mark ) - { - fseek( file, roffset, SEEK_END ); - valid = true; - break; - } + puts("...\n---\n"); + + emitter = createYAMLEmitter(this); + } else { + CV_Assert(fmt == FileStorage::FORMAT_JSON); + if (!append) + puts("{\n"); + else { + bool valid = false; + long roffset = 0; + for (; + fseek(file, roffset, SEEK_END) == 0; + roffset -= 1) { + const char end_mark = '}'; + if (fgetc(file) == end_mark) { + fseek(file, roffset, SEEK_END); + valid = true; + break; } + } - if ( valid ) - { - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, roffset, SEEK_END ); - fputs( ",", file ); - } - else - { - CV_Error( CV_StsError, "Could not find '}' in the end of file.\n" ); - } + if (valid) { + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, roffset, SEEK_END); + fputs(",", file); + } else { + CV_Error(cv::Error::StsError, "Could not find '}' in the end of file.\n"); } - write_stack.back().indent = 4; - emitter = createJSONEmitter(this); } - is_opened = true; + write_stack.back().indent = 4; + emitter = createJSONEmitter(this); + } + is_opened = true; + } else { + const size_t buf_size0 = 40; + buffer.resize(buf_size0); + if (mem_mode) { + strbuf = (char *) filename_or_buf; + strbufsize = strlen(strbuf); } + + const char *yaml_signature = "%YAML"; + const char *json_signature = "{"; + const char *xml_signature = "gets(16); + CV_Assert(buf); + char *bufPtr = cv_skip_BOM(buf); + size_t bufOffset = bufPtr - buf; + + if (strncmp(bufPtr, yaml_signature, strlen(yaml_signature)) == 0) + fmt = FileStorage::FORMAT_YAML; + else if (strncmp(bufPtr, json_signature, strlen(json_signature)) == 0) + fmt = FileStorage::FORMAT_JSON; + else if (strncmp(bufPtr, xml_signature, strlen(xml_signature)) == 0) + fmt = FileStorage::FORMAT_XML; + else if (strbufsize == bufOffset) + CV_Error(cv::Error::StsBadArg, "Input file is invalid"); else - { - const size_t buf_size0 = 40; - buffer.resize(buf_size0); - if( mem_mode ) - { - strbuf = (char*)filename_or_buf; - strbufsize = strlen(strbuf); - } + CV_Error(cv::Error::StsBadArg, "Unsupported file storage format"); - const char* yaml_signature = "%YAML"; - const char* json_signature = "{"; - const char* xml_signature = "gets(16); - CV_Assert(buf); - char* bufPtr = cv_skip_BOM(buf); - size_t bufOffset = bufPtr - buf; - - if(strncmp( bufPtr, yaml_signature, strlen(yaml_signature) ) == 0) - fmt = FileStorage::FORMAT_YAML; - else if(strncmp( bufPtr, json_signature, strlen(json_signature) ) == 0) - fmt = FileStorage::FORMAT_JSON; - else if(strncmp( bufPtr, xml_signature, strlen(xml_signature) ) == 0) - fmt = FileStorage::FORMAT_XML; - else if(strbufsize == bufOffset) - CV_Error(CV_BADARG_ERR, "Input file is invalid"); - else - CV_Error(CV_BADARG_ERR, "Unsupported file storage format"); + rewind(); + strbufpos = bufOffset; + bufofs = 0; - rewind(); - strbufpos = bufOffset; - bufofs = 0; + try { + char *ptr = bufferStart(); + ptr[0] = ptr[1] = ptr[2] = '\0'; + FileNode root_nodes(fs_ext, 0, 0); - try - { - char* ptr = bufferStart(); - ptr[0] = ptr[1] = ptr[2] = '\0'; - FileNode root_nodes(fs_ext, 0, 0); + uchar *rptr = reserveNodeSpace(root_nodes, 9); + *rptr = FileNode::SEQ; + writeInt(rptr + 1, 4); + writeInt(rptr + 5, 0); - uchar* rptr = reserveNodeSpace(root_nodes, 9); - *rptr = FileNode::SEQ; - writeInt(rptr + 1, 4); - writeInt(rptr + 5, 0); + roots.clear(); - roots.clear(); + switch (fmt) { + case FileStorage::FORMAT_XML: + parser = createXMLParser(this); + break; + case FileStorage::FORMAT_YAML: + parser = createYAMLParser(this); + break; + case FileStorage::FORMAT_JSON: + parser = createJSONParser(this); + break; + default: + parser = Ptr(); + } - switch (fmt) - { - case FileStorage::FORMAT_XML: parser = createXMLParser(this); break; - case FileStorage::FORMAT_YAML: parser = createYAMLParser(this); break; - case FileStorage::FORMAT_JSON: parser = createJSONParser(this); break; - default: parser = Ptr(); - } + if (!parser.empty()) { + ok = parser->parse(ptr); + if (ok) { + finalizeCollection(root_nodes); - if( !parser.empty() ) - { - ok = parser->parse(ptr); - if( ok ) - { - finalizeCollection(root_nodes); + CV_Assert(!fs_data_ptrs.empty()); + FileNode roots_node(fs_ext, 0, 0); + size_t i, nroots = roots_node.size(); + FileNodeIterator it = roots_node.begin(); - CV_Assert( !fs_data_ptrs.empty() ); - FileNode roots_node(fs_ext, 0, 0); - size_t i, nroots = roots_node.size(); - FileNodeIterator it = roots_node.begin(); - - for( i = 0; i < nroots; i++, ++it ) - roots.push_back(*it); - } + for (i = 0; i < nroots; i++, ++it) + roots.push_back(*it); } } - catch(...) - { - is_opened = true; - release(); - throw; - } - - // release resources that we do not need anymore - closeFile(); + } + catch (...) { is_opened = true; - std::vector tmpbuf; - std::swap(buffer, tmpbuf); - bufofs = 0; + release(); + throw; } - return ok; + + // release resources that we do not need anymore + closeFile(); + is_opened = true; + std::vector tmpbuf; + std::swap(buffer, tmpbuf); + bufofs = 0; } + return ok; +} - void puts( const char* str ) - { - CV_Assert( write_mode ); - if( mem_mode ) - std::copy(str, str + strlen(str), std::back_inserter(outbuf)); - else if( file ) - fputs( str, file ); +void FileStorage::Impl::puts(const char *str) { + CV_Assert(write_mode); + if (mem_mode) + std::copy(str, str + strlen(str), std::back_inserter(outbuf)); + else if (file) + fputs(str, file); #if USE_ZLIB - else if( gzfile ) - gzputs( gzfile, str ); + else if (gzfile) + gzputs(gzfile, str); #endif - else - CV_Error( CV_StsError, "The storage is not opened" ); - } - - char* getsFromFile( char* buf, int count ) - { - if( file ) - return fgets( buf, count, file ); - #if USE_ZLIB - if( gzfile ) - return gzgets( gzfile, buf, count ); - #endif - CV_Error(CV_StsError, "The storage is not opened"); - } + else + CV_Error(cv::Error::StsError, "The storage is not opened"); +} - char* gets( size_t maxCount ) - { - if( strbuf ) - { - size_t i = strbufpos, len = strbufsize; - const char* instr = strbuf; - for( ; i < len; i++ ) - { - char c = instr[i]; - if( c == '\0' || c == '\n' ) - { - if( c == '\n' ) - i++; - break; - } +char *FileStorage::Impl::getsFromFile(char *buf, int count) { + if (file) + return fgets(buf, count, file); +#if USE_ZLIB + if (gzfile) + return gzgets(gzfile, buf, count); +#endif + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::gets(size_t maxCount) { + if (strbuf) { + size_t i = strbufpos, len = strbufsize; + const char *instr = strbuf; + for (; i < len; i++) { + char c = instr[i]; + if (c == '\0' || c == '\n') { + if (c == '\n') + i++; + break; } - size_t count = i - strbufpos; - if( maxCount == 0 || maxCount > count ) - maxCount = count; - buffer.resize(std::max(buffer.size(), maxCount + 8)); - memcpy(&buffer[0], instr + strbufpos, maxCount); - buffer[maxCount] = '\0'; - strbufpos = i; - return maxCount > 0 ? &buffer[0] : 0; } + size_t count = i - strbufpos; + if (maxCount == 0 || maxCount > count) + maxCount = count; + buffer.resize(std::max(buffer.size(), maxCount + 8)); + memcpy(&buffer[0], instr + strbufpos, maxCount); + buffer[maxCount] = '\0'; + strbufpos = i; + return maxCount > 0 ? &buffer[0] : 0; + } + + const size_t MAX_BLOCK_SIZE = INT_MAX / 2; // hopefully, that will be enough + if (maxCount == 0) + maxCount = MAX_BLOCK_SIZE; + else + CV_Assert(maxCount < MAX_BLOCK_SIZE); + size_t ofs = 0; - const size_t MAX_BLOCK_SIZE = INT_MAX/2; // hopefully, that will be enough - if( maxCount == 0 ) - maxCount = MAX_BLOCK_SIZE; - else - CV_Assert(maxCount < MAX_BLOCK_SIZE); - size_t ofs = 0; - - for(;;) - { - int count = (int)std::min(buffer.size() - ofs - 16, maxCount); - char* ptr = getsFromFile( &buffer[ofs], count+1 ); - if( !ptr ) - break; - int delta = (int)strlen(ptr); - ofs += delta; - maxCount -= delta; - if( ptr[delta-1] == '\n' || maxCount == 0 ) - break; - if( delta == count ) - buffer.resize((size_t)(buffer.size()*1.5)); - } - return ofs > 0 ? &buffer[0] : 0; + for (;;) { + int count = (int) std::min(buffer.size() - ofs - 16, maxCount); + char *ptr = getsFromFile(&buffer[ofs], count + 1); + if (!ptr) + break; + int delta = (int) strlen(ptr); + ofs += delta; + maxCount -= delta; + if (ptr[delta - 1] == '\n' || maxCount == 0) + break; + if (delta == count) + buffer.resize((size_t) (buffer.size() * 1.5)); } + return ofs > 0 ? &buffer[0] : 0; +} - char* gets() - { - char* ptr = this->gets(0); - if( !ptr ) - { - ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? - *ptr = '\0'; - setEof(); - return 0; - } - else - { - size_t l = strlen(ptr); - if( l > 0 && ptr[l-1] != '\n' && ptr[l-1] != '\r' && !eof() ) - { - ptr[l] = '\n'; - ptr[l+1] = '\0'; - } +char *FileStorage::Impl::gets() { + char *ptr = this->gets(0); + if (!ptr) { + ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? + *ptr = '\0'; + setEof(); + return 0; + } else { + size_t l = strlen(ptr); + if (l > 0 && ptr[l - 1] != '\n' && ptr[l - 1] != '\r' && !eof()) { + ptr[l] = '\n'; + ptr[l + 1] = '\0'; } - lineno++; - return ptr; } + lineno++; + return ptr; +} - bool eof() - { - if( dummy_eof ) - return true; - if( strbuf ) - return strbufpos >= strbufsize; - if( file ) - return feof(file) != 0; +bool FileStorage::Impl::eof() { + if (dummy_eof) + return true; + if (strbuf) + return strbufpos >= strbufsize; + if (file) + return feof(file) != 0; #if USE_ZLIB - if( gzfile ) - return gzeof(gzfile) != 0; + if (gzfile) + return gzeof(gzfile) != 0; #endif - return false; - } + return false; +} - void setEof() - { - dummy_eof = true; - } +void FileStorage::Impl::setEof() { + dummy_eof = true; +} - void closeFile() - { - if( file ) - fclose( file ); +void FileStorage::Impl::closeFile() { + if (file) + fclose(file); #if USE_ZLIB - else if( gzfile ) - gzclose( gzfile ); + else if (gzfile) + gzclose(gzfile); #endif - file = 0; - gzfile = 0; - strbuf = 0; - strbufpos = 0; - is_opened = false; - } + file = 0; + gzfile = 0; + strbuf = 0; + strbufpos = 0; + is_opened = false; +} - void rewind() - { - if( file ) - ::rewind(file); +void FileStorage::Impl::rewind() { + if (file) + ::rewind(file); #if USE_ZLIB - else if( gzfile ) - gzrewind(gzfile); + else if (gzfile) + gzrewind(gzfile); #endif - strbufpos = 0; - } + strbufpos = 0; +} - char* resizeWriteBuffer( char* ptr, int len ) - { - const char* buffer_end = &buffer[0] + buffer.size(); - if( ptr + len < buffer_end ) - return ptr; +char *FileStorage::Impl::resizeWriteBuffer(char *ptr, int len) { + const char *buffer_end = &buffer[0] + buffer.size(); + if (ptr + len < buffer_end) + return ptr; - const char* buffer_start = &buffer[0]; - int written_len = (int)(ptr - buffer_start); + const char *buffer_start = &buffer[0]; + int written_len = (int) (ptr - buffer_start); - CV_Assert(written_len <= (int)buffer.size()); - int new_size = (int)((buffer_end - buffer_start)*3/2); - new_size = MAX( written_len + len, new_size ); - buffer.reserve( new_size + 256 ); - buffer.resize( new_size ); - bufofs = written_len; - return &buffer[0] + bufofs; + CV_Assert(written_len <= (int) buffer.size()); + int new_size = (int) ((buffer_end - buffer_start) * 3 / 2); + new_size = MAX(written_len + len, new_size); + buffer.reserve(new_size + 256); + buffer.resize(new_size); + bufofs = written_len; + return &buffer[0] + bufofs; +} + +char *FileStorage::Impl::flush() { + char *buffer_start = &buffer[0]; + char *ptr = buffer_start + bufofs; + + if (ptr > buffer_start + space) { + ptr[0] = '\n'; + ptr[1] = '\0'; + puts(buffer_start); + bufofs = 0; } - char* flush() - { - char* buffer_start = &buffer[0]; - char* ptr = buffer_start + bufofs; + int indent = write_stack.back().indent; - if( ptr > buffer_start + space ) - { - ptr[0] = '\n'; - ptr[1] = '\0'; - puts( buffer_start ); - bufofs = 0; - } + if (space != indent) { + memset(buffer_start, ' ', indent); + space = indent; + } + bufofs = space; + ptr = buffer_start + bufofs; - int indent = write_stack.back().indent; + return ptr; +} - if( space != indent ) - { - memset( buffer_start, ' ', indent ); - space = indent; - } - bufofs = space; - ptr = buffer_start + bufofs; +void FileStorage::Impl::endWriteStruct() { + CV_Assert(write_mode); - return ptr; - } + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); - void endWriteStruct() - { - CV_Assert( write_mode ); - CV_Assert( !write_stack.empty() ); + CV_Assert(!write_stack.empty()); - FStructData& current_struct = write_stack.back(); - if( fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1 ) - current_struct.indent = write_stack[write_stack.size() - 2].indent; + FStructData ¤t_struct = write_stack.back(); + if (fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1) + current_struct.indent = write_stack[write_stack.size() - 2].indent; - emitter->endWriteStruct(current_struct); + emitter->endWriteStruct(current_struct); - write_stack.pop_back(); - if( !write_stack.empty() ) - write_stack.back().flags &= ~FileNode::EMPTY; - } + write_stack.pop_back(); + if (!write_stack.empty()) + write_stack.back().flags &= ~FileNode::EMPTY; +} - void startWriteStruct( const char* key, int struct_flags, - const char* type_name ) - { - CV_Assert( write_mode ); +void FileStorage::Impl::startWriteStruct_helper(const char *key, int struct_flags, + const char *type_name) { + CV_Assert(write_mode); - struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; - if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, - "Some collection type: FileNode::SEQ or FileNode::MAP must be specified" ); + struct_flags = (struct_flags & (FileNode::TYPE_MASK | FileNode::FLOW)) | FileNode::EMPTY; + if (!FileNode::isCollection(struct_flags)) + CV_Error(cv::Error::StsBadArg, + "Some collection type: FileNode::SEQ or FileNode::MAP must be specified"); - if( type_name && type_name[0] == '\0' ) - type_name = 0; + if (type_name && type_name[0] == '\0') + type_name = 0; - FStructData s = emitter->startWriteStruct( write_stack.back(), key, struct_flags, type_name ); - write_stack.push_back(s); - size_t write_stack_size = write_stack.size(); - if( write_stack_size > 1 ) - write_stack[write_stack_size-2].flags &= ~FileNode::EMPTY; + FStructData s = emitter->startWriteStruct(write_stack.back(), key, struct_flags, type_name); - if( !FileNode::isFlow(s.flags) ) - flush(); + write_stack.push_back(s); + size_t write_stack_size = write_stack.size(); + if (write_stack_size > 1) + write_stack[write_stack_size - 2].flags &= ~FileNode::EMPTY; - if( fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) - { - emitter->write("type_id", type_name, false); - } - } + if (fmt != FileStorage::FORMAT_JSON && !FileNode::isFlow(s.flags)) + flush(); - void writeComment( const char* comment, bool eol_comment ) - { - CV_Assert(write_mode); - emitter->writeComment( comment, eol_comment ); + if (fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) { + emitter->write("type_id", type_name, false); } +} - void startNextStream() - { - CV_Assert(write_mode); - if( !empty_stream ) - { - while( !write_stack.empty() ) - endWriteStruct(); - flush(); - emitter->startNextStream(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); - bufofs = 0; - } - } +void FileStorage::Impl::startWriteStruct(const char *key, int struct_flags, + const char *type_name) { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 == FileStorage_API::NotUse) + switch_to_Base64_state(FileStorage_API::Uncertain); - void write( const String& key, int value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + if (state_of_writing_base64 == FileStorage_API::Uncertain && FileNode::isSeq(struct_flags) + && is_using_base64 && type_name == 0) { + /* Uncertain whether output Base64 data */ + make_write_struct_delayed(key, struct_flags, type_name); + } else if (type_name && memcmp(type_name, "binary", 6) == 0) { + /* Must output Base64 data */ + if ((FileNode::TYPE_MASK & struct_flags) != FileNode::SEQ) + CV_Error(cv::Error::StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if (state_of_writing_base64 != FileStorage_API::Uncertain) + CV_Error(cv::Error::StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); - void write( const String& key, double value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); + startWriteStruct_helper(key, struct_flags, "binary"); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + /* Won't output Base64 data */ + if (state_of_writing_base64 == FileStorage_API::InUse) + CV_Error(cv::Error::StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + startWriteStruct_helper(key, struct_flags, type_name); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); } +} - void write( const String& key, const String& value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value.c_str(), false); +void FileStorage::Impl::writeComment(const char *comment, bool eol_comment) { + CV_Assert(write_mode); + emitter->writeComment(comment, eol_comment); +} + +void FileStorage::Impl::startNextStream() { + CV_Assert(write_mode); + if (!empty_stream) { + while (!write_stack.empty()) + endWriteStruct(); + flush(); + emitter->startNextStream(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); + bufofs = 0; } +} - void writeRawData( const std::string& dt, const void* _data, size_t len ) - { - CV_Assert(write_mode); +void FileStorage::Impl::write(const String &key, int value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} - size_t elemSize = fs::calcStructSize(dt.c_str(), 0); - CV_Assert(elemSize); - CV_Assert( len % elemSize == 0 ); - len /= elemSize; +void FileStorage::Impl::write(const String &key, double value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} - bool explicitZero = fmt == FileStorage::FORMAT_JSON; - const uchar* data0 = (const uchar*)_data; - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; - char buf[256] = ""; +void FileStorage::Impl::write(const String &key, const String &value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value.c_str(), false); +} - fmt_pair_count = fs::decodeFormat( dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS ); +void FileStorage::Impl::writeRawData(const std::string &dt, const void *_data, size_t len) { + CV_Assert(write_mode); - if( !len ) - return; + if (is_using_base64 || state_of_writing_base64 == FileStorage_API::Base64State::InUse) { + writeRawDataBase64(_data, len, dt.c_str()); + return; + } else if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::NotUse); + } - if( !data0 ) - CV_Error( CV_StsNullPtr, "Null data pointer" ); + size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); + CV_Assert(len % elemSize == 0); + len /= elemSize; - if( fmt_pair_count == 1 ) - { - fmt_pairs[0] *= (int)len; - len = 1; - } + bool explicitZero = fmt == FileStorage::FORMAT_JSON; + const uchar *data0 = (const uchar *) _data; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2], k, fmt_pair_count; + char buf[256] = ""; - for(;len--; data0 += elemSize) - { - int offset = 0; - for( k = 0; k < fmt_pair_count; k++ ) - { - int i, count = fmt_pairs[k*2]; - int elem_type = fmt_pairs[k*2+1]; - int elem_size = CV_ELEM_SIZE(elem_type); - const char *ptr; + fmt_pair_count = fs::decodeFormat(dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS); - offset = cvAlign( offset, elem_size ); - const uchar* data = data0 + offset; + if (!len) + return; - for( i = 0; i < count; i++ ) - { - switch( elem_type ) - { + if (!data0) + CV_Error(cv::Error::StsNullPtr, "Null data pointer"); + + if (fmt_pair_count == 1) { + fmt_pairs[0] *= (int) len; + len = 1; + } + + for (; len--; data0 += elemSize) { + int offset = 0; + for (k = 0; k < fmt_pair_count; k++) { + int i, count = fmt_pairs[k * 2]; + int elem_type = fmt_pairs[k * 2 + 1]; + int elem_size = CV_ELEM_SIZE(elem_type); + const char *ptr; + + offset = cvAlign(offset, elem_size); + const uchar *data = data0 + offset; + + for (i = 0; i < count; i++) { + switch (elem_type) { case CV_8U: - ptr = fs::itoa( *(uchar*)data, buf, 10 ); + ptr = fs::itoa(*(uchar *) data, buf, 10); data++; break; case CV_8S: - ptr = fs::itoa( *(char*)data, buf, 10 ); + ptr = fs::itoa(*(char *) data, buf, 10); data++; break; case CV_16U: - ptr = fs::itoa( *(ushort*)data, buf, 10 ); + ptr = fs::itoa(*(ushort *) data, buf, 10); data += sizeof(ushort); break; case CV_16S: - ptr = fs::itoa( *(short*)data, buf, 10 ); + ptr = fs::itoa(*(short *) data, buf, 10); data += sizeof(short); break; case CV_32S: - ptr = fs::itoa( *(int*)data, buf, 10 ); + ptr = fs::itoa(*(int *) data, buf, 10); data += sizeof(int); break; case CV_32F: - ptr = fs::floatToString( buf, *(float*)data, false, explicitZero ); + ptr = fs::floatToString(buf, *(float *) data, false, explicitZero); data += sizeof(float); break; case CV_64F: - ptr = fs::doubleToString( buf, *(double*)data, explicitZero ); + ptr = fs::doubleToString(buf, *(double *) data, explicitZero); data += sizeof(double); break; case CV_16F: /* reference */ - ptr = fs::floatToString( buf, (float)*(float16_t*)data, true, explicitZero ); + ptr = fs::floatToString(buf, (float) *(float16_t *) data, true, explicitZero); data += sizeof(float16_t); break; default: - CV_Error( CV_StsUnsupportedFormat, "Unsupported type" ); + CV_Error(cv::Error::StsUnsupportedFormat, "Unsupported type"); return; - } - - emitter->writeScalar(0, ptr); } - offset = (int)(data - data0); + emitter->writeScalar(0, ptr); } + + offset = (int) (data - data0); } } +} - void writeRawDataBase64(const void* /*data*/, int /*len*/, const char* /*dt*/ ) - { +void FileStorage::Impl::workaround() { + check_if_write_struct_is_delayed(false); - } + if (state_of_writing_base64 != FileStorage_API::Base64State::Uncertain) + switch_to_Base64_state(FileStorage_API::Base64State::Uncertain); +} - String releaseAndGetString(); +void FileStorage::Impl::switch_to_Base64_state(FileStorage_API::Base64State new_state) { + const char *err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char *err_unable_to_switch = "Unexpected error, unable to switch to this state."; - FileNode getFirstTopLevelNode() const - { - return roots.empty() ? FileNode() : roots[0]; + /* like a finite state machine */ + switch (state_of_writing_base64) { + case FileStorage_API::Base64State::Uncertain: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + { + CV_DbgAssert(base64_writer == 0); + bool can_indent = (fmt != cv::FileStorage::Mode::FORMAT_JSON); + base64_writer = new base64::Base64Writer(*this, can_indent); + if (!can_indent) { + char *ptr = bufferPtr(); + *ptr++ = '\0'; + puts(bufferStart()); + setBufferPtr(bufferStart()); + memset(bufferStart(), 0, static_cast(space)); + puts("\"$base64$"); + } + break; + } + case FileStorage_API::Base64State::Uncertain: + break; + case FileStorage_API::Base64State::NotUse: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::InUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + delete base64_writer; + base64_writer = 0; + if ( fmt == cv::FileStorage::FORMAT_JSON ) + { + puts("\""); + setBufferPtr(bufferStart()); + flush(); + memset(bufferStart(), 0, static_cast(space) ); + setBufferPtr(bufferStart()); + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::NotUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; } - FileNode root(int streamIdx=0) const - { - return streamIdx >= 0 && streamIdx < (int)roots.size() ? roots[streamIdx] : FileNode(); - } + state_of_writing_base64 = new_state; +} - FileNode operator[](const String& nodename) const - { - return this->operator[](nodename.c_str()); +void FileStorage::Impl::make_write_struct_delayed(const char *key, int struct_flags, const char *type_name) { + CV_Assert(is_write_struct_delayed == false); + CV_DbgAssert(delayed_struct_key == nullptr); + CV_DbgAssert(delayed_struct_flags == 0); + CV_DbgAssert(delayed_type_name == nullptr); + + delayed_struct_flags = struct_flags; + + if (key != nullptr) { + delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(delayed_struct_key, key); } - FileNode operator[](const char* /*nodename*/) const - { - return FileNode(); + if (type_name != nullptr) { + delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(delayed_type_name, type_name); } - int getFormat() const { return fmt; } + is_write_struct_delayed = true; +} - char* bufferPtr() const { return (char*)(&buffer[0] + bufofs); } - char* bufferStart() const { return (char*)&buffer[0]; } - char* bufferEnd() const { return (char*)(&buffer[0] + buffer.size()); } - void setBufferPtr(char* ptr) - { - char* bufferstart = bufferStart(); - CV_Assert( ptr >= bufferstart && ptr <= bufferEnd() ); - bufofs = ptr - bufferstart; - } - int wrapMargin() const { return wrap_margin; } +void FileStorage::Impl::check_if_write_struct_is_delayed(bool change_type_to_base64) { + if (is_write_struct_delayed) { + /* save data to prevent recursive call errors */ + std::string struct_key; + std::string type_name; + int struct_flags = delayed_struct_flags; - FStructData& getCurrentStruct() - { - CV_Assert(!write_stack.empty()); - return write_stack.back(); + if (delayed_struct_key != nullptr && *delayed_struct_key != '\0') { + struct_key.assign(delayed_struct_key); + } + if (delayed_type_name != nullptr && *delayed_type_name != '\0') { + type_name.assign(delayed_type_name); + } + + /* reset */ + delete[] delayed_struct_key; + delete[] delayed_type_name; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + + is_write_struct_delayed = false; + + /* call */ + if (change_type_to_base64) { + startWriteStruct_helper(struct_key.c_str(), struct_flags, "binary"); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + startWriteStruct_helper(struct_key.c_str(), struct_flags, type_name.c_str()); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } } +} - void setNonEmpty() - { - empty_stream = false; +void FileStorage::Impl::writeRawDataBase64(const void *_data, size_t len, const char *dt) { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(true); + + if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::InUse); + } else if (state_of_writing_base64 != FileStorage_API::Base64State::InUse) { + CV_Error(cv::Error::StsError, "Base64 should not be used at present."); } - void processSpecialDouble( char* buf, double* value, char** endptr ) - { - FileStorage_API* fs = this; - char c = buf[0]; - int inf_hi = 0x7ff00000; + base64_writer->write(_data, len, dt); +} - if( c == '-' || c == '+' ) - { - inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; - c = *++buf; - } +FileNode FileStorage::Impl::getFirstTopLevelNode() const { + return roots.empty() ? FileNode() : roots[0]; +} - if( c != '.' ) - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); +FileNode FileStorage::Impl::root(int streamIdx) const { + return streamIdx >= 0 && streamIdx < (int) roots.size() ? roots[streamIdx] : FileNode(); +} - Cv64suf v; - v.f = 0.; - if( toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F' ) - v.u = (uint64)inf_hi << 32; - else if( toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N' ) - v.u = (uint64)-1; - else - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); - *value = v.f; - *endptr = buf + 4; - } +FileNode FileStorage::Impl::operator[](const String &nodename) const { + return this->operator[](nodename.c_str()); +} - double strtod( char* ptr, char** endptr ) - { - double fval = ::strtod( ptr, endptr ); - if( **endptr == '.' ) - { - char* dot_pos = *endptr; - *dot_pos = ','; - double fval2 = ::strtod( ptr, endptr ); - *dot_pos = '.'; - if( *endptr > dot_pos ) - fval = fval2; - else - *endptr = dot_pos; - } +FileNode FileStorage::Impl::operator[](const char * /*nodename*/) const { + return FileNode(); +} - if( *endptr == ptr || cv_isalpha(**endptr) ) - processSpecialDouble( ptr, &fval, endptr ); +int FileStorage::Impl::getFormat() const { return fmt; } - return fval; - } +char *FileStorage::Impl::bufferPtr() const { return (char *) (&buffer[0] + bufofs); } - void convertToCollection(int type, FileNode& node) - { - CV_Assert( type == FileNode::SEQ || type == FileNode::MAP ); +char *FileStorage::Impl::bufferStart() const { return (char *) &buffer[0]; } - int node_type = node.type(); - if( node_type == type ) - return; +char *FileStorage::Impl::bufferEnd() const { return (char *) (&buffer[0] + buffer.size()); } - bool named = node.isNamed(); - uchar* ptr = node.ptr() + 1 + (named ? 4 : 0); +void FileStorage::Impl::setBufferPtr(char *ptr) { + char *bufferstart = bufferStart(); + CV_Assert(ptr >= bufferstart && ptr <= bufferEnd()); + bufofs = ptr - bufferstart; +} - int ival = 0; - double fval = 0; - std::string sval; - bool add_first_scalar = false; +int FileStorage::Impl::wrapMargin() const { return wrap_margin; } - if( node_type != FileNode::NONE ) - { - // scalar nodes can only be converted to sequences, e.g. in XML: - // 5[parser_position]... => create 5 with name "a" - // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it - // - // otherwise we don't know where to get the element names from - CV_Assert( type == FileNode::SEQ ); - if( node_type == FileNode::INT ) - { - ival = readInt(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::REAL ) - { - fval = readReal(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::STRING ) - { - sval = std::string(node); - add_first_scalar = true; - } - else - CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); - } +FStructData &FileStorage::Impl::getCurrentStruct() { + CV_Assert(!write_stack.empty()); + return write_stack.back(); +} - ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); - *ptr++ = (uchar)(type | (named ? FileNode::NAMED : 0)); - // name has been copied automatically - if( named ) - ptr += 4; - // set raw_size(collection)==4, nelems(collection)==1 - writeInt(ptr, 4); - writeInt(ptr + 4, 0); - - if( add_first_scalar ) - addNode(node, std::string(), node_type, - node_type == FileNode::INT ? (const void*)&ival : - node_type == FileNode::REAL ? (const void*)&fval : - node_type == FileNode::STRING ? (const void*)sval.c_str() : 0, - -1); - } - - // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or - // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). - // If there is no enough space in the current block (it should be the last block added so far), - // the last block is shrunk so that it ends immediately before the reallocated node. Then, - // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. - // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. - // In the case (b) the existing tag and the name are copied automatically. - uchar* reserveNodeSpace(FileNode& node, size_t sz) - { - bool shrinkBlock = false; - size_t shrinkBlockIdx = 0, shrinkSize = 0; +void FileStorage::Impl::setNonEmpty() { + empty_stream = false; +} - uchar *ptr = 0, *blockEnd = 0; +void FileStorage::Impl::processSpecialDouble(char *buf, double *value, char **endptr) { + FileStorage_API *fs = this; + char c = buf[0]; + int inf_hi = 0x7ff00000; - if( !fs_data_ptrs.empty() ) - { - size_t blockIdx = node.blockIdx; - size_t ofs = node.ofs; - CV_Assert( blockIdx == fs_data_ptrs.size()-1 ); - CV_Assert( ofs <= fs_data_blksz[blockIdx] ); - CV_Assert( freeSpaceOfs <= fs_data_blksz[blockIdx] ); - //CV_Assert( freeSpaceOfs <= ofs + sz ); - - ptr = fs_data_ptrs[blockIdx] + ofs; - blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; - - CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); - if( ptr + sz <= blockEnd ) - { - freeSpaceOfs = ofs + sz; - return ptr; - } + if (c == '-' || c == '+') { + inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; + c = *++buf; + } - if (ofs == 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. - { - fs_data[blockIdx]->resize(sz); - ptr = &fs_data[blockIdx]->at(0); - fs_data_ptrs[blockIdx] = ptr; - fs_data_blksz[blockIdx] = sz; - freeSpaceOfs = sz; - return ptr; - } + if (c != '.') + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); - shrinkBlock = true; - shrinkBlockIdx = blockIdx; - shrinkSize = ofs; + Cv64suf v; + v.f = 0.; + if (toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F') + v.u = (uint64) inf_hi << 32; + else if (toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N') + v.u = (uint64) -1; + else + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + *value = v.f; + *endptr = buf + 4; +} + +double FileStorage::Impl::strtod(char *ptr, char **endptr) { + double fval = ::strtod(ptr, endptr); + if (**endptr == '.') { + char *dot_pos = *endptr; + *dot_pos = ','; + double fval2 = ::strtod(ptr, endptr); + *dot_pos = '.'; + if (*endptr > dot_pos) + fval = fval2; + else + *endptr = dot_pos; + } + + if (*endptr == ptr || cv_isalpha(**endptr)) + processSpecialDouble(ptr, &fval, endptr); + + return fval; +} + +void FileStorage::Impl::convertToCollection(int type, FileNode &node) { + CV_Assert(type == FileNode::SEQ || type == FileNode::MAP); + + int node_type = node.type(); + if (node_type == type) + return; + + bool named = node.isNamed(); + uchar *ptr = node.ptr() + 1 + (named ? 4 : 0); + + int ival = 0; + double fval = 0; + std::string sval; + bool add_first_scalar = false; + + if (node_type != FileNode::NONE) { + // scalar nodes can only be converted to sequences, e.g. in XML: + // 5[parser_position]... => create 5 with name "a" + // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it + // + // otherwise we don't know where to get the element names from + CV_Assert(type == FileNode::SEQ); + if (node_type == FileNode::INT) { + ival = readInt(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::REAL) { + fval = readReal(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::STRING) { + sval = std::string(node); + add_first_scalar = true; + } else + CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); + } + + ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); + *ptr++ = (uchar) (type | (named ? FileNode::NAMED : 0)); + // name has been copied automatically + if (named) + ptr += 4; + // set raw_size(collection)==4, nelems(collection)==1 + writeInt(ptr, 4); + writeInt(ptr + 4, 0); + + if (add_first_scalar) + addNode(node, std::string(), node_type, + node_type == FileNode::INT ? (const void *) &ival : + node_type == FileNode::REAL ? (const void *) &fval : + node_type == FileNode::STRING ? (const void *) sval.c_str() : 0, + -1); +} + +// a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or +// b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). +// If there is no enough space in the current block (it should be the last block added so far), +// the last block is shrunk so that it ends immediately before the reallocated node. Then, +// a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. +// The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. +// In the case (b) the existing tag and the name are copied automatically. +uchar *FileStorage::Impl::reserveNodeSpace(FileNode &node, size_t sz) { + bool shrinkBlock = false; + size_t shrinkBlockIdx = 0, shrinkSize = 0; + + uchar *ptr = 0, *blockEnd = 0; + + if (!fs_data_ptrs.empty()) { + size_t blockIdx = node.blockIdx; + size_t ofs = node.ofs; + CV_Assert(blockIdx == fs_data_ptrs.size() - 1); + CV_Assert(ofs <= fs_data_blksz[blockIdx]); + CV_Assert(freeSpaceOfs <= fs_data_blksz[blockIdx]); + //CV_Assert( freeSpaceOfs <= ofs + sz ); + + ptr = fs_data_ptrs[blockIdx] + ofs; + blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; + + CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); + if (ptr + sz <= blockEnd) { + freeSpaceOfs = ofs + sz; + return ptr; } - size_t blockSize = std::max((size_t)CV_FS_MAX_LEN*4 - 256, sz) + 256; - Ptr > pv = makePtr >(blockSize); - fs_data.push_back(pv); - uchar* new_ptr = &pv->at(0); - fs_data_ptrs.push_back(new_ptr); - fs_data_blksz.push_back(blockSize); - node.blockIdx = fs_data_ptrs.size()-1; - node.ofs = 0; - freeSpaceOfs = sz; - - if( ptr && ptr + 5 <= blockEnd ) + if (ofs == + 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. { - new_ptr[0] = ptr[0]; - if( ptr[0] & FileNode::NAMED ) - { - new_ptr[1] = ptr[1]; - new_ptr[2] = ptr[2]; - new_ptr[3] = ptr[3]; - new_ptr[4] = ptr[4]; - } + fs_data[blockIdx]->resize(sz); + ptr = &fs_data[blockIdx]->at(0); + fs_data_ptrs[blockIdx] = ptr; + fs_data_blksz[blockIdx] = sz; + freeSpaceOfs = sz; + return ptr; } - if (shrinkBlock) - { - fs_data[shrinkBlockIdx]->resize(shrinkSize); - fs_data_blksz[shrinkBlockIdx] = shrinkSize; + shrinkBlock = true; + shrinkBlockIdx = blockIdx; + shrinkSize = ofs; + } + + size_t blockSize = std::max((size_t) CV_FS_MAX_LEN * 4 - 256, sz) + 256; + Ptr > pv = makePtr >(blockSize); + fs_data.push_back(pv); + uchar *new_ptr = &pv->at(0); + fs_data_ptrs.push_back(new_ptr); + fs_data_blksz.push_back(blockSize); + node.blockIdx = fs_data_ptrs.size() - 1; + node.ofs = 0; + freeSpaceOfs = sz; + + if (ptr && ptr + 5 <= blockEnd) { + new_ptr[0] = ptr[0]; + if (ptr[0] & FileNode::NAMED) { + new_ptr[1] = ptr[1]; + new_ptr[2] = ptr[2]; + new_ptr[3] = ptr[3]; + new_ptr[4] = ptr[4]; } - - return new_ptr; } - unsigned getStringOfs( const std::string& key ) const - { - str_hash_t::const_iterator it = str_hash.find(key); - return it != str_hash.end() ? it->second : 0; + if (shrinkBlock) { + fs_data[shrinkBlockIdx]->resize(shrinkSize); + fs_data_blksz[shrinkBlockIdx] = shrinkSize; } - FileNode addNode( FileNode& collection, const std::string& key, - int elem_type, const void* value, int len ) - { - FileStorage_API* fs = this; - bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); - convertToCollection( noname ? FileNode::SEQ : FileNode::MAP, collection ); - - bool isseq = collection.empty() ? false : collection.isSeq(); - if( noname != isseq ) - CV_PARSE_ERROR_CPP( noname ? "Map element should have a name" : - "Sequence element should not have name (use <_>)" ); - unsigned strofs = 0; - if( !noname ) - { - strofs = getStringOfs(key); - if( !strofs ) - { - strofs = (unsigned)str_hash_data.size(); - size_t keysize = key.size() + 1; - str_hash_data.resize(strofs + keysize); - memcpy(&str_hash_data[0] + strofs, &key[0], keysize); - str_hash.insert(std::make_pair(key, strofs)); - } - } + return new_ptr; +} - uchar* cp = collection.ptr(); +unsigned FileStorage::Impl::getStringOfs(const std::string &key) const { + str_hash_t::const_iterator it = str_hash.find(key); + return it != str_hash.end() ? it->second : 0; +} - size_t blockIdx = fs_data_ptrs.size() - 1; - size_t ofs = freeSpaceOfs; - FileNode node(fs_ext, blockIdx, ofs); +FileNode FileStorage::Impl::addNode(FileNode &collection, const std::string &key, + int elem_type, const void *value, int len) { + FileStorage_API *fs = this; + bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); + convertToCollection(noname ? FileNode::SEQ : FileNode::MAP, collection); - size_t sz0 = 1 + (noname ? 0 : 4) + 8; - uchar* ptr = reserveNodeSpace(node, sz0); + bool isseq = collection.empty() ? false : collection.isSeq(); + if (noname != isseq) + CV_PARSE_ERROR_CPP(noname ? "Map element should have a name" : + "Sequence element should not have name (use <_>)"); + unsigned strofs = 0; + if (!noname) { + strofs = getStringOfs(key); + if (!strofs) { + strofs = (unsigned) str_hash_data.size(); + size_t keysize = key.size() + 1; + str_hash_data.resize(strofs + keysize); + memcpy(&str_hash_data[0] + strofs, &key[0], keysize); + str_hash.insert(std::make_pair(key, strofs)); + } + } - *ptr++ = (uchar)(elem_type | (noname ? 0 : FileNode::NAMED)); - if( elem_type == FileNode::NONE ) - freeSpaceOfs -= 8; + uchar *cp = collection.ptr(); - if( !noname ) - { - writeInt(ptr, (int)strofs); - ptr += 4; - } + size_t blockIdx = fs_data_ptrs.size() - 1; + size_t ofs = freeSpaceOfs; + FileNode node(fs_ext, blockIdx, ofs); - if( elem_type == FileNode::SEQ || elem_type == FileNode::MAP ) - { - writeInt(ptr, 4); - writeInt(ptr, 0); - } + size_t sz0 = 1 + (noname ? 0 : 4) + 8; + uchar *ptr = reserveNodeSpace(node, sz0); - if( value ) - node.setValue(elem_type, value, len); + *ptr++ = (uchar) (elem_type | (noname ? 0 : FileNode::NAMED)); + if (elem_type == FileNode::NONE) + freeSpaceOfs -= 8; - if( collection.isNamed() ) - cp += 4; - int nelems = readInt(cp + 5); - writeInt(cp + 5, nelems + 1); + if (!noname) { + writeInt(ptr, (int) strofs); + ptr += 4; + } - return node; + if (elem_type == FileNode::SEQ || elem_type == FileNode::MAP) { + writeInt(ptr, 4); + writeInt(ptr, 0); } - void finalizeCollection( FileNode& collection ) - { - if( !collection.isSeq() && !collection.isMap() ) - return; - uchar* ptr0 = collection.ptr(), *ptr = ptr0 + 1; - if( *ptr0 & FileNode::NAMED ) - ptr += 4; - size_t blockIdx = collection.blockIdx; - size_t ofs = collection.ofs + (size_t)(ptr + 8 - ptr0); - size_t rawSize = 4; - unsigned sz = (unsigned)readInt(ptr + 4); - if( sz > 0 ) - { - size_t lastBlockIdx = fs_data_ptrs.size() - 1; + if (value) + node.setValue(elem_type, value, len); - for( ; blockIdx < lastBlockIdx; blockIdx++ ) - { - rawSize += fs_data_blksz[blockIdx] - ofs; - ofs = 0; - } + if (collection.isNamed()) + cp += 4; + int nelems = readInt(cp + 5); + writeInt(cp + 5, nelems + 1); + + return node; +} + +void FileStorage::Impl::finalizeCollection(FileNode &collection) { + if (!collection.isSeq() && !collection.isMap()) + return; + uchar *ptr0 = collection.ptr(), *ptr = ptr0 + 1; + if (*ptr0 & FileNode::NAMED) + ptr += 4; + size_t blockIdx = collection.blockIdx; + size_t ofs = collection.ofs + (size_t) (ptr + 8 - ptr0); + size_t rawSize = 4; + unsigned sz = (unsigned) readInt(ptr + 4); + if (sz > 0) { + size_t lastBlockIdx = fs_data_ptrs.size() - 1; + + for (; blockIdx < lastBlockIdx; blockIdx++) { + rawSize += fs_data_blksz[blockIdx] - ofs; + ofs = 0; } - rawSize += freeSpaceOfs - ofs; - writeInt(ptr, (int)rawSize); } + rawSize += freeSpaceOfs - ofs; + writeInt(ptr, (int) rawSize); +} - void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const - { - while( ofs >= fs_data_blksz[blockIdx] ) - { - if( blockIdx == fs_data_blksz.size() - 1 ) - { - CV_Assert( ofs == fs_data_blksz[blockIdx] ); - break; - } - ofs -= fs_data_blksz[blockIdx]; - blockIdx++; +void FileStorage::Impl::normalizeNodeOfs(size_t &blockIdx, size_t &ofs) const { + while (ofs >= fs_data_blksz[blockIdx]) { + if (blockIdx == fs_data_blksz.size() - 1) { + CV_Assert(ofs == fs_data_blksz[blockIdx]); + break; } + ofs -= fs_data_blksz[blockIdx]; + blockIdx++; } +} - class Base64Decoder - { - public: - Base64Decoder() { ofs = 0; ptr = 0; indent = 0; totalchars = 0; eos = true; } - void init(Ptr& _parser, char* _ptr, int _indent) - { - parser = _parser; - ptr = _ptr; - indent = _indent; - encoded.clear(); - decoded.clear(); - ofs = 0; - totalchars = 0; - eos = false; - } +FileStorage::Impl::Base64State FileStorage::Impl::get_state_of_writing_base64() { + return state_of_writing_base64; +} - bool readMore(int needed) - { - static const uchar base64tab[] = +int FileStorage::Impl::get_space() { + return space; +} + + +FileStorage::Impl::Base64Decoder::Base64Decoder() { + ofs = 0; + ptr = 0; + indent = 0; + totalchars = 0; + eos = true; +} + +void FileStorage::Impl::Base64Decoder::init(Ptr &_parser, char *_ptr, int _indent) { + parser = _parser; + ptr = _ptr; + indent = _indent; + encoded.clear(); + decoded.clear(); + ofs = 0; + totalchars = 0; + eos = false; +} + +bool FileStorage::Impl::Base64Decoder::readMore(int needed) { + static const uchar base64tab[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if( eos ) - return false; - - size_t sz = decoded.size(); - CV_Assert( ofs <= sz ); - sz -= ofs; - for( size_t i = 0; i < sz; i++ ) - decoded[i] = decoded[ofs + i]; + if (eos) + return false; - decoded.resize(sz); - ofs = 0; + size_t sz = decoded.size(); + CV_Assert(ofs <= sz); + sz -= ofs; + for (size_t i = 0; i < sz; i++) + decoded[i] = decoded[ofs + i]; - CV_Assert( !parser.empty() && ptr ); - char *beg = 0, *end = 0; - bool ok = parser->getBase64Row(ptr, indent, beg, end); - ptr = end; - std::copy(beg, end, std::back_inserter(encoded)); - totalchars += end - beg; + decoded.resize(sz); + ofs = 0; - if( !ok || beg == end ) - { - // in the end of base64 sequence pad it with '=' characters so that - // its total length is multiple of - eos = true; - size_t tc = totalchars; - for( ; tc % 4 != 0; tc++ ) - encoded.push_back('='); - } + CV_Assert(!parser.empty() && ptr); + char *beg = 0, *end = 0; + bool ok = parser->getBase64Row(ptr, indent, beg, end); + ptr = end; + std::copy(beg, end, std::back_inserter(encoded)); + totalchars += end - beg; + + if (!ok || beg == end) { + // in the end of base64 sequence pad it with '=' characters so that + // its total length is multiple of + eos = true; + size_t tc = totalchars; + for (; tc % 4 != 0; tc++) + encoded.push_back('='); + } + + int i = 0, j, n = (int) encoded.size(); + if (n > 0) { + const uchar *tab = base64tab; + char *src = &encoded[0]; + + for (; i <= n - 4; i += 4) { + // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa + uchar d = tab[(int) (uchar) src[i]], c = tab[(int) (uchar) src[i + 1]]; + uchar b = tab[(int) (uchar) src[i + 2]], a = tab[(int) (uchar) src[i + 3]]; + + decoded.push_back((uchar) ((d << 2) | (c >> 4))); + decoded.push_back((uchar) ((c << 4) | (b >> 2))); + decoded.push_back((uchar) ((b << 6) | a)); + } + } - int i = 0, j, n = (int)encoded.size(); - if( n > 0 ) - { - const uchar* tab = base64tab; - char* src = &encoded[0]; + if (i > 0 && encoded[i - 1] == '=') { + if (i > 1 && encoded[i - 2] == '=' && !decoded.empty()) + decoded.pop_back(); + if (!decoded.empty()) + decoded.pop_back(); + } - for( ; i <= n - 4; i += 4 ) - { - // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa - uchar d = tab[(int)(uchar)src[i]], c = tab[(int)(uchar)src[i+1]]; - uchar b = tab[(int)(uchar)src[i+2]], a = tab[(int)(uchar)src[i+3]]; + n -= i; + for (j = 0; j < n; j++) + encoded[j] = encoded[i + j]; + encoded.resize(n); - decoded.push_back((uchar)((d << 2) | (c >> 4))); - decoded.push_back((uchar)((c << 4) | (b >> 2))); - decoded.push_back((uchar)((b << 6) | a)); - } - } + return (int) decoded.size() >= needed; +} - if( i > 0 && encoded[i-1] == '=' ) - { - if( i > 1 && encoded[i-2] == '=' && !decoded.empty() ) - decoded.pop_back(); - if( !decoded.empty() ) - decoded.pop_back(); - } +uchar FileStorage::Impl::Base64Decoder::getUInt8() { + size_t sz = decoded.size(); + if (ofs >= sz && !readMore(1)) + return (uchar) 0; + return decoded[ofs++]; +} - n -= i; - for( j = 0; j < n; j++ ) - encoded[j] = encoded[i + j]; - encoded.resize(n); +ushort FileStorage::Impl::Base64Decoder::getUInt16() { + size_t sz = decoded.size(); + if (ofs + 2 > sz && !readMore(2)) + return (ushort) 0; + ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); + ofs += 2; + return val; +} - return (int)decoded.size() >= needed; - } +int FileStorage::Impl::Base64Decoder::getInt32() { + size_t sz = decoded.size(); + if (ofs + 4 > sz && !readMore(4)) + return 0; + int ival = readInt(&decoded[ofs]); + ofs += 4; + return ival; +} - uchar getUInt8() - { - size_t sz = decoded.size(); - if( ofs >= sz && !readMore(1) ) - return (uchar)0; - return decoded[ofs++]; - } +double FileStorage::Impl::Base64Decoder::getFloat64() { + size_t sz = decoded.size(); + if (ofs + 8 > sz && !readMore(8)) + return 0; + double fval = readReal(&decoded[ofs]); + ofs += 8; + return fval; +} - ushort getUInt16() - { - size_t sz = decoded.size(); - if( ofs + 2 > sz && !readMore(2) ) - return (ushort)0; - ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); - ofs += 2; - return val; - } +bool FileStorage::Impl::Base64Decoder::endOfStream() const { return eos; } - int getInt32() - { - size_t sz = decoded.size(); - if( ofs + 4 > sz && !readMore(4) ) - return 0; - int ival = readInt(&decoded[ofs]); - ofs += 4; - return ival; - } +char *FileStorage::Impl::Base64Decoder::getPtr() const { return ptr; } - double getFloat64() - { - size_t sz = decoded.size(); - if( ofs + 8 > sz && !readMore(8) ) - return 0; - double fval = readReal(&decoded[ofs]); - ofs += 8; - return fval; - } - bool endOfStream() const { return eos; } - char* getPtr() const { return ptr; } - protected: - - Ptr parser; - char* ptr; - int indent; - std::vector encoded; - std::vector decoded; - size_t ofs; - size_t totalchars; - bool eos; - }; - - char* parseBase64(char* ptr, int indent, FileNode& collection) - { - const int BASE64_HDR_SIZE = 24; - char dt[BASE64_HDR_SIZE+1] = {0}; - base64decoder.init(parser, ptr, indent); +char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection) { + const int BASE64_HDR_SIZE = 24; + char dt[BASE64_HDR_SIZE + 1] = {0}; + base64decoder.init(parser, ptr, indent); - int i, k; + int i, k; - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - dt[i] = (char)base64decoder.getUInt8(); - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - if( isspace(dt[i])) - break; - dt[i] = '\0'; + for (i = 0; i < BASE64_HDR_SIZE; i++) + dt[i] = (char) base64decoder.getUInt8(); + for (i = 0; i < BASE64_HDR_SIZE; i++) + if (isspace(dt[i])) + break; + dt[i] = '\0'; - CV_Assert( !base64decoder.endOfStream() ); + CV_Assert(!base64decoder.endOfStream()); - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2]; - int fmt_pair_count = fs::decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - int ival = 0; - double fval = 0; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2]; + int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS); + int ival = 0; + double fval = 0; - for(;;) - { - for( k = 0; k < fmt_pair_count; k++ ) - { - int elem_type = fmt_pairs[k*2+1]; - int count = fmt_pairs[k*2]; + for (;;) { + for (k = 0; k < fmt_pair_count; k++) { + int elem_type = fmt_pairs[k * 2 + 1]; + int count = fmt_pairs[k * 2]; - for( i = 0; i < count; i++ ) - { - int node_type = FileNode::INT; - switch( elem_type ) - { + for (i = 0; i < count; i++) { + int node_type = FileNode::INT; + switch (elem_type) { case CV_8U: ival = base64decoder.getUInt8(); break; case CV_8S: - ival = (char)base64decoder.getUInt8(); + ival = (char) base64decoder.getUInt8(); break; case CV_16U: ival = base64decoder.getUInt16(); break; case CV_16S: - ival = (short)base64decoder.getUInt16(); + ival = (short) base64decoder.getUInt16(); break; case CV_32S: ival = base64decoder.getInt32(); break; - case CV_32F: - { + case CV_32F: { Cv32suf v; v.i = base64decoder.getInt32(); fval = v.f; node_type = FileNode::REAL; - } + } break; case CV_64F: fval = base64decoder.getFloat64(); node_type = FileNode::REAL; break; case CV_16F: - fval = (float)float16_t::fromBits(base64decoder.getUInt16()); + fval = (float) float16_t::fromBits(base64decoder.getUInt16()); node_type = FileNode::REAL; break; default: - CV_Error( Error::StsUnsupportedFormat, "Unsupported type" ); - } - - if( base64decoder.endOfStream() ) - break; - addNode(collection, std::string(), node_type, - node_type == FileNode::INT ? (void*)&ival : (void*)&fval, -1); + CV_Error(Error::StsUnsupportedFormat, "Unsupported type"); } + + if (base64decoder.endOfStream()) + break; + addNode(collection, std::string(), node_type, + node_type == FileNode::INT ? (void *) &ival : (void *) &fval, -1); } - if( base64decoder.endOfStream() ) - break; } - - finalizeCollection(collection); - return base64decoder.getPtr(); - } - - void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ) - { - std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); - error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line ); - } - - const uchar* getNodePtr(size_t blockIdx, size_t ofs) const - { - CV_Assert( blockIdx < fs_data_ptrs.size()); - CV_Assert( ofs < fs_data_blksz[blockIdx]); - - return fs_data_ptrs[blockIdx] + ofs; - } - - std::string getName( size_t nameofs ) const - { - CV_Assert( nameofs < str_hash_data.size() ); - return std::string(&str_hash_data[nameofs]); + if (base64decoder.endOfStream()) + break; } - FileStorage* getFS() { return fs_ext; } - - FileStorage* fs_ext; - - std::string filename; - int flags; - bool empty_stream; - - FILE* file; - gzFile gzfile; + finalizeCollection(collection); + return base64decoder.getPtr(); +} - bool is_opened; - bool dummy_eof; - bool write_mode; - bool mem_mode; - int fmt; +void FileStorage::Impl::parseError(const char *func_name, const std::string &err_msg, const char *source_file, + int source_line) { + std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); + error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line); +} - State state; //!< current state of the FileStorage (used only for writing) - int space, wrap_margin; - std::deque write_stack; - std::vector buffer; - size_t bufofs; +const uchar *FileStorage::Impl::getNodePtr(size_t blockIdx, size_t ofs) const { + CV_Assert(blockIdx < fs_data_ptrs.size()); + CV_Assert(ofs < fs_data_blksz[blockIdx]); - std::deque outbuf; + return fs_data_ptrs[blockIdx] + ofs; +} - Ptr emitter; - Ptr parser; - Base64Decoder base64decoder; +std::string FileStorage::Impl::getName(size_t nameofs) const { + CV_Assert(nameofs < str_hash_data.size()); + return std::string(&str_hash_data[nameofs]); +} - std::vector roots; - std::vector > > fs_data; - std::vector fs_data_ptrs; - std::vector fs_data_blksz; - size_t freeSpaceOfs; - typedef std::unordered_map str_hash_t; - str_hash_t str_hash; - std::vector str_hash_data; +FileStorage *FileStorage::Impl::getFS() { return fs_ext; } - std::vector strbufv; - char* strbuf; - size_t strbufsize; - size_t strbufpos; - int lineno; -}; FileStorage::FileStorage() : state(0) @@ -1807,7 +1842,7 @@ FileStorage::FileStorage(const String& filename, int flags, const String& encodi void FileStorage::startWriteStruct(const String& name, int struct_flags, const String& typeName) { - p->startWriteStruct(name.c_str(), struct_flags, typeName.c_str()); + p->startWriteStruct(name.size() ? name.c_str() : 0, struct_flags, typeName.size() ? typeName.c_str() : 0); elname = String(); if ((struct_flags & FileNode::TYPE_MASK) == FileNode::SEQ) state = FileStorage::VALUE_EXPECTED; @@ -1882,7 +1917,7 @@ std::string FileStorage::getDefaultObjectName(const std::string& _filename) } ptr++; if( ptr == ptr2 ) - CV_Error( CV_StsBadArg, "Invalid filename" ); + CV_Error( cv::Error::StsBadArg, "Invalid filename" ); char* name = name_buf.data(); @@ -2005,12 +2040,14 @@ FileStorage& operator << (FileStorage& fs, const String& str) if( c == '}' || c == ']' ) { if( fs_impl->write_stack.empty() ) - CV_Error_( CV_StsError, ("Extra closing '%c'", *_str) ); + CV_Error_( cv::Error::StsError, ("Extra closing '%c'", *_str) ); + + fs_impl->workaround(); int struct_flags = fs_impl->write_stack.back().flags; char expected_bracket = FileNode::isMap(struct_flags) ? '}' : ']'; if( c != expected_bracket ) - CV_Error_( CV_StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); + CV_Error_( cv::Error::StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); fs_impl->endWriteStruct(); CV_Assert(!fs_impl->write_stack.empty()); struct_flags = fs_impl->write_stack.back().flags; @@ -2020,7 +2057,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) else if( fs.state == NAME_EXPECTED + INSIDE_MAP ) { if (!cv_isalpha(c) && c != '_') - CV_Error_( CV_StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); + CV_Error_( cv::Error::StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); fs.elname = str; fs.state = VALUE_EXPECTED + INSIDE_MAP; } @@ -2049,7 +2086,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) } } else - CV_Error( CV_StsError, "Invalid fs.state" ); + CV_Error( cv::Error::StsError, "Invalid fs.state" ); return fs; } diff --git a/modules/core/src/persistence.hpp b/modules/core/src/persistence.hpp index 05c7adc..1a9dbec 100644 --- a/modules/core/src/persistence.hpp +++ b/modules/core/src/persistence.hpp @@ -163,6 +163,24 @@ public: CV_NORETURN virtual void parseError(const char* funcname, const std::string& msg, const char* filename, int lineno) = 0; + +private: + enum Base64State{ + Uncertain, + NotUse, + InUse, + }; + + friend class cv::FileStorage::Impl; + friend class cv::FileStorage; + friend class JSONEmitter; + friend class XMLEmitter; + friend class YAMLEmitter; + + virtual void check_if_write_struct_is_delayed(bool change_type_to_base64 = false) = 0; + virtual void switch_to_Base64_state(Base64State state) = 0; + virtual Base64State get_state_of_writing_base64() = 0; + virtual int get_space() = 0; }; class FileStorageEmitter diff --git a/modules/core/src/persistence_base64_encoding.cpp b/modules/core/src/persistence_base64_encoding.cpp new file mode 100644 index 0000000..7d90fd4 --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.cpp @@ -0,0 +1,370 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" + +namespace cv +{ + +class base64::Base64ContextEmitter +{ +public: + explicit Base64ContextEmitter(cv::FileStorage::Impl& fs, bool needs_indent_) + : file_storage(fs) + , needs_indent(needs_indent_) + , binary_buffer(BUFFER_LEN) + , base64_buffer(base64_encode_buffer_size(BUFFER_LEN)) + , src_beg(0) + , src_cur(0) + , src_end(0) + { + src_beg = binary_buffer.data(); + src_end = src_beg + BUFFER_LEN; + src_cur = src_beg; + + CV_Assert(fs.write_mode); + + if (needs_indent) + { + file_storage.flush(); + } + } + + ~Base64ContextEmitter() + { + /* cleaning */ + if (src_cur != src_beg) + flush(); /* encode the rest binary data to base64 buffer */ + } + + Base64ContextEmitter & write(const uchar * beg, const uchar * end) + { + if (beg >= end) + return *this; + + while (beg < end) { + /* collect binary data and copy to binary buffer */ + size_t len = std::min(end - beg, src_end - src_cur); + std::memcpy(src_cur, beg, len); + beg += len; + src_cur += len; + + if (src_cur >= src_end) { + /* binary buffer is full. */ + /* encode it to base64 and send result to fs */ + flush(); + } + } + + return *this; + } + + /* + * a convertor must provide : + * - `operator >> (uchar * & dst)` for writing current binary data to `dst` and moving to next data. + * - `operator bool` for checking if current loaction is valid and not the end. + */ + template inline + Base64ContextEmitter & write(_to_binary_convertor_t & convertor) + { + static const size_t BUFFER_MAX_LEN = 1024U; + + std::vector buffer(BUFFER_MAX_LEN); + uchar * beg = buffer.data(); + uchar * end = beg; + + while (convertor) { + convertor >> end; + write(beg, end); + end = beg; + } + + return *this; + } + + bool flush() + { + /* control line width, so on. */ + size_t len = base64_encode(src_beg, base64_buffer.data(), 0U, src_cur - src_beg); + if (len == 0U) + return false; + + src_cur = src_beg; + + if ( !needs_indent) + { + file_storage.puts((const char*)base64_buffer.data()); + } + else + { + const char newline[] = "\n"; + char space[80]; + int ident = file_storage.write_stack.back().indent; + memset(space, ' ', static_cast(ident)); + space[ident] = '\0'; + + file_storage.puts(space); + file_storage.puts((const char*)base64_buffer.data()); + file_storage.puts(newline); + file_storage.flush(); + } + + return true; + } + +private: + /* because of Base64, we must keep its length a multiple of 3 */ + static const size_t BUFFER_LEN = 48U; + // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); + +private: + cv::FileStorage::Impl& file_storage; + bool needs_indent; + + std::vector binary_buffer; + std::vector base64_buffer; + uchar * src_beg; + uchar * src_cur; + uchar * src_end; +}; + +std::string base64::make_base64_header(const char *dt) { + std::ostringstream oss; + oss << dt << ' '; + std::string buffer(oss.str()); + CV_Assert(buffer.size() < ::base64::HEADER_SIZE); + + buffer.reserve(::base64::HEADER_SIZE); + while (buffer.size() < ::base64::HEADER_SIZE) + buffer += ' '; + + return buffer; +} + +size_t base64::base64_encode(const uint8_t *src, uint8_t *dst, size_t off, size_t cnt) { + if (!src || !dst || !cnt) + return 0; + + /* initialize beginning and end */ + uint8_t * dst_beg = dst; + uint8_t * dst_cur = dst_beg; + + uint8_t const * src_beg = src + off; + uint8_t const * src_cur = src_beg; + uint8_t const * src_end = src_cur + cnt / 3U * 3U; + + /* integer multiples part */ + while (src_cur < src_end) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + uint8_t _0 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_1 & 0xF0U) >> 4U | (_2 & 0x03U) << 4U]; + *dst_cur++ = base64_mapping[(_0 & 0xC0U) >> 6U | (_1 & 0x0FU) << 2U]; + *dst_cur++ = base64_mapping[ _0 & 0x3FU]; + } + + /* remainder part */ + size_t rst = src_beg + cnt - src_cur; + if (rst == 1U) { + uint8_t _2 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U]; + } else if (rst == 2U) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U | (_1 & 0xF0U) >> 4U]; + *dst_cur++ = base64_mapping[(_1 & 0x0FU) << 2U]; + } + + /* padding */ + switch (rst) + { + case 1U: *dst_cur++ = base64_padding; + /* fallthrough */ + case 2U: *dst_cur++ = base64_padding; + /* fallthrough */ + default: *dst_cur = 0; + break; + } + + return static_cast(dst_cur - dst_beg); +} + +int base64::icvCalcStructSize(const char *dt, int initial_size) { + int size = cv::fs::calcElemSize( dt, initial_size ); + size_t elem_max_size = 0; + for ( const char * type = dt; *type != '\0'; type++ ) { + switch ( *type ) + { + case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } + case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } + case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; } + case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; } + case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } + case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } + case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } + default: break; + } + } + size = cvAlign( size, static_cast(elem_max_size) ); + return size; +} + +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; +} + +base64::Base64Writer::Base64Writer(cv::FileStorage::Impl& fs, bool can_indent) + : emitter(new Base64ContextEmitter(fs, can_indent)) + , data_type_string() +{ + CV_Assert(fs.write_mode); +} + +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); + RawDataToBinaryConvertor convertor(_data, static_cast(len), data_type_string); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( cv::Error::StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; + + /* output header */ + std::string buffer = make_base64_header(dt); + const uchar * beg = reinterpret_cast(buffer.data()); + const uchar * end = beg + buffer.size(); + + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( cv::Error::StsBadArg, "\'dt\' does not match." ); +} + +base64::RawDataToBinaryConvertor::RawDataToBinaryConvertor(const void* src, int len, const std::string & dt) + : beg(reinterpret_cast(src)) + , cur(0) + , end(0) +{ + CV_Assert(src); + CV_Assert(!dt.empty()); + CV_Assert(len > 0); + + /* calc step and to_binary_funcs */ + step_packed = make_to_binary_funcs(dt); + + end = beg; + cur = beg; + + step = icvCalcStructSize(dt.c_str(), 0); + end = beg + static_cast(len); +} + +inline base64::RawDataToBinaryConvertor& base64::RawDataToBinaryConvertor::operator >>(uchar * & dst) +{ + CV_DbgAssert(*this); + + for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) { + elem_to_binary_t & pack = to_binary_funcs[i]; + pack.func(cur + pack.offset, dst + pack.offset_packed); + } + cur += step; + dst += step_packed; + + return *this; +} + +inline base64::RawDataToBinaryConvertor::operator bool() const +{ + return cur < end; +} + +size_t base64::RawDataToBinaryConvertor::make_to_binary_funcs(const std::string &dt) +{ + size_t cnt = 0; + size_t offset = 0; + size_t offset_packed = 0; + char type = '\0'; + + std::istringstream iss(dt); + while (!iss.eof()) { + if (!(iss >> cnt)) { + iss.clear(); + cnt = 1; + } + CV_Assert(cnt > 0U); + if (!(iss >> type)) + break; + + while (cnt-- > 0) + { + elem_to_binary_t pack; + + size_t size = 0; + switch (type) + { + case 'u': + case 'c': + size = sizeof(uchar); + pack.func = to_binary; + break; + case 'w': + case 's': + size = sizeof(ushort); + pack.func = to_binary; + break; + case 'i': + size = sizeof(uint); + pack.func = to_binary; + break; + case 'f': + size = sizeof(float); + pack.func = to_binary; + break; + case 'd': + size = sizeof(double); + pack.func = to_binary; + break; + case 'r': + default: + CV_Error(cv::Error::StsError, "type is not supported"); + }; + + offset = static_cast(cvAlign(static_cast(offset), static_cast(size))); + pack.offset = offset; + offset += size; + + pack.offset_packed = offset_packed; + offset_packed += size; + + to_binary_funcs.push_back(pack); + } + } + + CV_Assert(iss.eof()); + return offset_packed; +} + +} \ No newline at end of file diff --git a/modules/core/src/persistence_base64_encoding.hpp b/modules/core/src/persistence_base64_encoding.hpp new file mode 100644 index 0000000..1ee5201 --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.hpp @@ -0,0 +1,127 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_BASE64_ENCODING_HPP +#define OPENCV_CORE_BASE64_ENCODING_HPP + +namespace cv +{ + +namespace base64 +{ +/* A decorator for CvFileStorage +* - no copyable +* - not safe for now +* - move constructor may be needed if C++11 +*/ +uint8_t const base64_mapping[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +uint8_t const base64_padding = '='; + +std::string make_base64_header(const char * dt); + +size_t base64_encode(uint8_t const * src, uint8_t * dst, size_t off, size_t cnt); + + +int icvCalcStructSize( const char* dt, int initial_size ); + +class Base64ContextEmitter; +class Impl; + +class Base64Writer +{ +public: + Base64Writer(cv::FileStorage::Impl& fs, bool can_indent); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + +private: + void check_dt(const char* dt); + +private: + // disable copy and assignment + Base64Writer(const Base64Writer &); + Base64Writer & operator=(const Base64Writer &); + +private: + + Base64ContextEmitter * emitter; + std::string data_type_string; +}; + +size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); + +template inline size_t +to_binary(_uint_t val, uchar * cur) +{ + size_t delta = CHAR_BIT; + size_t cnt = sizeof(_uint_t); + while (cnt --> static_cast(0U)) { + *cur++ = static_cast(val); + val >>= delta; + } + return sizeof(_uint_t); +} + +template<> inline size_t to_binary(double val, uchar * cur) +{ + Cv64suf bit64; + bit64.f = val; + return to_binary(bit64.u, cur); +} + +template<> inline size_t to_binary(float val, uchar * cur) +{ + Cv32suf bit32; + bit32.f = val; + return to_binary(bit32.u, cur); +} + +template inline size_t +to_binary(uchar const * val, uchar * cur) +{ + return to_binary<_primitive_t>(*reinterpret_cast<_primitive_t const *>(val), cur); +} + + + +class RawDataToBinaryConvertor +{ +public: + // NOTE: len is already multiplied by element size here + RawDataToBinaryConvertor(const void* src, int len, const std::string & dt); + + inline RawDataToBinaryConvertor & operator >>(uchar * & dst); + inline operator bool() const; + +private: + typedef size_t(*to_binary_t)(const uchar *, uchar *); + struct elem_to_binary_t + { + size_t offset; + size_t offset_packed; + to_binary_t func; + }; + +private: + size_t make_to_binary_funcs(const std::string &dt); + +private: + const uchar * beg; + const uchar * cur; + const uchar * end; + + size_t step; + size_t step_packed; + std::vector to_binary_funcs; +}; + +} + +} +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_impl.hpp b/modules/core/src/persistence_impl.hpp new file mode 100644 index 0000000..4ea2dc3 --- /dev/null +++ b/modules/core/src/persistence_impl.hpp @@ -0,0 +1,231 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_PERSISTENCE_IMPL_HPP +#define OPENCV_CORE_PERSISTENCE_IMPL_HPP + +#include "persistence.hpp" +#include "persistence_base64_encoding.hpp" +#include +#include + + +namespace cv +{ + +enum Base64State{ + Uncertain, + NotUse, + InUse, +}; + +class cv::FileStorage::Impl : public FileStorage_API +{ +public: + void init(); + + Impl(FileStorage* _fs); + + virtual ~Impl(); + + void release(String* out=0); + + void analyze_file_name( const std::string& file_name, std::vector& params ); + + bool open( const char* filename_or_buf, int _flags, const char* encoding ); + + void puts( const char* str ); + + char* getsFromFile( char* buf, int count ); + + char* gets( size_t maxCount ); + + char* gets(); + + bool eof(); + + void setEof(); + + void closeFile(); + + void rewind(); + + char* resizeWriteBuffer( char* ptr, int len ); + + char* flush(); + + void endWriteStruct(); + + void startWriteStruct_helper( const char* key, int struct_flags, + const char* type_name ); + + void startWriteStruct( const char* key, int struct_flags, + const char* type_name ); + + void writeComment( const char* comment, bool eol_comment ); + + void startNextStream(); + + void write( const String& key, int value ); + + void write( const String& key, double value ); + + void write( const String& key, const String& value ); + + void writeRawData( const std::string& dt, const void* _data, size_t len ); + + void workaround(); + + void switch_to_Base64_state( FileStorage_API::Base64State new_state); + + void make_write_struct_delayed( const char* key, int struct_flags, const char* type_name ); + + void check_if_write_struct_is_delayed( bool change_type_to_base64 ); + + void writeRawDataBase64(const void* _data, size_t len, const char* dt ); + + String releaseAndGetString(); + + FileNode getFirstTopLevelNode() const; + + FileNode root(int streamIdx=0) const; + + FileNode operator[](const String& nodename) const; + + FileNode operator[](const char* /*nodename*/) const; + + int getFormat() const; + + char* bufferPtr() const; + char* bufferStart() const; + char* bufferEnd() const; + void setBufferPtr(char* ptr); + int wrapMargin() const; + + FStructData& getCurrentStruct(); + + void setNonEmpty(); + + void processSpecialDouble( char* buf, double* value, char** endptr ); + + double strtod( char* ptr, char** endptr ); + + void convertToCollection(int type, FileNode& node); + + // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or + // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). + // If there is no enough space in the current block (it should be the last block added so far), + // the last block is shrunk so that it ends immediately before the reallocated node. Then, + // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. + // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. + // In the case (b) the existing tag and the name are copied automatically. + uchar* reserveNodeSpace(FileNode& node, size_t sz); + + unsigned getStringOfs( const std::string& key ) const; + + FileNode addNode( FileNode& collection, const std::string& key, + int elem_type, const void* value, int len ); + + void finalizeCollection( FileNode& collection ); + + void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const; + + Base64State get_state_of_writing_base64(); + + int get_space(); + + class Base64Decoder + { + public: + Base64Decoder(); + void init(Ptr& _parser, char* _ptr, int _indent); + + bool readMore(int needed); + + uchar getUInt8(); + + ushort getUInt16(); + + int getInt32(); + + double getFloat64(); + + bool endOfStream() const; + char* getPtr() const; + protected: + + Ptr parser; + char* ptr; + int indent; + std::vector encoded; + std::vector decoded; + size_t ofs; + size_t totalchars; + bool eos; + }; + + char* parseBase64(char* ptr, int indent, FileNode& collection); + + void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ); + + const uchar* getNodePtr(size_t blockIdx, size_t ofs) const; + + std::string getName( size_t nameofs ) const; + + FileStorage* getFS(); + + FileStorage* fs_ext; + + std::string filename; + int flags; + bool empty_stream; + + FILE* file; + gzFile gzfile; + + bool is_opened; + bool dummy_eof; + bool write_mode; + bool mem_mode; + int fmt; + + State state; //!< current state of the FileStorage (used only for writing) + bool is_using_base64; + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; + FileStorage_API::Base64State state_of_writing_base64; + + int space, wrap_margin; + std::deque write_stack; + std::vector buffer; + size_t bufofs; + + std::deque outbuf; + + Ptr emitter; + Ptr parser; + Base64Decoder base64decoder; + base64::Base64Writer* base64_writer; + + std::vector roots; + std::vector > > fs_data; + std::vector fs_data_ptrs; + std::vector fs_data_blksz; + size_t freeSpaceOfs; + typedef std::unordered_map str_hash_t; + str_hash_t str_hash; + std::vector str_hash_data; + + std::vector strbufv; + char* strbuf; + size_t strbufsize; + size_t strbufpos; + int lineno; +}; + +} + +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index 667895f..12a58e8 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -23,7 +23,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if( type_name && *type_name == '\0' ) @@ -53,29 +53,26 @@ public: void endWriteStruct(const FStructData& current_struct) { int struct_flags = current_struct.flags; - CV_Assert( FileNode::isCollection(struct_flags) ); - if( !FileNode::isFlow(struct_flags) ) - { -#if 0 - if ( fs->bufferPtr() <= fs->bufferStart() + fs->space ) - { - /* some bad code for base64_writer... */ - ptr = fs->bufferPtr(); - *ptr++ = '\n'; - *ptr++ = '\0'; - fs->puts( fs->bufferStart() ); - fs->setBufferPtr(fs->bufferStart()); + if (FileNode::isCollection(struct_flags)) { + if (!FileNode::isFlow(struct_flags)) { + if (fs->bufferPtr() <= fs->bufferStart() + fs->get_space()) { + /* some bad code for base64_writer... */ + char *ptr = fs->bufferPtr(); + *ptr++ = '\n'; + *ptr++ = '\0'; + fs->puts(fs->bufferStart()); + fs->setBufferPtr(fs->bufferStart()); + } + fs->flush(); } -#endif - fs->flush(); - } - char* ptr = fs->bufferPtr(); - if( ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags) ) - *ptr++ = ' '; - *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; - fs->setBufferPtr(ptr); + char *ptr = fs->bufferPtr(); + if (ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags)) + *ptr++ = ' '; + *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; + fs->setBufferPtr(ptr); + } } void write(const char* key, int value) @@ -97,11 +94,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -136,6 +133,20 @@ public: void writeScalar(const char* key, const char* data) { + /* check write_struct */ + + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + + /* check parameters */ + size_t key_len = 0u; if( key && *key == '\0' ) key = 0; @@ -143,9 +154,9 @@ public: { key_len = strlen(key); if ( key_len == 0u ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); else if ( static_cast(key_len) > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } size_t data_len = 0u; @@ -157,7 +168,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if ( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else { fs->setNonEmpty(); @@ -199,7 +210,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, static_cast(key_len) ); *ptr++ = '\"'; @@ -210,7 +221,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += key_len; @@ -233,7 +244,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = static_cast(strlen(comment)); char* ptr = fs->bufferPtr(); diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index 52b5374..62b7b1e 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -45,7 +45,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( FileNode::isMap(struct_flags) ^ (key != 0) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -61,26 +61,26 @@ public: if( !key ) key = "_"; else if( key[0] == '_' && key[1] == '\0' ) - CV_Error( CV_StsBadArg, "A single _ is a reserved tag name" ); + CV_Error( cv::Error::StsBadArg, "A single _ is a reserved tag name" ); len = (int)strlen( key ); *ptr++ = '<'; if( tag_type == CV_XML_CLOSING_TAG ) { if( !attrlist.empty() ) - CV_Error( CV_StsBadArg, "Closing tag should not include any attributes" ); + CV_Error( cv::Error::StsBadArg, "Closing tag should not include any attributes" ); *ptr++ = '/'; } if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key should start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key should start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, len ); for( i = 0; i < len; i++ ) { char c = key[i]; if( !cv_isalnum(c) && c != '_' && c != '-' ) - CV_Error( CV_StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); + CV_Error( cv::Error::StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); ptr[i] = c; } ptr += len; @@ -158,11 +158,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) { @@ -233,6 +233,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int len = (int)strlen(data); if( key && *key == '\0' ) key = 0; @@ -255,7 +265,7 @@ public: int new_offset = (int)(ptr - fs->bufferStart()) + len; if( key ) - CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + CV_Error( cv::Error::StsBadArg, "elements with keys can not be written to sequence" ); current_struct.flags = FileNode::SEQ; @@ -281,10 +291,10 @@ public: char* ptr; if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); if( strstr(comment, "--") != 0 ) - CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); + CV_Error( cv::Error::StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); len = (int)strlen(comment); eol = strchr(comment, '\n'); diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index 3f3742b..95db145 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -33,7 +33,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if (type_name && memcmp(type_name, "binary", 6) == 0) @@ -120,11 +120,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -174,6 +174,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; char* ptr; @@ -188,7 +198,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -201,10 +211,10 @@ public: { keylen = (int)strlen(key); if( keylen == 0 ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); if( keylen > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } if( data ) @@ -238,7 +248,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, keylen ); @@ -248,7 +258,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += keylen; @@ -271,7 +281,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = (int)strlen(comment); const char* eol = strchr(comment, '\n'); diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 82bd053..3712be9 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -586,6 +586,7 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CV_Assert(test_info); std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name); + std::string name_34 = string(cvtest::TS::ptr()->get_data_path()) + "io/3_4/" + name; if (!testReadWrite) name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name; @@ -661,7 +662,23 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo std::ifstream f(name.c_str(), std::ios::in|std::ios::binary); f.seekg(0, std::fstream::end); sz = (size_t)f.tellg(); + + f.seekg(0, std::ios::beg); + std::vector test_data(sz); + f.read(&test_data[0], sz); f.close(); + + std::ifstream reference(name_34.c_str(), std::ios::in|std::ios::binary); + ASSERT_TRUE(reference.is_open()); + reference.seekg(0, std::fstream::end); + size_t ref_sz = (size_t)reference.tellg(); + + reference.seekg(0, std::ios::beg); + std::vector reference_data(ref_sz); + reference.read(&reference_data[0], ref_sz); + reference.close(); + + EXPECT_EQ(reference_data, test_data); } std::cout << "Storage size: " << sz << std::endl; EXPECT_LE(sz, (size_t)6000); @@ -757,27 +774,27 @@ TEST(Core_InputOutput, filestorage_base64_basic_read_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_XML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_XML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true); } diff --git a/modules/python/test/test_filestorage_io.py b/modules/python/test/test_filestorage_io.py index 62b540d..01e0a72 100755 --- a/modules/python/test/test_filestorage_io.py +++ b/modules/python/test/test_filestorage_io.py @@ -1,6 +1,8 @@ #!/usr/bin/env python """Algorithm serialization test.""" from __future__ import print_function +import base64 +import json import tempfile import os import cv2 as cv @@ -109,5 +111,96 @@ class filestorage_io_test(NewOpenCVTests): def test_json(self): self.run_fs_test(".json") + def test_base64(self): + fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage_base64", suffix=".json") + os.close(fd) + np.random.seed(42) + self.write_base64_json(fname) + os.remove(fname) + + @staticmethod + def get_normal_2d_mat(): + rows = 10 + cols = 20 + cn = 3 + + image = np.zeros((rows, cols, cn), np.uint8) + image[:] = (1, 2, 127) + + for i in range(rows): + for j in range(cols): + image[i, j, 1] = (i + j) % 256 + + return image + + @staticmethod + def get_normal_nd_mat(): + shape = (2, 2, 1, 2) + cn = 4 + + image = np.zeros(shape + (cn,), np.float64) + image[:] = (0.888, 0.111, 0.666, 0.444) + + return image + + @staticmethod + def get_empty_2d_mat(): + shape = (0, 0) + cn = 1 + + image = np.zeros(shape + (cn,), np.uint8) + + return image + + @staticmethod + def get_random_mat(): + rows = 8 + cols = 16 + cn = 1 + + image = np.random.rand(rows, cols, cn) + + return image + + @staticmethod + def decode(data): + # strip $base64$ + encoded = data[8:] + + if len(encoded) == 0: + return b'' + + # strip info about datatype and padding + return base64.b64decode(encoded)[24:] + + def write_base64_json(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_WRITE_BASE64) + + mats = {'normal_2d_mat': self.get_normal_2d_mat(), + 'normal_nd_mat': self.get_normal_nd_mat(), + 'empty_2d_mat': self.get_empty_2d_mat(), + 'random_mat': self.get_random_mat()} + + for name, mat in mats.items(): + fs.write(name, mat) + + fs.release() + + data = {} + with open(fname) as file: + data = json.load(file) + + for name, mat in mats.items(): + buffer = b'' + + if mat.size != 0: + if hasattr(mat, 'tobytes'): + buffer = mat.tobytes() + else: + buffer = mat.tostring() + + self.assertEqual(buffer, self.decode(data[name]['data'])) + + if __name__ == '__main__': NewOpenCVTests.bootstrap() -- 2.7.4