From bc929a7d46e3baee7ade56b714ea782f6a5d3563 Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Mon, 28 May 2012 15:38:58 +0000 Subject: [PATCH] added API for storing OpenCV data structures to text string and reading them back --- modules/core/doc/xml_yaml_persistence.rst | 10 +- modules/core/include/opencv2/core/core.hpp | 11 +- modules/core/include/opencv2/core/types_c.h | 5 + modules/core/src/persistence.cpp | 279 +++++++++++++++++++--------- modules/core/test/test_io.cpp | 15 +- 5 files changed, 218 insertions(+), 102 deletions(-) diff --git a/modules/core/doc/xml_yaml_persistence.rst b/modules/core/doc/xml_yaml_persistence.rst index b3170d9..058f7ca 100644 --- a/modules/core/doc/xml_yaml_persistence.rst +++ b/modules/core/doc/xml_yaml_persistence.rst @@ -156,9 +156,9 @@ The constructors. .. ocv:function:: FileStorage::FileStorage() -.. ocv:function:: FileStorage::FileStorage(const string& filename, int flags, const string& encoding=string()) +.. ocv:function:: FileStorage::FileStorage(const string& source, int flags, const string& encoding=string()) - :param filename: Name of the file to open. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``. + :param source: Name of the file to open or the text string to read the data from. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``. If both ``FileStorage::WRITE`` and ``FileStorage::MEMORY`` flags are specified, ``source`` is used just to specify the output file format (e.g. ``mydata.xml``, ``.yml`` etc.). :param flags: Mode of operation. Possible values are: @@ -167,6 +167,8 @@ The constructors. * **FileStorage::WRITE** Open the file for writing. * **FileStorage::APPEND** Open the file for appending. + + * **FileStorage::MEMORY** Read data from ``source`` or write data to the internal buffer (which is returned by ``FileStorage::release``) :param encoding: Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it. @@ -197,9 +199,9 @@ FileStorage::release -------------------- Closes the file and releases all the memory buffers. -.. ocv:function:: void FileStorage::release() +.. ocv:function:: string FileStorage::release() -Call this method after all I/O operations with the storage are finished. +Call this method after all I/O operations with the storage are finished. If the storage was opened for writing data and ``FileStorage::WRITE`` was specified FileStorage::getFirstTopLevelNode diff --git a/modules/core/include/opencv2/core/core.hpp b/modules/core/include/opencv2/core/core.hpp index f6ed16a..c8c3284 100644 --- a/modules/core/include/opencv2/core/core.hpp +++ b/modules/core/include/opencv2/core/core.hpp @@ -3941,7 +3941,12 @@ public: { READ=0, //! read mode WRITE=1, //! write mode - APPEND=2 //! append mode + APPEND=2, //! append mode + MEMORY=4, + FORMAT_MASK=(7<<3), + FORMAT_AUTO=0, + FORMAT_XML=(1<<3), + FORMAT_YAML=(2<<3) }; enum { @@ -3953,7 +3958,7 @@ public: //! the default constructor CV_WRAP FileStorage(); //! the full constructor that opens file storage for reading or writing - CV_WRAP FileStorage(const string& filename, int flags, const string& encoding=string()); + CV_WRAP FileStorage(const string& source, int flags, const string& encoding=string()); //! the constructor that takes pointer to the C FileStorage structure FileStorage(CvFileStorage* fs); //! the destructor. calls release() @@ -3964,7 +3969,7 @@ public: //! returns true if the object is associated with currently opened file. CV_WRAP virtual bool isOpened() const; //! closes the file and releases all the memory buffers - CV_WRAP virtual void release(); + CV_WRAP virtual string release(); //! returns the first element of the top-level mapping CV_WRAP FileNode getFirstTopLevelNode() const; diff --git a/modules/core/include/opencv2/core/types_c.h b/modules/core/include/opencv2/core/types_c.h index a2bab32..e11f096 100644 --- a/modules/core/include/opencv2/core/types_c.h +++ b/modules/core/include/opencv2/core/types_c.h @@ -1740,6 +1740,11 @@ typedef struct CvFileStorage CvFileStorage; #define CV_STORAGE_WRITE_TEXT CV_STORAGE_WRITE #define CV_STORAGE_WRITE_BINARY CV_STORAGE_WRITE #define CV_STORAGE_APPEND 2 +#define CV_STORAGE_MEMORY 4 +#define CV_STORAGE_FORMAT_MASK (7<<3) +#define CV_STORAGE_FORMAT_AUTO 0 +#define CV_STORAGE_FORMAT_XML 8 +#define CV_STORAGE_FORMAT_YAML 16 /* List of attributes: */ typedef struct CvAttrList diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index c8c3664..a02c186 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -41,7 +41,10 @@ //M*/ #include "precomp.hpp" + #include +#include +#include #include #include @@ -208,7 +211,7 @@ typedef void (*CvStartNextStream)( struct CvFileStorage* fs ); typedef struct CvFileStorage { int flags; - int is_xml; + int fmt; int write_mode; int is_first; CvMemStorage* memstorage; @@ -240,52 +243,85 @@ typedef struct CvFileStorage CvWriteString write_string; CvWriteComment write_comment; CvStartNextStream start_next_stream; - //CvParse parse; + + const char* strbuf; + size_t strbufsize, strbufpos; + std::deque* outbuf; + + bool is_opened; } CvFileStorage; static void icvPuts( CvFileStorage* fs, const char* str ) { - CV_Assert( fs->file || fs->gzfile ); - if( fs->file ) + if( fs->outbuf ) + std::copy(str, str + strlen(str), std::back_inserter(*fs->outbuf)); + else if( fs->file ) fputs( str, fs->file ); - else + else if( fs->gzfile ) gzputs( fs->gzfile, str ); + else + CV_Error( CV_StsError, "The storage is not opened" ); } static char* icvGets( CvFileStorage* fs, char* str, int maxCount ) { - CV_Assert( fs->file || fs->gzfile ); + if( fs->strbuf ) + { + size_t i = fs->strbufpos, len = fs->strbufsize, j = 0; + const char* instr = fs->strbuf; + while( i < len && j < maxCount-1 ) + { + char c = instr[i++]; + if( c == '\0' ) + break; + str[j++] = c; + if( c == '\n' ) + break; + } + str[j++] = '\0'; + fs->strbufpos = i; + return j > 1 ? str : 0; + } if( fs->file ) return fgets( str, maxCount, fs->file ); - return gzgets( fs->gzfile, str, maxCount ); + if( fs->gzfile ) + return gzgets( fs->gzfile, str, maxCount ); + CV_Error( CV_StsError, "The storage is not opened" ); + return 0; } static int icvEof( CvFileStorage* fs ) { - CV_Assert( fs->file || fs->gzfile ); + if( fs->strbuf ) + return fs->strbufpos >= fs->strbufsize; if( fs->file ) return feof(fs->file); - return gzeof(fs->gzfile); + if( fs->gzfile ) + return gzeof(fs->gzfile); + return false; } -static void icvClose( CvFileStorage* fs ) +static void icvCloseFile( CvFileStorage* fs ) { if( fs->file ) fclose( fs->file ); - if( fs->gzfile ) + else if( fs->gzfile ) gzclose( fs->gzfile ); fs->file = 0; fs->gzfile = 0; + fs->strbuf = 0; + fs->strbufpos = 0; + fs->is_opened = false; } static void icvRewind( CvFileStorage* fs ) { - CV_Assert( fs->file || fs->gzfile ); if( fs->file ) rewind(fs->file); - else + else if( fs->gzfile ) gzrewind(fs->gzfile); + fs->strbufpos = 0; } #define CV_YML_INDENT 3 @@ -373,7 +409,7 @@ icvFSCreateCollection( CvFileStorage* fs, int tag, CvFileNode* collection ) { if( collection->tag != CV_NODE_NONE ) { - assert( fs->is_xml != 0 ); + assert( fs->fmt == CV_STORAGE_FORMAT_XML ); CV_PARSE_ERROR( "Sequence element should not have name (use <_>)" ); } @@ -477,19 +513,18 @@ icvFSFlush( CvFileStorage* fs ) } -/* closes file storage and deallocates buffers */ -CV_IMPL void -cvReleaseFileStorage( CvFileStorage** p_fs ) +static void +icvClose( CvFileStorage* fs, std::string* out ) { - if( !p_fs ) + if( out ) + out->clear(); + + if( !fs ) CV_Error( CV_StsNullPtr, "NULL double pointer to file storage" ); - - if( *p_fs ) + + if( fs->is_opened ) { - CvFileStorage* fs = *p_fs; - *p_fs = 0; - - if( fs->write_mode && (fs->file || fs->gzfile) ) + if( fs->write_mode && (fs->file || fs->gzfile || fs->outbuf) ) { if( fs->write_stack ) { @@ -497,19 +532,42 @@ cvReleaseFileStorage( CvFileStorage** p_fs ) cvEndWriteStruct(fs); } icvFSFlush(fs); - if( fs->is_xml ) + if( fs->fmt == CV_STORAGE_FORMAT_XML ) icvPuts( fs, "\n" ); } + + icvCloseFile(fs); + } + + if( fs->outbuf && out ) + { + out->resize(fs->outbuf->size()); + std::copy(fs->outbuf->begin(), fs->outbuf->end(), out->begin()); + } +} - //icvFSReleaseCollection( fs->roots ); // delete all the user types recursively - - icvClose(fs); - - cvReleaseMemStorage( &fs->strstorage ); +/* closes file storage and deallocates buffers */ +CV_IMPL void +cvReleaseFileStorage( CvFileStorage** p_fs ) +{ + if( !p_fs ) + CV_Error( CV_StsNullPtr, "NULL double pointer to file storage" ); + + if( *p_fs ) + { + CvFileStorage* fs = *p_fs; + *p_fs = 0; + + icvClose(fs, 0); + + cvReleaseMemStorage( &fs->strstorage ); cvFree( &fs->buffer_start ); cvReleaseMemStorage( &fs->memstorage ); - + + if( fs->outbuf ) + delete fs->outbuf; + memset( fs, 0, sizeof(*fs) ); cvFree( &fs ); } @@ -2601,10 +2659,22 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co char* xml_buf = 0; int default_block_size = 1 << 18; bool append = (flags & 3) == CV_STORAGE_APPEND; + bool mem = (flags & CV_STORAGE_MEMORY) != 0; + bool write_mode = (flags & 3) != 0; bool isGZ = false; + size_t fnamelen = 0; - if( !filename ) - CV_Error( CV_StsNullPtr, "NULL filename" ); + if( !filename || filename[0] == '\0' ) + { + if( !write_mode ) + CV_Error( CV_StsNullPtr, mem ? "NULL or empty filename" : "NULL or empty buffer" ); + mem = true; + } + else + fnamelen = strlen(filename); + + if( mem && append ) + CV_Error( CV_StsBadFlag, "CV_STORAGE_APPEND and CV_STORAGE_MEMORY are not currently compatible" ); fs = (CvFileStorage*)cvAlloc( sizeof(*fs) ); memset( fs, 0, sizeof(*fs)); @@ -2612,44 +2682,43 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co fs->memstorage = cvCreateMemStorage( default_block_size ); fs->dststorage = dststorage ? dststorage : fs->memstorage; - int fnamelen = (int)strlen(filename); - if( !fnamelen ) - CV_Error( CV_StsError, "Empty filename" ); - - fs->filename = (char*)cvMemStorageAlloc( fs->memstorage, fnamelen+1 ); - strcpy( fs->filename, filename ); - - char* dot_pos = strrchr(fs->filename, '.'); - char compression = '\0'; - - if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && - (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) + fs->flags = CV_FILE_STORAGE; + fs->write_mode = write_mode; + + if( !mem ) { - if( append ) - CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); - isGZ = true; - compression = dot_pos[3]; - if( compression ) - dot_pos[3] = '\0', fnamelen--; - } + fs->filename = (char*)cvMemStorageAlloc( fs->memstorage, fnamelen+1 ); + strcpy( fs->filename, filename ); - fs->flags = CV_FILE_STORAGE; - fs->write_mode = (flags & 3) != 0; + char* dot_pos = strrchr(fs->filename, '.'); + char compression = '\0'; - if( !isGZ ) - { - fs->file = fopen(fs->filename, !fs->write_mode ? "rt" : !append ? "wt" : "a+t" ); - if( !fs->file ) - goto _exit_; - } - else - { - char mode[] = { fs->write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; - fs->gzfile = gzopen(fs->filename, mode); - if( !fs->gzfile ) - goto _exit_; - } + if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && + (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) + { + if( append ) + CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); + isGZ = true; + compression = dot_pos[3]; + if( compression ) + dot_pos[3] = '\0', fnamelen--; + } + if( !isGZ ) + { + fs->file = fopen(fs->filename, !fs->write_mode ? "rt" : !append ? "wt" : "a+t" ); + if( !fs->file ) + goto _exit_; + } + else + { + char mode[] = { fs->write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; + fs->gzfile = gzopen(fs->filename, mode); + if( !fs->gzfile ) + goto _exit_; + } + } + fs->roots = 0; fs->struct_indent = 0; fs->struct_flags = 0; @@ -2657,27 +2726,38 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co if( fs->write_mode ) { + int fmt = flags & CV_STORAGE_FORMAT_MASK; + + if( mem ) + fs->outbuf = new std::deque; + + if( fmt == CV_STORAGE_FORMAT_AUTO && filename ) + { + const char* dot_pos = filename + fnamelen - (isGZ ? 7 : 4); + fs->fmt = (dot_pos >= filename && (memcmp( dot_pos, ".xml", 4) == 0 || + memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0)) ? + CV_STORAGE_FORMAT_XML : CV_STORAGE_FORMAT_YAML; + } + else + fs->fmt = fmt != CV_STORAGE_FORMAT_AUTO ? fmt : CV_STORAGE_FORMAT_XML; + // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) - int buf_size = CV_FS_MAX_LEN*(fs->is_xml ? 6 : 4) + 1024; - - dot_pos = fs->filename + fnamelen - (isGZ ? 7 : 4); - fs->is_xml = dot_pos > fs->filename && (memcmp( dot_pos, ".xml", 4) == 0 || - memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0); + int buf_size = CV_FS_MAX_LEN*(fs->fmt == CV_STORAGE_FORMAT_XML ? 6 : 4) + 1024; if( append ) fseek( fs->file, 0, SEEK_END ); - fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->is_xml ? + fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->fmt == CV_STORAGE_FORMAT_XML ? sizeof(CvXMLStackRecord) : sizeof(int), fs->memstorage ); fs->is_first = 1; fs->struct_indent = 0; fs->struct_flags = CV_NODE_EMPTY; fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 ); fs->buffer_end = fs->buffer_start + buf_size; - if( fs->is_xml ) + if( fs->fmt == CV_STORAGE_FORMAT_XML ) { - int file_size = fs->file ? (int)ftell( fs->file ) : 0; + size_t file_size = fs->file ? ftell( fs->file ) : (size_t)0; fs->strstorage = cvCreateChildMemStorage( fs->memstorage ); if( !append || file_size == 0 ) { @@ -2724,7 +2804,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co } if( last_occurence < 0 ) CV_Error( CV_StsError, "Could not find in the end of file.\n" ); - icvClose( fs ); + icvCloseFile( fs ); fs->file = fopen( fs->filename, "r+t" ); fseek( fs->file, last_occurence, SEEK_SET ); // replace the last "" with " ", which has the same length @@ -2757,18 +2837,30 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co } else { - int buf_size = 1 << 20; + if( mem ) + { + fs->strbuf = filename; + fs->strbufsize = fnamelen; + } + + size_t buf_size = 1 << 20; const char* yaml_signature = "%YAML:"; char buf[16]; icvGets( fs, buf, sizeof(buf)-2 ); - fs->is_xml = strncmp( buf, yaml_signature, strlen(yaml_signature) ) != 0; + fs->fmt = strncmp( buf, yaml_signature, strlen(yaml_signature) ) == 0 ? + CV_STORAGE_FORMAT_YAML : CV_STORAGE_FORMAT_XML; if( !isGZ ) { - fseek( fs->file, 0, SEEK_END ); - buf_size = ftell( fs->file ); - buf_size = MIN( buf_size, (1 << 20) ); - buf_size = MAX( buf_size, CV_FS_MAX_LEN*2 + 1024 ); + if( !mem ) + { + fseek( fs->file, 0, SEEK_END ); + buf_size = ftell( fs->file ); + } + else + buf_size = fs->strbufsize; + buf_size = MIN( buf_size, (size_t)(1 << 20) ); + buf_size = MAX( buf_size, (size_t)(CV_FS_MAX_LEN*2 + 1024) ); } icvRewind(fs); @@ -2785,7 +2877,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co //mode = cvGetErrMode(); //cvSetErrMode( CV_ErrModeSilent ); - if( fs->is_xml ) + if( fs->fmt == CV_STORAGE_FORMAT_XML ) icvXMLParse( fs ); else icvYMLParse( fs ); @@ -2795,16 +2887,21 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co cvFree( &fs->buffer_start ); fs->buffer = fs->buffer_end = 0; } + fs->is_opened = true; + _exit_: if( fs ) { - if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile) ) + if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile && !fs->outbuf && !fs->strbuf) ) { cvReleaseFileStorage( &fs ); } else if( !fs->write_mode ) { - icvClose(fs); + icvCloseFile(fs); + // we close the file since it's not needed anymore. But icvCloseFile() resets is_opened, + // which may be misleading. Since we restore the value of is_opened. + fs->is_opened = true; } } @@ -3057,7 +3154,7 @@ cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) return; } - if( fs->is_xml ) + if( fs->fmt == CV_STORAGE_FORMAT_XML ) { int buf_len = (int)strlen(ptr); icvXMLWriteScalar( fs, 0, ptr, buf_len ); @@ -5054,14 +5151,20 @@ bool FileStorage::open(const string& filename, int flags, const string& encoding bool FileStorage::isOpened() const { - return !fs.empty(); + return !fs.empty() && fs.obj->is_opened; } -void FileStorage::release() +string FileStorage::release() { + string buf; + + if( fs.obj && fs.obj->outbuf ) + icvClose(fs.obj, &buf); + fs.release(); structs.clear(); state = UNDEFINED; + return buf; } FileNode FileStorage::root(int streamidx) const diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 67be5d0..a74a7fb 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -91,7 +91,7 @@ protected: {-1000000, 1000000}, {-10, 10}, {-10, 10}}; RNG& rng = ts->get_rng(); RNG rng0; - test_case_count = 2; + test_case_count = 4; int progress = 0; MemStorage storage(cvCreateMemStorage(0)); @@ -102,9 +102,10 @@ protected: cvClearMemStorage(storage); + bool mem = (idx % 4) >= 2; string filename = tempfile(idx % 2 ? ".yml" : ".xml"); - FileStorage fs(filename.c_str(), FileStorage::WRITE); + FileStorage fs(filename, FileStorage::WRITE + (mem ? FileStorage::MEMORY : 0)); int test_int = (int)cvtest::randInt(rng); double test_real = (cvtest::randInt(rng)%2?1:-1)*exp(cvtest::randReal(rng)*18-9); @@ -179,11 +180,11 @@ protected: fs.writeObj("test_graph",graph); CvGraph* graph2 = (CvGraph*)cvClone(graph); - fs.release(); + string content = fs.release(); - if(!fs.open(filename.c_str(), FileStorage::READ)) + if(!fs.open(mem ? content : filename, FileStorage::READ + (mem ? FileStorage::MEMORY : 0))) { - ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", filename.c_str() ); + ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", !mem ? filename.c_str() : content.c_str()); ts->set_failed_test_info( cvtest::TS::FAIL_MISSING_TEST_DATA ); return; } @@ -310,7 +311,6 @@ protected: int real_width = (int)tm["width"]; int real_height = (int)tm["height"]; - int real_lbp_val = 0; FileNodeIterator it; it = tm_lbp.begin(); @@ -370,7 +370,8 @@ protected: } fs.release(); - remove(filename.c_str()); + if( !mem ) + remove(filename.c_str()); } } }; -- 2.7.4