From 08a961424a2816357558e059e924c7f1ace58eb4 Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 16 Jul 2013 20:39:15 +0700 Subject: [PATCH] #59 #72 --- tcejdb/Makefile.in | 4 +- tcejdb/bson.c | 108 +++++++++++++- tcejdb/bson.h | 9 +- tcejdb/configure | 2 +- tcejdb/configure.ac | 2 +- tcejdb/ejdb.c | 265 +++++++++++++++++++++++++---------- tcejdb/ejdb.h | 9 +- tcejdb/nxjson.c | 387 +++++++++++++++++++++++++++++++++++++++++++++++++++ tcejdb/nxjson.h | 65 +++++++++ tcejdb/tchdb.c | 22 +-- tcejdb/tchdb.h | 1 + tcejdb/tctdb.c | 7 +- tcejdb/tctdb.h | 2 + tcejdb/testejdb/t4.c | 8 +- 14 files changed, 794 insertions(+), 97 deletions(-) create mode 100644 tcejdb/nxjson.c create mode 100644 tcejdb/nxjson.h diff --git a/tcejdb/Makefile.in b/tcejdb/Makefile.in index 1fa404e..a98d2f9 100644 --- a/tcejdb/Makefile.in +++ b/tcejdb/Makefile.in @@ -853,7 +853,7 @@ tcatest.o tcamttest.o tcamgr.o tcawmgr.o : \ ejdb.o : basedefs.h myconf.h ejdb.h ejdb_private.h ejdbutl.h -bson.o : basedefs.h myconf.h bson.h +bson.o : basedefs.h myconf.h bson.h nxjson.h ejdbutl.o : basedefs.h tcutil.h ejdbutl.h @@ -867,4 +867,6 @@ tokyocabinet_all.o : myconf.h tcutil.h tchdb.h tcbdb.h tcfdb.h tctdb.h tcadb.h platform.o : basedefs.h platform.c win32/platform.c nix/platform.c +nxjson.o : nxjson.h + # END OF FILE diff --git a/tcejdb/bson.c b/tcejdb/bson.c index 90a3816..75af611 100644 --- a/tcejdb/bson.c +++ b/tcejdb/bson.c @@ -1669,7 +1669,6 @@ bson* bson_create_from_iterator(bson_iterator *from) { return bs; } - bson* bson_create_from_buffer(const void* buf, int bufsz) { return bson_create_from_buffer2(bson_create(), buf, bufsz); } @@ -1685,7 +1684,7 @@ bson* bson_create_from_buffer2(bson *rv, const void* buf, int bufsz) { } void bson_init_with_data(bson *bs, const void *bsdata) { - memset(bs, 0, sizeof(*bs)); + memset(bs, 0, sizeof (*bs)); bs->data = (char*) bsdata; bson_little_endian32(&bs->dataSize, bsdata); bs->finished = true; @@ -2057,8 +2056,9 @@ static int _bson2json(_BSON2JSONCTX *ctx, bson_iterator *it) { } const char *key = bson_iterator_key(it); BSPAD(0); + tcxstrcat2(out, "\""); _jsonxstrescaped(out, key); - tcxstrcat2(out, " : "); + tcxstrcat2(out, "\" : "); switch (bt) { case BSON_LONG: case BSON_INT: @@ -2072,7 +2072,9 @@ static int _bson2json(_BSON2JSONCTX *ctx, bson_iterator *it) { case BSON_STRING: case BSON_SYMBOL: { + tcxstrcat2(out, "\""); _jsonxstrescaped(out, bson_iterator_string(it)); + tcxstrcat2(out, "\""); break; } case BSON_OBJECT: @@ -2092,7 +2094,7 @@ static int _bson2json(_BSON2JSONCTX *ctx, bson_iterator *it) { bson_date_t t = bson_iterator_date(it); char dbuf[49]; tcdatestrwww(t, INT_MAX, dbuf); - tcxstrprintf(out, "%s", dbuf); + tcxstrprintf(out, "\"%s\"", dbuf); break; } case BSON_BOOL: @@ -2103,13 +2105,15 @@ static int _bson2json(_BSON2JSONCTX *ctx, bson_iterator *it) { char xoid[25]; bson_oid_t *oid = bson_iterator_oid(it); bson_oid_to_string(oid, xoid); - tcxstrprintf(out, "%s", xoid); + tcxstrprintf(out, "\"%s\"", xoid); break; } case BSON_REGEX: { - tcxstrprintf(out, "%s", bson_iterator_regex(it)); - break; + tcxstrcat2(out, "\""); + _jsonxstrescaped(out, bson_iterator_regex(it)); + tcxstrcat2(out, "\""); + break; } case BSON_BINDATA: { @@ -2155,3 +2159,93 @@ int bson2json(const char *bsdata, char **buf, int *sp) { return ret; } + +#include "nxjson.h" + +static void _json2bson(bson *out, const nx_json *json) { + const char *key = json->key; + switch (json->type) { + case NX_JSON_NULL: + assert(key); + bson_append_null(out, key); + break; + case NX_JSON_OBJECT: + { + if (key) { + bson_append_start_object(out, key); + } + for (nx_json* js = json->child; js; js = js->next) { + _json2bson(out, js); + } + if (key) { + bson_append_finish_object(out); + } + break; + } + case NX_JSON_ARRAY: + { + if (key) { + bson_append_start_array(out, key); + } + for (nx_json* js = json->child; js; js = js->next) { + _json2bson(out, js); + } + if (key) { + bson_append_finish_array(out); + } + break; + } + case NX_JSON_STRING: + assert(key); + bson_append_string(out, key, json->text_value); + break; + case NX_JSON_INTEGER: + assert(key); + if (json->int_value <= INT_MAX && json->int_value >= INT_MIN) { + bson_append_int(out, key, (int) json->int_value); + } else { + bson_append_long(out, key, json->int_value); + } + break; + case NX_JSON_DOUBLE: + assert(key); + bson_append_double(out, key, json->dbl_value); + break; + case NX_JSON_BOOL: + assert(key); + bson_append_bool(out, key, json->int_value ? true : false); + break; + default: + break; + } +} + +bson* json2bson(const char *jsonstr) { + bool err = false; + bson *out = NULL; + char *json = strdup(jsonstr); //nxjson uses inplace data modification + if (!json) { + return NULL; + } + out = bson_create(); + bson_init_as_query(out); + const nx_json *nxjson = nx_json_parse_utf8(json); + if (!nxjson) { + err = true; + goto finish; + } + _json2bson(out, nxjson); + bson_finish(out); + err = out->err; +finish: + free(json); + if (nxjson) { + nx_json_free(nxjson); + } + if (err && out) { + bson_del(out); + out = NULL; + } + return out; +} + diff --git a/tcejdb/bson.h b/tcejdb/bson.h index 20608aa..35c41de 100644 --- a/tcejdb/bson.h +++ b/tcejdb/bson.h @@ -1135,7 +1135,7 @@ EJDB_EXPORT int bson_merge_array_sets(const void *mbuf, const void *inbuf, bool /** - * Convert BSON into JSON buffer + * Convert BSON into JSON buffer. * @param src BSON data * @param buf Allocated buffer with resulting JSON data * @param sp JSON data length will be stored into @@ -1143,6 +1143,13 @@ EJDB_EXPORT int bson_merge_array_sets(const void *mbuf, const void *inbuf, bool */ EJDB_EXPORT int bson2json(const char *bsdata, char **buf, int *sp); +/** + * Convert JSON into BSON object. + * @param jsonstr NULL terminated JSON string + * @return Allocated BSON object filled with given JSON data or NULL on error + */ +EJDB_EXPORT bson* json2bson(const char *jsonstr); + EJDB_EXTERN_C_END #endif diff --git a/tcejdb/configure b/tcejdb/configure index 4400c07..5d50661 100755 --- a/tcejdb/configure +++ b/tcejdb/configure @@ -2196,7 +2196,7 @@ MYHEADERFILES="tcutil.h tchdb.h tcbdb.h tcfdb.h tctdb.h tcadb.h ejdb.h ejdb_priv MYLIBRARYFILES="libtcejdb.a" MYLIBOBJFILES="tcutil.o tchdb.o tcbdb.o tcfdb.o tctdb.o tcadb.o myconf.o md5.o ejdb.o \ -bson.o numbers.o encoding.o utf8proc.o ejdbutl.o platform.o" +bson.o numbers.o encoding.o utf8proc.o ejdbutl.o platform.o nxjson.o" MYCOMMANDFILES="tcutest tcumttest tcucodec tchtest tchmttest tchmgr" MYCOMMANDFILES="$MYCOMMANDFILES tcbtest tcbmttest tcbmgr tcftest tcfmttest tcfmgr" diff --git a/tcejdb/configure.ac b/tcejdb/configure.ac index 843c333..28e23ab 100644 --- a/tcejdb/configure.ac +++ b/tcejdb/configure.ac @@ -23,7 +23,7 @@ MYHEADERFILES="tcutil.h tchdb.h tcbdb.h tcfdb.h tctdb.h tcadb.h ejdb.h ejdb_priv MYLIBRARYFILES="libtcejdb.a" MYLIBOBJFILES="tcutil.o tchdb.o tcbdb.o tcfdb.o tctdb.o tcadb.o myconf.o md5.o ejdb.o \ -bson.o numbers.o encoding.o utf8proc.o ejdbutl.o platform.o" +bson.o numbers.o encoding.o utf8proc.o ejdbutl.o platform.o nxjson.o" MYCOMMANDFILES="tcutest tcumttest tcucodec tchtest tchmttest tchmgr" MYCOMMANDFILES="$MYCOMMANDFILES tcbtest tcbmttest tcbmgr tcftest tcfmttest tcfmgr" diff --git a/tcejdb/ejdb.c b/tcejdb/ejdb.c index 60bd4e6..63617d9 100644 --- a/tcejdb/ejdb.c +++ b/tcejdb/ejdb.c @@ -81,6 +81,7 @@ typedef struct { /* private function prototypes */ static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func); +static void _ejdbsetecode2(EJDB *jb, int ecode, const char *filename, int line, const char *func, bool notfatal); static bool _ejdbsetmutex(EJDB *ejdb); EJDB_INLINE bool _ejdblockmethod(EJDB *ejdb, bool wr); EJDB_INLINE bool _ejdbunlockmethod(EJDB *ejdb); @@ -129,8 +130,10 @@ EJDB_INLINE void _nufetch(_EJDBNUM *nu, const char *sval, bson_type bt); EJDB_INLINE int _nucmp(_EJDBNUM *nu, const char *sval, bson_type bt); EJDB_INLINE int _nucmp2(_EJDBNUM *nu1, _EJDBNUM *nu2, bson_type bt); static EJCOLL* _getcoll(EJDB *jb, const char *colname); -static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags); -static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags); +static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags, TCXSTR *log); +static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags, TCXSTR *log); +static EJCOLL* _createcollimpl(EJDB *jb, const char *colname, EJCOLLOPTS *opts); +static bool _rmcollimpl(EJDB *jb, EJCOLL *coll, bool unlinkfile); extern const char *utf8proc_errmsg(ssize_t errcode); @@ -163,7 +166,8 @@ const char* ejdberrmsg(int ecode) { case JBEQINCEXCL: return "$fields hint cannot mix include and exclude fields"; case JBEQACTKEY: return "action key in $do block can only be one of: $join"; case JBEMAXNUMCOLS: return "exceeded the maximum number of collections per database: 1024"; - + case JBEEJSONPARSE: return "JSON parsing failed"; + case JBEEI: return "data export/import failed"; default: return tcerrmsg(ecode); } } @@ -299,26 +303,9 @@ EJCOLL* ejdbcreatecoll(EJDB *jb, const char *colname, EJCOLLOPTS *opts) { if (coll) { return coll; } - if (!JBISVALCOLNAME(colname)) { - _ejdbsetecode(jb, JBEINVALIDCOLNAME, __FILE__, __LINE__, __func__); - return NULL; - } JBENSUREOPENLOCK(jb, true, NULL); - TCTDB *meta = jb->metadb; - char *row = tcsprintf("name\t%s", colname); - if (!tctdbput3(meta, colname, row)) { - goto finish; - } - if (!_addcoldb0(colname, jb, opts, &coll)) { - tctdbout2(meta, colname); //cleaning - goto finish; - } - _metasetopts(jb, colname, opts); -finish: + coll = _createcollimpl(jb, colname, opts); JBUNLOCKMETHOD(jb); - if (row) { - TCFREE(row); - } return coll; } @@ -331,38 +318,7 @@ bool ejdbrmcoll(EJDB *jb, const char *colname, bool unlinkfile) { goto finish; } if (!JBCLOCKMETHOD(coll, true)) return false; - tctdbout2(jb->metadb, colname); - tctdbvanish(coll->tdb); - TCLIST *paths = tclistnew2(10); - tclistpush2(paths, coll->tdb->hdb->path); - for (int j = 0; j < coll->tdb->inum; ++j) { - TDBIDX *idx = coll->tdb->idxs + j; - const char *ipath = tcbdbpath(idx->db); - if (ipath) { - tclistpush2(paths, ipath); - } - } - tctdbclose(coll->tdb); - if (unlinkfile) { - for (int i = 0; i < TCLISTNUM(paths); ++i) { - unlink(tclistval2(paths, i)); - } - } - tclistdel(paths); - jb->cdbsnum--; - for (int i = 0; i < EJDB_MAX_COLLECTIONS; ++i) { - if (jb->cdbs[i] == coll) { - jb->cdbs[i] = NULL; - break; - } - } - //collapse NULL hole - for (int i = 0; i < EJDB_MAX_COLLECTIONS - 1; ++i) { - if (!jb->cdbs[i] && jb->cdbs[i + 1]) { - jb->cdbs[i] = jb->cdbs[i + 1]; - jb->cdbs[i + 1] = NULL; - } - } + rv = _rmcollimpl(jb, coll, unlinkfile); JBCUNLOCKMETHOD(coll); _delcoldb(coll); TCFREE(coll); @@ -951,14 +907,14 @@ bson* ejdbmeta(EJDB *jb) { return bs; } -bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags) { +bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log) { assert(jb && path); bool err = false; bool isdir = false; tcstatfile(path, &isdir, NULL, NULL); if (!isdir) { if (mkdir(path, 00755)) { - _ejdbsetecode(jb, TCEMKDIR, __FILE__, __LINE__, __func__); + _ejdbsetecode2(jb, TCEMKDIR, __FILE__, __LINE__, __func__, true); return false; } } @@ -966,49 +922,54 @@ bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags) { if (_cnames == NULL) { _cnames = ejdbgetcolls(jb); if (_cnames == NULL) { + _ejdbsetecode2(jb, TCEINVALID, __FILE__, __LINE__, __func__, true); return false; } } + JBENSUREOPENLOCK(jb, false, false); for (int i = 0; i < TCLISTNUM(cnames); ++i) { const char *cn = TCLISTVALPTR(cnames, i); assert(cn); - EJCOLL *coll = ejdbgetcoll(jb, cn); + EJCOLL *coll = _getcoll(jb, cn); if (!coll) continue; if (!JBCLOCKMETHOD(coll, false)) { err = true; goto finish; } - if (!_exportcoll(coll, path, flags)) { + if (!_exportcoll(coll, path, flags, log)) { err = true; } JBCUNLOCKMETHOD(coll); } finish: + JBUNLOCKMETHOD(jb); if (_cnames != cnames) { tclistdel(_cnames); } return !err; } -bool ejdbimport(EJDB *jb, const char *path, TCLIST *cnames, int flags) { +bool ejdbimport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log) { assert(jb && path); bool err = false; bool isdir = false; if (!tcstatfile(path, &isdir, NULL, NULL) || !isdir) { - _ejdbsetecode(jb, TCENOFILE, __FILE__, __LINE__, __func__); + _ejdbsetecode2(jb, TCENOFILE, __FILE__, __LINE__, __func__, true); return false; } bool tail = (path[0] != '\0' && path[strlen(path) - 1] == MYPATHCHR); char *bsonpat = tail ? tcsprintf("%s*.bson") : tcsprintf("%s%c*.bson", path, MYPATHCHR); TCLIST *bspaths = tcglobpat(bsonpat); + JBENSUREOPENLOCK(jb, true, false); for (int i = 0; i < TCLISTNUM(bspaths); ++i) { const char* bspath = TCLISTVALPTR(bspaths, i); - if (!_importcoll(jb, bspath, cnames, flags)) { + if (!_importcoll(jb, bspath, cnames, flags, log)) { err = true; goto finish; } } finish: + JBUNLOCKMETHOD(jb); if (bsonpat) { TCFREE(bsonpat); } @@ -1023,26 +984,180 @@ finish: * private features *************************************************************************************************/ -static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags) { +/** + * In order to cleanup resources you need: + * _delcoldb(coll); + * TCFREE(coll); + * after this method completion with any return status. + */ +static bool _rmcollimpl(EJDB *jb, EJCOLL *coll, bool unlinkfile) { + assert(jb && coll); + bool rv = true; + tctdbout(jb->metadb, coll->cname, coll->cnamesz); + tctdbvanish(coll->tdb); + TCLIST *paths = tclistnew2(10); + tclistpush2(paths, coll->tdb->hdb->path); + for (int j = 0; j < coll->tdb->inum; ++j) { + TDBIDX *idx = coll->tdb->idxs + j; + const char *ipath = tcbdbpath(idx->db); + if (ipath) { + tclistpush2(paths, ipath); + } + } + tctdbclose(coll->tdb); + if (unlinkfile) { + for (int i = 0; i < TCLISTNUM(paths); ++i) { + unlink(tclistval2(paths, i)); + } + } + tclistdel(paths); + jb->cdbsnum--; + for (int i = 0; i < EJDB_MAX_COLLECTIONS; ++i) { + if (jb->cdbs[i] == coll) { + jb->cdbs[i] = NULL; + break; + } + } + //collapse the NULL hole + for (int i = 0; i < EJDB_MAX_COLLECTIONS - 1; ++i) { + if (!jb->cdbs[i] && jb->cdbs[i + 1]) { + jb->cdbs[i] = jb->cdbs[i + 1]; + jb->cdbs[i + 1] = NULL; + } + } + return rv; +} + +static EJCOLL* _createcollimpl(EJDB *jb, const char *colname, EJCOLLOPTS *opts) { + EJCOLL *coll; + if (!JBISVALCOLNAME(colname)) { + _ejdbsetecode(jb, JBEINVALIDCOLNAME, __FILE__, __LINE__, __func__); + return NULL; + } + TCTDB *meta = jb->metadb; + char *row = tcsprintf("name\t%s", colname); + if (!tctdbput3(meta, colname, row)) { + goto finish; + } + if (!_addcoldb0(colname, jb, opts, &coll)) { + tctdbout2(meta, colname); //cleaning + goto finish; + } + _metasetopts(jb, colname, opts); +finish: + if (row) { + TCFREE(row); + } + return coll; +} + +static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags, TCXSTR *log) { + if (log) { + tcxstrprintf(log, "\nImporting '%s'", bspath); + } bool err = false; // /foo/bar.bson char *dp = strrchr(bspath, '.'); char *pp = strrchr(bspath, MYPATHCHR); - if (pp > dp) { + if (!dp || !pp || pp > dp) { + if (log) { + tcxstrprintf(log, "ERROR: Invalid file path: '%s'", bspath); + } + _ejdbsetecode2(jb, JBEEI, __FILE__, __LINE__, __func__, true); return false; } + char *lastsep = pp; int i = 0; - char *cname; + char *cname, *mjson; + bson_type bt; + bson *mbson; //meta bson + bson_iterator mbsonit; + int sp; + EJCOLL *coll; + TCMALLOC(cname, dp - pp + 1); + TCXSTR *xmetapath = tcxstrnew(); + while (++pp != dp) { cname[i++] = *pp; } cname[i] = '\0'; - fprintf(stderr, "\ncname=%s", cname); + if (cnames != NULL) { + if (tclistlsearch(cnames, cname, i) == -1) { + goto finish; + } + } + tcxstrcat(xmetapath, bspath, lastsep - bspath + 1); + tcxstrprintf(xmetapath, "%s-meta.json", cname); + mjson = tcreadfile(tcxstrptr(xmetapath), 0, &sp); + if (!mjson) { + err = true; + if (log) { + tcxstrprintf(log, "ERROR: Error reading file: '%s'", tcxstrptr(xmetapath)); + } + _ejdbsetecode2(jb, TCEREAD, __FILE__, __LINE__, __func__, true); + goto finish; + } + mbson = json2bson(mjson); + if (!mbson) { + err = true; + if (log) { + tcxstrprintf(log, "ERROR: Invalid JSON in the file: '%s'", tcxstrptr(xmetapath)); + } + _ejdbsetecode2(jb, JBEEJSONPARSE, __FILE__, __LINE__, __func__, true); + } + coll = ejdbgetcoll(jb, cname); + if (coll && (flags & JBIMPORTREPLACE)) { + if (!ejdbrmcoll(jb, cname, true)) { + if (log) { + tcxstrprintf(log, "ERROR: Failed to remove collection: '%s'", cname); + } + err = true; + goto finish; + } + } + if (!coll) { + bson_iterator_init(&mbsonit, mbson); + EJCOLLOPTS cops = {0}; + if (bson_find_fieldpath_value("opts", &mbsonit) == BSON_OBJECT) { + bson_iterator sit; + bson_iterator_subiterator(&mbsonit, &sit); + while ((bt = bson_iterator_next(&sit)) != BSON_EOO) { + const char *key = bson_iterator_key(&sit); + if (strcmp("compressed", key) == 0 && bt == BSON_BOOL) { + cops.compressed = bson_iterator_bool(&sit); + } else if (strcmp("large", key) == 0 && bt == BSON_BOOL) { + cops.large = bson_iterator_bool(&sit); + } else if (strcmp("cachedrecords", key) == 0 && BSON_IS_NUM_TYPE(bt)) { + cops.cachedrecords = bson_iterator_int(&sit); + } else if (strcmp("records", key) == 0 && BSON_IS_NUM_TYPE(bt)) { + cops.records = bson_iterator_long(&sit); + } + } + } + coll = ejdbcreatecoll(jb, cname, &cops); + if (!coll) { + if (log) { + tcxstrprintf(log, "ERROR: Error creating collection: '%s'", cname); + } + } + } + + //todo lock coll & read bson collection data + +finish: + if (mbson) { + bson_del(mbson); + } + if (mjson) { + TCFREE(mjson); + } + tcxstrdel(xmetapath); + TCFREE(cname); return !err; } -static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags) { +static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags, TCXSTR *log) { bool err = false; char *fpath = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, (flags & JBJSONEXPORT) ? ".json" : ".bson"); char *fpathm = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, "-meta.json"); @@ -1064,7 +1179,7 @@ static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags) { NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); #endif if (INVALIDHANDLE(fd) || INVALIDHANDLE(fdm)) { - _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true); err = true; goto finish; } @@ -1079,7 +1194,7 @@ static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags) { int wsiz; if (flags & JBJSONEXPORT) { if (bson2json(TCXSTRPTR(bsbuf), &wbuf, &wsiz) != BSON_OK) { - _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__, true); goto wfinish; } } else { @@ -1087,7 +1202,7 @@ static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags) { wsiz = TCXSTRSIZE(bsbuf); } if (!tcwrite(fd, wbuf, wsiz)) { - _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true); goto wfinish; } wfinish: @@ -1124,25 +1239,25 @@ wfinish: int wsiz; if (bson2json(bson_data(&mbs), &wbuf, &wsiz) != BSON_OK) { err = true; - _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__, true); bson_destroy(&mbs); goto finish; } bson_destroy(&mbs); if (!tcwrite(fdm, wbuf, wsiz)) { err = true; - _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true); } TCFREE(wbuf); } finish: if (!INVALIDHANDLE(fd) && !CLOSEFH(fd)) { - _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true); err = true; } if (!INVALIDHANDLE(fdm) && !CLOSEFH(fdm)) { - _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true); err = true; } tcxstrdel(skbuf); @@ -1154,8 +1269,12 @@ finish: /* Set the error code of a table database object. */ static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func) { + _ejdbsetecode2(jb, ecode, filename, line, func, false); +} + +static void _ejdbsetecode2(EJDB *jb, int ecode, const char *filename, int line, const char *func, bool notfatal) { assert(jb && filename && line >= 1 && func); - tctdbsetecode(jb->metadb, ecode, filename, line, func); + tctdbsetecode2(jb->metadb, ecode, filename, line, func, notfatal); } static EJCOLL* _getcoll(EJDB *jb, const char *colname) { diff --git a/tcejdb/ejdb.h b/tcejdb/ejdb.h index 979631d..25ab13e 100644 --- a/tcejdb/ejdb.h +++ b/tcejdb/ejdb.h @@ -59,7 +59,8 @@ enum { /** Error codes */ JBEQINCEXCL = 9012, /**< $fields hint cannot mix include and exclude fields */ JBEQACTKEY = 9013, /**< action key in $do block can only be one of: $join */ JBEMAXNUMCOLS = 9014, /**< Exceeded the maximum number of collections per database */ - JBEEI = 9015 /**< EJDB export/import error */ + JBEEI = 9015, /**< EJDB export/import error */ + JBEEJSONPARSE = 9016 /**< JSON parsing failed */ }; enum { /** Database open modes */ @@ -514,9 +515,10 @@ enum { * @param path Directory name in which data will exported. * @param colnames List of collection names to export. If its value is `NULL` then all existing collections will be exported. * @param flags. Can be set to `JBJSONEXPORT` in order to export collection data into JSON instead of BSON. + * @param log Optional opration log buffer. * @return on sucess `true` */ -EJDB_EXPORT bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags); +EJDB_EXPORT bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log); /** * TODO @@ -524,9 +526,10 @@ EJDB_EXPORT bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flag * @param path * @param cnames * @param flags + * @param log Optional opration log buffer. * @return */ -EJDB_EXPORT bool ejdbimport(EJDB *jb, const char *path, TCLIST *cnames, int flags); +EJDB_EXPORT bool ejdbimport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log); EJDB_EXTERN_C_END diff --git a/tcejdb/nxjson.c b/tcejdb/nxjson.c new file mode 100644 index 0000000..2505ac1 --- /dev/null +++ b/tcejdb/nxjson.c @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2013 Yaroslav Stavnichiy + * + * This file is part of NXJSON. + * + * NXJSON is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + * NXJSON is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with NXJSON. If not, see . + */ + +// this file can be #included in your code +#ifndef NXJSON_C +#define NXJSON_C + +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include +#include +#include +#include + +#include "nxjson.h" + +// redefine NX_JSON_CALLOC & NX_JSON_FREE to use custom allocator +#ifndef NX_JSON_CALLOC +#define NX_JSON_CALLOC() calloc(1, sizeof(nx_json)) +#define NX_JSON_FREE(json) free((void*)(json)) +#endif + +// redefine NX_JSON_REPORT_ERROR to use custom error reporting +#ifndef NX_JSON_REPORT_ERROR +#define NX_JSON_REPORT_ERROR(msg, p) fprintf(stderr, "NXJSON PARSE ERROR (%d): " msg " at %s\n", __LINE__, p) +#endif + +#define IS_WHITESPACE(c) ((unsigned char)(c)<=(unsigned char)' ') + +static const nx_json dummy={ NX_JSON_NULL }; + +static nx_json* create_json(nx_json_type type, const char* key, nx_json* parent) { + nx_json* js=NX_JSON_CALLOC(); + assert(js); + js->type=type; + js->key=key; + if (!parent->last_child) { + parent->child=parent->last_child=js; + } + else { + parent->last_child->next=js; + parent->last_child=js; + } + parent->length++; + return js; +} + +void nx_json_free(const nx_json* js) { + nx_json* p=js->child; + nx_json* p1; + while (p) { + p1=p->next; + nx_json_free(p); + p=p1; + } + NX_JSON_FREE(js); +} + +static int unicode_to_utf8(unsigned int codepoint, char* p, char** endp) { + // code from http://stackoverflow.com/a/4609989/697313 + if (codepoint<0x80) *p++=codepoint; + else if (codepoint<0x800) *p++=192+codepoint/64, *p++=128+codepoint%64; + else if (codepoint-0xd800u<0x800) return 0; // surrogate must have been treated earlier + else if (codepoint<0x10000) *p++=224+codepoint/4096, *p++=128+codepoint/64%64, *p++=128+codepoint%64; + else if (codepoint<0x110000) *p++=240+codepoint/262144, *p++=128+codepoint/4096%64, *p++=128+codepoint/64%64, *p++=128+codepoint%64; + else return 0; // error + *endp=p; + return 1; +} + +nx_json_unicode_encoder nx_json_unicode_to_utf8=unicode_to_utf8; + +static inline int hex_val(char c) { + if (c>='0' && c<='9') return c-'0'; + if (c>='a' && c<='f') return c-'a'+10; + if (c>='A' && c<='F') return c-'A'+10; + return -1; +} + +static char* unescape_string(char* s, char** end, nx_json_unicode_encoder encoder) { + char* p=s; + char* d=s; + char c; + while ((c=*p++)) { + if (c=='"') { + *d='\0'; + *end=p; + return s; + } + else if (c=='\\') { + switch (*p) { + case '\\': + case '/': + case '"': + *d++=*p++; + break; + case 'b': + *d++='\b'; p++; + break; + case 'f': + *d++='\f'; p++; + break; + case 'n': + *d++='\n'; p++; + break; + case 'r': + *d++='\r'; p++; + break; + case 't': + *d++='\t'; p++; + break; + case 'u': // unicode + if (!encoder) { + // leave untouched + *d++=c; + break; + } + char* ps=p-1; + int h1, h2, h3, h4; + if ((h1=hex_val(p[1]))<0 || (h2=hex_val(p[2]))<0 || (h3=hex_val(p[3]))<0 || (h4=hex_val(p[4]))<0) { + NX_JSON_REPORT_ERROR("invalid unicode escape", p-1); + return 0; + } + unsigned int codepoint=h1<<12|h2<<8|h3<<4|h4; + if ((codepoint & 0xfc00)==0xd800) { // high surrogate; need one more unicode to succeed + p+=6; + if (p[-1]!='\\' || *p!='u' || (h1=hex_val(p[1]))<0 || (h2=hex_val(p[2]))<0 || (h3=hex_val(p[3]))<0 || (h4=hex_val(p[4]))<0) { + NX_JSON_REPORT_ERROR("invalid unicode surrogate", ps); + return 0; + } + unsigned int codepoint2=h1<<12|h2<<8|h3<<4|h4; + if ((codepoint2 & 0xfc00)!=0xdc00) { + NX_JSON_REPORT_ERROR("invalid unicode surrogate", ps); + return 0; + } + codepoint=0x10000+((codepoint-0xd800)<<10)+(codepoint2-0xdc00); + } + if (!encoder(codepoint, d, &d)) { + NX_JSON_REPORT_ERROR("invalid codepoint", ps); + return 0; + } + p+=5; + break; + default: + // leave untouched + *d++=c; + break; + } + } + else { + *d++=c; + } + } + NX_JSON_REPORT_ERROR("no closing quote for string", s); + return 0; +} + +static char* skip_block_comment(char* p) { + // assume p[-2]=='/' && p[-1]=='*' + char* ps=p-2; + if (!*p) { + NX_JSON_REPORT_ERROR("endless comment", ps); + return 0; + } + REPEAT: + p=strchr(p+1, '/'); + if (!p) { + NX_JSON_REPORT_ERROR("endless comment", ps); + return 0; + } + if (p[-1]!='*') goto REPEAT; + return p+1; +} + +static char* parse_key(const char** key, char* p, nx_json_unicode_encoder encoder) { + // on '}' return with *p=='}' + char c; + while ((c=*p++)) { + if (c=='"') { + *key=unescape_string(p, &p, encoder); + if (!*key) return 0; // propagate error + while (*p && IS_WHITESPACE(*p)) p++; + if (*p==':') return p+1; + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; + } + else if (IS_WHITESPACE(c) || c==',') { + // continue + } + else if (c=='}') { + return p-1; + } + else if (c=='/') { + if (*p=='/') { // line comment + char* ps=p-1; + p=strchr(p+1, '\n'); + if (!p) { + NX_JSON_REPORT_ERROR("endless comment", ps); + return 0; // error + } + p++; + } + else if (*p=='*') { // block comment + p=skip_block_comment(p+1); + if (!p) return 0; + } + else { + NX_JSON_REPORT_ERROR("unexpected chars", p-1); + return 0; // error + } + } + else { + NX_JSON_REPORT_ERROR("unexpected chars", p-1); + return 0; // error + } + } + NX_JSON_REPORT_ERROR("unexpected chars", p-1); + return 0; // error +} + +static char* parse_value(nx_json* parent, const char* key, char* p, nx_json_unicode_encoder encoder) { + nx_json* js; + while (1) { + switch (*p) { + case '\0': + NX_JSON_REPORT_ERROR("unexpected end of text", p); + return 0; // error + case ' ': case '\t': case '\n': case '\r': + case ',': + // skip + p++; + break; + case '{': + js=create_json(NX_JSON_OBJECT, key, parent); + p++; + while (1) { + const char* new_key; + p=parse_key(&new_key, p, encoder); + if (!p) return 0; // error + if (*p=='}') return p+1; // end of object + p=parse_value(js, new_key, p, encoder); + if (!p) return 0; // error + } + case '[': + js=create_json(NX_JSON_ARRAY, key, parent); + p++; + while (1) { + p=parse_value(js, 0, p, encoder); + if (!p) return 0; // error + if (*p==']') return p+1; // end of array + } + case ']': + return p; + case '"': + p++; + js=create_json(NX_JSON_STRING, key, parent); + js->text_value=unescape_string(p, &p, encoder); + if (!js->text_value) return 0; // propagate error + return p; + case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + { + js=create_json(NX_JSON_INTEGER, key, parent); + char* pe; + js->int_value=strtol(p, &pe, 0); + if (pe==p) { + NX_JSON_REPORT_ERROR("invalid number", p); + return 0; // error + } + if (*pe=='.' || *pe=='e' || *pe=='E') { // double value + js->type=NX_JSON_DOUBLE; + js->dbl_value=strtod(p, &pe); + if (pe==p) { + NX_JSON_REPORT_ERROR("invalid number", p); + return 0; // error + } + } + else { + js->dbl_value=js->int_value; + } + return pe; + } + case 't': + if (!strncmp(p, "true", 4)) { + js=create_json(NX_JSON_BOOL, key, parent); + js->int_value=1; + return p+4; + } + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; // error + case 'f': + if (!strncmp(p, "false", 5)) { + js=create_json(NX_JSON_BOOL, key, parent); + js->int_value=0; + return p+5; + } + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; // error + case 'n': + if (!strncmp(p, "null", 4)) { + create_json(NX_JSON_NULL, key, parent); + return p+4; + } + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; // error + case '/': // comment + if (p[1]=='/') { // line comment + char* ps=p; + p=strchr(p+2, '\n'); + if (!p) { + NX_JSON_REPORT_ERROR("endless comment", ps); + return 0; // error + } + p++; + } + else if (p[1]=='*') { // block comment + p=skip_block_comment(p+2); + if (!p) return 0; + } + else { + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; // error + } + break; + default: + NX_JSON_REPORT_ERROR("unexpected chars", p); + return 0; // error + } + } +} + +const nx_json* nx_json_parse_utf8(char* text) { + return nx_json_parse(text, unicode_to_utf8); +} + +const nx_json* nx_json_parse(char* text, nx_json_unicode_encoder encoder) { + nx_json js={0}; + if (!parse_value(&js, 0, text, encoder)) { + if (js.child) nx_json_free(js.child); + return 0; + } + return js.child; +} + +const nx_json* nx_json_get(const nx_json* json, const char* key) { + if (!json || !key) return &dummy; // never return null + nx_json* js; + for (js=json->child; js; js=js->next) { + if (js->key && !strcmp(js->key, key)) return js; + } + return &dummy; // never return null +} + +const nx_json* nx_json_item(const nx_json* json, int idx) { + if (!json) return &dummy; // never return null + nx_json* js; + for (js=json->child; js; js=js->next) { + if (!idx--) return js; + } + return &dummy; // never return null +} + + +#ifdef __cplusplus +} +#endif + +#endif /* NXJSON_C */ diff --git a/tcejdb/nxjson.h b/tcejdb/nxjson.h new file mode 100644 index 0000000..6b99726 --- /dev/null +++ b/tcejdb/nxjson.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2013 Yaroslav Stavnichiy + * + * This file is part of NXJSON. + * + * NXJSON is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + * NXJSON is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with NXJSON. If not, see . + */ + +#ifndef NXJSON_H +#define NXJSON_H + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef enum nx_json_type { + NX_JSON_NULL, // this is null value + NX_JSON_OBJECT, // this is an object; properties can be found in child nodes + NX_JSON_ARRAY, // this is an array; items can be found in child nodes + NX_JSON_STRING, // this is a string; value can be found in text_value field + NX_JSON_INTEGER, // this is an integer; value can be found in int_value field + NX_JSON_DOUBLE, // this is a double; value can be found in dbl_value field + NX_JSON_BOOL // this is a boolean; value can be found in int_value field +} nx_json_type; + +typedef struct nx_json { + nx_json_type type; // type of json node, see above + const char* key; // key of the property; for object's children only + const char* text_value; // text value of STRING node + long int_value; // the value of INTEGER or BOOL node + double dbl_value; // the value of DOUBLE node + int length; // number of children of OBJECT or ARRAY + struct nx_json* child; // points to first child + struct nx_json* next; // points to next child + struct nx_json* last_child; +} nx_json; + +typedef int (*nx_json_unicode_encoder)(unsigned int codepoint, char* p, char** endp); + +extern nx_json_unicode_encoder nx_json_unicode_to_utf8; + +const nx_json* nx_json_parse(char* text, nx_json_unicode_encoder encoder); +const nx_json* nx_json_parse_utf8(char* text); +void nx_json_free(const nx_json* js); +const nx_json* nx_json_get(const nx_json* json, const char* key); // get object's property by key +const nx_json* nx_json_item(const nx_json* json, int idx); // get array element by index + + +#ifdef __cplusplus +} +#endif + +#endif /* NXJSON_H */ diff --git a/tcejdb/tchdb.c b/tcejdb/tchdb.c index c21e642..b0f6b0b 100644 --- a/tcejdb/tchdb.c +++ b/tcejdb/tchdb.c @@ -864,9 +864,9 @@ TCHDBITER* tchdbiter2init(TCHDB *hdb) { hdb->iter2list = tclistnew2(16); } TCHDBITER *it; - TCMALLOC(it, sizeof(*it)); + TCMALLOC(it, sizeof (*it)); it->pos = hdb->frec; - TCLISTPUSH(hdb->iter2list, &it, sizeof(it)); + TCLISTPUSH(hdb->iter2list, &it, sizeof (it)); HDBUNLOCKMETHOD(hdb); return it; } @@ -1397,13 +1397,17 @@ uint64_t tchdbfsiz(TCHDB *hdb) { return rv; } - /************************************************************************************************* * features for experts *************************************************************************************************/ /* Set the error code of a hash database object. */ void tchdbsetecode(TCHDB *hdb, int ecode, const char *filename, int line, const char *func) { + tchdbsetecode2(hdb, ecode, filename, line, func, false); +} + +/* Set the error code of a hash database object. */ +void tchdbsetecode2(TCHDB *hdb, int ecode, const char *filename, int line, const char *func, bool notfatal) { assert(hdb && filename && line >= 1 && func); if (!hdb->fatal) { if (hdb->eckey) { @@ -1432,8 +1436,10 @@ void tchdbsetecode(TCHDB *hdb, int ecode, const char *filename, int line, const case TCEMKDIR: case TCERMDIR: { - hdb->fatal = true; - if (!INVALIDHANDLE(hdb->fd) && (hdb->omode & HDBOWRITER)) tchdbsetflag(hdb, HDBFFATAL, true); + if (!notfatal) { + hdb->fatal = true; + if (!INVALIDHANDLE(hdb->fd) && (hdb->omode & HDBOWRITER)) tchdbsetflag(hdb, HDBFFATAL, true); + } break; } case TCESUCCESS: @@ -2223,9 +2229,9 @@ static bool tchdbseekread2(TCHDB *hdb, off_t off, void *buf, size_t size, int op off += rb; } else if (rb == -1) { //if (errno != EINTR) { - tchdbsetecode(hdb, TCEREAD, __FILE__, __LINE__, __func__); - err = true; - break; + tchdbsetecode(hdb, TCEREAD, __FILE__, __LINE__, __func__); + err = true; + break; //} } else { if (size > 0) { diff --git a/tcejdb/tchdb.h b/tcejdb/tchdb.h index 719bc48..460b977 100644 --- a/tcejdb/tchdb.h +++ b/tcejdb/tchdb.h @@ -649,6 +649,7 @@ EJDB_EXPORT uint64_t tchdbfsiz(TCHDB *hdb); `line' specifies the line number of the code. `func' specifies the function name of the code. */ EJDB_EXPORT void tchdbsetecode(TCHDB *hdb, int ecode, const char *filename, int line, const char *func); +EJDB_EXPORT void tchdbsetecode2(TCHDB *hdb, int ecode, const char *filename, int line, const char *func, bool notfatal); /* Set the type of a hash database object. diff --git a/tcejdb/tctdb.c b/tcejdb/tctdb.c index d8d1221..a623d0a 100644 --- a/tcejdb/tctdb.c +++ b/tcejdb/tctdb.c @@ -1187,8 +1187,13 @@ TCLIST *tctdbmetasearch(TDBQRY **qrys, int num, int type) { /* Set the error code of a table database object. */ void tctdbsetecode(TCTDB *tdb, int ecode, const char *filename, int line, const char *func) { + tctdbsetecode2(tdb, ecode, filename, line, func, false); +} + +/* Set the error code of a table database object. */ +void tctdbsetecode2(TCTDB *tdb, int ecode, const char *filename, int line, const char *func, bool notfatal) { assert(tdb && filename && line >= 1 && func); - tchdbsetecode(tdb->hdb, ecode, filename, line, func); + tchdbsetecode2(tdb->hdb, ecode, filename, line, func, notfatal); } /* Set the file descriptor for debugging output. */ diff --git a/tcejdb/tctdb.h b/tcejdb/tctdb.h index 11fc4c5..d62a17e 100644 --- a/tcejdb/tctdb.h +++ b/tcejdb/tctdb.h @@ -797,6 +797,8 @@ EJDB_EXPORT TCLIST *tctdbmetasearch(TDBQRY **qrys, int num, int type); `func' specifies the function name of the code. */ EJDB_EXPORT void tctdbsetecode(TCTDB *tdb, int ecode, const char *filename, int line, const char *func); +EJDB_EXPORT void tctdbsetecode2(TCTDB *tdb, int ecode, const char *filename, int line, const char *func, bool notfatal); + /* Set the file descriptor for debugging output. `tdb' specifies the table database object. diff --git a/tcejdb/testejdb/t4.c b/tcejdb/testejdb/t4.c index 79151ff..02bc8f9 100644 --- a/tcejdb/testejdb/t4.c +++ b/tcejdb/testejdb/t4.c @@ -77,7 +77,7 @@ void testBSONExportImport() { tclistpush2(cnames, "col1"); tclistpush2(cnames, "col2"); - bool rv = ejdbexport(jb, "testBSONExportImport", cnames, 0); + bool rv = ejdbexport(jb, "testBSONExportImport", cnames, 0, log); if (!rv) { eprint(jb, __LINE__, "testBSONExportImport"); } @@ -90,8 +90,14 @@ void testBSONExportImport() { jb = ejdbnew(); CU_ASSERT_TRUE_FATAL(ejdbopen(jb, "dbt4_export", JBOWRITER | JBOCREAT | JBOTRUNC)); + TCXSTR *log = tcxstrnew(); + rv = ejdbimport(jb, "testBSONExportImport", cnames, 0, log); + CU_ASSERT_TRUE(rv); + + tcxstrdel(log); ejdbclose(jb); ejdbdel(jb); + tclistdel(cnames); } -- 2.7.4