From 4ccdd06b80704d8b281a9e4c577cf511040937a0 Mon Sep 17 00:00:00 2001 From: adam Date: Wed, 22 May 2013 20:35:25 +0700 Subject: [PATCH] working on json export --- tcejdb/bson.c | 170 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tcejdb/bson.h | 10 ++++ tcejdb/ejdb.c | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++++ tcejdb/ejdb.h | 19 ++++++- 4 files changed, 354 insertions(+), 1 deletion(-) diff --git a/tcejdb/bson.c b/tcejdb/bson.c index b64a726..7ada9a9 100644 --- a/tcejdb/bson.c +++ b/tcejdb/bson.c @@ -25,6 +25,7 @@ #include "bson.h" #include "encoding.h" #include "myconf.h" +#include "tcutil.h" #ifdef _MYBIGEND #define bson_little_endian64(out, in) ( bson_swap_endian64(out, in) ) @@ -1959,3 +1960,172 @@ int bson_merge_array_sets(const void *mbuf, const void *inbuf, bool pull, bool e } return ctx.ecode; } + +typedef struct { + int nlvl; //nesting level + TCXSTR *out; //output buffer +} _BSON2JSONCTX; + +static void _jsonxstrescaped(TCXSTR *xstr, const char *str) { + size_t sz = strlen(str); + int s = 0; + int e = 0; + char hb[7]; + hb[0] = '\\'; + hb[1] = 'u'; + hb[2] = '0'; + hb[3] = '0'; + hb[6] = '\0'; + while (e < sz) { + const char * ebuf = NULL; + switch (str[e]) { + case '\r': ebuf = "\\r"; + break; + case '\n': ebuf = "\\n"; + break; + case '\\': ebuf = "\\\\"; + break; + case '/': + break; + case '"': ebuf = "\\\""; + break; + case '\f': ebuf = "\\f"; + break; + case '\b': ebuf = "\\b"; + break; + case '\t': ebuf = "\\t"; + break; + default: + if ((unsigned char) str[e] < 0x20) { + static const char *hexchar = "0123456789ABCDEF"; + hb[4] = hexchar[str[e] >> 4]; + hb[5] = hexchar[str[e] & 0x0F]; + ebuf = hb; + } + break; + } + if (ebuf != NULL) { + if (e > s) { + tcxstrcat(xstr, str + s, e - s); + } + tcxstrcat2(xstr, ebuf); + s = ++e; + } else { + ++e; + } + } + tcxstrcat(xstr, (str + s), e - s); +} + +static int _bson2json(_BSON2JSONCTX *ctx, bson_iterator *it) { + +#define BSPAD(_n) \ + for (int i = 0; i < ctx->nlvl + (_n); ++i) tcxstrcat2(ctx->out, " ") + + bson_type bt; + TCXSTR *out = ctx->out; + BSPAD(0); + tcxstrcat2(ctx->out, "{\n"); + ctx->nlvl += 4; + int c = 0; + while ((bt = bson_iterator_next(it)) != BSON_EOO) { + if (c++ > 0) { + tcxstrcat2(out, ",\n"); + } + const char *key = bson_iterator_key(it); + BSPAD(0); + _jsonxstrescaped(out, key); + tcxstrcat2(out, " : "); + switch (bt) { + case BSON_LONG: + case BSON_INT: + tcxstrprintf(out, "%lld", (int64_t) bson_iterator_long_ext(it)); + break; + case BSON_DOUBLE: + { + tcxstrprintf(out, "%llf", bson_iterator_double(it)); + break; + } + case BSON_STRING: + case BSON_SYMBOL: + { + _jsonxstrescaped(out, bson_iterator_string(it)); + break; + } + case BSON_OBJECT: + case BSON_ARRAY: + { + bson_iterator sit; + bson_iterator_subiterator(it, &sit); + _bson2json(ctx, &sit); + } + case BSON_NULL: + tcxstrcat2(out, "null"); + case BSON_UNDEFINED: + break; + case BSON_DATE: + { + bson_date_t t = bson_iterator_date(it); + char dbuf[49]; + tcdatestrwww(t, INT_MAX, dbuf); + tcxstrprintf(out, "%s", dbuf); + break; + } + case BSON_BOOL: + tcxstrcat2(out, bson_iterator_bool(it) ? "true" : "false"); + break; + case BSON_OID: + { + char xoid[25]; + bson_oid_t *oid = bson_iterator_oid(it); + bson_oid_to_string(oid, xoid); + tcxstrprintf(out, "%s", xoid); + break; + } + case BSON_REGEX: + { + tcxstrprintf(out, "%s", bson_iterator_regex(it)); + break; + } + case BSON_BINDATA: + { + const char *buf = bson_iterator_bin_data(it); + int bsz = bson_iterator_bin_len(it); + char *b64data = tcbaseencode(buf, bsz); + tcxstrcat2(out, "\""); + tcxstrcat2(out, b64data); + tcxstrcat2(out, "\""); + TCFREE(b64data); + break; + } + default: + break; + } + } + BSPAD(-4); + tcxstrcat2(out, "\n}\n"); + return 0; +#undef BSPAD +} + +int bson2json(const char *bsdata, char **buf, int *sp) { + assert(bsdata && buf && sp); + bson_iterator it; + bson_iterator_from_buffer(&it, bsdata); + TCXSTR *out = tcxstrnew(); + _BSON2JSONCTX ctx = { + .nlvl = 0, + .out = out + }; + int ret = _bson2json(&ctx, &it); + if (ret == BSON_OK) { + *sp = TCXSTRSIZE(out); + *buf = tcxstrtomalloc(out); + } else { + *sp = 0; + *buf = NULL; + tcxstrclear(out); + } + return ret; +} + diff --git a/tcejdb/bson.h b/tcejdb/bson.h index c5e3778..592696e 100644 --- a/tcejdb/bson.h +++ b/tcejdb/bson.h @@ -1131,5 +1131,15 @@ EJDB_EXPORT bool bson_find_merged_array_sets(const void *mbuf, const void *inbuf EJDB_EXPORT int bson_merge_array_sets(const void *mbuf, const void *inbuf, bool pull, bool expandall, bson *bsout); +/** + * Convert BSON into JSON buffer + * @param src BSON data + * @param buf Allocated buffer with resulting JSON data + * @param sp JSON data length will be stored into + * @return BSON_OK or BSON_ERROR + */ +EJDB_EXPORT int bson2json(const char *bsdata, char **buf, int *sp); + + EJDB_EXTERN_C_END #endif diff --git a/tcejdb/ejdb.c b/tcejdb/ejdb.c index ceaa10a..92dd9de 100644 --- a/tcejdb/ejdb.c +++ b/tcejdb/ejdb.c @@ -49,6 +49,8 @@ /* Default size (16K) of tmp bson buffer on stack for field stripping in _pushstripbson() */ #define JBSBUFFERSZ 16384 +#define JBFILEMODE 00644 // permission of created files + /* string processing/conversion flags */ typedef enum { JBICASE = 1 @@ -74,6 +76,7 @@ typedef struct { } _DEFFEREDIDXCTX; /* private function prototypes */ +static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags); static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func); static bool _ejdbsetmutex(EJDB *ejdb); EJDB_INLINE bool _ejdblockmethod(EJDB *ejdb, bool wr); @@ -150,6 +153,7 @@ const char* ejdberrmsg(int ecode) { case JBEQINCEXCL: return "$fields hint cannot mix include and exclude fields"; case JBEQACTKEY: return "action key in $do block can only be one of: $join"; case JBEMAXNUMCOLS: return "exceeded the maximum number of collections per database: 1024"; + case JBEEI: return "export/import error"; default: return tcerrmsg(ecode); } @@ -776,10 +780,162 @@ bool ejdbtranstatus(EJCOLL *jcoll, bool *txactive) { return true; } +bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags) { + assert(jb && path); + bool err = false; + bool isdir = false; + if (!tcstatfile(path, &isdir, NULL, NULL)) { + return false; + } + if (!isdir) { + if (mkdir(path, 00755)) { + _ejdbsetecode(jb, TCEMKDIR, __FILE__, __LINE__, __func__); + return false; + } + } + TCLIST *_cnames = cnames; + if (_cnames == NULL) { + _cnames = ejdbgetcolls(jb); + if (_cnames == NULL) { + return false; + } + } + for (int i = 0; i < TCLISTNUM(cnames); ++i) { + const char *cn = TCLISTVALPTR(cnames, i); + assert(cn); + EJCOLL *coll = ejdbgetcoll(jb, cn); + if (!coll) continue; + if (!JBCLOCKMETHOD(coll, false)) { + err = true; + goto finish; + } + if (!_exportcoll(coll, path, flags)) { + err = true; + } + JBCUNLOCKMETHOD(coll); + } + +finish: + if (_cnames != cnames) { + tclistdel(_cnames); + } + return !err; +} + /************************************************************************************************* * private features *************************************************************************************************/ +static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags) { + bool err = false; + char *fpath = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, ".bson"); + char *fpathm = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, "-meta.json"); + TCHDB *hdb = coll->tdb->hdb; + TCXSTR *skbuf = tcxstrnew3(sizeof (bson_oid_t) + 1); + TCXSTR *colbuf = tcxstrnew3(1024); + TCXSTR *bsbuf = tcxstrnew3(1024); + int sz = 0; + uint64_t hdbiter; +#ifndef _WIN32 + HANDLE fd = open(fpath, O_RDWR | O_CREAT | O_TRUNC, JBFILEMODE); + HANDLE fdm = open(fpathm, O_RDWR | O_CREAT | O_TRUNC, JBFILEMODE); +#else + HANDLE fd = CreateFile(fpath, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + + HANDLE fdm = CreateFile(fpath, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); +#endif + if (INVALIDHANDLE(fd) || INVALIDHANDLE(fdm)) { + _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + err = true; + goto finish; + } + if (!tchdbiterinit4(hdb, &hdbiter)) { + goto finish; + } + while (!err && tchdbiternext4(hdb, &hdbiter, skbuf, colbuf)) { + sz = tcmaploadoneintoxstr(TCXSTRPTR(colbuf), TCXSTRSIZE(colbuf), JDBCOLBSON, JDBCOLBSONL, bsbuf); + if (sz > 0) { + char *wbuf = NULL; + int wsiz; + if (flags & JBJSONEXPORT) { + if (bson2json(TCXSTRPTR(bsbuf), &wbuf, &wsiz) != BSON_OK) { + _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__); + goto wfinish; + } + } else { + wbuf = TCXSTRPTR(bsbuf); + wsiz = TCXSTRSIZE(bsbuf); + } + if (!tcwrite(fd, wbuf, wsiz)) { + _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + goto wfinish; + } +wfinish: + if (wbuf && wbuf != TCXSTRPTR(bsbuf)) { + TCFREE(wbuf); + } + } + tcxstrclear(skbuf); + tcxstrclear(colbuf); + tcxstrclear(bsbuf); + } + + if (!err) { //export collection meta + TCMAP *cmeta = tctdbget(coll->jb->metadb, coll->cname, coll->cnamesz); + if (!cmeta) { + goto finish; + } + bson mbs; + bson_init(&mbs); + tcmapiterinit(cmeta); + const char *mkey = NULL; + while ((mkey = tcmapiternext2(cmeta)) != NULL) { + if (!mkey || (*mkey != 'i' && strcmp(mkey, "opts"))) { + continue; //allowing only index & opts meta bsons + } + bson *bs = _metagetbson(coll->jb, coll->cname, coll->cnamesz, mkey); + if (bs) { + bson_append_bson(&mbs, mkey, bs); + bson_del(bs); + } + } + bson_finish(&mbs); + char *wbuf = NULL; + int wsiz; + if (bson2json(bson_data(&mbs), &wbuf, &wsiz) != BSON_OK) { + err = true; + _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__); + bson_destroy(&mbs); + goto finish; + } + bson_destroy(&mbs); + if (!tcwrite(fdm, wbuf, wsiz)) { + err = true; + _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + } + TCFREE(wbuf); + } + +finish: + if (!INVALIDHANDLE(fd) && !CLOSEFH(fd)) { + _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + err = true; + } + if (!INVALIDHANDLE(fdm) && !CLOSEFH(fdm)) { + _ejdbsetecode(coll->jb, JBEEI, __FILE__, __LINE__, __func__); + err = true; + } + tcxstrdel(skbuf); + tcxstrdel(colbuf); + tcxstrdel(bsbuf); + TCFREE(fpath); + return !err; +} + /* Set the error code of a table database object. */ static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func) { assert(jb && filename && line >= 1 && func); diff --git a/tcejdb/ejdb.h b/tcejdb/ejdb.h index baeb4e3..fd8ecf5 100644 --- a/tcejdb/ejdb.h +++ b/tcejdb/ejdb.h @@ -56,7 +56,8 @@ enum { /** Error codes */ JBEQONEEMATCH = 9011, /**< Only one $elemMatch allowed in the fieldpath. */ JBEQINCEXCL = 9012, /**< $fields hint cannot mix include and exclude fields */ JBEQACTKEY = 9013, /**< action key in $do block can only be one of: $join */ - JBEMAXNUMCOLS = 9014 /**< Exceeded the maximum number of collections per database */ + JBEMAXNUMCOLS = 9014, /**< Exceeded the maximum number of collections per database */ + JBEEI = 9015 /**< EJDB export/import error */ }; enum { /** Database open modes */ @@ -435,6 +436,22 @@ EJDB_EXPORT bool ejdbtranabort(EJCOLL *coll); /** Get current transaction status, it will be placed into txActive*/ EJDB_EXPORT bool ejdbtranstatus(EJCOLL *jcoll, bool *txactive); + +/** Export settings */ +enum { + JBJSONEXPORT = 1 //If set json collection data will be exported into JSON instead of BSON +}; + +/** + * Export database data to the specified location. Read lock will be taken on each collection + * @param path Directory name in which data will exported. + * @param colnames List of collection names to export. If NULL all collections will be exported. + * @param flags. Reserved for future use. + * @return on sucess `true` + */ +EJDB_EXPORT bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags); + + EJDB_EXTERN_C_END #endif /* EJDB_H */ -- 2.7.4