From: adam Date: Thu, 27 Dec 2012 16:55:14 +0000 (+0700) Subject: Performance enhancements: memory allocs reduced to %20-30% X-Git-Tag: v1.2.12~538 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=e6b6f2fdeae4d0ee37790bef291807b7302df2c9;p=platform%2Fupstream%2Fejdb.git Performance enhancements: memory allocs reduced to %20-30% --- diff --git a/Changelog b/Changelog index 23a4895..535697d 100644 --- a/Changelog +++ b/Changelog @@ -1,8 +1,11 @@ +2012-12-28 Anton Adamansky. + * Performance enhancements, memory allocs reduced to %20-30% + - Release 1.0.37 + 2012-12-27 Anton Adamansky. * Fixed crash when using update qry with single indexed field(ticket: #23) - Release 1.0.36 - 2012-12-26 Anton Adamansky. * Fixed crash on collection remove (ticket: #18) * Incorrect dropping of all field indexes (ticket: #16) diff --git a/package.json b/package.json index 2bd5c0f..a400bd3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name" : "ejdb", - "version" : "1.0.36", + "version" : "1.0.37", "main" : "node/ejdb.js", "homepage" : "http://ejdb.org", "description" : "EJDB - Embedded JSON Database engine", diff --git a/tcejdb/configure b/tcejdb/configure index dfa9616..4b58166 100755 --- a/tcejdb/configure +++ b/tcejdb/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for tcejdb 1.0.36. +# Generated by GNU Autoconf 2.69 for tcejdb 1.0.37. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,8 +577,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='tcejdb' PACKAGE_TARNAME='tcejdb' -PACKAGE_VERSION='1.0.36' -PACKAGE_STRING='tcejdb 1.0.36' +PACKAGE_VERSION='1.0.37' +PACKAGE_STRING='tcejdb 1.0.37' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1257,7 +1257,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures tcejdb 1.0.36 to adapt to many kinds of systems. +\`configure' configures tcejdb 1.0.37 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1318,7 +1318,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of tcejdb 1.0.36:";; + short | recursive ) echo "Configuration of tcejdb 1.0.37:";; esac cat <<\_ACEOF @@ -1424,7 +1424,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -tcejdb configure 1.0.36 +tcejdb configure 1.0.37 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1722,7 +1722,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by tcejdb $as_me 1.0.36, which was +It was created by tcejdb $as_me 1.0.37, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4870,7 +4870,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by tcejdb $as_me 1.0.36, which was +This file was extended by tcejdb $as_me 1.0.37, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -4923,7 +4923,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -tcejdb config.status 1.0.36 +tcejdb config.status 1.0.37 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/tcejdb/configure.ac b/tcejdb/configure.ac index 6a091d7..b0dd933 100644 --- a/tcejdb/configure.ac +++ b/tcejdb/configure.ac @@ -11,7 +11,7 @@ test -n "$CPPFLAGS" && MYCPPFLAGS="$CPPFLAGS $MYCPPFLAGS" test -n "$LDFLAGS" && MYLDFLAGS="$LDFLAGS $MYLDFLAGS" # Package name -AC_INIT(tcejdb, 1.0.36) +AC_INIT(tcejdb, 1.0.37) # Package information MYLIBVER=9 diff --git a/tcejdb/ejdb.c b/tcejdb/ejdb.c index afa93ae..ca4fd28 100644 --- a/tcejdb/ejdb.c +++ b/tcejdb/ejdb.c @@ -97,9 +97,8 @@ static bool _qrycondcheckstrand(const char *vbuf, const TCLIST *tokens); static bool _qrycondcheckstror(const char *vbuf, const TCLIST *tokens); static bool _qrybsvalmatch(const EJQF *qf, bson_iterator *it, bool expandarrays); static bool _qrybsmatch(EJQF *qf, const void *bsbuf, int bsbufsz); -static bool _qryormatch(EJCOLL *jcoll, EJQ *ejq, const void *pkbuf, int pkbufsz, const void *bsbuf, int bsbufsz); -static bool _qryallcondsmatch(EJQ *ejq, int anum, EJCOLL *jcoll, EJQF **qfs, int qfsz, - const void *pkbuf, int pkbufsz, void **bsbuf, int *bsbufsz); +static bool _qryormatch(EJCOLL *jcoll, EJQ *ejq, const void *pkbuf, int pkbufsz); +static bool _qryallcondsmatch(EJQ *ejq, int anum, EJCOLL *jcoll, EJQF **qfs, int qfsz, const void *pkbuf, int pkbufsz); static void _qrydup(const EJQ *src, EJQ *target, uint32_t qflags); static void _qrydel(EJQ *q, bool freequery); static void _pushstripbson(TCLIST *rs, TCMAP *ifields, void *bsbuf, int bsbufsz); @@ -927,6 +926,14 @@ static void _qrydel(EJQ *q, bool freequery) { bson_del(q->hints); q->hints = NULL; } + if (q->colbuf) { + tcxstrdel(q->colbuf); + q->colbuf = NULL; + } + if (q->bsbuf) { + tcxstrdel(q->bsbuf); + q->bsbuf = NULL; + } if (freequery) { TCFREE(q); } @@ -1357,34 +1364,26 @@ static bool _qrybsmatch(EJQF *qf, const void *bsbuf, int bsbufsz) { return _qrybsrecurrmatch(qf, &ffpctx, 0); } -static bool _qryormatch(EJCOLL *jcoll, - EJQ *ejq, - const void *pkbuf, int pkbufsz, - const void *bsbuf, int bsbufsz) { +static bool _qryormatch(EJCOLL *jcoll, EJQ *ejq, const void *pkbuf, int pkbufsz) { if (ejq->orqobjsnum <= 0 || !ejq->orqobjs) { return true; } - bool rv = false; - void *nbsbuf = NULL; - if (bsbuf == NULL) { - int cbufsz; - int lbsbufsz; - char *cbuf = tchdbget(jcoll->tdb->hdb, pkbuf, pkbufsz, &cbufsz); - if (!cbuf) { + void *bsbuf = TCXSTRPTR(ejq->bsbuf); + int bsbufsz = TCXSTRSIZE(ejq->bsbuf); + if (!bsbuf || bsbufsz <= 0) { + tcxstrclear(ejq->colbuf); + tcxstrclear(ejq->bsbuf); + if (tchdbgetintoxstr(jcoll->tdb->hdb, pkbuf, pkbufsz, ejq->colbuf) <= 0) { return false; } - nbsbuf = tcmaploadone(cbuf, cbufsz, JDBCOLBSON, JDBCOLBSONL, &lbsbufsz); //BSON buffer - if (!nbsbuf) { - TCFREE(cbuf); + if (tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf) <= 0) { return false; } - bsbufsz = lbsbufsz; - bsbuf = nbsbuf; - TCFREE(cbuf); + bsbufsz = TCXSTRSIZE(ejq->bsbuf); + bsbuf = TCXSTRPTR(ejq->bsbuf); } if (ejq->lastmatchedorqf && _qrybsmatch(ejq->lastmatchedorqf, bsbuf, bsbufsz)) { - rv = true; - goto finish; + return true; } for (int i = 0; i < ejq->orqobjsnum; ++i) { const EJQ *oq = (ejq->orqobjs + i); @@ -1401,65 +1400,42 @@ static bool _qryormatch(EJCOLL *jcoll, } if (_qrybsmatch(qf, bsbuf, bsbufsz)) { ejq->lastmatchedorqf = qf; - rv = true; - goto finish; + return true; } } } -finish: - if (nbsbuf) { //BSON Buffer created by me - TCFREE(nbsbuf); - } - return rv; + return false; } -/** Caller must TCFREE(*bsbuf) if it return TRUE */ +/** Return true if all main query conditions matched */ static bool _qryallcondsmatch( EJQ *ejq, int anum, EJCOLL *jcoll, EJQF **qfs, int qfsz, - const void *pkbuf, int pkbufsz, - void **bsbuf, int *bsbufsz) { - + const void *pkbuf, int pkbufsz) { + assert(ejq->colbuf && ejq->bsbuf); if (!(ejq->flags & EJQUPDATING) && (ejq->flags & EJQONLYCOUNT) && anum < 1) { - *bsbuf = NULL; - *bsbufsz = 0; return true; } - bool rv = true; - //Load BSON - int cbufsz; - char *cbuf = tchdbget(jcoll->tdb->hdb, pkbuf, pkbufsz, &cbufsz); - if (!cbuf) { - *bsbuf = NULL; - *bsbufsz = 0; + tcxstrclear(ejq->colbuf); + tcxstrclear(ejq->bsbuf); + if (tchdbgetintoxstr(jcoll->tdb->hdb, pkbuf, pkbufsz, ejq->colbuf) <= 0) { return false; } - *bsbuf = tcmaploadone(cbuf, cbufsz, JDBCOLBSON, JDBCOLBSONL, bsbufsz); //BSON buffer - if (!*bsbuf) { - TCFREE(cbuf); - *bsbufsz = 0; + if (tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf) <= 0) { return false; } if (anum < 1) { - TCFREE(cbuf); return true; } for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags; //reset matching flags for (int i = 0; i < qfsz; ++i) { EJQF *qf = qfs[i]; if (qf->mflags & EJFEXCLUDED) continue; - if (!_qrybsmatch(qf, *bsbuf, *bsbufsz)) { - rv = false; - break; + if (!_qrybsmatch(qf, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf))) { + return false; } } - TCFREE(cbuf); - if (!rv) { - TCFREE(*bsbuf); - *bsbuf = NULL; - *bsbufsz = 0; - } - return rv; + return true; } typedef struct { @@ -1563,7 +1539,8 @@ static void _qrydup(const EJQ *src, EJQ *target, uint32_t qflags) { target->skip = src->skip; target->orqobjsnum = src->orqobjsnum; target->orqobjs = NULL; - + target->bsbuf = NULL; + target->colbuf = NULL; if (src->qobjlist) { target->qobjlist = tclistnew2(TCLISTNUM(src->qobjlist)); for (int i = 0; i < TCLISTNUM(src->qobjlist); ++i) { @@ -1662,7 +1639,7 @@ static bson_visitor_cmd_t _bsonstripvisitor(const char *ipath, int ipathlen, con /* push bson into rs with only fields listed in ifields */ static void _pushstripbson(TCLIST *rs, TCMAP *ifields, void *bsbuf, int bsbufsz) { if (!ifields || TCMAPRNUM(ifields) <= 0) { - tclistpushmalloc(rs, bsbuf, bsbufsz); + tclistpush(rs, bsbuf, bsbufsz); return; } char bstack[JBSBUFFERSZ]; @@ -1706,7 +1683,6 @@ static void _pushstripbson(TCLIST *rs, TCMAP *ifields, void *bsbuf, int bsbufsz) if (sdata && sdata != bstack) { TCFREE(sdata); } - TCFREE(bsbuf); } static bool _qryupdate(EJCOLL *jcoll, const EJQ *ejq, void *bsbuf, int bsbufsz, TCLIST *didxctx, TCXSTR *log) { @@ -1920,21 +1896,29 @@ finish: /** Query */ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int qflags, TCXSTR *log) { + *outcount = 0; + //Clone the query object EJQ *ejq; TCMALLOC(ejq, sizeof (*ejq)); _qrydup(q, ejq, EJQINTERNAL); - *outcount = 0; + //Temp tctdb column & bson buffers + assert(!ejq->colbuf); + assert(!ejq->bsbuf); + ejq->colbuf = tcxstrnew3(1024); + ejq->bsbuf = tcxstrnew3(1024); if (qflags & JBQRYCOUNT) { ejq->flags |= EJQONLYCOUNT; - }; + } + bool all = false; //need all records EJQF *mqf = NULL; //main indexed query condition if exists TCMAP *ifields = NULL; //field names included in result set TCLIST *res = (ejq->flags & EJQONLYCOUNT) ? NULL : tclistnew2(4096); _qrypreprocess(jcoll, ejq, qflags, &mqf, &ifields); + int sz = 0; //generic size var int anum = 0; //number of active conditions EJQF **ofs = NULL; //order fields int ofsz = 0; //order fields count @@ -1949,8 +1933,6 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int int kbufsz; const void *vbuf; int vbufsz; - void *bsbuf; //BSON buffer - int bsbufsz; //BSON buffer size static const bool yes = true; //Simple true const uint32_t count = 0; //current count @@ -2045,11 +2027,10 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (ifields) {\ _pushstripbson(res, ifields, (_bsbuf), (_bsbufsz)); \ } else { \ - tclistpushmalloc(res, (_bsbuf), _bsbufsz); \ + tclistpush(res, (_bsbuf), (_bsbufsz)); \ } \ - } else if ((_bsbuf)) { \ - TCFREE((_bsbuf)); \ } + //EOF #define JBQREGREC bool trim = (midx && *midx->name != '\0'); if (anum > 0 && !(mqf->flags & EJFEXCLUDED)) { @@ -2066,13 +2047,14 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int do { bson_oid_t oid; bson_oid_from_string(&oid, mqf->expr); - void *cbuf = tchdbget(jcoll->tdb->hdb, &oid, sizeof (oid), &bsbufsz); - if (!cbuf) { + tcxstrclear(ejq->colbuf); + tcxstrclear(ejq->bsbuf); + sz = tchdbgetintoxstr(jcoll->tdb->hdb, &oid, sizeof (oid), ejq->colbuf); + if (sz <= 0) { break; } - bsbuf = tcmaploadone(cbuf, bsbufsz, JDBCOLBSON, JDBCOLBSONL, &bsbufsz); - if (!bsbuf) { - TCFREE(cbuf); + sz = tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf); + if (sz <= 0) { break; } bool matched = true; @@ -2080,17 +2062,14 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int for (int i = 0; i < qfsz; ++i) { EJQF *qf = qfs[i]; if (qf->mflags & EJFEXCLUDED) continue; - if (!_qrybsmatch(qf, bsbuf, bsbufsz)) { + if (!_qrybsmatch(qf, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf))) { matched = false; break; } } - if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, &oid, sizeof (oid), bsbuf, bsbufsz))) { - JBQREGREC(&oid, sizeof (oid), bsbuf, bsbufsz); - } else { - TCFREE(bsbuf); + if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, &oid, sizeof (oid)))) { + JBQREGREC(&oid, sizeof (oid), TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } - TCFREE(cbuf); } while (false); } else if (mqf->tcop == TDBQCSTROREQ) { TCLIST *tokens = mqf->exprlist; @@ -2104,39 +2083,37 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int } int tnum = TCLISTNUM(tokens); for (int i = 0; (all || count < max) && i < tnum; i++) { + bool matched = true; + bson_oid_t oid; const char *token; int tsiz; TCLISTVAL(token, tokens, i, tsiz); if (tsiz < 1) { continue; } - bson_oid_t oid; bson_oid_from_string(&oid, token); - void *cdata = tchdbget(jcoll->tdb->hdb, &oid, sizeof (oid), &bsbufsz); - if (!cdata) { + tcxstrclear(ejq->bsbuf); + tcxstrclear(ejq->colbuf); + sz = tchdbgetintoxstr(jcoll->tdb->hdb, &oid, sizeof (oid), ejq->colbuf); + if (sz <= 0) { continue; } - bsbuf = tcmaploadone(cdata, bsbufsz, JDBCOLBSON, JDBCOLBSONL, &bsbufsz); - if (!bsbuf) { - TCFREE(cdata); - break; + sz = tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf); + if (sz <= 0) { + continue; } - bool matched = true; for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags; for (int i = 0; i < qfsz; ++i) { EJQF *qf = qfs[i]; if (qf->mflags & EJFEXCLUDED) continue; - if (!_qrybsmatch(qf, bsbuf, bsbufsz)) { + if (!_qrybsmatch(qf, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf))) { matched = false; break; } } - if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, &oid, sizeof (oid), bsbuf, bsbufsz))) { - JBQREGREC(&oid, sizeof (oid), bsbuf, bsbufsz); - } else { - TCFREE(bsbuf); + if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, &oid, sizeof (oid)))) { + JBQREGREC(&oid, sizeof (oid), TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } - TCFREE(cdata); } } else { assert(0); @@ -2151,8 +2128,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) { if (trim) kbufsz -= 3; vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz)) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } if (mqf->order >= 0) { tcbdbcurnext(cur); @@ -2171,9 +2149,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (trim) kbufsz -= 3; if (kbufsz == exprsz && !memcmp(kbuf, expr, exprsz)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2191,9 +2169,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (trim) kbufsz -= 3; if (kbufsz >= exprsz && !memcmp(kbuf, expr, exprsz)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2228,9 +2206,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (trim) kbufsz -= 3; if (kbufsz >= tsiz && !memcmp(kbuf, token, tsiz)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2266,9 +2244,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (trim) kbufsz -= 3; if (kbufsz == tsiz && !memcmp(kbuf, token, tsiz)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2288,9 +2266,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) { if (_nucmp(&num, kbuf, mqf->ftype) == 0) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2315,9 +2293,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (cmp < 0) break; if (cmp > 0 || (mqf->tcop == TDBQCNUMGE && cmp >= 0)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } tcbdbcurprev(cur); @@ -2330,9 +2308,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int int cmp = _nucmp2(&knum, &xnum, mqf->ftype); if (cmp > 0 || (mqf->tcop == TDBQCNUMGE && cmp >= 0)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } tcbdbcurnext(cur); @@ -2356,9 +2334,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int if (cmp > 0) break; if (cmp < 0 || (cmp <= 0 && mqf->tcop == TDBQCNUMLE)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } tcbdbcurnext(cur); @@ -2371,9 +2349,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int int cmp = _nucmp2(&knum, &xnum, mqf->ftype); if (cmp < 0 || (cmp <= 0 && mqf->tcop == TDBQCNUMLE)) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } tcbdbcurprev(cur); @@ -2402,9 +2380,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) { if (tctdbatof(kbuf) > upper) break; vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } tcbdbcurnext(cur); } @@ -2439,9 +2417,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) { if (tctdbatof(kbuf) == xnum) { vbuf = tcbdbcurval3(cur, &vbufsz); - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz, bsbuf, bsbufsz))) { - JBQREGREC(vbuf, vbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, vbuf, vbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, vbuf, vbufsz))) { + JBQREGREC(vbuf, vbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { break; @@ -2476,9 +2454,9 @@ static TCLIST* _qryexecute(EJCOLL *jcoll, const EJQ *q, uint32_t *outcount, int TCMAP *tres = tctdbidxgetbytokens(jcoll->tdb, midx, tokens, mqf->tcop, log); tcmapiterinit(tres); while ((all || count < max) && (kbuf = tcmapiternext(tres, &kbufsz)) != NULL) { - if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, kbuf, kbufsz, &bsbuf, &bsbufsz) && - (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, kbuf, kbufsz, bsbuf, bsbufsz))) { - JBQREGREC(kbuf, kbufsz, bsbuf, bsbufsz); + if (_qryallcondsmatch(ejq, anum, jcoll, qfs, qfsz, kbuf, kbufsz) && + (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, kbuf, kbufsz))) { + JBQREGREC(kbuf, kbufsz, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } tcmapdel(tres); @@ -2503,12 +2481,13 @@ fullscan: /* Full scan */ goto finish; } TCXSTR *skbuf = tcxstrnew3(sizeof (bson_oid_t) + 1); - TCXSTR *scbuf = tcxstrnew3(1024); + tcxstrclear(ejq->colbuf); + tcxstrclear(ejq->bsbuf); int rows = 0; - while ((all || count < max) && tchdbiternext4(hdb, &hdbiter, skbuf, scbuf)) { + while ((all || count < max) && tchdbiternext4(hdb, &hdbiter, skbuf, ejq->colbuf)) { ++rows; - void *bsbuf = tcmaploadone(TCXSTRPTR(scbuf), TCXSTRSIZE(scbuf), JDBCOLBSON, JDBCOLBSONL, &bsbufsz); - if (!bsbuf) { + sz = tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf); + if (sz <= 0) { goto wfinish; } bool matched = true; @@ -2518,33 +2497,29 @@ fullscan: /* Full scan */ if (qf->mflags & EJFEXCLUDED) { continue; } - if (!_qrybsmatch(qf, bsbuf, bsbufsz)) { + if (!_qrybsmatch(qf, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf))) { matched = false; break; } } - if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), bsbuf, bsbufsz))) { + if (matched && (ejq->orqobjsnum == 0 || _qryormatch(jcoll, ejq, TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf)))) { if (updkeys) { //we are in updating mode if (tcmapputkeep(updkeys, TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), &yes, sizeof (yes))) { - JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), bsbuf, bsbufsz); - } else { - TCFREE(bsbuf); + JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } } else { - JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), bsbuf, bsbufsz); + JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf)); } - } else { - TCFREE(bsbuf); } wfinish: tcxstrclear(skbuf); - tcxstrclear(scbuf); + tcxstrclear(ejq->colbuf); + tcxstrclear(ejq->bsbuf); } + tcxstrdel(skbuf); if (updkeys) { tcmapdel(updkeys); } - tcxstrdel(skbuf); - tcxstrdel(scbuf); sorting: /* Sorting resultset */ if (!res || aofsz <= 0) { //No sorting needed diff --git a/tcejdb/ejdb_private.h b/tcejdb/ejdb_private.h index 1cf5cb8..40d4d60 100644 --- a/tcejdb/ejdb_private.h +++ b/tcejdb/ejdb_private.h @@ -104,6 +104,10 @@ struct EJQ { /**> Query object. */ uint32_t max; /**> Max number of results */ uint32_t flags; /**> Control flags */ EJQF *lastmatchedorqf; /**> Reference to the last matched or query field */ + + //Temporal buffers used during query processing + TCXSTR *colbuf; /**> TCTDB current column buffer */ + TCXSTR *bsbuf; /**> current bson object */ }; #define JDBCOLBSON "$" /**> TCDB colname with BSON byte data */ diff --git a/tcejdb/tchdb.c b/tcejdb/tchdb.c index 4f11bea..11a971d 100644 --- a/tcejdb/tchdb.c +++ b/tcejdb/tchdb.c @@ -169,6 +169,8 @@ static char *tchdbgetimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, int *sp); static int tchdbgetintobuf(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, uint8_t hash, char *vbuf, int max); +static int tchdbgetintoxstrimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, uint8_t hash, + TCXSTR *xstr); static char *tchdbgetnextimpl(TCHDB *hdb, const char *kbuf, int ksiz, int *sp, const char **vbp, int *vsp); static int tchdbvsizimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, uint8_t hash); @@ -704,7 +706,7 @@ void *tchdbget(TCHDB *hdb, const void *kbuf, int ksiz, int *sp){ } if(!HDBLOCKRECORD(hdb, bidx, false)){ HDBUNLOCKMETHOD(hdb); - return false; + return NULL; } char *rv = tchdbgetimpl(hdb, kbuf, ksiz, bidx, hash, sp); HDBUNLOCKRECORD(hdb, bidx); @@ -712,6 +714,30 @@ void *tchdbget(TCHDB *hdb, const void *kbuf, int ksiz, int *sp){ return rv; } +int tchdbgetintoxstr(TCHDB *hdb, const void *kbuf, int ksiz, TCXSTR *xstr) { + assert(hdb && kbuf && ksiz >= 0 && xstr); + if (!HDBLOCKMETHOD(hdb, false)) return -1; + uint8_t hash; + uint64_t bidx = tchdbbidx(hdb, kbuf, ksiz, &hash); + if (hdb->fd < 0) { + tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__); + HDBUNLOCKMETHOD(hdb); + return -1; + } + if (hdb->async && !tchdbflushdrp(hdb)) { + HDBUNLOCKMETHOD(hdb); + return -1; + } + if (!HDBLOCKRECORD(hdb, bidx, false)) { + HDBUNLOCKMETHOD(hdb); + return -1; + } + int rv = tchdbgetintoxstrimpl(hdb, kbuf, ksiz, bidx, hash, xstr); + HDBUNLOCKRECORD(hdb, bidx); + HDBUNLOCKMETHOD(hdb); + return rv; + +} /* Retrieve a string record in a hash database object. */ char *tchdbget2(TCHDB *hdb, const char *kstr){ @@ -4097,6 +4123,90 @@ static char *tchdbgetimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, return NULL; } +static int tchdbgetintoxstrimpl(TCHDB *hdb, const char *kbuf, int ksiz, uint64_t bidx, uint8_t hash, TCXSTR *xstr) { + assert(hdb && kbuf && ksiz >= 0 && xstr); + if(hdb->recc){ + int tvsiz; + char *tvbuf = tcmdbget(hdb->recc, kbuf, ksiz, &tvsiz); + if(tvbuf){ + if(*tvbuf == '*'){ + tchdbsetecode(hdb, TCENOREC, __FILE__, __LINE__, __func__); + TCFREE(tvbuf); + return -1; + } + tvsiz = tvsiz - 1; + TCXSTRCAT(xstr, tvbuf + 1, tvsiz); + TCFREE(tvbuf); //todo + return tvsiz; + } + } + off_t off = tchdbgetbucket(hdb, bidx); + TCHREC rec; + char rbuf[HDBIOBUFSIZ]; + while(off > 0){ + rec.off = off; + if(!tchdbreadrec(hdb, &rec, rbuf)) return -1; + if(hash > rec.hash){ + off = rec.left; + } else if(hash < rec.hash){ + off = rec.right; + } else { + if(!rec.kbuf && !tchdbreadrecbody(hdb, &rec)) return -1; + int kcmp = tcreckeycmp(kbuf, ksiz, rec.kbuf, rec.ksiz); + if(kcmp > 0){ + off = rec.left; + TCFREE(rec.bbuf); + rec.kbuf = NULL; + rec.bbuf = NULL; + } else if(kcmp < 0){ + off = rec.right; + TCFREE(rec.bbuf); + rec.kbuf = NULL; + rec.bbuf = NULL; + } else { + if(!rec.vbuf && !tchdbreadrecbody(hdb, &rec)) return -1; + if(hdb->zmode){ + int zsiz; + char *zbuf; + if(hdb->opts & HDBTDEFLATE){ + zbuf = _tc_inflate(rec.vbuf, rec.vsiz, &zsiz, _TCZMRAW); + } else if(hdb->opts & HDBTBZIP){ + zbuf = _tc_bzdecompress(rec.vbuf, rec.vsiz, &zsiz); + } else if(hdb->opts & HDBTTCBS){ + zbuf = tcbsdecode(rec.vbuf, rec.vsiz, &zsiz); + } else { + zbuf = hdb->dec(rec.vbuf, rec.vsiz, &zsiz, hdb->decop); + } + TCFREE(rec.bbuf); + if(!zbuf){ + tchdbsetecode(hdb, TCEMISC, __FILE__, __LINE__, __func__); + return -1; + } + if(hdb->recc){ + if(tcmdbrnum(hdb->recc) >= hdb->rcnum) tchdbcacheadjust(hdb); + tcmdbput4(hdb->recc, kbuf, ksiz, "=", 1, zbuf, zsiz); + } + TCXSTRCAT(xstr, zbuf, zsiz); + TCFREE(zbuf); + return zsiz; + } + if(hdb->recc){ + if(tcmdbrnum(hdb->recc) >= hdb->rcnum) tchdbcacheadjust(hdb); + tcmdbput4(hdb->recc, kbuf, ksiz, "=", 1, rec.vbuf, rec.vsiz); + } + TCXSTRCAT(xstr, rec.vbuf, rec.vsiz); + TCFREE(rec.bbuf); + return rec.vsiz; + } + } + } + if(hdb->recc){ + if(tcmdbrnum(hdb->recc) >= hdb->rcnum) tchdbcacheadjust(hdb); + tcmdbput(hdb->recc, kbuf, ksiz, "*", 1); + } + tchdbsetecode(hdb, TCENOREC, __FILE__, __LINE__, __func__); + return -1; +} /* Retrieve a record in a hash database object and write the value into a buffer. `hdb' specifies the hash database object. diff --git a/tcejdb/tchdb.h b/tcejdb/tchdb.h index b352372..e61772b 100644 --- a/tcejdb/tchdb.h +++ b/tcejdb/tchdb.h @@ -353,6 +353,17 @@ bool tchdbout2(TCHDB *hdb, const char *kstr); void *tchdbget(TCHDB *hdb, const void *kbuf, int ksiz, int *sp); +/** + * Retrieve a record into extensibble string. + * + * @param hdb specifies the hash database object. + * @param kbuf specifies the pointer to the region of the key. + * @param xstr specifies extensible string object data will be stored into. + * @return + */ +int tchdbgetintoxstr(TCHDB *hdb, const void *kbuf, int ksiz, TCXSTR *xstr); + + /* Retrieve a string record in a hash database object. `hdb' specifies the hash database object. `kstr' specifies the string of the key. diff --git a/tcejdb/tcutil.c b/tcejdb/tcutil.c index 627a8fa..e638daa 100644 --- a/tcejdb/tcutil.c +++ b/tcejdb/tcutil.c @@ -2304,6 +2304,30 @@ void *tcmaploadone(const void *ptr, int size, const void *kbuf, int ksiz, int *s return NULL; } +/* Extract a map record from a serialized byte array into extensible string object. */ +int tcmaploadoneintoxstr(const void *ptr, int size, const void *kbuf, int ksiz, TCXSTR *xstr) { + assert(ptr && size >= 0 && kbuf && ksiz >= 0 && xstr); + const char *rp = ptr; + const char *ep = (char *) ptr + size; + while (rp < ep) { + int step, rsiz; + TCREADVNUMBUF(rp, rsiz, step); + rp += step; + if (rsiz == ksiz && !memcmp(kbuf, rp, rsiz)) { + rp += rsiz; + TCREADVNUMBUF(rp, rsiz, step); + rp += step; + TCXSTRCAT(xstr, rp, rsiz); + return rsiz; + } + rp += rsiz; + TCREADVNUMBUF(rp, rsiz, step); + rp += step; + rp += rsiz; + } + return -1; +} + /* Perform formatted output into a map object. */ void tcmapprintf(TCMAP *map, const char *kstr, const char *format, ...){ diff --git a/tcejdb/tcutil.h b/tcejdb/tcutil.h index e663221..dfa3117 100644 --- a/tcejdb/tcutil.h +++ b/tcejdb/tcutil.h @@ -1004,6 +1004,17 @@ const char **tcmapvals2(const TCMAP *map, int *np); void *tcmaploadone(const void *ptr, int size, const void *kbuf, int ksiz, int *sp); +/* Extract a map record from a serialized byte array into extensible string object. + `ptr' specifies the pointer to the region of serialized byte array. + `size' specifies the size of the region. + `kbuf' specifies the pointer to the region of the key. + `ksiz' specifies the size of the region of the key. + `xstr' specifies extensible string object data will be placed into. + If successful, the return value is >= 0 + corresponding record. 0 is returned when no record corresponds.*/ +int tcmaploadoneintoxstr(const void *ptr, int size, const void *kbuf, int ksiz, TCXSTR *xstr); + + /* Perform formatted output into a map object. `map' specifies the map object. `kstr' specifies the string of the key.