1 /**************************************************************************************************
2 * EJDB database library http://ejdb.org
3 * Copyright (C) 2012-2015 Softmotions Ltd <info@softmotions.com>
5 * This file is part of EJDB.
6 * EJDB is free software; you can redistribute it and/or modify it under the terms of
7 * the GNU Lesser General Public License as published by the Free Software Foundation; either
8 * version 2.1 of the License or any later version. EJDB is distributed in the hope
9 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
12 * You should have received a copy of the GNU Lesser General Public License along with EJDB;
13 * if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
14 * Boston, MA 02111-1307 USA.
15 *************************************************************************************************/
19 #include "ejdb_private.h"
23 #define JBLOCKMETHOD(JB_ejdb, JB_wr) \
24 ((JB_ejdb)->mmtx ? _ejdblockmethod((JB_ejdb), (JB_wr)) : true)
25 #define JBUNLOCKMETHOD(JB_ejdb) \
26 ((JB_ejdb)->mmtx ? _ejdbunlockmethod(JB_ejdb) : true)
28 #define JBCLOCKMETHOD(JB_col, JB_wr) \
29 ((JB_col)->mmtx ? _ejcollockmethod((JB_col), (JB_wr)) : true)
30 #define JBCUNLOCKMETHOD(JB_col) \
31 ((JB_col)->mmtx ? _ejcollunlockmethod(JB_col) : true)
33 #define JBISOPEN(JB_jb) ((JB_jb) && (JB_jb)->metadb && (JB_jb)->metadb->open) ? true : false
35 #define JBISVALCOLNAME(JB_cname) ((JB_cname) && \
36 strlen(JB_cname) < JBMAXCOLNAMELEN && \
37 !strchr((JB_cname), '.') && \
38 !strchr((JB_cname), '$'))
40 #define JBENSUREOPENLOCK(JB_jb, JB_lock, JB_ret) \
42 if (!JBLOCKMETHOD((JB_jb), (JB_lock))) return JB_ret; \
43 if (!JBISOPEN(JB_jb)) { \
44 _ejdbsetecode((JB_jb), TCEINVALID, __FILE__, __LINE__, __func__); \
45 JBUNLOCKMETHOD(JB_jb); \
49 /* Default size of stack allocated buffer for string conversions eg. tcicaseformat() */
50 #define JBSTRINOPBUFFERSZ 512
52 /* Default size (16K) of tmp bson buffer on stack for field stripping in _pushstripbson() */
53 #define JBSBUFFERSZ 16384
55 #define JBFILEMODE 00644 // permission of created files
57 /* string processing/conversion flags */
68 /* opaque data for `_bsonipathrowldr()` and `_bsonfpathrowldr()` functions */
70 EJCOLL *coll; //current collection
71 bool icase; //ignore case normalization
75 /* Maximum number of objects keeped to update deffered indexes */
76 #define JBMAXDEFFEREDIDXNUM 512
78 /* context of deffered index updates. See `_updatebsonidx()` */
85 /* query execution context. See `_qryexecute()`*/
87 bool imode; //if true ifields are included otherwise excluded
88 int qflags; //query flags (JBQRYCOUNT|JBQRYFINDONE) passed into `ejdbqryexecute()` method
89 EJCOLL *coll; //collection
90 EJQ *q; //query object
91 EJQF *mqf; //main indexed query condition if exists
92 TCMAP *ifields; //include/exclude $fields
93 TCMAP *dfields; //$do fields
94 TCLIST *res; //result set
95 TCXSTR *log; //query debug log buffer
96 TCLIST *didxctx; //deffered indexes context
100 /* private function prototypes */
101 static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func);
102 static void _ejdbsetecode2(EJDB *jb, int ecode, const char *filename, int line, const char *func, bool notfatal);
103 static bool _ejdbsetmutex(EJDB *ejdb);
104 EJDB_INLINE bool _ejdblockmethod(EJDB *ejdb, bool wr);
105 EJDB_INLINE bool _ejdbunlockmethod(EJDB *ejdb);
106 EJDB_INLINE bool _ejdbcolsetmutex(EJCOLL *coll);
107 EJDB_INLINE bool _ejcollockmethod(EJCOLL *coll, bool wr);
108 EJDB_INLINE bool _ejcollunlockmethod(EJCOLL *coll);
109 static bson_type _bsonoidkey(bson *bs, bson_oid_t *oid);
110 static char* _bsonitstrval(EJDB *jb, bson_iterator *it, int *vsz, TCLIST *tokens, txtflags_t flags);
111 static char* _bsonipathrowldr(TCLIST *tokens, const char *pkbuf, int pksz, const char *rowdata, int rowdatasz,
112 const char *ipath, int ipathsz, void *op, int *vsz);
113 static char* _bsonfpathrowldr(TCLIST *tokens, const char *rowdata, int rowdatasz,
114 const char *fpath, int fpathsz, void *op, int *vsz);
115 static bool _createcoldb(const char *colname, EJDB *jb, EJCOLLOPTS *opts, TCTDB** res);
116 static bool _addcoldb0(const char *colname, EJDB *jb, EJCOLLOPTS *opts, EJCOLL **res);
117 static void _delcoldb(EJCOLL *cdb);
118 static void _delqfdata(const EJQ *q, const EJQF *ejqf);
119 static bool _ejdbsavebsonimpl(EJCOLL *coll, bson *bs, bson_oid_t *oid, bool merge);
120 static bool _updatebsonidx(EJCOLL *coll, const bson_oid_t *oid, const bson *bs,
121 const void *obsdata, int obsdatasz, TCLIST *dlist);
122 static bool _metasetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts);
123 static bool _metagetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts);
124 static bson* _metagetbson(EJDB *jb, const char *colname, int colnamesz, const char *mkey);
125 static bson* _metagetbson2(EJCOLL *coll, const char *mkey) __attribute__((unused));
126 static bool _metasetbson(EJDB *jb, const char *colname, int colnamesz,
127 const char *mkey, bson *val, bool merge, bool mergeoverwrt);
128 static bool _metasetbson2(EJCOLL *coll, const char *mkey, bson *val, bool merge, bool mergeoverwrt);
129 static bson* _imetaidx(EJCOLL *coll, const char *ipath);
130 static bool _qrypreprocess(_QRYCTX *ctx);
131 static TCLIST* _parseqobj(EJDB *jb, EJQ *q, bson *qspec);
132 static TCLIST* _parseqobj2(EJDB *jb, EJQ *q, const void *qspecbsdata);
133 static int _parse_qobj_impl(EJDB *jb, EJQ *q, bson_iterator *it, TCLIST *qlist, TCLIST *pathStack, EJQF *pqf, int mgrp);
134 static int _ejdbsoncmp(const TCLISTDATUM *d1, const TCLISTDATUM *d2, void *opaque);
135 static bool _qrycondcheckstrand(const char *vbuf, const TCLIST *tokens);
136 static bool _qrycondcheckstror(const char *vbuf, const TCLIST *tokens);
137 static bool _qrybsvalmatch(const EJQF *qf, bson_iterator *it, bool expandarrays, int *arridx);
138 static bool _qrybsmatch(EJQF *qf, const void *bsbuf, int bsbufsz);
139 static bool _qry_and_or_match(EJCOLL *coll, EJQ *ejq, const void *pkbuf, int pkbufsz);
140 static bool _qryormatch2(EJCOLL *coll, EJQ *ejq, const void *bsbuf, int bsbufsz);
141 static bool _qryormatch3(EJCOLL *coll, EJQ *ejq, EJQ *oq, const void *bsbuf, int bsbufsz);
142 static bool _qryandmatch2(EJCOLL *coll, EJQ *ejq, const void *bsbuf, int bsbufsz);
143 static bool _qryallcondsmatch(EJQ *ejq, int anum, EJCOLL *coll, EJQF **qfs, int qfsz, const void *pkbuf, int pkbufsz);
144 static EJQ* _qryaddand(EJDB *jb, EJQ *q, const void *andbsdata);
145 static bool _qrydup(const EJQ *src, EJQ *target, uint32_t qflags);
146 static void _qrydel(EJQ *q, bool freequery);
147 static bool _pushprocessedbson(_QRYCTX *ctx, const void *bsbuf, int bsbufsz);
148 static bool _exec_do(_QRYCTX *ctx, const void *bsbuf, bson *bsout);
149 static void _qryctxclear(_QRYCTX *ctx);
150 static TCLIST* _qryexecute(EJCOLL *coll, const EJQ *q, uint32_t *count, int qflags, TCXSTR *log);
151 EJDB_INLINE void _nufetch(_EJDBNUM *nu, const char *sval, bson_type bt);
152 EJDB_INLINE int _nucmp(_EJDBNUM *nu, const char *sval, bson_type bt);
153 EJDB_INLINE int _nucmp2(_EJDBNUM *nu1, _EJDBNUM *nu2, bson_type bt);
154 static EJCOLL* _getcoll(EJDB *jb, const char *colname);
155 static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags, TCXSTR *log);
156 static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags, TCXSTR *log);
157 static EJCOLL* _createcollimpl(EJDB *jb, const char *colname, EJCOLLOPTS *opts);
158 static bool _rmcollimpl(EJDB *jb, EJCOLL *coll, bool unlinkfile);
159 static bool _setindeximpl(EJCOLL *coll, const char *fpath, int flags, bool nolock);
161 extern const char *utf8proc_errmsg(ssize_t errcode);
163 static const bool yes = true;
165 const char *ejdbversion() {
169 const char* ejdberrmsg(int ecode) {
170 if (ecode > -6 && ecode < 0) { //Hook for negative error codes of utf8proc library
171 return utf8proc_errmsg(ecode);
174 case JBEINVALIDCOLNAME:
175 return "invalid collection name";
177 return "invalid bson object";
178 case JBEQINVALIDQCONTROL:
179 return "invalid query control field starting with '$'";
180 case JBEQINOPNOTARRAY:
181 return "$strand, $stror, $in, $nin, $bt keys require not empty array value";
183 return "inconsistent database metadata";
184 case JBEFPATHINVALID:
185 return "invalid JSEJDB_EXPORT const char *ejdbversion();ON field path value";
187 return "invalid query regexp value";
189 return "result set sorting error";
191 return "invalid query";
193 return "bson record update failed";
194 case JBEINVALIDBSONPK:
195 return "invalid bson _id field";
197 return "only one $elemMatch allowed in the fieldpath";
199 return "$fields hint cannot mix include and exclude fields";
201 return "action key in $do block can be one of: $join, $slice";
203 return "exceeded the maximum number of collections per database: 1024";
205 return "JSON parsing failed";
207 return "data export/import failed";
209 return "bson size exceeds the maximum allowed size limit";
211 return "invalid ejdb command specified";
213 return tcerrmsg(ecode);
217 bool ejdbisvalidoidstr(const char *oid) {
222 for (; oid[i] != '\0' &&
223 ((oid[i] >= 0x30 && oid[i] <= 0x39) || /* 1 - 9 */
224 (oid[i] >= 0x61 && oid[i] <= 0x66)); ++i); /* a - f */
228 /* Get the last happened error code of a database object. */
229 int ejdbecode(EJDB *jb) {
230 assert(jb && jb->metadb);
231 return tctdbecode(jb->metadb);
234 EJDB* ejdbnew(void) {
236 TCCALLOC(jb, 1, sizeof (*jb));
237 jb->metadb = tctdbnew();
238 tctdbsetmutex(jb->metadb);
239 tctdbsetcache(jb->metadb, 1024, 0, 0);
240 if (!_ejdbsetmutex(jb)) {
241 tctdbdel(jb->metadb);
248 void ejdbdel(EJDB *jb) {
249 assert(jb && jb->metadb);
250 if (JBISOPEN(jb)) ejdbclose(jb);
251 for (int i = 0; i < jb->cdbsnum; ++i) {
253 _delcoldb(jb->cdbs[i]);
259 pthread_rwlock_destroy(jb->mmtx);
262 tctdbdel(jb->metadb);
266 bool ejdbclose(EJDB *jb) {
267 JBENSUREOPENLOCK(jb, true, false);
269 for (int i = 0; i < jb->cdbsnum; ++i) {
271 JBCLOCKMETHOD(jb->cdbs[i], true);
272 if (!tctdbclose(jb->cdbs[i]->tdb)) {
275 JBCUNLOCKMETHOD(jb->cdbs[i]);
277 if (!tctdbclose(jb->metadb)) {
284 bool ejdbisopen(EJDB *jb) {
288 bool ejdbopen(EJDB *jb, const char *path, int mode) {
289 assert(jb && path && jb->metadb);
290 if (!JBLOCKMETHOD(jb, true)) return false;
292 _ejdbsetecode(jb, TCEINVALID, __FILE__, __LINE__, __func__);
296 bool rv = tctdbopen(jb->metadb, path, mode);
301 TCTDB *mdb = jb->metadb;
302 rv = tctdbiterinit(mdb);
306 char *colname = NULL;
307 for (int i = 0; i < mdb->hdb->rnum && (colname = tctdbiternext2(mdb)) != NULL; ++i) {
310 _metagetopts(jb, colname, &opts);
311 _addcoldb0(colname, jb, &opts, &cdb);
319 EJCOLL* ejdbgetcoll(EJDB *jb, const char *colname) {
322 JBENSUREOPENLOCK(jb, false, NULL);
323 coll = _getcoll(jb, colname);
328 TCLIST* ejdbgetcolls(EJDB *jb) {
331 JBENSUREOPENLOCK(jb, false, NULL);
332 TCLIST *ret = tclistnew2(jb->cdbsnum);
333 for (int i = 0; i < jb->cdbsnum; ++i) {
335 TCLISTPUSH(ret, coll, sizeof (*coll));
341 EJCOLL* ejdbcreatecoll(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
343 EJCOLL *coll = ejdbgetcoll(jb, colname);
347 JBENSUREOPENLOCK(jb, true, NULL);
348 coll = _createcollimpl(jb, colname, opts);
353 bool ejdbrmcoll(EJDB *jb, const char *colname, bool unlinkfile) {
355 JBENSUREOPENLOCK(jb, true, false);
357 EJCOLL *coll = _getcoll(jb, colname);
361 if (!JBCLOCKMETHOD(coll, true)) return false;
362 rv = _rmcollimpl(jb, coll, unlinkfile);
363 JBCUNLOCKMETHOD(coll);
371 /* Save/Update BSON */
372 bool ejdbsavebson(EJCOLL *coll, bson *bs, bson_oid_t *oid) {
373 return ejdbsavebson2(coll, bs, oid, false);
376 bool ejdbsavebson2(EJCOLL *coll, bson *bs, bson_oid_t *oid, bool merge) {
378 if (!bs || bs->err || !bs->finished) {
379 _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
382 if (!JBISOPEN(coll->jb)) {
383 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
386 if (!JBCLOCKMETHOD(coll, true)) return false;
387 bool rv = _ejdbsavebsonimpl(coll, bs, oid, merge);
388 JBCUNLOCKMETHOD(coll);
392 bool ejdbsavebson3(EJCOLL *coll, const void *bsdata, bson_oid_t *oid, bool merge) {
394 bson_init_with_data(&bs, bsdata);
395 return ejdbsavebson2(coll, &bs, oid, merge);
398 bool ejdbrmbson(EJCOLL *coll, bson_oid_t *oid) {
400 if (!JBISOPEN(coll->jb)) {
401 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
404 JBCLOCKMETHOD(coll, true);
408 TCMAP *rmap = tctdbget(coll->tdb, oid, sizeof (*oid));
412 olddata = tcmapget3(rmap, JDBCOLBSON, JDBCOLBSONL, &olddatasz);
413 if (!_updatebsonidx(coll, oid, NULL, olddata, olddatasz, NULL) ||
414 !tctdbout(coll->tdb, oid, sizeof (*oid))) {
418 JBCUNLOCKMETHOD(coll);
425 bson* ejdbloadbson(EJCOLL *coll, const bson_oid_t *oid) {
427 if (!JBISOPEN(coll->jb)) {
428 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
431 JBCLOCKMETHOD(coll, false);
434 void *cdata = tchdbget(coll->tdb->hdb, oid, sizeof (*oid), &datasz);
438 void *bsdata = tcmaploadone(cdata, datasz, JDBCOLBSON, JDBCOLBSONL, &datasz);
447 bson_init_finished_data(ret, bsdata);
449 JBCUNLOCKMETHOD(coll);
456 EJQ* ejdbcreatequery(EJDB *jb, bson *qobj, bson *orqobjs, int orqobjsnum, bson *hints) {
458 if (!qobj || qobj->err || !qobj->finished) {
459 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
463 TCCALLOC(q, 1, sizeof (*q));
465 q->qflist = _parseqobj(jb, q, qobj);
470 if (orqobjs && orqobjsnum > 0) {
471 for (int i = 0; i < orqobjsnum; ++i) {
472 bson *oqb = (orqobjs + i);
474 if (ejdbqueryaddor(jb, q, bson_data(oqb)) == NULL) {
480 if (hints->err || !hints->finished) {
481 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
484 q->hints = bson_create();
485 if (bson_copy(q->hints, hints)) {
495 EJQ* ejdbcreatequery2(EJDB *jb, const void *qbsdata) {
498 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
502 TCCALLOC(q, 1, sizeof (*q));
503 q->qflist = _parseqobj2(jb, q, qbsdata);
513 EJQ* ejdbqueryaddor(EJDB *jb, EJQ *q, const void *orbsdata) {
514 assert(jb && q && orbsdata);
516 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
519 EJQ *oq = ejdbcreatequery2(jb, orbsdata);
523 if (q->orqlist == NULL) {
524 q->orqlist = tclistnew2(TCLISTINYNUM);
526 TCLISTPUSH(q->orqlist, &oq, sizeof(oq));
530 EJQ* ejdbqueryhints(EJDB *jb, EJQ *q, const void *hintsbsdata) {
533 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
537 BSON_ITERATOR_FROM_BUFFER(&it, hintsbsdata);
538 bson *bs = bson_create_from_iterator(&it);
541 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
549 tcmapdel(q->ifields);
556 void ejdbquerydel(EJQ *q) {
561 bool ejdbsetindex(EJCOLL *coll, const char *fpath, int flags) {
562 return _setindeximpl(coll, fpath, flags, false);
565 uint32_t ejdbupdate(EJCOLL *coll, bson *qobj, bson *orqobjs, int orqobjsnum, bson *hints, TCXSTR *log) {
568 EJQ *q = ejdbcreatequery(coll->jb, qobj, orqobjs, orqobjsnum, hints);
572 ejdbqryexecute(coll, q, &count, JBQRYCOUNT, log);
577 EJQRESULT ejdbqryexecute(EJCOLL *coll, const EJQ *q, uint32_t *count, int qflags, TCXSTR *log) {
578 assert(coll && q && q->qflist);
579 if (!JBISOPEN(coll->jb)) {
580 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
583 JBCLOCKMETHOD(coll, (q->flags & EJQUPDATING) ? true : false);
584 _ejdbsetecode(coll->jb, TCESUCCESS, __FILE__, __LINE__, __func__);
585 if (ejdbecode(coll->jb) != TCESUCCESS) { //we are not in fatal state
586 JBCUNLOCKMETHOD(coll);
589 TCLIST *res = _qryexecute(coll, q, count, qflags, log);
590 JBCUNLOCKMETHOD(coll);
594 bson* ejdbqrydistinct(EJCOLL *coll, const char *fpath, bson *qobj, bson *orqobjs, int orqobjsnum, uint32_t *count, TCXSTR *log) {
600 bson_init_as_query(&hints);
601 bson_append_start_object(&hints, "$fields");
602 bson_append_int(&hints, fpath, 1);
603 bson_append_finish_object(&hints);
604 bson_append_start_object(&hints, "$orderby");
605 bson_append_int(&hints, fpath, 1);
606 bson_append_finish_object(&hints);
614 rqobj = bson_create();
618 q = ejdbcreatequery(coll->jb, rqobj, orqobjs, orqobjsnum, &hints);
622 if (q->flags & EJQUPDATING) {
623 _ejdbsetecode(coll->jb, JBEQERROR, __FILE__, __LINE__, __func__);
626 TCLIST *res = ejdbqryexecute(coll, q, &icount, 0, log);
627 rres = bson_create();
630 bson_iterator bsi[2];
631 bson_iterator *prev = NULL, *cur;
633 char biindstr[TCNUMBUFSIZ];
634 memset(biindstr, '\0', 32);
635 int fplen = strlen(fpath);
636 for(int i = 0; i < TCLISTNUM(res); ++i) {
637 cur = bsi + (biind & 1);
638 BSON_ITERATOR_FROM_BUFFER(cur, TCLISTVALPTR(res, i));
639 bson_type bt = bson_find_fieldpath_value2(fpath, fplen, cur);
640 if (bt == BSON_EOO) {
644 if (prev == NULL || bson_compare_it_current(prev, cur)) {
645 bson_numstrn(biindstr, TCNUMBUFSIZ, biind);
646 bson_append_field_from_iterator2(biindstr, cur, rres);
663 bson_destroy(&hints);
667 int ejdbqresultnum(EJQRESULT qr) {
668 return qr ? tclistnum(qr) : 0;
671 const void* ejdbqresultbsondata(EJQRESULT qr, int pos, int *size) {
672 if (!qr || pos < 0) {
676 const void *bsdata = tclistval2(qr, pos);
677 *size = (bsdata != NULL) ? bson_size2(bsdata) : 0;
681 void ejdbqresultdispose(EJQRESULT qr) {
687 bool ejdbsyncoll(EJCOLL *coll) {
689 if (!JBISOPEN(coll->jb)) {
690 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
694 if (!JBCLOCKMETHOD(coll, true)) return false;
695 rv = tctdbsync(coll->tdb);
696 JBCUNLOCKMETHOD(coll);
700 bool ejdbsyncdb(EJDB *jb) {
702 JBENSUREOPENLOCK(jb, true, false);
704 for (int i = 0; i < jb->cdbsnum; ++i) {
706 rv = JBCLOCKMETHOD(jb->cdbs[i], true);
708 rv = tctdbsync(jb->cdbs[i]->tdb);
709 JBCUNLOCKMETHOD(jb->cdbs[i]);
716 bool ejdbtranbegin(EJCOLL *coll) {
718 if (!JBISOPEN(coll->jb)) {
719 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
722 for (double wsec = 1.0 / sysconf_SC_CLK_TCK; true; wsec *= 2) {
723 if (!JBCLOCKMETHOD(coll, true)) return false;
724 if (!coll->tdb->open || !coll->tdb->wmode) {
725 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
726 JBCUNLOCKMETHOD(coll);
729 if (!coll->tdb->tran) break;
730 JBCUNLOCKMETHOD(coll);
731 if (wsec > 1.0) wsec = 1.0;
734 if (!tctdbtranbeginimpl(coll->tdb)) {
735 JBCUNLOCKMETHOD(coll);
738 coll->tdb->tran = true;
739 JBCUNLOCKMETHOD(coll);
743 bool ejdbtrancommit(EJCOLL *coll) {
745 if (!JBISOPEN(coll->jb)) {
746 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
749 if (!JBCLOCKMETHOD(coll, true)) return false;
750 if (!coll->tdb->open || !coll->tdb->wmode || !coll->tdb->tran) {
751 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
752 JBCUNLOCKMETHOD(coll);
755 coll->tdb->tran = false;
757 if (!tctdbtrancommitimpl(coll->tdb)) err = true;
758 JBCUNLOCKMETHOD(coll);
762 bool ejdbtranabort(EJCOLL *coll) {
764 if (!JBISOPEN(coll->jb)) {
765 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
768 if (!JBCLOCKMETHOD(coll, true)) return false;
769 if (!coll->tdb->open || !coll->tdb->wmode || !coll->tdb->tran) {
770 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
771 JBCUNLOCKMETHOD(coll);
774 coll->tdb->tran = false;
776 if (!tctdbtranabortimpl(coll->tdb)) err = true;
777 JBCUNLOCKMETHOD(coll);
781 bool ejdbtranstatus(EJCOLL *coll, bool *txactive) {
782 assert(coll && txactive);
783 if (!JBISOPEN(coll->jb)) {
784 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
787 if (!JBCLOCKMETHOD(coll, true)) return false;
788 *txactive = coll->tdb->tran;
789 JBCUNLOCKMETHOD(coll);
793 static int _cmpcolls(const TCLISTDATUM *d1, const TCLISTDATUM *d2) {
794 EJCOLL *c1 = (EJCOLL*) d1->ptr;
795 EJCOLL *c2 = (EJCOLL*) d2->ptr;
796 return memcmp(c1->cname, c2->cname, MIN(c1->cnamesz, c2->cnamesz));
799 bson* ejdbmeta(EJDB *jb) {
800 JBENSUREOPENLOCK(jb, false, NULL);
801 char nbuff[TCNUMBUFSIZ];
802 bson *bs = bson_create();
804 bson_append_string(bs, "file", jb->metadb->hdb->path);
805 bson_append_start_array(bs, "collections"); //collections
807 TCLIST *cols = ejdbgetcolls(jb);
808 tclistsortex(cols, _cmpcolls);
810 for (int i = 0; i < TCLISTNUM(cols); ++i) {
811 EJCOLL *coll = (EJCOLL*) TCLISTVALPTR(cols, i);
812 if (!JBCLOCKMETHOD(coll, false)) {
818 bson_numstrn(nbuff, TCNUMBUFSIZ, i);
819 bson_append_start_object(bs, nbuff); //coll obj
820 bson_append_string_n(bs, "name", coll->cname, coll->cnamesz);
821 bson_append_string(bs, "file", coll->tdb->hdb->path);
822 bson_append_long(bs, "records", coll->tdb->hdb->rnum);
824 bson_append_start_object(bs, "options"); //coll.options
825 bson_append_long(bs, "buckets", coll->tdb->hdb->bnum);
826 bson_append_long(bs, "cachedrecords", coll->tdb->hdb->rcnum);
827 bson_append_bool(bs, "large", (coll->tdb->opts & TDBTLARGE));
828 bson_append_bool(bs, "compressed", (coll->tdb->opts & TDBTDEFLATE));
829 bson_append_finish_object(bs); //eof coll.options
831 bson_append_start_array(bs, "indexes"); //coll.indexes[]
832 for (int j = 0; j < coll->tdb->inum; ++j) {
833 TDBIDX *idx = (coll->tdb->idxs + j);
834 if (idx->type != TDBITLEXICAL &&
835 idx->type != TDBITDECIMAL &&
836 idx->type != TDBITTOKEN) {
839 bson_numstrn(nbuff, TCNUMBUFSIZ, j);
840 bson_append_start_object(bs, nbuff); //coll.indexes.index
841 bson_append_string(bs, "field", idx->name + 1);
842 bson_append_string(bs, "iname", idx->name);
845 bson_append_string(bs, "type", "lexical");
848 bson_append_string(bs, "type", "decimal");
851 bson_append_string(bs, "type", "token");
854 TCBDB *idb = (TCBDB*) idx->db;
856 bson_append_long(bs, "records", idb->rnum);
857 bson_append_string(bs, "file", idb->hdb->path);
859 bson_append_finish_object(bs); //eof coll.indexes.index
861 bson_append_finish_array(bs); //eof coll.indexes[]
862 bson_append_finish_object(bs); //eof coll
863 JBCUNLOCKMETHOD(coll);
865 bson_append_finish_array(bs); //eof collections
870 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
877 bool ejdbexport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log) {
881 tcstatfile(path, &isdir, NULL, NULL);
883 if (mkdir(path, 00755)) {
884 _ejdbsetecode2(jb, TCEMKDIR, __FILE__, __LINE__, __func__, true);
888 JBENSUREOPENLOCK(jb, false, false);
889 TCLIST *_cnames = cnames;
890 if (_cnames == NULL) {
891 _cnames = tclistnew2(jb->cdbsnum);
892 for (int i = 0; i < jb->cdbsnum; ++i) {
893 EJCOLL *c = jb->cdbs[i];
894 TCLISTPUSH(_cnames, c->cname, c->cnamesz);
897 for (int i = 0; i < TCLISTNUM(_cnames); ++i) {
898 const char *cn = TCLISTVALPTR(_cnames, i);
900 EJCOLL *coll = _getcoll(jb, cn);
902 if (!JBCLOCKMETHOD(coll, false)) {
906 if (!_exportcoll(coll, path, flags, log)) {
909 JBCUNLOCKMETHOD(coll);
913 if (_cnames != cnames) {
919 bool ejdbimport(EJDB *jb, const char *path, TCLIST *cnames, int flags, TCXSTR *log) {
924 flags |= JBIMPORTUPDATE;
926 if (!tcstatfile(path, &isdir, NULL, NULL) || !isdir) {
927 _ejdbsetecode2(jb, TCENOFILE, __FILE__, __LINE__, __func__, true);
930 bool tail = (path[0] != '\0' && path[strlen(path) - 1] == MYPATHCHR);
931 char *bsonpat = tail ? tcsprintf("%s*.bson") : tcsprintf("%s%c*.bson", path, MYPATHCHR);
932 TCLIST *bspaths = tcglobpat(bsonpat);
933 JBENSUREOPENLOCK(jb, true, false);
934 for (int i = 0; i < TCLISTNUM(bspaths); ++i) {
935 const char* bspath = TCLISTVALPTR(bspaths, i);
936 if (!_importcoll(jb, bspath, cnames, flags, log)) {
952 bson* ejdbcommand2(EJDB *jb, void *cmdbsondata) {
954 bson_init_with_data(&cmd, cmdbsondata);
955 bson *bret = ejdbcommand(jb, &cmd);
959 bson* ejdbcommand(EJDB *jb, bson *cmd) {
960 bson *ret = bson_create();
962 const char *err = NULL;
964 TCLIST *cnames = NULL;
970 BSON_ITERATOR_INIT(&it, cmd);
972 while ((bt = bson_iterator_next(&it)) != BSON_EOO) {
973 const char *key = BSON_ITERATOR_KEY(&it);
974 if (!strcmp("export", key) || !strcmp("import", key)) {
979 BSON_ITERATOR_SUBITERATOR(&it, &sit);
980 if (bson_find_fieldpath_value("path", &sit) == BSON_STRING) {
981 path = strdup(bson_iterator_string(&sit));
983 BSON_ITERATOR_SUBITERATOR(&it, &sit);
984 if (bson_find_fieldpath_value("mode", &sit) == BSON_INT) {
985 flags = bson_iterator_int(&sit);
987 BSON_ITERATOR_SUBITERATOR(&it, &sit);
988 if (bson_find_fieldpath_value("cnames", &sit) == BSON_ARRAY) {
990 BSON_ITERATOR_SUBITERATOR(&sit, &ait);
991 while ((bt = bson_iterator_next(&ait)) != BSON_EOO) {
992 if (bt == BSON_STRING) {
993 if (cnames == NULL) {
994 cnames = tclistnew();
996 const char *sv = bson_iterator_string(&ait);
997 TCLISTPUSH(cnames, sv, strlen(sv));
1002 err = "Missing required 'path' field";
1003 ecode = JBEINVALIDCMD;
1006 if (!strcmp("export", key)) {
1007 rv = ejdbexport(jb, path, cnames, flags, xlog);
1009 rv = ejdbimport(jb, path, cnames, flags, xlog);
1012 ecode = ejdbecode(jb);
1013 err = ejdberrmsg(ecode);
1016 } else if (!strcmp("ping", key)) {
1018 tcxstrprintf(xlog, "pong");
1020 err = "Unknown command";
1021 ecode = JBEINVALIDCMD;
1027 bson_append_string(ret, "error", err);
1028 bson_append_int(ret, "errorCode", ecode);
1031 bson_append_string(ret, "log", TCXSTRPTR(xlog));
1041 /*************************************************************************************************
1043 *************************************************************************************************/
1045 static bool _setindeximpl(EJCOLL *coll, const char *fpath, int flags, bool nolock) {
1046 assert(coll && fpath);
1050 int tcitype = 0; //TCDB index type
1051 int oldiflags = 0; //Old index flags stored in meta
1052 bool ibld = (flags & JBIDXREBLD);
1054 flags &= ~JBIDXREBLD;
1056 bool idrop = (flags & JBIDXDROP);
1058 flags &= ~JBIDXDROP;
1060 bool idropall = (flags & JBIDXDROPALL);
1063 flags &= ~JBIDXDROPALL;
1065 bool iop = (flags & JBIDXOP);
1069 char ipath[BSON_MAX_FPATH_LEN + 2]; //add 2 bytes for one char prefix and '\0'term
1070 char ikey[BSON_MAX_FPATH_LEN + 2]; //add 2 bytes for one char prefix and '\0'term
1071 int fpathlen = strlen(fpath);
1072 if (fpathlen > BSON_MAX_FPATH_LEN) {
1073 _ejdbsetecode(coll->jb, JBEFPATHINVALID, __FILE__, __LINE__, __func__);
1077 memmove(ikey + 1, fpath, fpathlen + 1);
1079 memmove(ipath + 1, fpath, fpathlen + 1);
1080 ipath[0] = 's'; //defaulting to string index type
1083 JBENSUREOPENLOCK(coll->jb, true, false);
1085 imeta = _imetaidx(coll, fpath);
1087 if (idrop) { //Cannot drop/optimize not existent index;
1089 JBUNLOCKMETHOD(coll->jb);
1094 iop = false; //New index will be created
1096 imeta = bson_create();
1098 bson_append_string(imeta, "ipath", fpath);
1099 bson_append_int(imeta, "iflags", flags);
1101 rv = _metasetbson2(coll, ikey, imeta, false, false);
1104 JBUNLOCKMETHOD(coll->jb);
1109 if (bson_find(&it, imeta, "iflags") != BSON_EOO) {
1110 oldiflags = bson_iterator_int(&it);
1112 if (!idrop && oldiflags != flags) { //Update index meta
1114 bson_init(&imetadelta);
1115 bson_append_int(&imetadelta, "iflags", (flags | oldiflags));
1116 bson_finish(&imetadelta);
1117 rv = _metasetbson2(coll, ikey, &imetadelta, true, true);
1118 bson_destroy(&imetadelta);
1121 JBUNLOCKMETHOD(coll->jb);
1128 JBUNLOCKMETHOD(coll->jb);
1131 tcitype = TDBITVOID;
1132 if (idropall && oldiflags) {
1133 flags = oldiflags; //Drop index only for existing types
1138 flags = oldiflags; //Optimize index for all existing types
1143 if (!JBCLOCKMETHOD(coll, true)) {
1148 _BSONIPATHROWLDR op;
1152 if (flags & JBIDXSTR) {
1154 rv = tctdbsetindexrldr(coll->tdb, ipath, tcitype, _bsonipathrowldr, &op);
1156 if (flags & JBIDXISTR) {
1159 rv = tctdbsetindexrldr(coll->tdb, ipath, tcitype, _bsonipathrowldr, &op);
1161 if (rv && (flags & JBIDXNUM)) {
1163 rv = tctdbsetindexrldr(coll->tdb, ipath, tcitype, _bsonipathrowldr, &op);
1165 if (rv && (flags & JBIDXARR)) {
1167 rv = tctdbsetindexrldr(coll->tdb, ipath, tcitype, _bsonipathrowldr, &op);
1169 if (idrop) { //Update index meta on drop
1170 oldiflags &= ~flags;
1171 if (oldiflags) { //Index dropped only for some types
1173 bson_init(&imetadelta);
1174 bson_append_int(&imetadelta, "iflags", oldiflags);
1175 bson_finish(&imetadelta);
1176 rv = _metasetbson2(coll, ikey, &imetadelta, true, true);
1177 bson_destroy(&imetadelta);
1178 } else { //Index dropped completely
1179 rv = _metasetbson2(coll, ikey, NULL, false, false);
1183 if ((flags & JBIDXSTR) && (ibld || !(oldiflags & JBIDXSTR))) {
1185 rv = tctdbsetindexrldr(coll->tdb, ipath, TDBITLEXICAL, _bsonipathrowldr, &op);
1187 if ((flags & JBIDXISTR) && (ibld || !(oldiflags & JBIDXISTR))) {
1190 rv = tctdbsetindexrldr(coll->tdb, ipath, TDBITLEXICAL, _bsonipathrowldr, &op);
1192 if (rv && (flags & JBIDXNUM) && (ibld || !(oldiflags & JBIDXNUM))) {
1194 rv = tctdbsetindexrldr(coll->tdb, ipath, TDBITDECIMAL, _bsonipathrowldr, &op);
1196 if (rv && (flags & JBIDXARR) && (ibld || !(oldiflags & JBIDXARR))) {
1198 rv = tctdbsetindexrldr(coll->tdb, ipath, TDBITTOKEN, _bsonipathrowldr, &op);
1202 JBCUNLOCKMETHOD(coll);
1212 * In order to cleanup resources you need:
1215 * after this method completion with any return status.
1217 static bool _rmcollimpl(EJDB *jb, EJCOLL *coll, bool unlinkfile) {
1220 tctdbout(jb->metadb, coll->cname, coll->cnamesz);
1221 tctdbvanish(coll->tdb);
1222 TCLIST *paths = tclistnew2(10);
1223 tclistpush2(paths, coll->tdb->hdb->path);
1224 for (int j = 0; j < coll->tdb->inum; ++j) {
1225 TDBIDX *idx = coll->tdb->idxs + j;
1226 const char *ipath = tcbdbpath(idx->db);
1228 tclistpush2(paths, ipath);
1231 tctdbclose(coll->tdb);
1233 for (int i = 0; i < TCLISTNUM(paths); ++i) {
1234 unlink(tclistval2(paths, i));
1239 for (int i = 0; i < EJDB_MAX_COLLECTIONS; ++i) {
1240 if (jb->cdbs[i] == coll) {
1245 //collapse the NULL hole
1246 for (int i = 0; i < EJDB_MAX_COLLECTIONS - 1; ++i) {
1247 if (!jb->cdbs[i] && jb->cdbs[i + 1]) {
1248 jb->cdbs[i] = jb->cdbs[i + 1];
1249 jb->cdbs[i + 1] = NULL;
1255 static EJCOLL* _createcollimpl(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
1256 EJCOLL *coll = NULL;
1257 if (!JBISVALCOLNAME(colname)) {
1258 _ejdbsetecode(jb, JBEINVALIDCOLNAME, __FILE__, __LINE__, __func__);
1261 TCTDB *meta = jb->metadb;
1262 char *row = tcsprintf("name\t%s", colname);
1263 if (!tctdbput3(meta, colname, row)) {
1266 if (!_addcoldb0(colname, jb, opts, &coll)) {
1267 tctdbout2(meta, colname); //cleaning
1270 _metasetopts(jb, colname, opts);
1278 static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags, TCXSTR *log) {
1280 tcxstrprintf(log, "\n\nReading '%s'", bspath);
1284 char *dp = strrchr(bspath, '.');
1285 char *pp = strrchr(bspath, MYPATHCHR);
1286 if (!dp || !pp || pp > dp) {
1288 tcxstrprintf(log, "\nERROR: Invalid file path: '%s'", bspath);
1290 _ejdbsetecode2(jb, JBEEI, __FILE__, __LINE__, __func__, true);
1295 char *cname, *mjson = NULL;
1297 bson *mbson = NULL; //meta bson
1298 bson_iterator mbsonit;
1302 TCMALLOC(cname, dp - pp + 1);
1303 TCXSTR *xmetapath = tcxstrnew();
1305 while (++pp != dp) {
1309 if (cnames != NULL) {
1310 if (tclistlsearch(cnames, cname, i) == -1) {
1314 tcxstrcat(xmetapath, bspath, lastsep - bspath + 1);
1315 tcxstrprintf(xmetapath, "%s-meta.json", cname);
1316 mjson = tcreadfile(tcxstrptr(xmetapath), 0, &sp);
1320 tcxstrprintf(log, "\nERROR: Error reading the file: '%s'", tcxstrptr(xmetapath));
1322 _ejdbsetecode2(jb, TCEREAD, __FILE__, __LINE__, __func__, true);
1325 mbson = json2bson(mjson);
1329 tcxstrprintf(log, "\nERROR: Invalid JSON in the file: '%s'", tcxstrptr(xmetapath));
1331 _ejdbsetecode2(jb, JBEEJSONPARSE, __FILE__, __LINE__, __func__, true);
1333 coll = _getcoll(jb, cname);
1334 if (coll && (flags & JBIMPORTREPLACE)) {
1336 tcxstrprintf(log, "\nReplacing all data in '%s'", cname);
1338 err = !(_rmcollimpl(jb, coll, true));
1344 tcxstrprintf(log, "\nERROR: Failed to remove collection: '%s'", cname);
1350 //Build collection options
1351 BSON_ITERATOR_INIT(&mbsonit, mbson);
1352 EJCOLLOPTS cops = {0};
1353 if (bson_find_fieldpath_value("opts", &mbsonit) == BSON_OBJECT) {
1355 BSON_ITERATOR_SUBITERATOR(&mbsonit, &sit);
1356 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
1357 const char *key = BSON_ITERATOR_KEY(&sit);
1358 if (strcmp("compressed", key) == 0 && bt == BSON_BOOL) {
1359 cops.compressed = bson_iterator_bool(&sit);
1360 } else if (strcmp("large", key) == 0 && bt == BSON_BOOL) {
1361 cops.large = bson_iterator_bool(&sit);
1362 } else if (strcmp("cachedrecords", key) == 0 && BSON_IS_NUM_TYPE(bt)) {
1363 cops.cachedrecords = bson_iterator_int(&sit);
1364 } else if (strcmp("records", key) == 0 && BSON_IS_NUM_TYPE(bt)) {
1365 cops.records = bson_iterator_long(&sit);
1369 coll = _createcollimpl(jb, cname, &cops);
1373 tcxstrprintf(log, "\nERROR: Error creating collection: '%s'", cname);
1379 //Register collection indexes
1380 BSON_ITERATOR_INIT(&mbsonit, mbson);
1381 while ((bt = bson_iterator_next(&mbsonit)) != BSON_EOO) {
1382 const char *key = BSON_ITERATOR_KEY(&mbsonit);
1383 if (bt != BSON_OBJECT || strlen(key) < 2 || key[0] != 'i') {
1390 BSON_ITERATOR_SUBITERATOR(&mbsonit, &sit);
1391 bt = bson_find_fieldpath_value("ipath", &sit);
1392 if (bt == BSON_STRING) {
1393 ipath = strdup(bson_iterator_string(&sit));
1396 BSON_ITERATOR_SUBITERATOR(&mbsonit, &sit);
1397 bt = bson_find_fieldpath_value("iflags", &sit);
1398 if (bt == BSON_INT || bt == BSON_LONG) {
1399 iflags = bson_iterator_int(&sit);
1402 if (!_setindeximpl(coll, ipath, iflags, true)) {
1405 tcxstrprintf(log, "\nERROR: Error creating collection index. Collection: '%s' Field: '%s'", cname, ipath);
1413 int fd = open(bspath, O_RDONLY, TCFILEMODE);
1417 tcxstrprintf(log, "\nERROR: Error reading file: '%s'", bspath);
1419 _ejdbsetecode2(jb, TCEREAD, __FILE__, __LINE__, __func__, true);
1422 if (!JBCLOCKMETHOD(coll, true)) {
1426 int32_t maxdocsiz = 0, docsiz = 0, numdocs = 0;
1428 TCMALLOC(docbuf, 4);
1430 sp = read(fd, docbuf, 4);
1434 memcpy(&docsiz, docbuf, 4);
1435 docsiz = TCHTOIL(docsiz);
1436 if (docsiz > EJDB_MAX_IMPORTED_BSON_SIZE) {
1439 tcxstrprintf(log, "\nERROR: BSON document size: %d exceeds the maximum allowed size limit: %d for import operation",
1440 docsiz, EJDB_MAX_IMPORTED_BSON_SIZE);
1442 _ejdbsetecode2(jb, JBETOOBIGBSON, __FILE__, __LINE__, __func__, true);
1448 if (maxdocsiz < docsiz) {
1450 TCREALLOC(docbuf, docbuf, maxdocsiz);
1452 sp = read(fd, docbuf + 4, docsiz - 4);
1453 if (sp < docsiz - 4) {
1458 bson_init_with_data(&savebs, docbuf);
1459 if (docbuf[docsiz - 1] != '\0' || savebs.err || !savebs.finished) {
1461 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
1464 if (!_ejdbsavebsonimpl(coll, &savebs, &oid, false)) {
1473 if (!tctdbsync(coll->tdb)) {
1477 tcxstrprintf(log, "\n%d objects imported into '%s'", numdocs, cname);
1479 JBCUNLOCKMETHOD(coll);
1487 tcxstrdel(xmetapath);
1489 tcxstrprintf(log, "\nERROR: Importing data into: '%s' failed with error: '%s'", cname, (ejdbecode(jb) != 0) ? ejdberrmsg(ejdbecode(jb)) : "Unknown");
1495 static bool _exportcoll(EJCOLL *coll, const char *dpath, int flags, TCXSTR *log) {
1497 char *fpath = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, (flags & JBJSONEXPORT) ? ".json" : ".bson");
1498 char *fpathm = tcsprintf("%s%c%s%s", dpath, MYPATHCHR, coll->cname, "-meta.json");
1499 TCHDB *hdb = coll->tdb->hdb;
1500 TCXSTR *skbuf = tcxstrnew3(sizeof (bson_oid_t) + 1);
1501 TCXSTR *colbuf = tcxstrnew3(1024);
1502 TCXSTR *bsbuf = tcxstrnew3(1024);
1505 HANDLE fd = open(fpath, O_RDWR | O_CREAT | O_TRUNC, JBFILEMODE);
1506 HANDLE fdm = open(fpathm, O_RDWR | O_CREAT | O_TRUNC, JBFILEMODE);
1508 HANDLE fd = CreateFile(fpath, GENERIC_READ | GENERIC_WRITE,
1509 FILE_SHARE_READ | FILE_SHARE_WRITE,
1510 NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
1512 HANDLE fdm = CreateFile(fpathm, GENERIC_READ | GENERIC_WRITE,
1513 FILE_SHARE_READ | FILE_SHARE_WRITE,
1514 NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
1516 if (INVALIDHANDLE(fd) || INVALIDHANDLE(fdm)) {
1517 _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true);
1521 TCHDBITER *it = tchdbiter2init(hdb);
1525 while (!err && tchdbiter2next(hdb, it, skbuf, colbuf)) {
1526 sz = tcmaploadoneintoxstr(TCXSTRPTR(colbuf), TCXSTRSIZE(colbuf), JDBCOLBSON, JDBCOLBSONL, bsbuf);
1530 if (flags & JBJSONEXPORT) {
1531 if (bson2json(TCXSTRPTR(bsbuf), &wbuf, &wsiz) != BSON_OK) {
1532 _ejdbsetecode2(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__, true);
1536 wbuf = TCXSTRPTR(bsbuf);
1537 wsiz = TCXSTRSIZE(bsbuf);
1539 if (!tcwrite(fd, wbuf, wsiz)) {
1540 _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true);
1544 if (wbuf && wbuf != TCXSTRPTR(bsbuf)) {
1549 tcxstrclear(colbuf);
1553 if (!err) { //export collection meta
1554 TCMAP *cmeta = tctdbget(coll->jb->metadb, coll->cname, coll->cnamesz);
1560 tcmapiterinit(cmeta);
1561 const char *mkey = NULL;
1562 while ((mkey = tcmapiternext2(cmeta)) != NULL) {
1563 if (!mkey || (*mkey != 'i' && strcmp(mkey, "opts"))) {
1564 continue; //allowing only index & opts meta bsons
1566 bson *bs = _metagetbson(coll->jb, coll->cname, coll->cnamesz, mkey);
1568 bson_append_bson(&mbs, mkey, bs);
1577 if (bson2json(bson_data(&mbs), &wbuf, &wsiz) != BSON_OK) {
1579 _ejdbsetecode2(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__, true);
1584 if (!tcwrite(fdm, wbuf, wsiz)) {
1586 _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true);
1592 if (!INVALIDHANDLE(fd) && !CLOSEFH(fd)) {
1593 _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true);
1596 if (!INVALIDHANDLE(fdm) && !CLOSEFH(fdm)) {
1597 _ejdbsetecode2(coll->jb, JBEEI, __FILE__, __LINE__, __func__, true);
1608 /* Set the error code of a table database object. */
1609 static void _ejdbsetecode(EJDB *jb, int ecode, const char *filename, int line, const char *func) {
1610 _ejdbsetecode2(jb, ecode, filename, line, func, false);
1613 static void _ejdbsetecode2(EJDB *jb, int ecode, const char *filename, int line, const char *func, bool notfatal) {
1614 assert(jb && filename && line >= 1 && func);
1615 tctdbsetecode2(jb->metadb, ecode, filename, line, func, notfatal);
1618 static EJCOLL* _getcoll(EJDB *jb, const char *colname) {
1620 for (int i = 0; i < jb->cdbsnum; ++i) {
1621 assert(jb->cdbs[i]);
1622 if (!strcmp(colname, jb->cdbs[i]->cname)) {
1629 /* Set mutual exclusion control of a table database object for threading. */
1630 static bool _ejdbsetmutex(EJDB *ejdb) {
1632 if (ejdb->mmtx || JBISOPEN(ejdb)) {
1633 _ejdbsetecode(ejdb, TCEINVALID, __FILE__, __LINE__, __func__);
1636 TCMALLOC(ejdb->mmtx, sizeof (pthread_rwlock_t));
1638 if (pthread_rwlock_init(ejdb->mmtx, NULL) != 0) err = true;
1647 /* Lock a method of the table database object.
1648 `tdb' specifies the table database object.
1649 `wr' specifies whether the lock is writer or not.
1650 If successful, the return value is true, else, it is false. */
1651 EJDB_INLINE bool _ejdblockmethod(EJDB *ejdb, bool wr) {
1653 if (wr ? pthread_rwlock_wrlock(ejdb->mmtx) != 0 : pthread_rwlock_rdlock(ejdb->mmtx) != 0) {
1654 _ejdbsetecode(ejdb, TCETHREAD, __FILE__, __LINE__, __func__);
1661 /* Unlock a method of the table database object.
1662 `tdb' specifies the table database object.
1663 If successful, the return value is true, else, it is false. */
1664 EJDB_INLINE bool _ejdbunlockmethod(EJDB *ejdb) {
1666 if (pthread_rwlock_unlock(ejdb->mmtx) != 0) {
1667 _ejdbsetecode(ejdb, TCETHREAD, __FILE__, __LINE__, __func__);
1674 EJDB_INLINE bool _ejdbcolsetmutex(EJCOLL *coll) {
1675 assert(coll && coll->jb);
1677 _ejdbsetecode(coll->jb, TCEINVALID, __FILE__, __LINE__, __func__);
1680 TCMALLOC(coll->mmtx, sizeof (pthread_rwlock_t));
1682 if (pthread_rwlock_init(coll->mmtx, NULL) != 0) err = true;
1691 EJDB_INLINE bool _ejcollockmethod(EJCOLL *coll, bool wr) {
1692 assert(coll && coll->jb);
1693 if (wr ? pthread_rwlock_wrlock(coll->mmtx) != 0 : pthread_rwlock_rdlock(coll->mmtx) != 0) {
1694 _ejdbsetecode(coll->jb, TCETHREAD, __FILE__, __LINE__, __func__);
1698 return (coll->tdb && coll->tdb->open);
1701 EJDB_INLINE bool _ejcollunlockmethod(EJCOLL *coll) {
1702 assert(coll && coll->jb);
1703 if (pthread_rwlock_unlock(coll->mmtx) != 0) {
1704 _ejdbsetecode(coll->jb, TCETHREAD, __FILE__, __LINE__, __func__);
1711 bool ejcollockmethod(EJCOLL *coll, bool wr) {
1712 return _ejcollockmethod(coll, wr);
1715 bool ejcollunlockmethod(EJCOLL *coll) {
1716 return _ejcollunlockmethod(coll);
1719 static void _qrydel(EJQ *q, bool freequery) {
1723 const EJQF *qf = NULL;
1725 for (int i = 0; i < TCLISTNUM(q->qflist); ++i) {
1726 qf = TCLISTVALPTR(q->qflist, i);
1730 tclistdel(q->qflist);
1735 for (int i = 0; i < TCLISTNUM(q->orqlist); ++i) {
1736 EJQ *oq = *((EJQ**) TCLISTVALPTR(q->orqlist, i));
1739 tclistdel(q->orqlist);
1744 for (int i = 0; i < TCLISTNUM(q->andqlist); ++i) {
1745 EJQ *aq = *((EJQ**) TCLISTVALPTR(q->andqlist, i));
1748 tclistdel(q->andqlist);
1757 tcmapdel(q->ifields);
1761 tcxstrdel(q->colbuf);
1765 tcxstrdel(q->bsbuf);
1769 tcxstrdel(q->tmpbuf);
1772 if (q->allqfields) {
1773 TCFREE(q->allqfields);
1774 q->allqfields = NULL;
1781 static bool _qrybsvalmatch(const EJQF *qf, bson_iterator *it, bool expandarrays, int *arridx) {
1782 if (qf->tcop == TDBQTRUE) {
1783 return (true == !qf->negate);
1786 bson_type bt = BSON_ITERATOR_TYPE(it);
1787 const char *expr = qf->expr;
1788 int exprsz = qf->exprsz;
1789 char sbuf[JBSTRINOPBUFFERSZ]; //buffer for icase comparisons
1790 char oidbuf[25]; //OID buffer
1797 // Feature #129: Handle BSON_SYMBOL like BSON_STRING
1798 #define _FETCHSTRFVAL() \
1800 fvalsz = (BSON_IS_STRING_TYPE(bt)) ? bson_iterator_string_len(it) : 1; \
1801 fval = (BSON_IS_STRING_TYPE(bt)) ? bson_iterator_string(it) : ""; \
1802 if (bt == BSON_OID) { \
1803 bson_oid_to_string(bson_iterator_oid(it), oidbuf); \
1810 if (bt == BSON_ARRAY && expandarrays) { //Iterate over array
1812 BSON_ITERATOR_SUBITERATOR(it, &sit);
1814 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
1815 if (_qrybsvalmatch(qf, &sit, false, arridx)) {
1823 return (false == !qf->negate);
1828 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1829 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1830 if (cbufstrlen < 0) {
1831 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1834 rv = (exprsz == cbufstrlen) && (exprsz == 0 || !memcmp(expr, cbuf, exprsz));
1836 if (cbuf && cbuf != sbuf) {
1840 rv = (exprsz == fvalsz - 1) && (exprsz == 0 || !memcmp(expr, fval, exprsz));
1846 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1847 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1848 if (cbufstrlen < 0) {
1849 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1852 rv = (exprsz <= cbufstrlen) && strstr(cbuf, expr);
1854 if (cbuf && cbuf != sbuf) {
1858 rv = (exprsz <= fvalsz) && strstr(fval, expr);
1864 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1865 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1866 if (cbufstrlen < 0) {
1867 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1870 rv = tcstrfwm(cbuf, expr);
1872 if (cbuf && cbuf != sbuf) {
1876 rv = tcstrfwm(fval, expr);
1882 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1883 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1884 if (cbufstrlen < 0) {
1885 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1888 rv = tcstrbwm(cbuf, expr);
1890 if (cbuf && cbuf != sbuf) {
1894 rv = tcstrbwm(fval, expr);
1899 TCLIST *tokens = qf->exprlist;
1902 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1903 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1904 if (cbufstrlen < 0) {
1905 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1908 rv = _qrycondcheckstrand(cbuf, tokens);
1910 if (cbuf && cbuf != sbuf) {
1914 rv = _qrycondcheckstrand(fval, tokens);
1919 TCLIST *tokens = qf->exprlist;
1922 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1923 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1924 if (cbufstrlen < 0) {
1925 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1928 rv = _qrycondcheckstror(cbuf, tokens);
1930 if (cbuf && cbuf != sbuf) {
1934 rv = _qrycondcheckstror(fval, tokens);
1938 case TDBQCSTROREQ: {
1939 TCLIST *tokens = qf->exprlist;
1942 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1943 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1944 if (cbufstrlen < 0) {
1945 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1949 if (tcmapget(qf->exprmap, cbuf, cbufstrlen, &sp) != NULL) {
1954 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
1955 const char *token = TCLISTVALPTR(tokens, i);
1956 int tokensz = TCLISTVALSIZ(tokens, i);
1957 if (tokensz == cbufstrlen && !strncmp(token, cbuf, tokensz)) {
1964 if (cbuf && cbuf != sbuf) {
1969 if (tcmapget3(qf->exprmap, fval, (fvalsz - 1), &sp) != NULL) {
1974 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
1975 const char *token = TCLISTVALPTR(tokens, i);
1976 int tokensz = TCLISTVALSIZ(tokens, i);
1977 if (tokensz == (fvalsz - 1) && !strncmp(token, fval, tokensz)) {
1986 case TDBQCSTRORBW: {
1987 TCLIST *tokens = qf->exprlist;
1990 if ((qf->flags & EJCONDICASE) && (bt == BSON_STRING)) {
1991 cbufstrlen = tcicaseformat(fval, fvalsz - 1, sbuf, JBSTRINOPBUFFERSZ, &cbuf);
1992 if (cbufstrlen < 0) {
1993 _ejdbsetecode(qf->jb, cbufstrlen, __FILE__, __LINE__, __func__);
1996 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
1997 const char *token = TCLISTVALPTR(tokens, i);
1998 int tokensz = TCLISTVALSIZ(tokens, i);
1999 if (tokensz <= cbufstrlen && !strncmp(token, cbuf, tokensz)) {
2005 if (cbuf && cbuf != sbuf) {
2009 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
2010 const char *token = TCLISTVALPTR(tokens, i);
2011 int tokensz = TCLISTVALSIZ(tokens, i);
2012 if (tokensz <= (fvalsz - 1) && !strncmp(token, fval, tokensz)) {
2022 rv = qf->regex && (regexec((regex_t *) qf->regex, fval, 0, NULL, 0) == 0);
2026 if (bt == BSON_DOUBLE) {
2027 rv = (qf->exprdblval == bson_iterator_double_raw(it));
2028 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2029 rv = (qf->exprlongval == bson_iterator_long(it));
2036 if (bt == BSON_DOUBLE) {
2037 rv = (qf->exprdblval < bson_iterator_double_raw(it));
2038 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2039 rv = (qf->exprlongval < bson_iterator_long(it));
2046 if (bt == BSON_DOUBLE) {
2047 rv = (qf->exprdblval <= bson_iterator_double_raw(it));
2048 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2049 rv = (qf->exprlongval <= bson_iterator_long(it));
2056 if (bt == BSON_DOUBLE) {
2057 rv = (qf->exprdblval > bson_iterator_double_raw(it));
2058 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2059 rv = (qf->exprlongval > bson_iterator_long(it));
2066 if (bt == BSON_DOUBLE) {
2067 rv = (qf->exprdblval >= bson_iterator_double_raw(it));
2068 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2069 rv = (qf->exprlongval >= bson_iterator_long(it));
2076 assert(qf->ftype == BSON_ARRAY);
2077 TCLIST *tokens = qf->exprlist;
2079 assert(TCLISTNUM(tokens) == 2);
2080 if (BSON_ITERATOR_TYPE(it) == BSON_DOUBLE) {
2081 double v1 = tcatof(tclistval2(tokens, 0));
2082 double v2 = tcatof(tclistval2(tokens, 1));
2083 double val = bson_iterator_double(it);
2084 rv = (v2 > v1) ? (v2 >= val && v1 <= val) : (v2 <= val && v1 >= val);
2086 int64_t v1 = tcatoi(tclistval2(tokens, 0));
2087 int64_t v2 = tcatoi(tclistval2(tokens, 1));
2088 int64_t val = bson_iterator_long(it);
2089 rv = (v2 > v1) ? (v2 >= val && v1 <= val) : (v2 <= val && v1 >= val);
2093 case TDBQCNUMOREQ: {
2094 TCLIST *tokens = qf->exprlist;
2096 if (bt == BSON_DOUBLE) {
2097 double nval = bson_iterator_double_raw(it);
2098 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
2099 if (tcatof(TCLISTVALPTR(tokens, i)) == nval) {
2104 } else if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2105 int64_t nval = bson_iterator_long(it);
2106 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
2107 if (tcatoi(TCLISTVALPTR(tokens, i)) == nval) {
2116 return (rv == !qf->negate);
2118 #undef _FETCHSTRFVAL
2122 //Fills `ffpctx` and `qf->uslots`
2123 static void _qrysetarrayidx(FFPCTX *ffpctx, EJQF *qf, int dpos, int mpos) {
2124 if (ffpctx->dpos == dpos && ffpctx->mpos == -1) { //single ctx matching
2125 ffpctx->mpos = mpos;
2128 for (int i = 0; i < TCLISTNUM(qf->uslots); ++i) {
2129 USLOT *us = TCLISTVALPTR(qf->uslots, i);
2131 if (us->dpos == dpos && us->mpos == -1) {
2138 static bool _qrybsrecurrmatch(EJQF *qf, FFPCTX *ffpctx, int currpos) {
2139 assert(qf && ffpctx && ffpctx->stopnestedarr);
2140 bson_type bt = bson_find_fieldpath_value3(ffpctx);
2141 if (bt == BSON_ARRAY && ffpctx->stopos < ffpctx->fplen) { //a bit of complicated code in this case =)
2142 //we just stepped in some array in middle of our fieldpath, so have to perform recursive nested iterations
2143 //$elemMatch active in this context
2144 while (ffpctx->fpath[ffpctx->stopos] == '.' && ffpctx->stopos < ffpctx->fplen) ffpctx->stopos++;
2145 ffpctx->fplen = ffpctx->fplen - ffpctx->stopos;
2146 assert(ffpctx->fplen > 0);
2147 ffpctx->fpath = ffpctx->fpath + ffpctx->stopos;
2148 currpos += ffpctx->stopos; //adjust cumulative field position
2151 BSON_ITERATOR_SUBITERATOR(ffpctx->input, &sit);
2152 for (int arr_idx = 0;(bt = bson_iterator_next(&sit)) != BSON_EOO; ++arr_idx) {
2153 if (bt != BSON_OBJECT && bt != BSON_ARRAY)
2157 BSON_ITERATOR_SUBITERATOR(&sit, &sit2);
2159 ffpctx->input = &sit2;
2161 // Match using context initialised above.
2162 if (_qrybsrecurrmatch(qf, ffpctx, currpos) == qf->negate) {
2167 if (qf->elmatchgrp > 0 && qf->elmatchpos == currpos) { //$elemMatch matching group exists at right place
2168 // Match all sub-queries on current field pos. Early exit (break) on failure.
2169 for (int i = TCLISTNUM(qf->q->qflist) - 1; i >= 0; --i) {
2170 EJQF *eqf = TCLISTVALPTR(qf->q->qflist, i);
2171 if (eqf == qf || (eqf->mflags & EJFEXCLUDED) || eqf->elmatchgrp != qf->elmatchgrp) {
2174 eqf->mflags |= EJFEXCLUDED;
2175 BSON_ITERATOR_SUBITERATOR(&sit, &sit2);
2176 FFPCTX nffpctx = *ffpctx;
2177 nffpctx.fplen = eqf->fpathsz - eqf->elmatchpos;
2178 if (nffpctx.fplen <= 0) { //should never happen if query construction is correct
2183 nffpctx.fpath = eqf->fpath + eqf->elmatchpos;
2184 nffpctx.input = &sit2;
2187 // Match sub-query at current field pos.
2188 // Ignores outer negate (qf) on inner query (eqf).
2189 if (_qrybsrecurrmatch(eqf, &nffpctx, eqf->elmatchpos) == qf->negate) {
2190 // Skip all remaining sub-queries on this field. Go to next element, if any.
2197 _qrysetarrayidx(ffpctx, qf, (currpos - 1), arr_idx);
2198 // Only return success at this point.
2199 // An failure here may precede a later success so proceed to next element, if any.
2200 return ret != qf->negate;
2205 if (bt == BSON_EOO || bt == BSON_UNDEFINED || bt == BSON_NULL) {
2206 return qf->negate; //Field missing
2207 } else if (qf->tcop == TDBQCEXIST) {
2211 bool ret = _qrybsvalmatch(qf, ffpctx->input, true, &mpos);
2212 if (ret && currpos == 0 && bt == BSON_ARRAY) { //save $(projection)
2213 _qrysetarrayidx(ffpctx, qf, ffpctx->fplen, mpos);
2219 static bool _qrybsmatch(EJQF *qf, const void *bsbuf, int bsbufsz) {
2220 if (qf->tcop == TDBQTRUE) {
2224 BSON_ITERATOR_FROM_BUFFER(&it, bsbuf);
2227 .fplen = qf->fpathsz,
2229 .stopnestedarr = true,
2235 for (int i = 0; i < TCLISTNUM(qf->uslots); ++i) {
2236 ((USLOT*) (TCLISTVALPTR(qf->uslots, i)))->mpos = -1;
2239 return _qrybsrecurrmatch(qf, &ffpctx, 0);
2242 static bool _qry_and_or_match(EJCOLL *coll, EJQ *ejq, const void *pkbuf, int pkbufsz) {
2243 bool isor = (ejq->orqlist && TCLISTNUM(ejq->orqlist) > 0);
2244 bool isand = (ejq->andqlist && TCLISTNUM(ejq->andqlist) > 0);
2245 if (!isor && !isand) {
2248 void *bsbuf = TCXSTRPTR(ejq->bsbuf);
2249 int bsbufsz = TCXSTRSIZE(ejq->bsbuf);
2251 if (!bsbuf || bsbufsz <= 0) {
2252 tcxstrclear(ejq->colbuf);
2253 tcxstrclear(ejq->bsbuf);
2254 if (tchdbgetintoxstr(coll->tdb->hdb, pkbuf, pkbufsz, ejq->colbuf) <= 0) {
2257 if (tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf) <= 0) {
2260 bsbufsz = TCXSTRSIZE(ejq->bsbuf);
2261 bsbuf = TCXSTRPTR(ejq->bsbuf);
2263 if (isand && !_qryandmatch2(coll, ejq, bsbuf, bsbufsz)) {
2266 return _qryormatch2(coll, ejq, bsbuf, bsbufsz);
2272 static bool _qryormatch3(EJCOLL *coll, EJQ *ejq, EJQ *oq, const void *bsbuf, int bsbufsz) {
2274 int jm = TCLISTNUM(oq->qflist);
2275 for (; j < jm; ++j) {
2276 EJQF *qf = TCLISTVALPTR(oq->qflist, j);
2278 qf->mflags = qf->flags;
2279 if (qf->mflags & EJFEXCLUDED) {
2282 if (!_qrybsmatch(qf, bsbuf, bsbufsz)) {
2286 if (j == jm) { //all fields in oq are matched
2287 if (oq->andqlist && TCLISTNUM(oq->andqlist) > 0 &&
2288 !_qryandmatch2(coll, oq, bsbuf, bsbufsz)) { //we have nested $and fields
2291 if (oq->orqlist && TCLISTNUM(oq->orqlist) &&
2292 !_qryormatch2(coll, oq, bsbuf, bsbufsz)) { //we have nested $or fields
2300 static bool _qryormatch2(EJCOLL *coll, EJQ *ejq, const void *bsbuf, int bsbufsz) {
2301 if (!ejq->orqlist || TCLISTNUM(ejq->orqlist) < 1) {
2304 if (ejq->lastmatchedorq && _qryormatch3(coll, ejq, ejq->lastmatchedorq, bsbuf, bsbufsz)) {
2307 for (int i = 0; i < TCLISTNUM(ejq->orqlist); ++i) {
2308 EJQ *oq = *((EJQ**) TCLISTVALPTR(ejq->orqlist, i));
2309 assert(oq && oq->qflist);
2310 if (ejq->lastmatchedorq == oq) {
2313 if (_qryormatch3(coll, ejq, oq, bsbuf, bsbufsz)) {
2314 ejq->lastmatchedorq = oq;
2321 static bool _qryandmatch2(EJCOLL *coll, EJQ *ejq, const void *bsbuf, int bsbufsz) {
2322 if (!ejq->andqlist || TCLISTNUM(ejq->andqlist) < 1) {
2325 for (int i = 0; i < TCLISTNUM(ejq->andqlist); ++i) {
2326 EJQ *aq = *((EJQ**) TCLISTVALPTR(ejq->andqlist, i));
2327 assert(aq && aq->qflist);
2328 for (int j = 0; j < TCLISTNUM(aq->qflist); ++j) {
2329 EJQF *qf = TCLISTVALPTR(aq->qflist, j);
2331 qf->mflags = qf->flags;
2332 if (qf->mflags & EJFEXCLUDED) {
2335 if (!_qrybsmatch(qf, bsbuf, bsbufsz)) {
2339 if (aq->andqlist && TCLISTNUM(aq->andqlist) > 0 &&
2340 !_qryandmatch2(coll, aq, bsbuf, bsbufsz)) { //we have nested $and fields
2343 if (aq->orqlist && TCLISTNUM(aq->orqlist) > 0 &&
2344 !_qryormatch2(coll, aq, bsbuf, bsbufsz)) { //we have nested $or fields
2351 /** Return true if all main query conditions matched */
2352 static bool _qryallcondsmatch(
2354 EJCOLL *coll, EJQF **qfs, int qfsz,
2355 const void *pkbuf, int pkbufsz) {
2356 assert(ejq->colbuf && ejq->bsbuf);
2357 if (!(ejq->flags & EJQUPDATING) && (ejq->flags & EJQONLYCOUNT) && anum < 1) {
2360 tcxstrclear(ejq->colbuf);
2361 tcxstrclear(ejq->bsbuf);
2362 if (tchdbgetintoxstr(coll->tdb->hdb, pkbuf, pkbufsz, ejq->colbuf) <= 0) {
2365 if (tcmaploadoneintoxstr(TCXSTRPTR(ejq->colbuf), TCXSTRSIZE(ejq->colbuf), JDBCOLBSON, JDBCOLBSONL, ejq->bsbuf) <= 0) {
2371 for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags; //reset matching flags
2372 for (int i = 0; i < qfsz; ++i) {
2374 if (qf->mflags & EJFEXCLUDED) continue;
2375 if (!_qrybsmatch(qf, TCXSTRPTR(ejq->bsbuf), TCXSTRSIZE(ejq->bsbuf))) {
2382 static EJQ* _qryaddand(EJDB *jb, EJQ *q, const void *andbsdata) {
2383 assert(jb && q && andbsdata);
2385 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
2388 EJQ *oq = ejdbcreatequery2(jb, andbsdata);
2392 if (q->andqlist == NULL) {
2393 q->andqlist = tclistnew2(TCLISTINYNUM);
2395 tclistpush(q->andqlist, &oq, sizeof(oq));
2404 /* RS sorting comparison func */
2405 static int _ejdbsoncmp(const TCLISTDATUM *d1, const TCLISTDATUM *d2, void *opaque) {
2406 _EJBSORTCTX *ctx = opaque;
2409 for (int i = 0; !res && i < ctx->ofsz; ++i) {
2410 const EJQF *qf = ctx->ofs[i];
2411 if (qf->flags & EJFORDERUSED) {
2414 res = bson_compare(d1->ptr, d2->ptr, qf->fpath, qf->fpathsz) * (qf->order >= 0 ? 1 : -1);
2419 EJDB_INLINE void _nufetch(_EJDBNUM *nu, const char *sval, bson_type bt) {
2420 if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2421 nu->inum = tcatoi(sval);
2422 } else if (bt == BSON_DOUBLE) {
2423 nu->dnum = tcatof(sval);
2430 EJDB_INLINE int _nucmp(_EJDBNUM *nu, const char *sval, bson_type bt) {
2431 if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2432 int64_t v = tcatoi(sval);
2433 return (nu->inum > v) ? 1 : (nu->inum < v ? -1 : 0);
2434 } else if (bt == BSON_DOUBLE) {
2435 double v = tcatof(sval);
2436 return (nu->dnum > v) ? 1 : (nu->dnum < v ? -1 : 0);
2443 EJDB_INLINE int _nucmp2(_EJDBNUM *nu1, _EJDBNUM *nu2, bson_type bt) {
2444 if (bt == BSON_INT || bt == BSON_LONG || bt == BSON_BOOL || bt == BSON_DATE) {
2445 return (nu1->inum > nu2->inum) ? 1 : (nu1->inum < nu2->inum ? -1 : 0);
2446 } else if (bt == BSON_DOUBLE) {
2447 return (nu1->dnum > nu2->dnum) ? 1 : (nu1->dnum < nu2->dnum ? -1 : 0);
2454 static void _qryfieldup(const EJQF *src, EJQF *target, uint32_t qflags) {
2455 assert(src && target);
2456 memset(target, 0, sizeof (*target));
2457 target->exprdblval = src->exprdblval;
2458 target->exprlongval = src->exprlongval;
2459 target->flags = src->flags;
2460 target->ftype = src->ftype;
2461 target->negate = src->negate;
2462 target->order = src->order;
2463 target->orderseq = src->orderseq;
2464 target->tcop = src->tcop;
2465 target->elmatchgrp = src->elmatchgrp;
2466 target->elmatchpos = src->elmatchpos;
2469 TCMEMDUP(target->expr, src->expr, src->exprsz);
2470 target->exprsz = src->exprsz;
2473 TCMEMDUP(target->fpath, src->fpath, src->fpathsz);
2474 target->fpathsz = src->fpathsz;
2476 if (src->regex && (EJQINTERNAL & qflags)) {
2477 //We cannot do deep copy of regex_t so do shallow copy only for internal query objects
2478 target->regex = src->regex;
2480 if (src->exprlist) {
2481 target->exprlist = tclistdup(src->exprlist);
2484 target->exprmap = tcmapdup(src->exprmap);
2486 if (src->updateobj) {
2487 target->updateobj = bson_dup(src->updateobj);
2490 target->ufields = tclistdup(src->ufields);
2493 target->uslots = tclistdup(src->uslots);
2497 /* Clone query object */
2498 static bool _qrydup(const EJQ *src, EJQ *target, uint32_t qflags) {
2499 assert(src && target);
2500 memset(target, 0, sizeof (*target));
2501 target->flags = src->flags | qflags;
2502 target->max = src->max;
2503 target->skip = src->skip;
2505 target->qflist = tclistnew2(TCLISTNUM(src->qflist));
2506 for (int i = 0; i < TCLISTNUM(src->qflist); ++i) {
2508 _qryfieldup(TCLISTVALPTR(src->qflist, i), &qf, qflags);
2510 TCLISTPUSH(target->qflist, &qf, sizeof (qf));
2514 target->hints = bson_dup(src->hints);
2517 target->ifields = tcmapdup(src->ifields);
2520 target->orqlist = tclistnew2(TCLISTNUM(src->orqlist));
2521 for (int i = 0; i < TCLISTNUM(src->orqlist); ++i) {
2523 TCMALLOC(q, sizeof(*q));
2524 if (_qrydup(*((EJQ**) TCLISTVALPTR(src->orqlist, i)), q, qflags)) {
2525 TCLISTPUSH(target->orqlist, &q, sizeof (q));
2531 if (src->andqlist) {
2532 target->andqlist = tclistnew2(TCLISTNUM(src->andqlist));
2533 for (int i = 0; i < TCLISTNUM(src->andqlist); ++i) {
2535 TCMALLOC(q, sizeof(*q));
2536 if (_qrydup(*((EJQ**) TCLISTVALPTR(src->andqlist, i)), q, qflags)) {
2537 tclistpush(target->andqlist, &q, sizeof (q));
2546 typedef struct { /**> $do action visitor context */
2551 } _BSONDOVISITORCTX;
2553 static bson_visitor_cmd_t _bsondovisitor(const char *ipath, int ipathlen, const char *key, int keylen,
2554 const bson_iterator *it, bool after, void *op) {
2557 _BSONDOVISITORCTX *ictx = op;
2559 TCMAP *dfields = ictx->dfields;
2560 bson_type lbt = BSON_ITERATOR_TYPE(it), bt;
2561 bson_iterator doit, bufit, sit;
2564 const EJQF *dofield;
2565 bson_visitor_cmd_t rv = BSON_VCMD_SKIP_AFTER;
2569 bson_append_field_from_iterator(it, ictx->sbson);
2570 rv = (BSON_VCMD_SKIP_AFTER | BSON_VCMD_SKIP_NESTED);
2576 dofield = tcmapget(dfields, ipath, ipathlen, &sp);
2578 if (lbt == BSON_ARRAY) {
2579 bson_append_field_from_iterator(it, ictx->sbson);
2580 rv = (BSON_VCMD_SKIP_AFTER | BSON_VCMD_SKIP_NESTED);
2582 bson_append_field_from_iterator(it, ictx->sbson);
2586 assert(dofield->updateobj && (dofield->flags & EJCONDOIT));
2587 BSON_ITERATOR_INIT(&doit, dofield->updateobj);
2588 while ((bt = bson_iterator_next(&doit)) != BSON_EOO) {
2589 const char *dofname = BSON_ITERATOR_KEY(&doit);
2591 if (bt == BSON_STRING && !strcmp("$join", dofname)) {
2592 coll = _getcoll(ictx->jb, bson_iterator_string(&doit));
2595 if (lbt == BSON_STRING) {
2596 sval = bson_iterator_string(it);
2597 if (!ejdbisvalidoidstr(sval)) break;
2598 bson_oid_from_string(&loid, sval);
2599 } else if (lbt == BSON_OID) {
2600 loid = *(bson_iterator_oid(it));
2602 if (lbt == BSON_STRING || lbt == BSON_OID) {
2603 tcxstrclear(ictx->q->colbuf);
2604 tcxstrclear(ictx->q->tmpbuf);
2605 if (!tchdbgetintoxstr(coll->tdb->hdb, &loid, sizeof (loid), ictx->q->colbuf) ||
2606 !tcmaploadoneintoxstr(TCXSTRPTR(ictx->q->colbuf), TCXSTRSIZE(ictx->q->colbuf),
2607 JDBCOLBSON, JDBCOLBSONL, ictx->q->tmpbuf)) {
2610 BSON_ITERATOR_FROM_BUFFER(&bufit, TCXSTRPTR(ictx->q->tmpbuf));
2611 bson_append_object_from_iterator(BSON_ITERATOR_KEY(it), &bufit, ictx->sbson);
2614 if (lbt == BSON_ARRAY) {
2615 BSON_ITERATOR_SUBITERATOR(it, &sit);
2616 bson_append_start_array(ictx->sbson, BSON_ITERATOR_KEY(it));
2617 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
2618 if (bt != BSON_STRING && bt != BSON_OID) {
2619 bson_append_field_from_iterator(&sit, ictx->sbson);
2622 if (bt == BSON_STRING) {
2623 sval = bson_iterator_string(&sit);
2624 if (!ejdbisvalidoidstr(sval)) break;
2625 bson_oid_from_string(&loid, sval);
2626 } else if (bt == BSON_OID) {
2627 loid = *(bson_iterator_oid(&sit));
2629 tcxstrclear(ictx->q->colbuf);
2630 tcxstrclear(ictx->q->tmpbuf);
2631 if (!tchdbgetintoxstr(coll->tdb->hdb, &loid, sizeof (loid), ictx->q->colbuf) ||
2632 !tcmaploadoneintoxstr(TCXSTRPTR(ictx->q->colbuf), TCXSTRSIZE(ictx->q->colbuf),
2633 JDBCOLBSON, JDBCOLBSONL, ictx->q->tmpbuf)) {
2634 bson_append_field_from_iterator(&sit, ictx->sbson);
2637 BSON_ITERATOR_FROM_BUFFER(&bufit, TCXSTRPTR(ictx->q->tmpbuf));
2638 bson_append_object_from_iterator(BSON_ITERATOR_KEY(&sit), &bufit, ictx->sbson);
2640 bson_append_finish_array(ictx->sbson);
2641 rv = (BSON_VCMD_SKIP_AFTER | BSON_VCMD_SKIP_NESTED);
2644 } else if (lbt == BSON_ARRAY &&
2645 (bt == BSON_ARRAY || BSON_IS_NUM_TYPE(bt)) &&
2646 !strcmp("$slice", dofname)) {
2648 bson_append_start_array(ictx->sbson, BSON_ITERATOR_KEY(it));
2649 int skip = 0, limit, idx = 0, i;
2650 char nbuff[TCNUMBUFSIZ];
2652 if (bt == BSON_ARRAY) { // $slice : [skip, limit]
2655 BSON_ITERATOR_SUBITERATOR(&doit, &sit2);
2657 bt2 = bson_find_fieldpath_value2("0", 1, &sit2);
2658 if (!BSON_IS_NUM_TYPE(bt2)) {
2659 bson_append_field_from_iterator(it, ictx->sbson);
2662 skip = bson_iterator_int(&sit2);
2664 bt2 = bson_find_fieldpath_value2("1", 1, &sit2);
2665 if (!BSON_IS_NUM_TYPE(bt2)) {
2666 bson_append_field_from_iterator(it, ictx->sbson);
2669 limit = abs(bson_iterator_int(&sit2));
2670 } else { // $slice : limit
2671 limit = abs(bson_iterator_int(&doit));
2676 BSON_ITERATOR_SUBITERATOR(it, &sit);
2677 while (bson_iterator_next(&sit) != BSON_EOO) ++cnt;
2678 skip = cnt + skip % cnt;
2684 limit = (limit <= INT_MAX - skip) ? limit + skip : INT_MAX;
2685 BSON_ITERATOR_SUBITERATOR(it, &sit);
2686 for (i = 0; idx < limit && (bt = bson_iterator_next(&sit)) != BSON_EOO; ++idx) {
2688 bson_numstrn(nbuff, TCNUMBUFSIZ, i++);
2689 bson_append_field_from_iterator2(nbuff, &sit, ictx->sbson);
2692 bson_append_finish_array(ictx->sbson);
2693 rv = (BSON_VCMD_SKIP_AFTER | BSON_VCMD_SKIP_NESTED);
2699 bson_append_field_from_iterator(it, ictx->sbson);
2705 static bool _pushprocessedbson(_QRYCTX *ctx, const void *bsbuf, int bsbufsz) {
2706 assert(bsbuf && bsbufsz);
2707 if (!ctx->dfields && !ctx->ifields && !ctx->q->ifields) { //Trivial case: no $do operations or $fields
2708 tclistpush(ctx->res, bsbuf, bsbufsz);
2712 EJDB *jb = ctx->coll->jb;
2714 TCMAP *ifields = ctx->ifields;
2716 char bstack[JBSBUFFERSZ];
2718 bson_init_on_stack(&bsout, bstack, bsbufsz, JBSBUFFERSZ);
2720 if (ctx->dfields) { //$do fields exists
2721 rv = _exec_do(ctx, bsbuf, &bsout);
2724 if (rv && (ifields || q->ifields)) { //$fields hints
2725 TCMAP *_ifields = ifields;
2726 TCMAP *_fkfields = NULL; //Fields with overriden keys
2727 char* inbuf = (bsout.finished) ? bsout.data : (char*) bsbuf;
2728 if (bsout.finished) {
2729 bson_init_size(&bsout, bson_size(&bsout));
2731 if (q->ifields) { //we have positional $(projection)
2732 assert(ctx->imode == true); //ensure we are in include mode
2734 _ifields = tcmapnew2(q->ifields->bnum);
2736 _ifields = tcmapdup(ifields);
2738 _fkfields = tcmapnew2(TCMAPTINYBNUM);
2739 for (int i = 0; i < TCLISTNUM(q->qflist); ++i) {
2740 EJQF *qf = TCLISTVALPTR(q->qflist, i);
2741 const char *dfpath = tcmapget(q->ifields, qf->fpath, qf->fpathsz, &sp);
2743 TCXSTR *ifield = tcxstrnew3(sp + 10);
2745 BSON_ITERATOR_FROM_BUFFER(&it, inbuf);
2748 .fplen = qf->fpathsz,
2750 .stopnestedarr = true,
2755 const char *dpos = strchr(dfpath, '$');
2757 ctx.dpos = (dpos - dfpath) - 1;
2758 qf->mflags = (qf->flags & ~EJFEXCLUDED);
2759 if (!_qrybsrecurrmatch(qf, &ctx, 0)) {
2760 assert(false); //something wrong, it should never be happen
2761 } else if (ctx.mpos >= 0) {
2762 tcxstrcat(ifield, dfpath, (dpos - dfpath));
2763 tcxstrprintf(ifield, "%d", ctx.mpos);
2764 tcmapput(_fkfields, TCXSTRPTR(ifield), TCXSTRSIZE(ifield), "0", strlen("0"));
2765 tcxstrcat(ifield, dpos + 1, sp - (dpos - dfpath) - 1);
2766 tcmapput(_ifields, TCXSTRPTR(ifield), TCXSTRSIZE(ifield), &yes, sizeof (yes));
2768 assert(false); //something wrong, it should never be happen
2774 BSONSTRIPCTX sctx = {
2775 .ifields = _ifields,
2776 .fkfields = _fkfields,
2777 .imode = ctx->imode,
2782 if (bson_strip2(&sctx) != BSON_OK) {
2783 _ejdbsetecode(jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
2785 if (inbuf != bsbuf && inbuf != bstack) {
2788 if (_ifields != ifields) {
2792 tcmapdel(_fkfields);
2797 assert(bsout.finished);
2798 if (bsout.flags & BSON_FLAG_STACK_ALLOCATED) {
2799 TCLISTPUSH(ctx->res, bsout.data, bson_size(&bsout));
2801 tclistpushmalloc(ctx->res, bsout.data, bson_size(&bsout));
2804 bson_destroy(&bsout);
2809 static bool _exec_do(_QRYCTX *ctx, const void *bsbuf, bson *bsout) {
2810 assert(ctx && ctx->dfields);
2811 _BSONDOVISITORCTX ictx = {
2813 .jb = ctx->coll->jb,
2814 .dfields = ctx->dfields,
2818 BSON_ITERATOR_FROM_BUFFER(&it, bsbuf);
2819 bson_visit_fields(&it, 0, _bsondovisitor, &ictx);
2820 if (bson_finish(bsout) != BSON_OK) {
2821 _ejdbsetecode(ctx->coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
2827 //Create update BSON object for $set/$unset/$inc operations
2828 static bson* _qfgetupdateobj(const EJQF *qf) {
2829 assert(qf->updateobj);
2830 if (!qf->ufields || TCLISTNUM(qf->ufields) < 1) { //we do not ref $(query) fields.
2831 return qf->updateobj;
2833 const EJQ *q = qf->q;
2834 char pbuf[BSON_MAX_FPATH_LEN + 1];
2838 bson *ret = bson_create();
2840 for (int i = 0; i < TCLISTNUM(qf->ufields); ++i) {
2841 const char *uf = TCLISTVALPTR(qf->ufields, i);
2842 for (int j = 0; *(q->allqfields + j) != '\0'; ++j) {
2843 const EJQF *kqf = *(q->allqfields + j);
2844 if (kqf == qf || kqf->uslots == NULL || TCLISTNUM(kqf->uslots) < 1) {
2847 for (int k = 0; k < TCLISTNUM(kqf->uslots); ++k) {
2848 USLOT *uslot = TCLISTVALPTR(kqf->uslots, k);
2849 if (uslot->op == uf && uslot->mpos >= 0) {
2850 char *dp = strchr(uf, '$');
2853 assert(ppos == uslot->dpos + 1);
2854 if (ppos < 1 || ppos >= BSON_MAX_FPATH_LEN - 1) {
2857 memcpy(pbuf, uf, ppos);
2858 int wl = bson_numstrn(pbuf + ppos, (BSON_MAX_FPATH_LEN - ppos), uslot->mpos);
2859 if (wl >= BSON_MAX_FPATH_LEN - ppos) { //output is truncated
2864 for (int fpos = (dp - uf) + 1; ppos < BSON_MAX_FPATH_LEN && *(uf + fpos) != '\0';) {
2865 pbuf[ppos++] = *(uf + fpos++);
2867 assert(ppos <= BSON_MAX_FPATH_LEN);
2870 bt = bson_find(&it, qf->updateobj, uf);
2871 if (bt == BSON_EOO) {
2875 bson_append_field_from_iterator2(pbuf, &it, ret);
2881 BSON_ITERATOR_INIT(&it, qf->updateobj);
2882 while ((bt = bson_iterator_next(&it)) != BSON_EOO) {
2883 const char *key = bson_iterator_key(&it);
2884 if (strchr(key, '$') == NULL) {
2885 bson_append_field_from_iterator2(key, &it, ret);
2892 static bool _qryupdate(_QRYCTX *ctx, void *bsbuf, int bsbufsz) {
2893 assert(ctx && ctx->q && (ctx->q->flags & EJQUPDATING) && bsbuf && ctx->didxctx);
2897 EJCOLL *coll = ctx->coll;
2901 bson_iterator it, it2;
2904 if (q->flags & EJQDROPALL) { //Records will be dropped
2905 bt = bson_find_from_buffer(&it, bsbuf, JDBIDKEYNAME);
2906 if (bt != BSON_OID) {
2907 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
2910 oid = bson_iterator_oid(&it);
2914 bson_oid_to_string(oid, xoid);
2915 tcxstrprintf(ctx->log, "$DROPALL ON: %s\n", xoid);
2917 const void *olddata;
2919 TCMAP *rmap = tctdbget(coll->tdb, oid, sizeof (*oid));
2921 olddata = tcmapget3(rmap, JDBCOLBSON, JDBCOLBSONL, &olddatasz);
2922 if (!_updatebsonidx(coll, oid, NULL, olddata, olddatasz, ctx->didxctx) ||
2923 !tctdbout(coll->tdb, oid, sizeof (*oid))) {
2931 //Apply update operation
2937 const EJQF *setqf = NULL; /*$set*/
2938 const EJQF *unsetqf = NULL; /*$unset*/
2939 const EJQF *incqf = NULL; /*$inc*/
2940 const EJQF *renameqf = NULL; /*$rename*/
2941 const EJQF *addsetqf[2] = {NULL}; /*$addToSet, $addToSetAll*/
2942 const EJQF *pushqf[2] = {NULL}; /*$push, $pushAll */
2943 const EJQF *pullqf[2] = {NULL}; /*$pull, $pullAll*/
2945 for (int i = 0; i < TCLISTNUM(q->qflist); ++i) {
2947 const EJQF *qf = TCLISTVALPTR(q->qflist, i);
2948 const uint32_t flags = qf->flags;
2950 if (qf->updateobj == NULL) {
2953 if (flags & EJCONDSET) { //$set
2955 } else if (flags & EJCONDUNSET) { //$unset
2957 } else if (flags & EJCONDINC) { //$inc
2959 } else if (flags & EJCONDRENAME) { //$rename
2961 } else if (flags & EJCONDADDSET) { //$addToSet, $addToSetAll
2962 if (flags & EJCONDALL) {
2967 } else if (flags & EJCONDPUSH) { //$push, $pushAll
2968 if (flags & EJCONDALL) {
2973 } else if (flags & EJCONDPULL) { //$pull, $pullAll
2974 if (flags & EJCONDALL) {
2983 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
2984 if (bsout.finished) {
2985 //reinit `bsout`, `inbuf` already points to `bsout.data` and will be freed later
2986 bson_init_size(&bsout, bson_size(&bsout));
2988 assert(bsout.data == NULL);
2989 bson_init_size(&bsout, bsbufsz);
2991 bson *updobj = _qfgetupdateobj(renameqf);
2992 TCMAP *rfields = tcmapnew2(TCMAPTINYBNUM);
2993 BSON_ITERATOR_INIT(&it, updobj);
2994 while ((bt = bson_iterator_next(&it)) != BSON_EOO) {
2995 if (bt != BSON_STRING) {
2998 const char *nfpath = bson_iterator_string(&it);
2999 int nlen = bson_iterator_string_len(&it);
3003 tcmapputkeep(rfields,
3004 BSON_ITERATOR_KEY(&it), strlen(BSON_ITERATOR_KEY(&it)),
3008 if (bson_rename(rfields, inbuf, &bsout, &rencnt) != BSON_OK) {
3010 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3016 bson_finish(&bsout);
3017 if (inbuf != bsbuf) {
3020 if (updobj != renameqf->updateobj) {
3028 if (unsetqf) { //$unset
3029 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
3030 if (bsout.finished) {
3031 bson_init_size(&bsout, bson_size(&bsout));
3033 assert(bsout.data == NULL);
3034 bson_init_size(&bsout, bsbufsz);
3037 bson *updobj = _qfgetupdateobj(unsetqf);
3038 TCMAP *ifields = tcmapnew2(TCMAPTINYBNUM);
3039 BSON_ITERATOR_INIT(&it, updobj);
3040 while ((bt = bson_iterator_next(&it)) != BSON_EOO) {
3041 const char *fpath = BSON_ITERATOR_KEY(&it);
3042 tcmapput(ifields, fpath, strlen(fpath), &yes, sizeof(yes));
3044 if (bson_strip(ifields, false, inbuf, &bsout, &matched) != BSON_OK) {
3046 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3052 bson_finish(&bsout);
3053 if (inbuf != bsbuf) {
3056 if (updobj != unsetqf->updateobj) {
3066 bson *updobj = _qfgetupdateobj(setqf);
3067 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
3068 if (bsout.finished) {
3069 bson_init_size(&bsout, bson_size(&bsout));
3071 assert(bsout.data == NULL);
3072 bson_init_size(&bsout, bsbufsz);
3074 int err = bson_merge_fieldpaths(bsbuf, bson_data(updobj), &bsout);
3077 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3079 bson_finish(&bsout);
3080 if (inbuf != bsbuf) {
3083 if (updobj != setqf->updateobj) {
3092 bson *updobj = _qfgetupdateobj(incqf);
3094 bson_create_from_buffer2(&bsout, bsbuf, bsbufsz);
3096 BSON_ITERATOR_INIT(&it, updobj);
3097 while ((bt = bson_iterator_next(&it)) != BSON_EOO) {
3098 if (!BSON_IS_NUM_TYPE(bt)) {
3101 BSON_ITERATOR_INIT(&it2, &bsout);
3102 bt2 = bson_find_fieldpath_value(BSON_ITERATOR_KEY(&it), &it2);
3103 if (!BSON_IS_NUM_TYPE(bt2)) {
3106 if (bt2 == BSON_DOUBLE) {
3107 double v = bson_iterator_double(&it2);
3108 if (bt == BSON_DOUBLE) {
3109 v += bson_iterator_double(&it);
3111 v += bson_iterator_long(&it);
3113 if (bson_inplace_set_double(&it2, v)) {
3115 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3120 int64_t v = bson_iterator_long(&it2);
3121 v += bson_iterator_long(&it);
3122 if (bson_inplace_set_long(&it2, v)) {
3124 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3130 if (updobj != incqf->updateobj) {
3138 for (int i = 0; i < 2; ++i) { //$pull $pullAll
3139 const EJQF *qf = pullqf[i];
3141 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
3142 if (bson_find_merged_arrays(bson_data(qf->updateobj), inbuf, (qf->flags & EJCONDALL))) {
3143 if (bsout.finished) {
3144 bson_init_size(&bsout, bson_size(&bsout));
3146 assert(bsout.data == NULL);
3147 bson_init_size(&bsout, bsbufsz);
3149 //$pull $pullAll merge
3150 if (bson_merge_arrays(bson_data(qf->updateobj), inbuf,
3151 BSON_MERGE_ARRAY_PULL, (qf->flags & EJCONDALL), &bsout)) {
3153 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3155 if (inbuf != bsbuf) {
3158 bson_finish(&bsout);
3166 for (int i = 0; i < 2; ++i) { //$push $pushAll
3167 const EJQF *qf = pushqf[i];
3169 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
3170 if (bsout.finished) {
3171 bson_init_size(&bsout, bson_size(&bsout));
3173 assert(bsout.data == NULL);
3174 bson_init_size(&bsout, bsbufsz);
3176 //$push $pushAll merge
3177 if (bson_merge_arrays(bson_data(qf->updateobj), inbuf,
3178 BSON_MERGE_ARRAY_PUSH, (qf->flags & EJCONDALL), &bsout)) {
3180 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3182 if (inbuf != bsbuf) {
3185 bson_finish(&bsout);
3189 for (int i = 0; i < 2; ++i) { //$addToSet $addToSetAll
3190 const EJQF *qf = addsetqf[i];
3192 char *inbuf = (bsout.finished) ? bsout.data : bsbuf;
3193 if ((qf->flags & EJCONDALL) || bson_find_unmerged_arrays(bson_data(qf->updateobj), inbuf)) {
3194 //Missing $addToSet element in some array field found
3195 if (bsout.finished) {
3196 bson_init_size(&bsout, bson_size(&bsout));
3198 assert(bsout.data == NULL);
3199 bson_init_size(&bsout, bsbufsz);
3201 //$addToSet $addToSetAll merge
3202 if (bson_merge_arrays(bson_data(qf->updateobj), inbuf,
3203 BSON_MERGE_ARRAY_ADDSET, (qf->flags & EJCONDALL), &bsout)) {
3205 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3207 if (inbuf != bsbuf) {
3210 bson_finish(&bsout);
3218 //Finishing document update
3219 if (!update || !rv) {
3222 if (bsout.err) { //Resulting BSON is OK?
3224 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3227 if (bson_size(&bsout) == 0) { //Record was not updated
3231 bt = bson_find_from_buffer(&it, bsbuf, JDBIDKEYNAME);
3232 if (bt != BSON_OID) {
3234 _ejdbsetecode(coll->jb, JBEQUPDFAILED, __FILE__, __LINE__, __func__);
3237 oid = bson_iterator_oid(&it);
3238 rowm = tcmapnew2(TCMAPTINYBNUM);
3239 tcmapput(rowm, JDBCOLBSON, JDBCOLBSONL, bson_data(&bsout), bson_size(&bsout));
3240 rv = tctdbput(coll->tdb, oid, sizeof (*oid), rowm);
3242 rv = _updatebsonidx(coll, oid, &bsout, bsbuf, bsbufsz, ctx->didxctx);
3246 bson_destroy(&bsout);
3254 static TCLIST* _qryexecute(EJCOLL *coll, const EJQ *_q, uint32_t *outcount, int qflags, TCXSTR *log) {
3255 assert(coll && coll->tdb && coll->tdb->hdb);
3258 _QRYCTX ctx = {NULL};
3259 EJQ *q; //Clone the query object
3260 TCMALLOC(q, sizeof (*q));
3261 if (!_qrydup(_q, q, EJQINTERNAL)) {
3267 ctx.qflags = qflags;
3269 if (!_qrypreprocess(&ctx)) {
3273 bool all = false; //if True we need all records to fetch (sorting)
3274 TCHDB *hdb = coll->tdb->hdb;
3275 TCLIST *res = ctx.res;
3276 EJQF *mqf = ctx.mqf;
3278 int sz = 0; //generic size var
3279 int anum = 0; //number of active conditions
3280 int ofsz = 0; //order fields count
3281 int aofsz = 0; //active order fields count
3282 const int qfsz = TCLISTNUM(q->qflist); //number of all condition fields
3283 EJQF **ofs = NULL; //order fields
3284 EJQF **qfs = NULL; //condition fields array
3286 TCMALLOC(qfs, qfsz * sizeof (EJQF*));
3294 uint32_t count = 0; //current count
3295 uint32_t max = (q->max > 0) ? q->max : UINT_MAX;
3296 uint32_t skip = q->skip;
3297 const TDBIDX *midx = mqf ? mqf->idx : NULL;
3299 if (midx) { //Main index used for ordering
3300 if (mqf->orderseq == 1 &&
3301 !(mqf->tcop == TDBQCSTRAND || mqf->tcop == TDBQCSTROR || mqf->tcop == TDBQCSTRNUMOR)) {
3302 mqf->flags |= EJFORDERUSED;
3305 for (int i = 0; i < qfsz; ++i) {
3306 EJQF *qf = TCLISTVALPTR(q->qflist, i);
3310 tcxstrprintf(log, "USING HASH TOKENS IN: %s\n", qf->fpath);
3312 if (qf->flags & EJCONDOIT) {
3313 tcxstrprintf(log, "FIELD: %s HAS $do OPERATION\n", qf->fpath);
3318 if (qf->fpathsz > 0 && !(qf->flags & EJFEXCLUDED)) {
3323 if (q->flags & EJQONLYCOUNT) {
3324 qf->flags |= EJFORDERUSED;
3328 if (ofsz > 0) { //Collect order fields array
3329 TCMALLOC(ofs, ofsz * sizeof (EJQF*));
3330 for (int i = 0; i < ofsz; ++i) {
3331 for (int j = 0; j < qfsz; ++j) {
3332 if (qfs[j]->orderseq == i + 1) { //orderseq starts with 1
3334 if (!(ofs[i]->flags & EJFORDERUSED)) {
3336 if (ctx.ifields) { //Force order field to be included in result set
3337 if (ctx.imode) { //add field to the included set
3338 tcmapputkeep(ctx.ifields, ofs[i]->fpath, ofs[i]->fpathsz, &yes, sizeof (yes));
3339 } else { //remove field from excluded
3340 tcmapout(ctx.ifields, ofs[i]->fpath, ofs[i]->fpathsz);
3348 for (int i = 0; i < ofsz; ++i) assert(ofs[i] != NULL);
3351 if ((q->flags & EJQONLYCOUNT) && qfsz == 0 &&
3352 (q->orqlist == NULL || TCLISTNUM(q->orqlist) < 1) &&
3353 (q->andqlist == NULL || TCLISTNUM(q->andqlist) < 1)) { //primitive count(*) query
3354 count = coll->tdb->hdb->rnum;
3356 tcxstrprintf(log, "SIMPLE COUNT(*): %u\n", count);
3361 if (!(q->flags & EJQONLYCOUNT) && aofsz > 0 && (!midx || mqf->orderseq != 1)) { //Main index is not the main order field
3362 all = true; //Need all records for ordering for some other fields
3366 tcxstrprintf(log, "UPDATING MODE: %s\n", (q->flags & EJQUPDATING) ? "YES" : "NO");
3367 tcxstrprintf(log, "MAX: %u\n", max);
3368 tcxstrprintf(log, "SKIP: %u\n", skip);
3369 tcxstrprintf(log, "COUNT ONLY: %s\n", (q->flags & EJQONLYCOUNT) ? "YES" : "NO");
3370 tcxstrprintf(log, "MAIN IDX: '%s'\n", midx ? midx->name : "NONE");
3371 tcxstrprintf(log, "ORDER FIELDS: %d\n", ofsz);
3372 tcxstrprintf(log, "ACTIVE CONDITIONS: %d\n", anum);
3373 tcxstrprintf(log, "ROOT $OR QUERIES: %d\n", ((q->orqlist) ? TCLISTNUM(q->orqlist) : 0));
3374 tcxstrprintf(log, "ROOT $AND QUERIES: %d\n", ((q->andqlist) ? TCLISTNUM(q->andqlist) : 0));
3375 tcxstrprintf(log, "FETCH ALL: %s\n", all ? "YES" : "NO");
3377 if (max < UINT_MAX - skip) {
3383 if (!midx && (!mqf || !(mqf->flags & EJFPKMATCHING))) { //Missing main index & no PK matching
3387 tcxstrprintf(log, "MAIN IDX TCOP: %d\n", mqf->tcop);
3390 #define JBQREGREC(_pkbuf, _pkbufsz, _bsbuf, _bsbufsz) \
3392 if (q->flags & EJQUPDATING) { \
3393 _qryupdate(&ctx, (_bsbuf), (_bsbufsz)); \
3395 if (!(q->flags & EJQONLYCOUNT) && (all || count > skip)) { \
3396 _pushprocessedbson(&ctx, (_bsbuf), (_bsbufsz)); \
3398 //EOF #define JBQREGREC
3400 bool trim = (midx && *midx->name != '\0');
3401 if (anum > 0 && !(mqf->flags & EJFEXCLUDED) && !(mqf->uslots && TCLISTNUM(mqf->uslots) > 0)) {
3403 mqf->flags |= EJFEXCLUDED;
3406 if (mqf->flags & EJFPKMATCHING) { //PK matching
3408 tcxstrprintf(log, "PRIMARY KEY MATCHING: TRUE\n");
3411 if (mqf->tcop == TDBQCSTREQ) {
3414 bson_oid_from_string(&oid, mqf->expr);
3415 tcxstrclear(q->colbuf);
3416 tcxstrclear(q->bsbuf);
3417 sz = tchdbgetintoxstr(coll->tdb->hdb, &oid, sizeof (oid), q->colbuf);
3421 sz = tcmaploadoneintoxstr(TCXSTRPTR(q->colbuf), TCXSTRSIZE(q->colbuf), JDBCOLBSON, JDBCOLBSONL, q->bsbuf);
3425 bool matched = true;
3426 for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags;
3427 for (int i = 0; i < qfsz; ++i) {
3429 if (qf->mflags & EJFEXCLUDED) continue;
3430 if (!_qrybsmatch(qf, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf))) {
3435 if (matched && _qry_and_or_match(coll, q, &oid, sizeof (oid))) {
3436 JBQREGREC(&oid, sizeof (oid), TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3439 } else if (mqf->tcop == TDBQCSTROREQ) {
3440 TCLIST *tokens = mqf->exprlist;
3443 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3444 if (!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))) {
3445 TCFREE(tclistremove2(tokens, i));
3449 int tnum = TCLISTNUM(tokens);
3450 for (int i = 0; (all || count < max) && i < tnum; i++) {
3451 bool matched = true;
3455 TCLISTVAL(token, tokens, i, tsiz);
3459 bson_oid_from_string(&oid, token);
3460 tcxstrclear(q->bsbuf);
3461 tcxstrclear(q->colbuf);
3462 sz = tchdbgetintoxstr(coll->tdb->hdb, &oid, sizeof (oid), q->colbuf);
3466 sz = tcmaploadoneintoxstr(TCXSTRPTR(q->colbuf), TCXSTRSIZE(q->colbuf), JDBCOLBSON, JDBCOLBSONL, q->bsbuf);
3470 for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags;
3471 for (int i = 0; i < qfsz; ++i) {
3473 if (qf->mflags & EJFEXCLUDED) continue;
3474 if (!_qrybsmatch(qf, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf))) {
3479 if (matched && _qry_and_or_match(coll, q, &oid, sizeof (oid))) {
3480 JBQREGREC(&oid, sizeof (oid), TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3486 } else if (mqf->tcop == TDBQTRUE) {
3487 BDBCUR *cur = tcbdbcurnew(midx->db);
3488 if (mqf->order >= 0) {
3493 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3494 if (trim) kbufsz -= 3;
3495 vbuf = tcbdbcurval3(cur, &vbufsz);
3496 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3497 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3499 if (mqf->order >= 0) {
3506 } else if (mqf->tcop == TDBQCSTREQ) { /* string is equal to */
3507 assert(midx->type == TDBITLEXICAL);
3508 char *expr = mqf->expr;
3509 int exprsz = mqf->exprsz;
3510 BDBCUR *cur = tcbdbcurnew(midx->db);
3511 tcbdbcurjump(cur, expr, exprsz + trim);
3512 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3513 if (trim) kbufsz -= 3;
3514 if (kbufsz == exprsz && !memcmp(kbuf, expr, exprsz)) {
3515 vbuf = tcbdbcurval3(cur, &vbufsz);
3516 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3517 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3525 } else if (mqf->tcop == TDBQCSTRBW) { /* string begins with */
3526 assert(midx->type == TDBITLEXICAL);
3527 char *expr = mqf->expr;
3528 int exprsz = mqf->exprsz;
3529 BDBCUR *cur = tcbdbcurnew(midx->db);
3530 tcbdbcurjump(cur, expr, exprsz + trim);
3531 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3532 if (trim) kbufsz -= 3;
3533 if (kbufsz >= exprsz && !memcmp(kbuf, expr, exprsz)) {
3534 vbuf = tcbdbcurval3(cur, &vbufsz);
3535 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3536 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3544 } else if (mqf->tcop == TDBQCSTRORBW) { /* string begins with one token in */
3545 assert(mqf->ftype == BSON_ARRAY);
3546 assert(midx->type == TDBITLEXICAL);
3547 BDBCUR *cur = tcbdbcurnew(midx->db);
3548 TCLIST *tokens = mqf->exprlist;
3551 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3552 if (!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))) {
3553 TCFREE(tclistremove2(tokens, i));
3557 if (mqf->order < 0 && (mqf->flags & EJFORDERUSED)) {
3558 tclistinvert(tokens);
3560 int tnum = TCLISTNUM(tokens);
3561 for (int i = 0; (all || count < max) && i < tnum; i++) {
3564 TCLISTVAL(token, tokens, i, tsiz);
3565 if (tsiz < 1) continue;
3566 tcbdbcurjump(cur, token, tsiz + trim);
3567 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3568 if (trim) kbufsz -= 3;
3569 if (kbufsz >= tsiz && !memcmp(kbuf, token, tsiz)) {
3570 vbuf = tcbdbcurval3(cur, &vbufsz);
3571 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3572 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3581 } else if (mqf->tcop == TDBQCSTROREQ) { /* string is equal to at least one token in */
3582 assert(mqf->ftype == BSON_ARRAY);
3583 assert(midx->type == TDBITLEXICAL);
3584 BDBCUR *cur = tcbdbcurnew(midx->db);
3585 TCLIST *tokens = mqf->exprlist;
3588 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3589 if (!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))) {
3590 TCFREE(tclistremove2(tokens, i));
3594 if (mqf->order < 0 && (mqf->flags & EJFORDERUSED)) {
3595 tclistinvert(tokens);
3597 int tnum = TCLISTNUM(tokens);
3598 for (int i = 0; (all || count < max) && i < tnum; i++) {
3601 TCLISTVAL(token, tokens, i, tsiz);
3602 if (tsiz < 1) continue;
3603 tcbdbcurjump(cur, token, tsiz + trim);
3604 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3605 if (trim) kbufsz -= 3;
3606 if (kbufsz == tsiz && !memcmp(kbuf, token, tsiz)) {
3607 vbuf = tcbdbcurval3(cur, &vbufsz);
3608 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3609 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3618 } else if (mqf->tcop == TDBQCNUMEQ) { /* number is equal to */
3619 assert(midx->type == TDBITDECIMAL);
3620 char *expr = mqf->expr;
3621 int exprsz = mqf->exprsz;
3622 BDBCUR *cur = tcbdbcurnew(midx->db);
3624 _nufetch(&num, expr, mqf->ftype);
3625 tctdbqryidxcurjumpnum(cur, expr, exprsz, true);
3626 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3627 if (_nucmp(&num, kbuf, mqf->ftype) == 0) {
3628 vbuf = tcbdbcurval3(cur, &vbufsz);
3629 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3630 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3638 } else if (mqf->tcop == TDBQCNUMGT || mqf->tcop == TDBQCNUMGE) {
3639 /* number is greater than | number is greater than or equal to */
3640 assert(midx->type == TDBITDECIMAL);
3641 char *expr = mqf->expr;
3642 int exprsz = mqf->exprsz;
3643 BDBCUR *cur = tcbdbcurnew(midx->db);
3645 _nufetch(&xnum, expr, mqf->ftype);
3646 if (mqf->order < 0 && (mqf->flags & EJFORDERUSED)) { //DESC
3648 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3650 _nufetch(&knum, kbuf, mqf->ftype);
3651 int cmp = _nucmp2(&knum, &xnum, mqf->ftype);
3653 if (cmp > 0 || (mqf->tcop == TDBQCNUMGE && cmp >= 0)) {
3654 vbuf = tcbdbcurval3(cur, &vbufsz);
3655 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3656 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3662 tctdbqryidxcurjumpnum(cur, expr, exprsz, true);
3663 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3665 _nufetch(&knum, kbuf, mqf->ftype);
3666 int cmp = _nucmp2(&knum, &xnum, mqf->ftype);
3667 if (cmp > 0 || (mqf->tcop == TDBQCNUMGE && cmp >= 0)) {
3668 vbuf = tcbdbcurval3(cur, &vbufsz);
3669 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3670 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3677 } else if (mqf->tcop == TDBQCNUMLT || mqf->tcop == TDBQCNUMLE) {
3678 /* number is less than | number is less than or equal to */
3679 assert(midx->type == TDBITDECIMAL);
3680 char *expr = mqf->expr;
3681 int exprsz = mqf->exprsz;
3682 BDBCUR *cur = tcbdbcurnew(midx->db);
3684 _nufetch(&xnum, expr, mqf->ftype);
3685 if (mqf->order >= 0) { //ASC
3687 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3689 _nufetch(&knum, kbuf, mqf->ftype);
3690 int cmp = _nucmp2(&knum, &xnum, mqf->ftype);
3692 if (cmp < 0 || (cmp <= 0 && mqf->tcop == TDBQCNUMLE)) {
3693 vbuf = tcbdbcurval3(cur, &vbufsz);
3694 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3695 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3701 tctdbqryidxcurjumpnum(cur, expr, exprsz, false);
3702 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3704 _nufetch(&knum, kbuf, mqf->ftype);
3705 int cmp = _nucmp2(&knum, &xnum, mqf->ftype);
3706 if (cmp < 0 || (cmp <= 0 && mqf->tcop == TDBQCNUMLE)) {
3707 vbuf = tcbdbcurval3(cur, &vbufsz);
3708 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3709 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3716 } else if (mqf->tcop == TDBQCNUMBT) { /* number is between two tokens of */
3717 assert(mqf->ftype == BSON_ARRAY);
3718 assert(midx->type == TDBITDECIMAL);
3719 assert(mqf->exprlist);
3720 TCLIST *tokens = mqf->exprlist;
3721 assert(TCLISTNUM(tokens) == 2);
3724 long double lower = tcatof2(tclistval2(tokens, 0));
3725 long double upper = tcatof2(tclistval2(tokens, 1));
3726 expr = tclistval2(tokens, (lower > upper) ? 1 : 0);
3727 exprsz = strlen(expr);
3728 if (lower > upper) {
3729 long double swap = lower;
3733 BDBCUR *cur = tcbdbcurnew(midx->db);
3734 tctdbqryidxcurjumpnum(cur, expr, exprsz, true);
3735 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3736 if (tcatof2(kbuf) > upper) break;
3737 vbuf = tcbdbcurval3(cur, &vbufsz);
3738 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3739 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3744 if (!all && !(q->flags & EJQONLYCOUNT) && mqf->order < 0 && (mqf->flags & EJFORDERUSED)) { //DESC
3747 } else if (mqf->tcop == TDBQCNUMOREQ) { /* number is equal to at least one token in */
3748 assert(mqf->ftype == BSON_ARRAY);
3749 assert(midx->type == TDBITDECIMAL);
3750 BDBCUR *cur = tcbdbcurnew(midx->db);
3751 TCLIST *tokens = mqf->exprlist;
3753 tclistsortex(tokens, tdbcmppkeynumasc);
3754 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3755 if (tcatof2(TCLISTVALPTR(tokens, i)) == tcatof2(TCLISTVALPTR(tokens, i - 1))) {
3756 TCFREE(tclistremove2(tokens, i));
3760 if (mqf->order < 0 && (mqf->flags & EJFORDERUSED)) {
3761 tclistinvert(tokens);
3763 int tnum = TCLISTNUM(tokens);
3764 for (int i = 0; (all || count < max) && i < tnum; i++) {
3767 TCLISTVAL(token, tokens, i, tsiz);
3768 if (tsiz < 1) continue;
3769 long double xnum = tcatof2(token);
3770 tctdbqryidxcurjumpnum(cur, token, tsiz, true);
3771 while ((all || count < max) && (kbuf = tcbdbcurkey3(cur, &kbufsz)) != NULL) {
3772 if (tcatof2(kbuf) == xnum) {
3773 vbuf = tcbdbcurval3(cur, &vbufsz);
3774 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, vbuf, vbufsz) && _qry_and_or_match(coll, q, vbuf, vbufsz)) {
3775 JBQREGREC(vbuf, vbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3784 } else if (mqf->tcop == TDBQCSTRAND || mqf->tcop == TDBQCSTROR || mqf->tcop == TDBQCSTRNUMOR) {
3785 /* string includes all tokens in | string includes at least one token in */
3786 assert(midx->type == TDBITTOKEN);
3787 assert(mqf->ftype == BSON_ARRAY);
3788 TCLIST *tokens = mqf->exprlist;
3790 if (mqf->tcop == TDBQCSTRNUMOR) {
3791 tclistsortex(tokens, tdbcmppkeynumasc);
3792 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3793 if (tcatof2(TCLISTVALPTR(tokens, i)) == tcatof2(TCLISTVALPTR(tokens, i - 1))) {
3794 TCFREE(tclistremove2(tokens, i));
3800 for (int i = 1; i < TCLISTNUM(tokens); i++) {
3801 if (!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))) {
3802 TCFREE(tclistremove2(tokens, i));
3807 TCMAP *tres = tctdbidxgetbytokens(coll->tdb, midx, tokens, mqf->tcop, log);
3808 tcmapiterinit(tres);
3809 while ((all || count < max) && (kbuf = tcmapiternext(tres, &kbufsz)) != NULL) {
3810 if (_qryallcondsmatch(q, anum, coll, qfs, qfsz, kbuf, kbufsz) && _qry_and_or_match(coll, q, kbuf, kbufsz)) {
3811 JBQREGREC(kbuf, kbufsz, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3817 if (q->flags & EJQONLYCOUNT) {
3823 fullscan: /* Full scan */
3825 assert(!res || TCLISTNUM(res) == 0);
3827 if ((q->flags & EJQDROPALL) && (q->flags & EJQONLYCOUNT)) {
3828 //if we are in primitive $dropall case. Query: {$dropall:true}
3829 if (qfsz == 1 && qfs[0]->tcop == TDBQTRUE) { //single $dropall field
3831 tcxstrprintf(log, "VANISH WHOLE COLLECTION ON $dropall\n");
3833 //write lock already acquired so use impl
3834 count = coll->tdb->hdb->rnum;
3835 if (!tctdbvanish(coll->tdb)) {
3843 tcxstrprintf(log, "RUN FULLSCAN\n");
3845 TCMAP *updkeys = (q->flags & EJQUPDATING) ? tcmapnew2(100 * 1024) : NULL;
3846 TCHDBITER *hdbiter = tchdbiter2init(hdb);
3850 TCXSTR *skbuf = tcxstrnew3(sizeof (bson_oid_t) + 1);
3851 tcxstrclear(q->colbuf);
3852 tcxstrclear(q->bsbuf);
3854 while ((all || count < max) && tchdbiter2next(hdb, hdbiter, skbuf, q->colbuf)) {
3856 sz = tcmaploadoneintoxstr(TCXSTRPTR(q->colbuf), TCXSTRSIZE(q->colbuf), JDBCOLBSON, JDBCOLBSONL, q->bsbuf);
3860 bool matched = true;
3861 for (int i = 0; i < qfsz; ++i) qfs[i]->mflags = qfs[i]->flags;
3862 for (int i = 0; i < qfsz; ++i) {
3864 if (qf->mflags & EJFEXCLUDED) {
3867 if (!_qrybsmatch(qf, TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf))) {
3872 if (matched && _qry_and_or_match(coll, q, TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf))) {
3873 if (updkeys) { //we are in updating mode
3874 if (tcmapputkeep(updkeys, TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), &yes, sizeof (yes))) {
3875 JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3878 JBQREGREC(TCXSTRPTR(skbuf), TCXSTRSIZE(skbuf), TCXSTRPTR(q->bsbuf), TCXSTRSIZE(q->bsbuf));
3883 tcxstrclear(q->colbuf);
3884 tcxstrclear(q->bsbuf);
3886 tchdbiter2dispose(hdb, hdbiter);
3892 sorting: /* Sorting resultset */
3893 if (!res || aofsz <= 0) { //No sorting needed
3896 _EJBSORTCTX sctx; //sorting context
3899 ejdbqsortlist(res, _ejdbsoncmp, &sctx);
3902 //check $upsert operation
3903 if (count == 0 && (q->flags & EJQUPDATING)) { //finding $upsert qf if no updates maden
3904 for (int i = 0; i < qfsz; ++i) {
3905 if (qfs[i]->flags & EJCONDUPSERT) {
3906 bson *updateobj = qfs[i]->updateobj;
3909 if (_ejdbsavebsonimpl(coll, updateobj, &oid, false)) {
3910 bson *nbs = bson_create();
3911 bson_init_size(nbs, bson_size(updateobj) + (strlen(JDBIDKEYNAME) + 1/*key*/ + 1/*type*/ + sizeof (oid)));
3912 bson_append_oid(nbs, JDBIDKEYNAME, &oid);
3913 bson_ensure_space(nbs, bson_size(updateobj) - 4);
3914 bson_append(nbs, bson_data(updateobj) + 4, bson_size(updateobj) - (4 + 1/*BSON_EOO*/));
3917 _ejdbsetecode(coll->jb, JBEINVALIDBSON, __FILE__, __LINE__, __func__);
3921 if (!(q->flags & EJQONLYCOUNT) && (all || count > skip)) {
3922 _pushprocessedbson(&ctx, bson_data(nbs), bson_size(nbs));
3932 if (max < UINT_MAX && max > skip) {
3936 if (all) { //skipping results after full sorting with skip > 0
3937 for (int i = 0; i < skip && res->num > 0; ++i) {
3938 TCFREE(res->array[res->start].ptr);
3943 if ((res->start & 0xff) == 0 && res->start > (res->num >> 1)) {
3944 memmove(res->array, res->array + res->start, res->num * sizeof (res->array[0]));
3947 if (TCLISTNUM(res) > max) { //truncate results if max specified
3948 int end = res->start + res->num;
3949 TCLISTDATUM *array = res->array;
3950 for (int i = (res->start + max); i < end; i++) {
3951 TCFREE(array[i].ptr);
3956 count = (skip < count) ? count - skip : 0;
3962 tcxstrprintf(log, "RS COUNT: %u\n", count);
3963 tcxstrprintf(log, "RS SIZE: %d\n", (res ? TCLISTNUM(res) : 0));
3964 tcxstrprintf(log, "FINAL SORTING: %s\n", ((q->flags & EJQONLYCOUNT) || aofsz <= 0) ? "NO" : "YES");
3967 //Apply deffered index changes
3969 for (int i = TCLISTNUM(ctx.didxctx) - 1; i >= 0; --i) {
3970 _DEFFEREDIDXCTX *di = TCLISTVALPTR(ctx.didxctx, i);
3973 tctdbidxout2(coll->tdb, &(di->oid), sizeof (di->oid), di->rmap);
3977 tctdbidxput2(coll->tdb, &(di->oid), sizeof (di->oid), di->imap);
3989 ctx.res = NULL; //save res from deleting in `_qryctxclear()`
3995 static void _qryctxclear(_QRYCTX *ctx) {
3997 tcmapdel(ctx->dfields);
4000 tcmapdel(ctx->ifields);
4003 ejdbquerydel(ctx->q);
4006 tclistdel(ctx->res);
4009 tclistdel(ctx->didxctx);
4011 memset(ctx, 0, sizeof(*ctx));
4014 static TDBIDX* _qryfindidx(EJCOLL *coll, EJQF *qf, bson *idxmeta) {
4015 TCTDB *tdb = coll->tdb;
4022 p = (qf->flags & EJCONDICASE) ? 'i' : 's'; //lexical string index
4031 p = 'n'; //number index
4035 p = 'a'; //token index
4038 p = 'o'; //take first appropriate index
4041 if (p == '\0' || !qf->fpath || !qf->fpathsz) {
4044 for (int i = 0; i < tdb->inum; ++i) {
4045 TDBIDX *idx = tdb->idxs + i;
4048 if (*idx->name == 'a' || *idx->name == 'i') { //token index or icase index not the best solution here
4051 } else if (*idx->name != p) {
4054 if (!strcmp(qf->fpath, idx->name + 1)) {
4058 //No direct operation index. Any alternatives?
4060 !(qf->flags & EJCONDICASE) && //if not case insensitive query
4062 qf->tcop == TDBQCSTREQ ||
4063 qf->tcop == TDBQCSTROREQ ||
4064 qf->tcop == TDBQCNUMOREQ ||
4065 qf->tcop == TDBQCNUMEQ)
4068 bson_type bt = bson_find(&it, idxmeta, "iflags");
4069 if (bt != BSON_INT) {
4072 int iflags = bson_iterator_int(&it);
4073 if (iflags & JBIDXARR) { //array token index exists so convert qf into TDBQCSTROR
4074 for (int i = 0; i < tdb->inum; ++i) {
4075 TDBIDX *idx = tdb->idxs + i;
4076 if (!strcmp(qf->fpath, idx->name + 1)) {
4077 if (qf->tcop == TDBQCSTREQ) {
4078 qf->tcop = TDBQCSTROR;
4079 qf->exprlist = tclistnew2(1);
4080 TCLISTPUSH(qf->exprlist, qf->expr, qf->exprsz);
4081 if (qf->expr) TCFREE(qf->expr);
4082 qf->expr = tclistdump(qf->exprlist, &qf->exprsz);
4083 qf->ftype = BSON_ARRAY;
4085 } else if (qf->tcop == TDBQCNUMEQ) {
4086 qf->tcop = TDBQCSTRNUMOR;
4087 qf->exprlist = tclistnew2(1);
4088 TCLISTPUSH(qf->exprlist, qf->expr, qf->exprsz);
4089 if (qf->expr) TCFREE(qf->expr);
4090 qf->expr = tclistdump(qf->exprlist, &qf->exprsz);
4091 qf->ftype = BSON_ARRAY;
4093 } else if (qf->tcop == TDBQCSTROREQ) {
4094 assert(qf->ftype == BSON_ARRAY);
4095 qf->tcop = TDBQCSTROR;
4097 } else if (qf->tcop == TDBQCNUMOREQ) {
4098 assert(qf->ftype == BSON_ARRAY);
4099 qf->tcop = TDBQCSTRNUMOR;
4109 static void _registerallqfields(TCLIST *reg, EJQ *q) {
4110 for (int i = 0; i < TCLISTNUM(q->qflist); ++i) {
4111 EJQF *qf = TCLISTVALPTR(q->qflist, i);
4113 tclistpush(reg, &qf, sizeof(qf));
4115 for (int i = 0; q->andqlist && i < TCLISTNUM(q->andqlist); ++i) {
4116 _registerallqfields(reg, *((EJQ**) TCLISTVALPTR(q->andqlist, i)));
4118 for (int i = 0; q->orqlist && i < TCLISTNUM(q->orqlist); ++i) {
4119 _registerallqfields(reg, *((EJQ**) TCLISTVALPTR(q->orqlist, i)));
4123 static bool _qrypreprocess(_QRYCTX *ctx) {
4124 assert(ctx->coll && ctx->q && ctx->q->qflist);
4127 //Fill the NULL terminated registry of all *EQF fields including all $and $or QF
4128 assert(!q->allqfields);
4129 TCLIST *alist = tclistnew2(TCLISTINYNUM);
4130 _registerallqfields(alist, q);
4131 TCMALLOC(q->allqfields, sizeof(EJQF*) * (TCLISTNUM(alist) + 1));
4132 for (int i = 0; i < TCLISTNUM(alist); ++i) {
4133 EJQF **qfp = TCLISTVALPTR(alist, i);
4134 q->allqfields[i] = *qfp; //*EJQF
4136 q->allqfields[TCLISTNUM(alist)] = NULL;
4140 if (ctx->qflags & JBQRYCOUNT) { //sync the user JBQRYCOUNT flag with internal
4141 q->flags |= EJQONLYCOUNT;
4143 EJQF *oqf = NULL; //Order condition
4144 TCLIST *qflist = q->qflist;
4150 bson_iterator it, sit;
4152 bt = bson_find(&it, q->hints, "$orderby");
4153 if (bt == BSON_OBJECT) {
4155 BSON_ITERATOR_SUBITERATOR(&it, &sit);
4156 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
4157 if (!BSON_IS_NUM_TYPE(bt)) {
4160 const char *ofield = BSON_ITERATOR_KEY(&sit);
4161 int odir = bson_iterator_int(&sit);
4162 odir = (odir > 0) ? 1 : (odir < 0 ? -1 : 0);
4168 for (int i = 0; i < TCLISTNUM(qflist); ++i) {
4169 if (!strcmp(ofield, ((EJQF*) TCLISTVALPTR(qflist, i))->fpath)) {
4170 qf = TCLISTVALPTR(qflist, i);
4175 if (qf == NULL) { //Create syntetic query field for orderby ops
4176 memset(&nqf, 0, sizeof (EJQF));
4177 nqf.fpath = tcstrdup(ofield);
4178 nqf.fpathsz = strlen(nqf.fpath);
4179 nqf.expr = tcstrdup("");
4181 nqf.tcop = TDBQTRUE; //disable any TC matching operation
4182 nqf.ftype = BSON_OBJECT;
4183 nqf.orderseq = orderseq++;
4185 nqf.flags |= EJFEXCLUDED; //field excluded from matching
4187 TCLISTPUSH(qflist, qf, sizeof (*qf));
4189 qf->orderseq = orderseq++;
4194 bt = bson_find(&it, q->hints, "$skip");
4195 if (BSON_IS_NUM_TYPE(bt)) {
4196 int64_t v = bson_iterator_long(&it);
4197 q->skip = (uint32_t) ((v < 0) ? 0 : v);
4199 bt = bson_find(&it, q->hints, "$max");
4200 if (ctx->qflags & JBQRYFINDONE) {
4201 q->max = (uint32_t) 1;
4202 } else if (BSON_IS_NUM_TYPE(bt)) {
4203 int64_t v = bson_iterator_long(&it);
4204 q->max = (uint32_t) ((v < 0) ? 0 : v);
4206 if (!(ctx->qflags & JBQRYCOUNT)) {
4207 bt = bson_find(&it, q->hints, "$fields"); //Collect required fields
4208 if (bt == BSON_OBJECT) {
4209 TCMAP *fmap = tcmapnew2(TCMAPTINYBNUM);
4210 BSON_ITERATOR_SUBITERATOR(&it, &sit);
4211 for (int i = 0; (bt = bson_iterator_next(&sit)) != BSON_EOO; ++i) {
4212 if (!BSON_IS_NUM_TYPE(bt)) {
4215 bool inc = (bson_iterator_int(&sit) > 0 ? true : false);
4216 if (i > 0 && inc != ctx->imode) { //$fields hint cannot mix include and exclude fields
4218 _ejdbsetecode(ctx->coll->jb, JBEQINCEXCL, __FILE__, __LINE__, __func__);
4222 const char *key = BSON_ITERATOR_KEY(&sit);
4224 //Checking the $(projection) operator.
4225 if (inc && (pptr = strstr(key, ".$")) && (*(pptr + 2) == '\0' || *(pptr + 2) == '.')) {// '.$' || '.$.'
4227 for (int i = 0; *(key + i) != '\0'; ++i) {
4228 if (*(key + i) == '$' && (dc++ > 0)) break;
4230 if (dc != 1) { //More than one '$' chars in projection, it is invalid
4233 for (int i = 0; i < TCLISTNUM(qflist); ++i) {
4234 EJQF *qf = TCLISTVALPTR(qflist, i);
4236 for (j = 0; *(key + j) != '\0' && *(key + j) == *(qf->fpath + j); ++j);
4237 if (key + j == pptr || key + j == pptr + 1) { //existing QF matched the $(projection) prefix
4239 q->ifields = tcmapnew2(TCMAPTINYBNUM);
4241 tcmapput(q->ifields, qf->fpath, qf->fpathsz, key, strlen(key));
4245 continue; //skip registering this fields in the fmap
4247 tcmapputkeep(fmap, key, strlen(key), &yes, sizeof (yes));
4249 if (TCMAPRNUM(fmap) == 0) { //if {$fields : {}} we will force {$fields : {_id:1}}
4251 tcmapputkeep(fmap, JDBIDKEYNAME, JDBIDKEYNAMEL, &yes, sizeof (yes));
4253 ctx->ifields = fmap;
4258 const int scoreexact = 100;
4259 const int scoregtlt = 50;
4260 int maxiscore = 0; //Maximum index score
4261 int maxselectivity = 0;
4263 uint32_t skipflags = (//skip field flags
4272 for (int i = 0; i < TCLISTNUM(qflist); ++i) {
4274 EJQF *qf = (EJQF*) TCLISTVALPTR(qflist, i);
4275 assert(qf && qf->fpath);
4277 if (qf->flags & EJCONDOIT) { //$do field
4278 TCMAP *dmap = ctx->dfields;
4280 dmap = tcmapnew2(TCMAPTINYBNUM);
4281 ctx->dfields = dmap;
4283 tcmapputkeep(dmap, qf->fpath, qf->fpathsz, qf, sizeof (*qf));
4286 if (qf->flags & skipflags) {
4290 if (!qf->negate && (qf->tcop == TDBQCSTREQ || qf->tcop == TDBQCSTROREQ) && !strcmp(JDBIDKEYNAME, qf->fpath)) {
4291 qf->flags |= EJFPKMATCHING;
4296 bool firstorderqf = false;
4297 qf->idxmeta = _imetaidx(ctx->coll, qf->fpath);
4298 qf->idx = _qryfindidx(ctx->coll, qf, qf->idxmeta);
4299 if (qf->order && qf->orderseq == 1) { //Index for first 'orderby' exists
4301 firstorderqf = true;
4303 if (!qf->idx || !qf->idxmeta) {
4305 bson_del(qf->idxmeta);
4311 if (qf->tcop == TDBQTRUE || qf->negate) {
4315 int selectivity = -1;
4316 bt = bson_find(&it, qf->idxmeta, "selectivity");
4317 if (bt == BSON_DOUBLE) {
4318 selectivity = (int) ((double) bson_iterator_double(&it) * 100); //Selectivity percent
4320 bt = bson_find(&it, qf->idxmeta, "avgreclen");
4321 if (bt == BSON_DOUBLE) {
4322 avgreclen = (int) bson_iterator_double(&it);
4324 if (selectivity > 0) {
4325 if (selectivity <= 20) { //Not using index at all if selectivity lesser than 20%
4328 iscore += selectivity;
4331 iscore += (maxselectivity - selectivity) / 2;
4333 if (selectivity > maxselectivity) {
4334 maxselectivity = selectivity;
4341 iscore += scoreexact;
4345 if (avgreclen > 0 && qf->exprsz > avgreclen) {
4346 iscore += scoreexact;
4354 iscore += scoreexact;
4356 iscore += scoregtlt;
4360 if (iscore >= maxiscore) {
4365 if (ctx->mqf == NULL && (oqf && oqf->idx && !oqf->negate)) {
4369 if (q->flags & EJQHASUQUERY) { //check update $(query) projection then sync inter-qf refs #91
4370 for (int i = 0; *(q->allqfields + i) != '\0'; ++i) {
4371 EJQF *qf = q->allqfields[i];
4372 if (!qf->ufields) continue;
4373 TCLIST *uflist = qf->ufields;
4374 for (int j = 0; j < TCLISTNUM(uflist); ++j) {
4375 const char *ukey = TCLISTVALPTR(uflist, j);
4376 char *pptr = strstr(ukey, ".$");
4378 for (int k = 0; *(q->allqfields + k) != '\0'; ++k) {
4380 EJQF *kqf = q->allqfields[k];
4381 if (kqf == qf || !kqf->fpath) { //do not process itself
4384 for (l = 0; *(ukey + l) != '\0' && *(ukey + l) == *(kqf->fpath + l); ++l);
4385 if (ukey + l == pptr || ukey + l == pptr + 1) { //existing QF matched the $(query) prefix
4387 kqf->uslots = tclistnew2(TCLISTINYNUM);
4391 .dpos = (pptr - ukey),
4394 tclistpush(kqf->uslots, &uslot, sizeof(uslot));
4401 //Init query processing buffers
4402 assert(!q->colbuf && !q->bsbuf);
4403 q->colbuf = tcxstrnew3(1024);
4404 q->bsbuf = tcxstrnew3(1024);
4405 q->tmpbuf = tcxstrnew();
4406 ctx->didxctx = (q->flags & EJQUPDATING) ? tclistnew() : NULL;
4407 ctx->res = (q->flags & EJQONLYCOUNT) ? NULL : tclistnew2(4096);
4411 static bool _metasetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
4414 return _metasetbson(jb, colname, strlen(colname), "opts", NULL, false, false);
4416 bson *bsopts = bson_create();
4418 bson_append_bool(bsopts, "compressed", opts->compressed);
4419 bson_append_bool(bsopts, "large", opts->large);
4420 bson_append_int(bsopts, "cachedrecords", opts->cachedrecords);
4421 bson_append_int(bsopts, "records", opts->records);
4422 bson_finish(bsopts);
4423 rv = _metasetbson(jb, colname, strlen(colname), "opts", bsopts, false, false);
4428 static bool _metagetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
4431 memset(opts, 0, sizeof (*opts));
4432 bson *bsopts = _metagetbson(jb, colname, strlen(colname), "opts");
4437 bson_type bt = bson_find(&it, bsopts, "compressed");
4438 if (bt == BSON_BOOL) {
4439 opts->compressed = bson_iterator_bool(&it);
4441 bt = bson_find(&it, bsopts, "large");
4442 if (bt == BSON_BOOL) {
4443 opts->large = bson_iterator_bool(&it);
4445 bt = bson_find(&it, bsopts, "cachedrecords");
4446 if (BSON_IS_NUM_TYPE(bt)) {
4447 opts->cachedrecords = bson_iterator_long(&it);
4449 bt = bson_find(&it, bsopts, "records");
4450 if (BSON_IS_NUM_TYPE(bt)) {
4451 opts->records = bson_iterator_long(&it);
4457 static bool _metasetbson(EJDB *jb, const char *colname, int colnamesz,
4458 const char *mkey, bson *val, bool merge, bool mergeoverwrt) {
4459 assert(jb && colname && mkey);
4462 bson *oldval = NULL;
4465 TCMAP *cmeta = tctdbget(jb->metadb, colname, colnamesz);
4467 _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
4472 if (tcmapout2(cmeta, mkey)) {
4473 rv = tctdbput(jb->metadb, colname, colnamesz, cmeta);
4478 if (merge) { //Merged
4479 oldval = _metagetbson(jb, colname, colnamesz, mkey);
4481 bson_init(&mresult);
4482 bson_merge(oldval, val, mergeoverwrt, &mresult);
4483 bson_finish(&mresult);
4493 tcmapput(cmeta, mkey, strlen(mkey), bson_data(bsave), bson_size(bsave));
4494 rv = tctdbput(jb->metadb, colname, colnamesz, cmeta);
4498 bson_destroy(bsave);
4505 tctdbsync(jb->metadb);
4509 static bool _metasetbson2(EJCOLL *coll, const char *mkey, bson *val, bool merge, bool mergeoverwrt) {
4511 return _metasetbson(coll->jb, coll->cname, coll->cnamesz, mkey, val, merge, mergeoverwrt);
4514 /**Returned meta BSON data must be freed by 'bson_del' */
4515 static bson* _metagetbson(EJDB *jb, const char *colname, int colnamesz, const char *mkey) {
4516 assert(jb && colname && mkey);
4518 TCMAP *cmeta = tctdbget(jb->metadb, colname, colnamesz);
4520 _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
4524 const void *raw = tcmapget(cmeta, mkey, strlen(mkey), &bsz);
4525 if (!raw || bsz == 0) {
4529 bson_init_size(rv, bsz);
4530 bson_ensure_space(rv, bsz - 4);
4531 bson_append(rv, ((char*) raw) + 4, bsz - (4 + 1/*BSON_EOO*/));
4538 static bson* _metagetbson2(EJCOLL *coll, const char *mkey) {
4540 return _metagetbson(coll->jb, coll->cname, coll->cnamesz, mkey);
4543 /**Returned index meta if not NULL it must be freed by 'bson_del' */
4544 static bson* _imetaidx(EJCOLL *coll, const char *ipath) {
4545 assert(coll && ipath);
4546 if (*ipath == '\0') {
4550 char fpathkey[BSON_MAX_FPATH_LEN + 1];
4551 TCMAP *cmeta = tctdbget(coll->jb->metadb, coll->cname, coll->cnamesz);
4553 _ejdbsetecode(coll->jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
4556 int klen = snprintf(fpathkey, BSON_MAX_FPATH_LEN + 1, "i%s", ipath); //'i' prefix for all columns with index meta
4557 if (klen > BSON_MAX_FPATH_LEN) {
4558 _ejdbsetecode(coll->jb, JBEFPATHINVALID, __FILE__, __LINE__, __func__);
4562 const void *bsdata = tcmapget(cmeta, fpathkey, klen, &bsz);
4565 bson_init_size(rv, bsz);
4566 bson_ensure_space(rv, bsz - 4);
4567 bson_append(rv, ((char*) bsdata) + 4, bsz - (4 + 1));
4577 /** Free EJQF field **/
4578 static void _delqfdata(const EJQ *q, const EJQF *qf) {
4587 bson_del(qf->idxmeta);
4589 if (qf->updateobj) {
4590 bson_del(qf->updateobj);
4593 tclistdel(qf->ufields);
4596 tclistdel(qf->uslots);
4598 if (qf->regex && !(EJQINTERNAL & q->flags)) {
4599 //We do not clear regex_t data because it not deep copy in internal queries
4600 regfree((regex_t *) qf->regex);
4604 tclistdel(qf->exprlist);
4607 tcmapdel(qf->exprmap);
4611 static bool _ejdbsavebsonimpl(EJCOLL *coll, bson *bs, bson_oid_t *oid, bool merge) {
4614 bson_type oidt = _bsonoidkey(bs, oid);
4615 if (oidt == BSON_EOO) { //missing _id so generate a new _id
4617 nbs = bson_create();
4618 bson_init_size(nbs, bson_size(bs) + (strlen(JDBIDKEYNAME) + 1/*key*/ + 1/*type*/ + sizeof (*oid)));
4619 bson_append_oid(nbs, JDBIDKEYNAME, oid);
4620 bson_ensure_space(nbs, bson_size(bs) - 4);
4621 bson_append(nbs, bson_data(bs) + 4, bson_size(bs) - (4 + 1/*BSON_EOO*/));
4625 } else if (oidt != BSON_OID) { //_oid presented by it is not BSON_OID
4626 _ejdbsetecode(coll->jb, JBEINVALIDBSONPK, __FILE__, __LINE__, __func__);
4629 TCTDB *tdb = coll->tdb;
4630 TCMAP *rowm = (tdb->hdb->rnum > 0) ? tctdbget(tdb, oid, sizeof (*oid)) : NULL;
4631 char *obsdata = NULL; //Old bson
4633 if (rowm) { //Save the copy of old bson data
4634 const void *obs = tcmapget(rowm, JDBCOLBSON, JDBCOLBSONL, &obsdatasz);
4635 if (obs && obsdatasz > 0) {
4636 TCMALLOC(obsdata, obsdatasz);
4637 memmove(obsdata, obs, obsdatasz);
4640 rowm = tcmapnew2(TCMAPTINYBNUM);
4642 if (merge && !nbs && obsdata) {
4643 nbs = bson_create();
4644 bson_init_size(nbs, MAX(obsdatasz, bson_size(bs)));
4645 bson_merge2(obsdata, bson_data(bs), true, nbs);
4650 tcmapput(rowm, JDBCOLBSON, JDBCOLBSONL, bson_data(bs), bson_size(bs));
4651 if (!tctdbput(tdb, oid, sizeof (*oid), rowm)) {
4655 rv = _updatebsonidx(coll, oid, bs, obsdata, obsdatasz, NULL);
4670 * Copy BSON array into new TCLIST. TCLIST must be freed by 'tclistdel'.
4671 * @param it BSON iterator
4672 * @param type[out] Detected BSON type of last element
4674 static TCLIST* _fetch_bson_str_array(EJDB *jb, bson_iterator *it, bson_type *type, txtflags_t tflags) {
4675 TCLIST *res = tclistnew();
4678 for (int i = 0; (ftype = bson_iterator_next(it)) != BSON_EOO; ++i) {
4682 if (tflags & JBICASE) { //ignore case
4684 char sbuf[JBSTRINOPBUFFERSZ];
4685 int len = tcicaseformat(bson_iterator_string(it), bson_iterator_string_len(it) - 1, sbuf, JBSTRINOPBUFFERSZ, &buf);
4687 _ejdbsetecode(jb, len, __FILE__, __LINE__, __func__);
4690 tclistpush2(res, buf);
4691 if (buf && buf != sbuf) {
4695 tclistpush2(res, bson_iterator_string(it));
4703 tclistprintf(res, "%" PRId64, bson_iterator_long(it));
4707 tclistprintf(res, "%f", bson_iterator_double(it));
4712 bson_oid_to_string(bson_iterator_oid(it), xoid);
4713 tclistprintf(res, "%s", xoid);
4722 /** result must be freed by TCFREE */
4723 static char* _fetch_bson_str_array2(EJDB *jb, bson_iterator *it, bson_type *type, txtflags_t tflags) {
4724 TCLIST *res = _fetch_bson_str_array(jb, it, type, tflags);
4725 char *tokens = tcstrjoin(res, ',');
4730 static int _parse_qobj_impl(EJDB *jb, EJQ *q, bson_iterator *it, TCLIST *qlist, TCLIST *pathStack, EJQF *pqf, int elmatchgrp) {
4731 assert(it && qlist && pathStack);
4733 bson_type ftype, bt;
4735 while ((ftype = bson_iterator_next(it)) != BSON_EOO) {
4736 const char *fkey = BSON_ITERATOR_KEY(it);
4737 bool isckey = (*fkey == '$'); //Key is a control key: $in, $nin, $not, $all, ...
4739 memset(&qf, 0, sizeof (qf));
4742 qf.elmatchgrp = pqf->elmatchgrp;
4743 qf.elmatchpos = pqf->elmatchpos;
4744 qf.flags = pqf->flags;
4746 qf.negate = pqf->negate;
4750 //Push key on top of path stack
4751 tclistpush2(pathStack, fkey);
4754 if (!strcmp("$or", fkey) || //Both levels operators.
4755 !strcmp("$and", fkey)) {
4757 } else if (!strcmp("$set", fkey) ||
4758 !strcmp("$inc", fkey) ||
4759 !strcmp("$dropall", fkey) ||
4760 !strcmp("$addToSet", fkey) ||
4761 !strcmp("$addToSetAll", fkey) ||
4762 !strcmp("$push", fkey) ||
4763 !strcmp("$pushAll", fkey) ||
4764 !strcmp("$pull", fkey) ||
4765 !strcmp("$pullAll", fkey) ||
4766 !strcmp("$upsert", fkey) ||
4767 !strcmp("$do", fkey) ||
4768 !strcmp("$unset", fkey) ||
4769 !strcmp("$rename", fkey)
4771 if (pqf) { //Top level ops
4773 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4777 if (!pqf) { //Require parent query object
4779 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4783 if (!strcmp("$not", fkey)) {
4784 qf.negate = !qf.negate;
4785 } else if (!strcmp("$gt", fkey)) {
4786 qf.flags |= EJCOMPGT;
4787 } else if (!strcmp("$gte", fkey)) {
4788 qf.flags |= EJCOMPGTE;
4789 } else if (!strcmp("$lt", fkey)) {
4790 qf.flags |= EJCOMPLT;
4791 } else if (!strcmp("$lte", fkey)) {
4792 qf.flags |= EJCOMPLTE;
4793 } else if (!strcmp("$begin", fkey)) {
4794 qf.flags |= EJCONDSTARTWITH;
4795 } else if (!strcmp("$icase", fkey)) {
4796 qf.flags |= EJCONDICASE;
4804 if (!strcmp("$and", fkey)) {
4806 BSON_ITERATOR_SUBITERATOR(it, &sit);
4807 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
4808 if (bt != BSON_OBJECT) {
4811 if (_qryaddand(jb, q, bson_iterator_value(&sit)) == NULL) {
4816 } else if (!strcmp("$or", fkey)) {
4818 BSON_ITERATOR_SUBITERATOR(it, &sit);
4819 while ((bt = bson_iterator_next(&sit)) != BSON_EOO) {
4820 if (bt != BSON_OBJECT) {
4823 if (ejdbqueryaddor(jb, q, bson_iterator_value(&sit)) == NULL) {
4830 BSON_ITERATOR_SUBITERATOR(it, &sit);
4831 bson_type atype = 0;
4832 TCLIST *tokens = _fetch_bson_str_array(jb, &sit, &atype, (qf.flags & EJCONDICASE) ? JBICASE : 0);
4834 ret = JBEQINOPNOTARRAY;
4835 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4839 assert(!qf.expr && !qf.fpath);
4840 qf.ftype = BSON_ARRAY;
4841 qf.expr = tclistdump(tokens, &qf.exprsz);
4842 qf.exprlist = tokens;
4843 if (!strcmp("$in", fkey) || !strcmp("$nin", fkey)) {
4844 if (!strcmp("$nin", fkey)) {
4847 if (BSON_IS_NUM_TYPE(atype) || atype == BSON_BOOL || atype == BSON_DATE) {
4848 qf.tcop = TDBQCNUMOREQ;
4850 qf.tcop = TDBQCSTROREQ;
4851 if (TCLISTNUM(tokens) >= JBINOPTMAPTHRESHOLD) {
4852 assert(!qf.exprmap);
4853 qf.exprmap = tcmapnew2(TCLISTNUM(tokens));
4854 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
4855 tcmapputkeep(qf.exprmap, TCLISTVALPTR(tokens, i), TCLISTVALSIZ(tokens, i), &yes, sizeof (yes));
4859 } else if (!strcmp("$bt", fkey)) { //between
4860 qf.tcop = TDBQCNUMBT;
4861 if (TCLISTNUM(tokens) != 2) {
4862 ret = JBEQINOPNOTARRAY;
4863 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4865 tclistdel(qf.exprlist);
4868 } else if (!strcmp("$strand", fkey)) { //$strand
4869 qf.tcop = TDBQCSTRAND;
4870 } else if (!strcmp("$stror", fkey)) { //$stror
4871 qf.tcop = TDBQCSTROR;
4872 } else if (qf.flags & EJCONDSTARTWITH) { //$begin with some token
4873 qf.tcop = TDBQCSTRORBW;
4875 ret = JBEQINVALIDQCONTROL;
4876 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4878 tclistdel(qf.exprlist);
4881 qf.fpath = tcstrjoin(pathStack, '.');
4882 qf.fpathsz = strlen(qf.fpath);
4883 TCLISTPUSH(qlist, &qf, sizeof (qf));
4888 BSON_ITERATOR_SUBITERATOR(it, &sit);
4889 ret = _parse_qobj_impl(jb, q, &sit, qlist, pathStack, &qf, elmatchgrp);
4896 if (!strcmp("$inc", fkey)) {
4897 qf.flags |= EJCONDINC;
4898 } else if (!pqf) { //top level op
4899 if (!strcmp("$do", fkey)) {
4900 qf.flags |= EJCONDOIT;
4901 } else if (!strcmp("$set", fkey)) { //top level set OP
4902 qf.flags |= EJCONDSET;
4903 } else if (!strcmp("$addToSet", fkey)) {
4904 qf.flags |= EJCONDADDSET;
4905 } else if (!strcmp("$addToSetAll", fkey)) {
4906 qf.flags |= EJCONDADDSET;
4907 qf.flags |= EJCONDALL;
4908 } else if (!strcmp("$push", fkey)) {
4909 qf.flags |= EJCONDPUSH;
4910 } else if (!strcmp("$pushAll", fkey)) {
4911 qf.flags |= EJCONDPUSH;
4912 qf.flags |= EJCONDALL;
4913 } else if (!strcmp("$pull", fkey)) {
4914 qf.flags |= EJCONDPULL;
4915 } else if (!strcmp("$pullAll", fkey)) {
4916 qf.flags |= EJCONDPULL;
4917 qf.flags |= EJCONDALL;
4918 } else if (!strcmp("$upsert", fkey)) {
4919 qf.flags |= EJCONDSET;
4920 qf.flags |= EJCONDUPSERT;
4921 } else if (!strcmp("$unset", fkey)) {
4922 qf.flags |= EJCONDUNSET;
4923 } else if (!strcmp("$rename", fkey)) {
4924 qf.flags |= EJCONDRENAME;
4929 (EJCONDSET | EJCONDINC | EJCONDADDSET | EJCONDPULL | EJCONDPUSH |
4930 EJCONDUNSET | EJCONDRENAME))) {
4932 assert(qf.updateobj == NULL);
4933 qf.q->flags |= EJQUPDATING;
4934 qf.updateobj = bson_create();
4935 bson_init_as_query(qf.updateobj);
4938 BSON_ITERATOR_SUBITERATOR(it, &sit);
4939 while ((sbt = bson_iterator_next(&sit)) != BSON_EOO) {
4940 if ((qf.flags & EJCONDALL) && sbt != BSON_ARRAY) {
4941 //addToSet, push, pull accept only an arrays
4944 if (!(qf.flags & EJCONDALL) &&
4945 (qf.flags & (EJCONDSET | EJCONDINC | EJCONDUNSET | EJCONDRENAME))) { //Checking the $(query) positional operator.
4946 const char* ukey = BSON_ITERATOR_KEY(&sit);
4948 if ((pptr = strstr(ukey, ".$")) && pptr && (*(pptr + 2) == '\0' || *(pptr + 2) == '.')) {// '.$' || '.$.'
4950 for (int i = 0; *(ukey + i) != '\0'; ++i) {
4951 if (*(ukey + i) == '$' && (dc++ > 0)) break;
4953 if (dc != 1) { //More than one '$' chars in projection, it is invalid
4956 // Now just store only [$(query) key] into the qf->ufields
4958 qf.ufields = tclistnew2(TCLISTINYNUM);
4960 q->flags |= EJQHASUQUERY;
4961 tclistpush(qf.ufields, ukey, strlen(ukey));
4964 bson_append_field_from_iterator(&sit, qf.updateobj);
4966 bson_finish(qf.updateobj);
4967 if (qf.updateobj->err) {
4969 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4972 qf.fpath = strdup(fkey);
4973 qf.fpathsz = strlen(qf.fpath);
4975 qf.flags |= EJFEXCLUDED;
4976 TCLISTPUSH(qlist, &qf, sizeof (qf));
4980 if (!strcmp("$elemMatch", fkey)) {
4981 if (qf.elmatchgrp) { //only one $elemMatch allowed in query field
4983 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
4986 qf.elmatchgrp = ++elmatchgrp;
4987 char *fpath = tcstrjoin(pathStack, '.');
4988 qf.elmatchpos = strlen(fpath) + 1; //+ 1 to skip next dot '.'
4992 if (qf.flags & EJCONDOIT) {
4993 qf.updateobj = bson_create();
4994 bson_init_as_query(qf.updateobj);
4997 BSON_ITERATOR_SUBITERATOR(it, &sit);
4999 while ((sbt = bson_iterator_next(&sit)) != BSON_EOO) {
5000 const char *akey = BSON_ITERATOR_KEY(&sit);
5001 if (!strcmp("$join", akey) || !strcmp("$slice", akey)) {
5002 bson_append_field_from_iterator(&sit, qf.updateobj);
5006 bson_finish(qf.updateobj);
5007 if (qf.updateobj->err) {
5009 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
5014 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
5017 qf.fpath = strdup(fkey);
5018 qf.fpathsz = strlen(qf.fpath);
5020 qf.flags |= EJFEXCLUDED;
5021 TCLISTPUSH(qlist, &qf, sizeof (qf));
5026 BSON_ITERATOR_SUBITERATOR(it, &sit);
5027 ret = _parse_qobj_impl(jb, q, &sit, qlist, pathStack, &qf, elmatchgrp);
5032 assert(!qf.fpath && !qf.expr);
5034 TCMALLOC(qf.expr, 25 * sizeof (char));
5035 bson_oid_to_string(bson_iterator_oid(it), qf.expr);
5037 qf.fpath = tcstrjoin(pathStack, '.');
5038 qf.fpathsz = strlen(qf.fpath);
5039 qf.tcop = TDBQCSTREQ;
5040 TCLISTPUSH(qlist, &qf, sizeof (qf));
5044 assert(!qf.fpath && !qf.expr);
5046 if (qf.flags & EJCONDICASE) {
5047 qf.exprsz = tcicaseformat(bson_iterator_string(it), bson_iterator_string_len(it) - 1, NULL, 0, &qf.expr);
5048 if (qf.exprsz < 0) {
5050 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
5055 qf.expr = tcstrdup(bson_iterator_string(it));
5056 qf.exprsz = strlen(qf.expr);
5059 qf.fpath = tcstrjoin(pathStack, '.');
5060 qf.fpathsz = strlen(qf.fpath);
5061 if (qf.flags & EJCONDSTARTWITH) {
5062 qf.tcop = TDBQCSTRBW;
5064 qf.tcop = TDBQCSTREQ;
5065 if (!strcmp(JDBIDKEYNAME, fkey)) { //_id
5066 qf.ftype = BSON_OID;
5069 TCLISTPUSH(qlist, &qf, sizeof (qf));
5076 assert(!qf.fpath && !qf.expr);
5078 qf.fpath = tcstrjoin(pathStack, '.');
5079 qf.fpathsz = strlen(qf.fpath);
5080 if (ftype == BSON_LONG || ftype == BSON_INT || ftype == BSON_DATE) {
5081 qf.exprlongval = bson_iterator_long(it);
5082 qf.exprdblval = qf.exprlongval;
5083 // 2015-04-14: Change to use standard format string for int64_t
5084 qf.expr = tcsprintf("%" PRId64, qf.exprlongval);
5086 qf.exprdblval = bson_iterator_double(it);
5087 qf.exprlongval = (int64_t) qf.exprdblval;
5088 qf.expr = tcsprintf("%f", qf.exprdblval);
5090 qf.exprsz = strlen(qf.expr);
5091 if (qf.flags & EJCOMPGT) {
5092 qf.tcop = TDBQCNUMGT;
5093 } else if (qf.flags & EJCOMPGTE) {
5094 qf.tcop = TDBQCNUMGE;
5095 } else if (qf.flags & EJCOMPLT) {
5096 qf.tcop = TDBQCNUMLT;
5097 } else if (qf.flags & EJCOMPLTE) {
5098 qf.tcop = TDBQCNUMLE;
5100 qf.tcop = TDBQCNUMEQ;
5102 TCLISTPUSH(qlist, &qf, sizeof (qf));
5106 assert(!qf.fpath && !qf.expr);
5108 qf.tcop = TDBQCSTRRX;
5109 char *re = tcstrdup(bson_iterator_regex(it));
5110 const char *opts = bson_iterator_regex_opts(it);
5111 qf.fpath = tcstrjoin(pathStack, '.');
5112 qf.fpathsz = strlen(qf.fpath);
5114 qf.exprsz = strlen(qf.expr);
5115 const char *rxstr = qf.expr;
5117 int rxopt = REG_EXTENDED | REG_NOSUB;
5118 if (strchr(opts, 'i')) {
5121 if (regcomp(&rxbuf, rxstr, rxopt) == 0) {
5122 TCMALLOC(qf.regex, sizeof (rxbuf));
5123 memcpy(qf.regex, &rxbuf, sizeof (rxbuf));
5125 ret = JBEQINVALIDQRX;
5126 _ejdbsetecode(jb, ret, __FILE__, __LINE__, __func__);
5131 TCLISTPUSH(qlist, &qf, sizeof (qf));
5135 case BSON_UNDEFINED:
5137 qf.tcop = TDBQCEXIST;
5138 qf.negate = !qf.negate;
5139 qf.expr = tcstrdup(""); //Empty string as expr
5141 qf.fpath = tcstrjoin(pathStack, '.');
5142 qf.fpathsz = strlen(qf.fpath);
5143 TCLISTPUSH(qlist, &qf, sizeof (qf));
5146 case BSON_BOOL: { //boolean converted into number
5147 bool bv = bson_iterator_bool_raw(it);
5149 if (!strcmp("$dropall", fkey) && bv) {
5150 qf.flags |= EJFEXCLUDED;
5151 qf.fpath = tcstrjoin(pathStack, '.');
5152 qf.fpathsz = strlen(qf.fpath);
5154 qf.q->flags |= EJQUPDATING;
5155 qf.q->flags |= EJQDROPALL;
5156 qf.expr = tcstrdup(""); //Empty string as expr
5158 TCLISTPUSH(qlist, &qf, sizeof (qf));
5161 if (!strcmp("$exists", fkey)) {
5162 qf.tcop = TDBQCEXIST;
5163 qf.fpath = tcstrjoin(pathStack, '.');
5164 qf.fpathsz = strlen(qf.fpath);
5165 qf.expr = tcstrdup(""); //Empty string as expr
5168 qf.negate = !qf.negate;
5170 TCLISTPUSH(qlist, &qf, sizeof (qf));
5174 qf.tcop = TDBQCNUMEQ;
5175 qf.fpath = tcstrjoin(pathStack, '.');
5176 qf.fpathsz = strlen(qf.fpath);
5177 qf.exprlongval = (bv ? 1 : 0);
5178 qf.exprdblval = qf.exprlongval;
5179 qf.expr = strdup(bv ? "1" : "0");
5181 TCLISTPUSH(qlist, &qf, sizeof (qf));
5189 assert(pathStack->num > 0);
5190 TCFREE(tclistpop2(pathStack));
5192 if (ret) { //cleanup on error condition
5200 * Convert bson query spec into field path -> EJQF instance.
5201 * Created map instance must be freed by `tcmapdel`.
5202 * Each element of map must be freed by `ejdbquerydel`.
5204 static TCLIST* _parseqobj(EJDB *jb, EJQ *q, bson *qspec) {
5206 return _parseqobj2(jb, q, bson_data(qspec));
5209 static TCLIST* _parseqobj2(EJDB *jb, EJQ *q, const void *qspecbsdata) {
5210 assert(qspecbsdata);
5212 TCLIST *res = tclistnew2(TCLISTINYNUM);
5213 TCLIST *pathStack = tclistnew2(TCLISTINYNUM);
5215 BSON_ITERATOR_FROM_BUFFER(&it, qspecbsdata);
5216 rv = _parse_qobj_impl(jb, q, &it, res, pathStack, NULL, 0);
5221 assert(!pathStack->num);
5222 tclistdel(pathStack);
5227 * Get OID value from the '_id' field of specified bson object.
5228 * @param bson[in] BSON object
5229 * @param oid[out] Pointer to OID type
5230 * @return True if OID value is found int _id field of bson object otherwise False.
5232 static bson_type _bsonoidkey(bson *bs, bson_oid_t *oid) {
5234 bson_type bt = bson_find(&it, bs, JDBIDKEYNAME);
5235 if (bt == BSON_OID) {
5236 *oid = *bson_iterator_oid(&it);
5242 * Return string value representation of value pointed by 'it'.
5243 * Resulting value size stored into 'vsz'.
5244 * If returned value is not NULL it must be freed by TCFREE.
5246 static char* _bsonitstrval(EJDB *jb, bson_iterator *it, int *vsz, TCLIST *tokens, txtflags_t tflags) {
5249 bson_type btype = BSON_ITERATOR_TYPE(it);
5250 if (btype == BSON_STRING) {
5251 if (tokens) { //split string into tokens and push it into 'tokens' list
5252 const unsigned char *sp = (unsigned char *) bson_iterator_string(it);
5253 while (*sp != '\0') {
5254 while ((*sp != '\0' && *sp <= ' ') || *sp == ',') {
5257 const unsigned char *ep = sp;
5258 while (*ep > ' ' && *ep != ',') {
5262 if (tflags & JBICASE) { //ignore case mode
5264 char sbuf[JBSTRINOPBUFFERSZ];
5265 int len = tcicaseformat((const char*) sp, ep - sp, sbuf, JBSTRINOPBUFFERSZ, &buf);
5266 if (len >= 0) { //success
5267 TCLISTPUSH(tokens, buf, len);
5269 _ejdbsetecode(jb, len, __FILE__, __LINE__, __func__);
5271 if (buf && buf != sbuf) {
5275 TCLISTPUSH(tokens, sp, ep - sp);
5281 retlen = bson_iterator_string_len(it) - 1;
5282 if (tflags & JBICASE) {
5283 retlen = tcicaseformat(bson_iterator_string(it), retlen, NULL, 0, &ret);
5285 ret = tcmemdup(bson_iterator_string(it), retlen);
5288 } else if (BSON_IS_NUM_TYPE(btype) || btype == BSON_BOOL || btype == BSON_DATE) {
5289 char nbuff[TCNUMBUFSIZ];
5290 if (btype == BSON_INT || btype == BSON_LONG || btype == BSON_BOOL || btype == BSON_DATE) {
5291 retlen = bson_numstrn(nbuff, TCNUMBUFSIZ, bson_iterator_long(it));
5292 if (retlen >= TCNUMBUFSIZ) {
5293 retlen = TCNUMBUFSIZ - 1;
5295 } else if (btype == BSON_DOUBLE) {
5296 retlen = tcftoa(bson_iterator_double(it), nbuff, TCNUMBUFSIZ, 6);
5297 if (retlen >= TCNUMBUFSIZ) {
5298 retlen = TCNUMBUFSIZ - 1;
5301 if (tflags & JBICASE) {
5302 retlen = tcicaseformat(nbuff, retlen, NULL, 0, &ret);
5304 ret = tcmemdup(nbuff, retlen);
5306 } else if (btype == BSON_ARRAY) {
5307 bson_type eltype; //last element bson type
5309 BSON_ITERATOR_SUBITERATOR(it, &sit);
5311 while ((eltype = bson_iterator_next(&sit)) != BSON_EOO) {
5313 char *v = _bsonitstrval(jb, &sit, &vz, NULL, tflags);
5315 TCLISTPUSH(tokens, v, vz);
5320 //Array elements are joined with ',' delimeter.
5321 ret = _fetch_bson_str_array2(jb, &sit, &eltype, tflags);
5322 retlen = strlen(ret);
5326 _ejdbsetecode(jb, retlen, __FILE__, __LINE__, __func__);
5334 static char* _bsonipathrowldr(
5336 const char *pkbuf, int pksz,
5337 const char *rowdata, int rowdatasz,
5338 const char *ipath, int ipathsz, void *op, int *vsz) {
5341 if (ipath && *ipath == '\0') { //PK TODO review
5343 const unsigned char *sp = (unsigned char *) pkbuf;
5344 while (*sp != '\0') {
5345 while ((*sp != '\0' && *sp <= ' ') || *sp == ',') {
5348 const unsigned char *ep = sp;
5349 while (*ep > ' ' && *ep != ',') {
5352 if (ep > sp) TCLISTPUSH(tokens, sp, ep - sp);
5358 TCMEMDUP(res, pkbuf, pksz);
5363 if (!ipath || ipathsz < 2 || *(ipath + 1) == '\0' || strchr("snai", *ipath) == NULL) {
5366 //skip index type prefix char with (fpath + 1)
5367 res = _bsonfpathrowldr(tokens, rowdata, rowdatasz, ipath + 1, ipathsz - 1, op, vsz);
5368 if (*vsz == 0) { //Do not allow empty strings for index opration
5369 if (res) TCFREE(res);
5375 static char* _bsonfpathrowldr(TCLIST *tokens, const char *rowdata, int rowdatasz,
5376 const char *fpath, int fpathsz, void *op, int *vsz) {
5377 _BSONIPATHROWLDR *odata = (_BSONIPATHROWLDR*) op;
5378 assert(odata && odata->coll);
5382 char *bsdata = tcmaploadone(rowdata, rowdatasz, JDBCOLBSON, JDBCOLBSONL, &bsize);
5387 BSON_ITERATOR_FROM_BUFFER(&it, bsdata);
5388 bson_find_fieldpath_value2(fpath, fpathsz, &it);
5389 ret = _bsonitstrval(odata->coll->jb, &it, vsz, tokens, (odata->icase ? JBICASE : 0));
5394 static bool _updatebsonidx(EJCOLL *coll, const bson_oid_t *oid, const bson *bs,
5395 const void *obsdata, int obsdatasz, TCLIST *dlist) {
5397 TCMAP *cmeta = tctdbget(coll->jb->metadb, coll->cname, coll->cnamesz);
5399 _ejdbsetecode(coll->jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
5402 TCMAP *imap = NULL; //New index map
5403 TCMAP *rimap = NULL; //Remove index map
5404 bson_type mt = BSON_EOO;
5405 bson_type ft = BSON_EOO;
5406 bson_type oft = BSON_EOO;
5407 bson_iterator fit, oit, mit;
5409 char ikey[BSON_MAX_FPATH_LEN + 2];
5413 tcmapiterinit(cmeta);
5414 while ((mkey = tcmapiternext(cmeta, &mkeysz)) != NULL && mkeysz > 0) {
5415 if (*mkey != 'i' || mkeysz > BSON_MAX_FPATH_LEN + 1) {
5418 const void *mraw = tcmapget(cmeta, mkey, mkeysz, &bsz);
5419 if (!mraw || !bsz || (mt = bson_find_from_buffer(&mit, mraw, "iflags")) != BSON_INT) {
5422 int iflags = bson_iterator_int(&mit);
5423 //OK then process index keys
5424 memcpy(ikey + 1, mkey + 1, mkeysz - 1);
5425 ikey[mkeysz] = '\0';
5428 char *fvalue = NULL;
5430 char *ofvalue = NULL;
5431 txtflags_t textflags = (iflags & JBIDXISTR) ? JBICASE : 0;
5433 if (obsdata && obsdatasz > 0) {
5434 BSON_ITERATOR_FROM_BUFFER(&oit, obsdata);
5435 oft = bson_find_fieldpath_value2(mkey + 1, mkeysz - 1, &oit);
5436 TCLIST *tokens = (oft == BSON_ARRAY || (oft == BSON_STRING && (iflags & JBIDXARR))) ? tclistnew() : NULL;
5437 ofvalue = BSON_IS_IDXSUPPORTED_TYPE(oft) ? _bsonitstrval(coll->jb, &oit, &ofvaluesz, tokens, textflags) : NULL;
5440 ofvalue = tclistdump(tokens, &ofvaluesz);
5445 BSON_ITERATOR_INIT(&fit, bs);
5446 ft = bson_find_fieldpath_value2(mkey + 1, mkeysz - 1, &fit);
5447 TCLIST *tokens = (ft == BSON_ARRAY || (ft == BSON_STRING && (iflags & JBIDXARR))) ? tclistnew() : NULL;
5448 fvalue = BSON_IS_IDXSUPPORTED_TYPE(ft) ? _bsonitstrval(coll->jb, &fit, &fvaluesz, tokens, textflags) : NULL;
5451 fvalue = tclistdump(tokens, &fvaluesz);
5455 if (!fvalue && !ofvalue) {
5459 imap = tcmapnew2(TCMAPTINYBNUM);
5460 rimap = tcmapnew2(TCMAPTINYBNUM);
5462 for (int i = 4; i <= 7; ++i) { /* JBIDXNUM, JBIDXSTR, JBIDXARR, JBIDXISTR */
5464 int itype = (1 << i);
5465 if (itype == JBIDXNUM && (JBIDXNUM & iflags)) {
5467 } else if (itype == JBIDXSTR && (JBIDXSTR & iflags)) {
5469 } else if (itype == JBIDXISTR && (JBIDXISTR & iflags)) {
5471 } else if (itype == JBIDXARR && (JBIDXARR & iflags)) {
5473 if (ofvalue && oft == BSON_ARRAY &&
5474 (!fvalue || ft != oft || fvaluesz != ofvaluesz || memcmp(fvalue, ofvalue, fvaluesz))) {
5475 tcmapput(rimap, ikey, mkeysz, ofvalue, ofvaluesz);
5478 if (fvalue && fvaluesz > 0 && ft == BSON_ARRAY && (!ofvalue || rm)) {
5479 tcmapput(imap, ikey, mkeysz, fvalue, fvaluesz);
5485 if (ofvalue && oft != BSON_ARRAY &&
5486 (!fvalue || ft != oft || fvaluesz != ofvaluesz || memcmp(fvalue, ofvalue, fvaluesz))) {
5487 tcmapput(rimap, ikey, mkeysz, ofvalue, ofvaluesz);
5490 if (fvalue && fvaluesz > 0 && ft != BSON_ARRAY && (!ofvalue || rm)) {
5491 tcmapput(imap, ikey, mkeysz, fvalue, fvaluesz);
5494 if (fvalue) TCFREE(fvalue);
5495 if (ofvalue) TCFREE(ofvalue);
5499 if (dlist) { //storage for deffered index ops provided, save changes into
5500 _DEFFEREDIDXCTX dctx;
5502 dctx.rmap = (rimap && TCMAPRNUM(rimap) > 0) ? tcmapdup(rimap) : NULL;
5503 dctx.imap = (imap && TCMAPRNUM(imap) > 0) ? tcmapdup(imap) : NULL;
5504 if (dctx.imap || dctx.rmap) {
5505 TCLISTPUSH(dlist, &dctx, sizeof (dctx));
5507 //flush deffered indexes if number pending objects greater JBMAXDEFFEREDIDXNUM
5508 if (TCLISTNUM(dlist) >= JBMAXDEFFEREDIDXNUM) {
5509 for (int i = 0; i < TCLISTNUM(dlist); ++i) {
5510 _DEFFEREDIDXCTX *di = TCLISTVALPTR(dlist, i);
5513 tctdbidxout2(coll->tdb, &(di->oid), sizeof (di->oid), di->rmap);
5517 tctdbidxput2(coll->tdb, &(di->oid), sizeof (di->oid), di->imap);
5521 TCLISTTRUNC(dlist, 0);
5523 } else { //apply index changes immediately
5524 if (rimap && !tctdbidxout2(coll->tdb, oid, sizeof (*oid), rimap)) rv = false;
5525 if (imap && !tctdbidxput2(coll->tdb, oid, sizeof (*oid), imap)) rv = false;
5527 if (imap) tcmapdel(imap);
5528 if (rimap) tcmapdel(rimap);
5532 static void _delcoldb(EJCOLL *coll) {
5534 tctdbdel(coll->tdb);
5538 TCFREE(coll->cname);
5540 pthread_rwlock_destroy(coll->mmtx);
5545 static bool _addcoldb0(const char *cname, EJDB *jb, EJCOLLOPTS *opts, EJCOLL **res) {
5550 for (i = 0; i < EJDB_MAX_COLLECTIONS && jb->cdbs[i]; ++i);
5551 if (i == EJDB_MAX_COLLECTIONS) {
5552 _ejdbsetecode(jb, JBEMAXNUMCOLS, __FILE__, __LINE__, __func__);
5555 rv = _createcoldb(cname, jb, opts, &cdb);
5561 TCCALLOC(coll, 1, sizeof (*coll));
5564 coll->cname = tcstrdup(cname);
5565 coll->cnamesz = strlen(cname);
5569 _ejdbcolsetmutex(coll);
5574 static bool _createcoldb(const char *colname, EJDB *jb, EJCOLLOPTS *opts, TCTDB **res) {
5575 assert(jb && jb->metadb);
5576 if (!JBISVALCOLNAME(colname)) {
5577 _ejdbsetecode(jb, JBEINVALIDCOLNAME, __FILE__, __LINE__, __func__);
5582 TCTDB *cdb = tctdbnew();
5585 if (opts->cachedrecords > 0) {
5586 tctdbsetcache(cdb, opts->cachedrecords, 0, 0);
5590 if (opts->records > 0) {
5591 bnum = tclmax(opts->records * 2 + 1, TDBDEFBNUM);
5594 tflags |= TDBTLARGE;
5596 if (opts->compressed) {
5597 tflags |= TDBTDEFLATE;
5599 tctdbtune(cdb, bnum, 0, 0, tflags);
5601 const char *mdbpath = jb->metadb->hdb->path;
5603 TCXSTR *cxpath = tcxstrnew2(mdbpath);
5604 tcxstrcat2(cxpath, "_");
5605 tcxstrcat2(cxpath, colname);
5606 uint32_t mode = jb->metadb->hdb->omode;
5607 if (mode & (JBOWRITER | JBOCREAT)) {
5610 rv = tctdbopen(cdb, tcxstrptr(cxpath), mode);
5611 *res = rv ? cdb : NULL;
5616 /* Check whether a string includes all tokens in another string.*/
5617 static bool _qrycondcheckstrand(const char *vbuf, const TCLIST *tokens) {
5618 assert(vbuf && tokens);
5619 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
5620 const char *token = TCLISTVALPTR(tokens, i);
5621 int tokensz = TCLISTVALSIZ(tokens, i);
5623 const char *str = vbuf;
5625 const char *sp = str;
5626 while (*str != '\0' && !strchr(", ", *str)) {
5629 if (tokensz == (str - sp) && !strncmp(token, sp, tokensz)) { //Token matched
5633 if (*str == '\0') break;
5643 /* Check whether a string includes at least one tokens in another string.*/
5644 static bool _qrycondcheckstror(const char *vbuf, const TCLIST *tokens) {
5645 for (int i = 0; i < TCLISTNUM(tokens); ++i) {
5646 const char *token = TCLISTVALPTR(tokens, i);
5647 int tokensz = TCLISTVALSIZ(tokens, i);
5649 const char *str = vbuf;
5651 const char *sp = str;
5652 while (*str != '\0' && !strchr(", ", *str)) {
5655 if (tokensz == (str - sp) && !strncmp(token, sp, tokensz)) { //Token matched
5659 if (*str == '\0') break;