#59 #72

diff --git a/tcejdb/ejdb.c b/tcejdb/ejdb.c
index 63617d9..091083c 100644 (file)
--- a/tcejdb/ejdb.c
+++ b/tcejdb/ejdb.c
@@ -168,6 +168,7 @@ const char* ejdberrmsg(int ecode) {
         case JBEMAXNUMCOLS: return "exceeded the maximum number of collections per database: 1024";
         case JBEEJSONPARSE: return "JSON parsing failed";
         case JBEEI: return "data export/import failed";
+        case JBETOOBIGBSON: return "bson size exceeds the maximum allowed size limit";
         default: return tcerrmsg(ecode);
     }
 }
@@ -1061,7 +1062,7 @@ static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags,
     char *pp = strrchr(bspath, MYPATHCHR);
     if (!dp || !pp || pp > dp) {
         if (log) {
-            tcxstrprintf(log, "ERROR: Invalid file path: '%s'", bspath);
+            tcxstrprintf(log, "\nERROR: Invalid file path: '%s'", bspath);
         }
         _ejdbsetecode2(jb, JBEEI, __FILE__, __LINE__, __func__, true);
         return false;
@@ -1093,7 +1094,7 @@ static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags,
     if (!mjson) {
         err = true;
         if (log) {
-            tcxstrprintf(log, "ERROR: Error reading file: '%s'", tcxstrptr(xmetapath));
+            tcxstrprintf(log, "\nERROR: Error reading file: '%s'", tcxstrptr(xmetapath));
         }
         _ejdbsetecode2(jb, TCEREAD, __FILE__, __LINE__, __func__, true);
         goto finish;
@@ -1102,17 +1103,20 @@ static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags,
     if (!mbson) {
         err = true;
         if (log) {
-            tcxstrprintf(log, "ERROR: Invalid JSON in the file: '%s'", tcxstrptr(xmetapath));
+            tcxstrprintf(log, "\nERROR: Invalid JSON in the file: '%s'", tcxstrptr(xmetapath));
         }
         _ejdbsetecode2(jb, JBEEJSONPARSE, __FILE__, __LINE__, __func__, true);
     }
     coll = ejdbgetcoll(jb, cname);
     if (coll && (flags & JBIMPORTREPLACE)) {
-        if (!ejdbrmcoll(jb, cname, true)) {
+        err = !(_rmcollimpl(jb, coll, true));
+        _delcoldb(coll);
+        TCFREE(coll);
+        coll = NULL;
+        if (err) {
             if (log) {
-                tcxstrprintf(log, "ERROR: Failed to remove collection: '%s'", cname);
+                tcxstrprintf(log, "\nERROR: Failed to remove collection: '%s'", cname);
             }
-            err = true;
             goto finish;
         }
     }
@@ -1135,16 +1139,71 @@ static bool _importcoll(EJDB *jb, const char *bspath, TCLIST *cnames, int flags,
                 }
             }
         }
-        coll = ejdbcreatecoll(jb, cname, &cops);
+        coll = _createcollimpl(jb, cname, &cops);
         if (!coll) {
+            err = true;
             if (log) {
-                tcxstrprintf(log, "ERROR: Error creating collection: '%s'", cname);
+                tcxstrprintf(log, "\nERROR: Error creating collection: '%s'", cname);
             }
+            goto finish;
         }
     }
-
-    //todo lock coll & read bson collection data
-
+    int fd = open(bspath, O_RDONLY, TCFILEMODE);
+    if (fd == -1) {
+        err = true;
+        if (log) {
+            tcxstrprintf(log, "\nERROR: Error reading file: '%s'", bspath);
+        }
+        _ejdbsetecode2(jb, TCEREAD, __FILE__, __LINE__, __func__, true);
+        goto finish;
+    }
+    if (!JBCLOCKMETHOD(coll, true)) {
+        err = true;
+        goto finish;
+    }
+    int maxdocsiz = 0, docsiz = 0, numdocs = 0;
+    void *docbuf;
+    TCMALLOC(docbuf, 1);
+    while (true) {
+        numdocs = 0;
+        sp = read(fd, &docsiz, 4);
+        if (sp < 4) {
+            break;
+        }
+        docsiz = TCHTOIL(docsiz);
+        if (docsiz > EJDB_MAX_IMPORTED_BSON_SIZE) {
+            err = true;
+            _ejdbsetecode2(jb, JBETOOBIGBSON, __FILE__, __LINE__, __func__, true);
+            break;
+        }
+        if (docsiz < 4) {
+            break;
+        }
+        if (maxdocsiz < docsiz) {
+            maxdocsiz = docsiz;
+            TCREALLOC(docbuf, docbuf, maxdocsiz);
+        }
+        sp = read(fd, docbuf, docsiz);
+        if (sp < docsiz) {
+            break;
+        }
+        bson_oid_t oid;
+        if (!ejdbsavebson3(coll, docbuf, &oid, false)) {
+            err = true;
+            break;
+        }
+        ++numdocs;
+    }
+    if (docbuf) {
+        TCFREE(docbuf);
+    }
+    if (!ejdbsyncoll(coll)) {
+        err = true;
+    }
+    if (!err && log) {
+        tcxstrprintf(log, "\n%s imported %d entries", cname, numdocs);
+    }
+    JBCUNLOCKMETHOD(coll);
 finish:
     if (mbson) {
         bson_del(mbson);
@@ -1154,6 +1213,9 @@ finish:
     }
     tcxstrdel(xmetapath);
     TCFREE(cname);
+    if (err && log) {
+         tcxstrprintf(log, "\nERROR: Importing data into: '%s' failed with error: '%s'", cname, ejdberrmsg(ejdbecode(jb)));
+    }
     return !err;
 }
 

diff --git a/tcejdb/ejdb.h b/tcejdb/ejdb.h
index 25ab13e..a33dc17 100644 (file)
--- a/tcejdb/ejdb.h
+++ b/tcejdb/ejdb.h
@@ -22,6 +22,11 @@
 
 EJDB_EXTERN_C_START
 
+#ifndef EJDB_MAX_IMPORTED_BSON_SIZE
+//64 MB is the default maximum size of bson object imported with `ejdbimport()`
+#define EJDB_MAX_IMPORTED_BSON_SIZE 67108864
+#endif
+
 struct EJDB; /**< EJDB database object. */
 typedef struct EJDB EJDB;
 
@@ -60,7 +65,8 @@ enum { /** Error codes */
     JBEQACTKEY = 9013, /**< action key in $do block can only be one of: $join */
     JBEMAXNUMCOLS = 9014, /**< Exceeded the maximum number of collections per database */
     JBEEI = 9015, /**< EJDB export/import error */
-    JBEEJSONPARSE = 9016 /**< JSON parsing failed */
+    JBEEJSONPARSE = 9016, /**< JSON parsing failed */
+    JBETOOBIGBSON = 9017 /**< BSON size is too big */
 };
 
 enum { /** Database open modes */

diff --git a/tcejdb/tcutil.c b/tcejdb/tcutil.c
index 70ff9da..9fe9cbf 100644 (file)
--- a/tcejdb/tcutil.c
+++ b/tcejdb/tcutil.c
@@ -6925,15 +6925,6 @@ static int tcchidxcmp(const void *a, const void *b) {
  *************************************************************************************************/
 
 
-#define TCFILEMODE     00644             // permission of a creating file
-#ifdef _WIN32
-#define TCIOBUFSIZ     65536             // size of an I/O buffer
-#else
-#define TCIOBUFSIZ     16384             // size of an I/O buffer
-#endif
-
-
-
 /* Get the canonicalized absolute path of a file. */
 #ifndef _WIN32
 

diff --git a/tcejdb/tcutil.h b/tcejdb/tcutil.h
index 87c150b..6c40852 100644 (file)
--- a/tcejdb/tcutil.h
+++ b/tcejdb/tcutil.h
@@ -2834,6 +2834,13 @@ EJDB_EXPORT int tcchidxhash(TCCHIDX *chidx, const void *ptr, int size);
  * filesystem utilities
  *************************************************************************************************/
 
+#define TCFILEMODE     00644             // permission of a creating file
+#ifdef _WIN32
+#define TCIOBUFSIZ     65536             // size of an I/O buffer
+#else
+#define TCIOBUFSIZ     16384             // size of an I/O buffer
+#endif
+
 
 /* Get the canonicalized absolute path of a file.
    `path' specifies the path of the file.

diff --git a/tcejdb/testejdb/t4.c b/tcejdb/testejdb/t4.c
index 02bc8f9..1229363 100644 (file)
--- a/tcejdb/testejdb/t4.c
+++ b/tcejdb/testejdb/t4.c
@@ -73,10 +73,12 @@ void testBSONExportImport() {
     bson_destroy(&bv1);
 
 
-    TCLIST* cnames = tclistnew();
+    TCXSTR *log = tcxstrnew();
+    TCLIST *cnames = tclistnew();
     tclistpush2(cnames, "col1");
     tclistpush2(cnames, "col2");
 
+
     bool rv = ejdbexport(jb, "testBSONExportImport", cnames, 0, log);
     if (!rv) {
         eprint(jb, __LINE__, "testBSONExportImport");
@@ -90,7 +92,6 @@ void testBSONExportImport() {
     jb = ejdbnew();
     CU_ASSERT_TRUE_FATAL(ejdbopen(jb, "dbt4_export", JBOWRITER | JBOCREAT | JBOTRUNC));
 
-    TCXSTR *log = tcxstrnew();
     rv = ejdbimport(jb, "testBSONExportImport", cnames, 0, log);
     CU_ASSERT_TRUE(rv);