SDB: refine and refactoring pull/push process

When sdb gets remote directory entries, it also gets their stats.
Therefore, we do not have to request stat after requesting directory entries.
Also, SYNC_INFO structure is implemented

Change-Id: I976b1cff79afff60e92f4509b80932211e15466a

diff --git a/src/command_function.c b/src/command_function.c
index 32acb56e2c202571cb939e35dbdffb952a067358..f1c26cfef1adf33ecb7ae64461e9aa75787ac3ef 100644 (file)
--- a/src/command_function.c
+++ b/src/command_function.c
@@ -462,17 +462,34 @@ int push(int argc, char ** argv) {
         --argc;
     }
 
+    SYNC_INFO info = {
+            .srcF = (FILE_FUNC*)&LOCAL_FILE_FUNC,
+            .dstF = (FILE_FUNC*)&REMOTE_FILE_FUNC,
+            .tag = {'p', 'u', 's', 'h', 'e', 'd', '\0'},
+    };
     for (i=1; i<argc-1; i++) {
-        do_sync_copy(argv[i], argv[argc-1], (FILE_FUNC*)&LOCAL_FILE_FUNC, (FILE_FUNC*)&REMOTE_FILE_FUNC, utf8);
+        info.copied = 0;
+        info.skipped = 0;
+        info.total_bytes = 0;
+        do_sync_copy(argv[i], argv[argc-1], &info, utf8);
     }
     return 0;
 }
 
 int pull(int argc, char ** argv) {
+
+    SYNC_INFO info = {
+            .srcF = (FILE_FUNC*)&REMOTE_FILE_FUNC,
+            .dstF = (FILE_FUNC*)&LOCAL_FILE_FUNC,
+            .copied = 0,
+            .skipped = 0,
+            .total_bytes = 0,
+            .tag = {'p', 'u', 'l', 'l', 'e', 'd', '\0'},
+    };
     if (argc == 2) {
-        return do_sync_copy(argv[1], ".", (FILE_FUNC*)&REMOTE_FILE_FUNC, (FILE_FUNC*)&LOCAL_FILE_FUNC, 0);
+        return do_sync_copy(argv[1], ".", &info, 0);
     }
-    return do_sync_copy(argv[1], argv[2], (FILE_FUNC*)&REMOTE_FILE_FUNC, (FILE_FUNC*)&LOCAL_FILE_FUNC, 0);
+    return do_sync_copy(argv[1], argv[2], &info, 0);
 }
 
 int shell(int argc, char ** argv) {
@@ -552,7 +569,15 @@ int install(int argc, char **argv) {
     }
 
     D("Push file: %s to %s\n", srcpath, destination);
-    if(do_sync_copy(srcpath, destination, (FILE_FUNC*)&LOCAL_FILE_FUNC, (FILE_FUNC*)&REMOTE_FILE_FUNC, 0)) {
+    SYNC_INFO info = {
+            .srcF = (FILE_FUNC*)&LOCAL_FILE_FUNC,
+            .dstF = (FILE_FUNC*)&REMOTE_FILE_FUNC,
+            .copied = 0,
+            .skipped = 0,
+            .total_bytes = 0,
+            .tag = {'p', 'u', 'l', 'l', 'e', 'd', '\0'},
+    };
+    if(do_sync_copy(srcpath, destination, &info, 0)) {
         return 1;
     }
 

diff --git a/src/file_sync_client.c b/src/file_sync_client.c
index fd7904c53141c4e33317b4b2816dc8e960b6bcd1..4e54246bf7dd7e9501d9bd87f89d243488365949 100644 (file)
--- a/src/file_sync_client.c
+++ b/src/file_sync_client.c
@@ -37,24 +37,30 @@
 #include "fdevent.h"
 #include "log.h"
 
-static __inline__ void finalize(int srcfd, int dstfd, FILE_FUNC* srcF, FILE_FUNC* dstF);
+static __inline__ void finalize(int srcfd, int dstfd, SYNC_INFO* sync_info);
 
-void create_copy_info(COPY_INFO** info, char* srcp, char* dstp) {
+void create_copy_info(COPY_INFO** info, char* srcp, char* dstp, struct stat* src_stat) {
     *info = (COPY_INFO*)malloc(sizeof(COPY_INFO));
     (*info)->src = srcp;
     (*info)->dst = dstp;
+    (*info)->_stat = *src_stat;
 }
 
-static __inline__ void finalize(int srcfd, int dstfd, FILE_FUNC* srcF, FILE_FUNC* dstF) {
-    srcF->finalize(srcfd);
-    dstF->finalize(dstfd);
+static __inline__ void finalize(int srcfd, int dstfd, SYNC_INFO* sync_info) {
+    sync_info->srcF->finalize(srcfd);
+    sync_info->dstF->finalize(dstfd);
 }
 
-static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, unsigned* total_bytes, struct stat* src_stat, char* copy_flag) {
+static int file_copy(int src_fd, int dst_fd, COPY_INFO* copy_info, SYNC_INFO* sync_info) {
+    char* srcp = copy_info->src;
+    char* dstp = copy_info->dst;
+    struct stat* src_stat = &(copy_info->_stat);
     D("file is copied from 'fd:%d' '%s' to 'fd:%d' '%s'\n", src_fd, srcp, dst_fd, dstp);
 
     unsigned file_byte = src_stat->st_size;
     char* file_name = get_filename(srcp);
+    FILE_FUNC* srcF = sync_info->srcF;
+    FILE_FUNC* dstF = sync_info->dstF;
 
     unsigned flag_size = 1;
     char byte_flag[3] = {' ', 'B', '\0'};
@@ -78,7 +84,7 @@ static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC*
      * remote channel is created and protocol confliction is made
      * it is ok to other pull or push process if remote connection is not made
      */
-    if(srcF->local) {
+    if(!strcmp(sync_info->tag, "pushed")) {
         src_fd = srcF->readopen(src_fd, srcp, src_stat);
         if(src_fd < 0) {
             return -1;
@@ -110,7 +116,7 @@ static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC*
             break;
         }
         else if(ret == 1) {
-            if(dstF->writefile(dst_fd, dstp, &srcbuf, &written_byte)) {
+            if(dstF->writefile(dst_fd, dstp, &srcbuf, sync_info)) {
                 goto error;
             }
         }
@@ -118,7 +124,7 @@ static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC*
             continue;
         }
         else if(ret == 3) {
-            if(dstF->writefile(dst_fd, dstp, &srcbuf, &written_byte)) {
+            if(dstF->writefile(dst_fd, dstp, &srcbuf, sync_info)) {
                 goto error;
             }
             break;
@@ -134,7 +140,7 @@ static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC*
         int progress_byte = written_byte / flag_size;
 
         if(store_byte != progress_byte) {
-            fprintf(stderr,"%s %30s\t%3d%%\t%7d%s\r\r", copy_flag, file_name, percent, progress_byte, byte_flag);
+            fprintf(stderr,"%s %30s\t%3d%%\t%7d%s\r\r", sync_info->tag, file_name, percent, progress_byte, byte_flag);
             store_byte = progress_byte;
         }
     }
@@ -142,8 +148,8 @@ static int file_copy(int src_fd, int dst_fd, char* srcp, char* dstp, FILE_FUNC*
         return -1;
     }
 
-    fprintf(stderr,"%s %30s\t100%%\t%7d%s\n", copy_flag, file_name, file_byte / flag_size, byte_flag);
-    *total_bytes = *total_bytes + written_byte;
+    fprintf(stderr,"%s %30s\t100%%\t%7d%s\n", sync_info->tag, file_name, file_byte / flag_size, byte_flag);
+    sync_info->total_bytes = sync_info->total_bytes + written_byte;
     return 0;
 
 error:
@@ -168,25 +174,17 @@ static void free_copyinfo(void* data) {
     }
 }
 
-int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int is_utf8) {
-
-    char copy_flag[7];
-    if(srcF->local) {
-        snprintf(copy_flag, sizeof(copy_flag), "%s", "pushed");
-    }
-    else {
-        snprintf(copy_flag, sizeof(copy_flag), "%s", "pulled");
-    }
+int do_sync_copy(char* srcp, char* dstp, SYNC_INFO* sync_info, int is_utf8) {
 
     D("copy %s to the %s\n", srcp, dstp);
-    unsigned total_bytes = 0;
     long long start_time = NOW();
 
     int src_fd = 0;
     int dst_fd = 0;
 
-    int pushed = 0;
-    int skiped = 0;
+    FILE_FUNC* srcF = sync_info->srcF;
+    FILE_FUNC* dstF = sync_info->dstF;
+
     src_fd = srcF->initialize(srcp);
     dst_fd = dstF->initialize(dstp);
     if(src_fd < 0 || dst_fd < 0) {
@@ -240,8 +238,12 @@ int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int i
                 dstp = full_dstpath;
             }
         }
-        if(!file_copy(src_fd, dst_fd, srcp, dstp, srcF, dstF, &total_bytes, &src_stat, copy_flag)) {
-            pushed++;
+        COPY_INFO copy_info;
+        copy_info.src = srcp;
+        copy_info.dst = dstp;
+        copy_info._stat = src_stat;
+        if(!file_copy(src_fd, dst_fd, &copy_info, sync_info)) {
+            sync_info->copied++;
         }
         else {
             goto error;
@@ -260,15 +262,15 @@ int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int i
         s_strncpy(_dstp, dstp, len+1);
 
         COPY_INFO* __info;
-        create_copy_info(&__info, _srcp, _dstp);
+        create_copy_info(&__info, _srcp, _dstp, &src_stat);
         append(&dir_list, __info);
 
         while(dir_list != NULL) {
             LIST_NODE* entry_list = NULL;
             COPY_INFO* _info = (COPY_INFO*)dir_list->data;
-            if(srcF->get_dirlist(src_fd, _info->src, _info->dst, &entry_list)) {
+            if(srcF->get_dirlist(src_fd, _info->src, _info->dst, &entry_list, sync_info)) {
                 fprintf(stderr,"skipped: %s -> %s\n", _info->src, _info->dst);
-                skiped++;
+                sync_info->skipped++;
                 free_list(entry_list, NULL);
                 remove_first(&dir_list, free_copyinfo);
                 continue;
@@ -277,28 +279,24 @@ int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int i
             LIST_NODE* curptr = entry_list;
 
             while(curptr != NULL) {
-                COPY_INFO* info = (COPY_INFO*)curptr->data;
+                COPY_INFO* copy_info = (COPY_INFO*)curptr->data;
                 curptr = curptr->next_ptr;
-                char* src_p = (char*)info->src;
-                char* dst_p = (char*)info->dst;
-
-                if(srcF->_stat(src_fd, src_p, &src_stat, 1)) {
-                    goto skip_in;
-                }
+                char* src_p = (char*)copy_info->src;
+                char* dst_p = (char*)copy_info->dst;
 
-                src_dir = srcF->is_dir(src_p, &src_stat);
+                src_dir = srcF->is_dir(src_p, &(copy_info->_stat));
                 if(src_dir < 0) {
                     fprintf(stderr,ERR_REASON_SYNC_NOT_FILE, src_p);
                     goto skip_in;
                 }
                 if(src_dir == 1) {
-                    append(&dir_list, info);
+                    append(&dir_list, copy_info);
                     continue;
                 }
                 else {
-                    if(!file_copy(src_fd, dst_fd, src_p, dst_p, srcF, dstF, &total_bytes, &src_stat, copy_flag)) {
-                        pushed++;
-                        free(info);
+                    if(!file_copy(src_fd, dst_fd, copy_info, sync_info)) {
+                        sync_info->copied++;
+                        free(copy_info);
                         free(src_p);
                         free(dst_p);
                         continue;
@@ -306,8 +304,8 @@ int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int i
                 }
 skip_in:
                 fprintf(stderr,"skipped: %s -> %s\n", src_p, dst_p);
-                skiped++;
-                free(info);
+                sync_info->skipped++;
+                free(copy_info);
                 free(src_p);
                 free(dst_p);
             }
@@ -316,21 +314,21 @@ skip_in:
     }
 
     fprintf(stderr,"%d file(s) %s. %d file(s) skipped.\n",
-            pushed, copy_flag, skiped);
+            sync_info->copied, sync_info->tag, sync_info->skipped);
 
     long long end_time = NOW() - start_time;
 
     if(end_time != 0) {
         fprintf(stderr,"%-30s   %lld KB/s (%lld bytes in %lld.%03llds)\n",
                 srcp,
-                ((((long long) total_bytes) * 1000000LL) / end_time) / 1024LL,
-                (long long) total_bytes, (end_time / 1000000LL), (end_time % 1000000LL) / 1000LL);
+                ((((long long) sync_info->total_bytes) * 1000000LL) / end_time) / 1024LL,
+                (long long) sync_info->total_bytes, (end_time / 1000000LL), (end_time % 1000000LL) / 1000LL);
     }
 
-    finalize(src_fd, dst_fd, srcF, dstF);
+    finalize(src_fd, dst_fd, sync_info);
     return 0;
 
 error:
-    finalize(src_fd, dst_fd, srcF, dstF);
+    finalize(src_fd, dst_fd, sync_info);
     return 1;
 }

diff --git a/src/file_sync_client.h b/src/file_sync_client.h
index c169ce5d79638dd059282bf226cdf078abc8f873..949f66186a94dfc21f6f27a13c2ef1aaacc835fb 100644 (file)
--- a/src/file_sync_client.h
+++ b/src/file_sync_client.h
@@ -29,7 +29,7 @@
 
 #include "file_sync_functions.h"
 
-void create_copy_info(COPY_INFO** info, char* srcp, char* dstp);
-int do_sync_copy(char* srcp, char* dstp, FILE_FUNC* srcF, FILE_FUNC* dstF, int is_utf8);
+void create_copy_info(COPY_INFO** info, char* srcp, char* dstp, struct stat* src_stat);
+int do_sync_copy(char* srcp, char* dstp, SYNC_INFO* sync_info, int is_utf8);
 
 #endif /* FILE_SYNC_CLIENT_H_ */

diff --git a/src/file_sync_functions.c b/src/file_sync_functions.c
index 56963bb756a4ae459ffa4d0c5c952129fb5a19ea..b87981cad1304a5554bfd3ed338c44a24f8c68ac 100644 (file)
--- a/src/file_sync_functions.c
+++ b/src/file_sync_functions.c
@@ -48,7 +48,6 @@ const unsigned sync_fail = MKSYNC('F','A','I','L');
 const unsigned sync_quit = MKSYNC('Q','U','I','T');
 
 const struct file_function LOCAL_FILE_FUNC = {
-        .local = 1,
         .initialize=initialize_local,
         .finalize=finalize_local,
         ._stat=_stat_local,
@@ -63,7 +62,6 @@ const struct file_function LOCAL_FILE_FUNC = {
 };
 
 const struct file_function REMOTE_FILE_FUNC = {
-        .local=0,
         .initialize=initialize_remote,
         .finalize=finalize_remote,
         ._stat=_stat_remote,
@@ -426,7 +424,7 @@ int readfile_remote(int fd, char* srcpath, struct stat* st, FILE_BUFFER* buffer)
     return 1;
 }
 
-int writefile_local(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_bytes) {
+int writefile_local(int fd, char* dstp, FILE_BUFFER* sbuf, SYNC_INFO* sync_info) {
     D("write local file 'fd:%d' '%s'\n", fd, dstp);
     char* data = sbuf->data;
     unsigned len = sbuf->size;
@@ -439,11 +437,11 @@ int writefile_local(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_bytes
         print_error(1, ERR_SITU_SYNC_WRITE_FILE, ERR_REASON_GENERAL_CONNECTION_FAIL, dstp, strerror(errno));
     }
 
-    *total_bytes += len;
+    sync_info->total_bytes += len;
     return 0;
 }
 
-int writefile_remote(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_bytes) {
+int writefile_remote(int fd, char* dstp, FILE_BUFFER* sbuf, SYNC_INFO* sync_info) {
     D("write remote file 'fd:%d' '%s'\n", fd, dstp);
     int size = ltohl(sbuf->size);
 
@@ -451,11 +449,11 @@ int writefile_remote(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_byte
         print_error(1, ERR_SITU_SYNC_WRITE_FILE, ERR_REASON_GENERAL_CONNECTION_FAIL, dstp, strerror(errno));
     }
 
-    *total_bytes += size;
+    sync_info->total_bytes += size;
     return 0;
 }
 
-int getdirlist_local(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist) {
+int getdirlist_local(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist, SYNC_INFO* sync_info) {
     D("get list of local file 'fd:%d' '%s'\n", fd, src_dir);
     DIR* d;
 
@@ -487,16 +485,25 @@ int getdirlist_local(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist)
         char* dst_full_path = (char*)malloc(sizeof(char)*len);
         append_file(dst_full_path, dst_dir, file_name, len);
 
-        COPY_INFO* info;
-        create_copy_info(&info, src_full_path, dst_full_path);
-        prepend(dirlist, info);
+        struct stat src_stat;
+        if(!_stat_local(fd, src_full_path, &src_stat, 1)) {
+            COPY_INFO* info;
+            create_copy_info(&info, src_full_path, dst_full_path, &src_stat);
+            prepend(dirlist, info);
+        }
+        else {
+            fprintf(stderr,"skipped: %s -> %s\n", src_full_path, dst_full_path);
+            sync_info->skipped++;
+            free(src_full_path);
+            free(dst_full_path);
+        }
     }
 
     closedir(d);
     return 0;
 }
 
-int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist) {
+int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist, SYNC_INFO* sync_info) {
     D("get list of remote file 'fd:%d' '%s'\n", fd, src_dir);
     SYNC_MSG msg;
     int len;
@@ -535,6 +542,8 @@ int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist)
         len = ltohl(msg.dent.namelen);
         if(len > 256) {
             fprintf(stderr,"error: name of a file in the remote directory '%s' exceeds 256\n", src_dir);
+            fprintf(stderr,"skipped: %s/? -> %s/?\n", src_dir, dst_dir);
+            sync_info->skipped++;
             continue;
         }
 
@@ -561,8 +570,23 @@ int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist)
         char* dst_full_path = (char*)malloc(sizeof(char)*len);
         append_file(dst_full_path, dst_dir, file_name, len);
 
+        struct stat st;
+        st.st_mode = ltohl(msg.dent.mode);
+        /**
+         * FIXME
+         * SDBD does not send error reason if remote stat fails.
+         * We cannot know the reason before we change sync protocol.
+         */
+        if(!st.st_mode) {
+            print_error(0, ERR_SITU_SYNC_STAT_FILE, ERR_REASON_GENERAL_UNKNOWN, file_name);
+            fprintf(stderr,"skipped: %s -> %s\n", src_full_path, dst_full_path);
+            sync_info->skipped++;
+            continue;
+        }
+        st.st_size = ltohl(msg.dent.size);
+
         COPY_INFO* info;
-        create_copy_info(&info, src_full_path, dst_full_path);
+        create_copy_info(&info, src_full_path, dst_full_path, &st);
         prepend(dirlist, info);
     }
     return 0;

diff --git a/src/file_sync_functions.h b/src/file_sync_functions.h
index 9d358d6db51487f1ec45f94796d58400f1418797..acdd7a9ded3f99e0e852795b7a085b357335de0c 100644 (file)
--- a/src/file_sync_functions.h
+++ b/src/file_sync_functions.h
@@ -33,9 +33,20 @@
 #include "file_sync_service.h"
 #include "linkedlist.h"
 
+struct sync_info {
+    struct file_function* srcF;
+    struct file_function* dstF;
+    int copied;
+    int skipped;
+    unsigned int total_bytes;
+    char tag[10];
+};
+typedef struct sync_info SYNC_INFO;
+
 struct copy_info {
     char* src;
     char* dst;
+    struct stat _stat;
 };
 typedef struct copy_info COPY_INFO;
 
@@ -72,14 +83,13 @@ int writeclose_remote(int fd, char* dstp, struct stat* st);
 int readfile_local(int lfd, char* srcpath, struct stat* st, FILE_BUFFER* sbuf);
 int readfile_remote(int fd, char* srcpath, struct stat* st, FILE_BUFFER* buffer);
 
-int writefile_local(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_bytes);
-int writefile_remote(int fd, char* dstp, FILE_BUFFER* sbuf, unsigned* total_bytes);
+int writefile_local(int fd, char* dstp, FILE_BUFFER* sbuf, SYNC_INFO* info);
+int writefile_remote(int fd, char* dstp, FILE_BUFFER* sbuf, SYNC_INFO* info);
 
-int getdirlist_local(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist);
-int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist);
+int getdirlist_local(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist, SYNC_INFO* info);
+int getdirlist_remote(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist, SYNC_INFO* info);
 
 struct file_function {
-    int local;
     int(*initialize)(char* path);
     void(*finalize)(int fd);
 
@@ -90,8 +100,8 @@ struct file_function {
     int(*writeopen)(int fd, char* dstp, struct stat* st);
     int(*writeclose)(int fd, char* dstp, struct stat* st);
     int(*readfile)(int fd, char* path, struct stat* st, FILE_BUFFER* buf);
-    int(*writefile)(int fd, char* path, FILE_BUFFER* buf, unsigned* total_bytes);
-    int(*get_dirlist)(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist);
+    int(*writefile)(int fd, char* dstp, FILE_BUFFER* sbuf, SYNC_INFO* sync_info);
+    int(*get_dirlist)(int fd, char* src_dir, char* dst_dir, LIST_NODE** dirlist, SYNC_INFO* sync_info);
 };
 typedef struct file_function FILE_FUNC;
 

diff --git a/src/transport.c b/src/transport.c
index 06cfcd70d5e14075727cb4bc4bcfa6a9b4d00040..d5e691c2c8afc14d8bd34c1d351443584c56b8e1 100755 (executable)
--- a/src/transport.c
+++ b/src/transport.c
@@ -368,6 +368,7 @@ static void *transport_thread(void *_t)
 read_loop:
 
     D("T(%s), FD(%d)\n", t->serial, t->sfd);
+
     t->connection_state = CS_WAITCNXN;
     send_cmd(A_VERSION, MAX_PAYLOAD, A_CNXN, "host::", t);
     t->connection_state = CS_OFFLINE;
@@ -666,7 +667,7 @@ int readx(int fd, void *ptr, size_t len)
             return -1;
         }
         if( r == 0) {
-            D("FD(%d) disconnected\n", fd);
+            LOG_ERROR("FD(%d) disconnected\n", fd);
             return -1;
         }
         len -= r;