Add support for large files

Currently session size is send over USB as integer 32b.
So max size of all files is 2GB.
Some fixed uboot versions treat this value as unsigned integer 32b.
So there max size of all files is 4GB.

Let's provide initial support for large files in lthor by
counting all size using off_t which thanks to define is 64b
on all platforms.

This patch also add a suitable warning if image is larger than
2GB and reports an error if image is larger than 4GB.

Change-Id: I402b3684fa96c7531edb580e38f109ecc8a5ebd0
Signed-off-by: Krzysztof Opasiak <k.opasiak@samsung.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c6678716a37e5a8e222cb9e0eaf86d4bf2a43724..901d0d26af232a7e1ccf70d31f47de26468f7ea1 100755 (executable)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -50,6 +50,7 @@ ADD_DEFINITIONS("-DPACKAGE=\"${PACKAGE}\"")
 ADD_DEFINITIONS("-DPACKAGE_NAME=\"${PKGNAME}\"")
 ADD_DEFINITIONS("-DPREFIX=\"${PREFIX}\"")
 ADD_DEFINITIONS("-DPACKAGE_VERSION=\"${PACKAGE_VERSION}\"")
+ADD_DEFINITIONS("-D_FILE_OFFSET_BITS=64")
 
 #SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed")
 IF(APPLE)

diff --git a/libthor/thor.c b/libthor/thor.c
index 6a15155d5fb6992b34b6e3e031a6f4518d526e8d..a670f4d69d88d89e66d78f6f0dc39c51247f00dd 100644 (file)
--- a/libthor/thor.c
+++ b/libthor/thor.c
@@ -157,7 +157,7 @@ static int t_thor_exec_cmd(thor_device_handle *th,  request_type req_id,
                                    NULL, 0, NULL);
 }
 
-int thor_start_session(thor_device_handle *th, size_t total)
+int thor_start_session(thor_device_handle *th, off_t total)
 {
        int ret;
 
@@ -200,7 +200,7 @@ out:
 static int t_thor_prep_next_chunk(struct t_thor_data_chunk *chunk,
                                  struct t_thor_data_transfer *transfer_data)
 {
-       size_t to_read;
+       off_t to_read;
        int ret;
 
        to_read = transfer_data->data_left - transfer_data->data_in_progress;
@@ -319,7 +319,7 @@ complete_all:
 
 static int t_thor_init_chunk(struct t_thor_data_chunk *chunk,
                             thor_device_handle *th,
-                            size_t trans_unit_size,
+                            off_t trans_unit_size,
                             void *user_data)
 {
        int ret;
@@ -376,7 +376,7 @@ static inline void t_thor_cancel_chunk(struct t_thor_data_chunk *chunk)
 
 static int t_thor_send_raw_data(thor_device_handle *th,
                                struct thor_data_src *data,
-                               size_t trans_unit_size,
+                               off_t trans_unit_size,
                                thor_progress_cb report_progress,
                                void *user_data)
 {
@@ -441,11 +441,11 @@ int thor_send_data(thor_device_handle *th, struct thor_data_src *data,
                   void *user_data, thor_next_entry_cb report_next_entry,
                   void *ne_cb_data)
 {
-       size_t filesize;
+       off_t filesize;
        const char *filename;
        struct res_pkt resp;
        int32_t int_data[2];
-       size_t trans_unit_size;
+       off_t trans_unit_size;
        int ret;
 
        while (1) {

diff --git a/libthor/thor.h b/libthor/thor.h
index dceb273d63d8600b4ae9a34b4f563bf7cc498718..9bc99300988ed08bb00561da368a1cd5ac71f5f1 100644 (file)
--- a/libthor/thor.h
+++ b/libthor/thor.h
@@ -37,9 +37,9 @@ enum thor_data_type {
 };
 
 struct thor_data_src {
-       size_t (*get_file_length)(struct thor_data_src *src);
-       size_t (*get_size)(struct thor_data_src *src);
-       size_t (*get_block)(struct thor_data_src *src, void *data, size_t len);
+       off_t (*get_file_length)(struct thor_data_src *src);
+       off_t (*get_size)(struct thor_data_src *src);
+       off_t (*get_block)(struct thor_data_src *src, void *data, off_t len);
        const char* (*get_name)(struct thor_data_src *src);
        int (*next_file)(struct thor_data_src *src);
        void (*release)(struct thor_data_src *src);
@@ -76,7 +76,7 @@ int thor_open(struct thor_device_id *dev_id, int wait,
 void thor_close(thor_device_handle *th);
 
 /* Start thor "session" */
-int thor_start_session(thor_device_handle *th, size_t total);
+int thor_start_session(thor_device_handle *th, off_t total);
 
 /* Send a butch of data to the target */
 int thor_send_data(thor_device_handle *th, struct thor_data_src *data,

diff --git a/libthor/thor_internal.h b/libthor/thor_internal.h
index 06f44d82505427ccb59aba9e25d2f50b294cb031..ec3d80f4981493f76a24d3863102c6a347b435ef 100644 (file)
--- a/libthor/thor_internal.h
+++ b/libthor/thor_internal.h
@@ -51,7 +51,7 @@ typedef void (*t_usb_transfer_cb)(struct t_usb_transfer *);
 struct t_usb_transfer {
        struct libusb_transfer *ltransfer;
        t_usb_transfer_cb transfer_finished;
-       size_t size;
+       off_t size;
        int ret;
        int cancelled;
 };
@@ -60,10 +60,10 @@ struct t_thor_data_chunk {
        struct t_usb_transfer data_transfer;
        struct t_usb_transfer resp_transfer;
        void *user_data;
-       size_t useful_size;
+       off_t useful_size;
        struct data_res_pkt resp;
        unsigned char *buf;
-       size_t trans_unit_size;
+       off_t trans_unit_size;
        int chunk_number;
        int data_finished;
        int resp_finished;
@@ -74,9 +74,9 @@ struct t_thor_data_transfer {
        struct thor_data_src *data;
        thor_progress_cb report_progress;
        void *user_data;
-       size_t data_left;
-       size_t data_sent;
-       size_t data_in_progress;
+       off_t data_left;
+       off_t data_sent;
+       off_t data_in_progress;
        int chunk_number;
        int completed;
        int ret;
@@ -88,7 +88,7 @@ int t_usb_handle_events_completed(int *completed);
 int t_usb_init_transfer(struct t_usb_transfer *t,
                        libusb_device_handle *devh,
                        unsigned char ep,
-                       unsigned char *buf, size_t size,
+                       unsigned char *buf, off_t size,
                        t_usb_transfer_cb transfer_finished,
                        unsigned int timeout);
 
@@ -99,7 +99,7 @@ static inline void t_usb_cleanup_transfer(struct t_usb_transfer *t)
 
 static inline int t_usb_init_in_transfer(struct t_usb_transfer *t,
                           struct thor_device_handle *th,
-                          unsigned char *buf, size_t size,
+                          unsigned char *buf, off_t size,
                           t_usb_transfer_cb transfer_finished,
                           unsigned int timeout)
 {
@@ -109,7 +109,7 @@ static inline int t_usb_init_in_transfer(struct t_usb_transfer *t,
 
 static inline int t_usb_init_out_transfer(struct t_usb_transfer *t,
                           struct thor_device_handle *th,
-                          unsigned char *buf, size_t size,
+                          unsigned char *buf, off_t size,
                           t_usb_transfer_cb transfer_finished,
                           unsigned int timeout)
 {
@@ -132,10 +132,10 @@ int t_file_get_data_src(const char *path, struct thor_data_src **data);
 int t_tar_get_data_src(const char *path, struct thor_data_src **data);
 
 int t_usb_send(struct thor_device_handle *th, unsigned char *buf,
-              size_t count, int timeout);
+              off_t count, int timeout);
 
 int t_usb_recv(struct thor_device_handle *th, unsigned char *buf,
-              size_t count, int timeout);
+              off_t count, int timeout);
 
 int t_usb_send_req(struct thor_device_handle *th, request_type req_id,
                   int req_sub_id, int *idata, int icnt, char **sdata,

diff --git a/libthor/thor_raw_file.c b/libthor/thor_raw_file.c
index 1ec1385718ed338de5c850b61731c096bde4e2af..d4177bb433ebb0684953890f4d26bf0da28a9bd2 100644 (file)
--- a/libthor/thor_raw_file.c
+++ b/libthor/thor_raw_file.c
@@ -31,11 +31,11 @@ struct file_data_src {
        struct thor_data_src src;
        int fd;
        const char* filename;
-       size_t filesize;
+       off_t filesize;
        int pos;
 };
 
-static size_t file_get_file_length(struct thor_data_src *src)
+static off_t file_get_file_length(struct thor_data_src *src)
 {
        struct file_data_src *filedata =
                container_of(src, struct file_data_src, src);
@@ -43,8 +43,8 @@ static size_t file_get_file_length(struct thor_data_src *src)
        return filedata->filesize;
 }
 
-static size_t file_get_data_block(struct thor_data_src *src,
-                                 void *data, size_t len)
+static off_t file_get_data_block(struct thor_data_src *src,
+                                 void *data, off_t len)
 {
        struct file_data_src *filedata =
                container_of(src, struct file_data_src, src);

diff --git a/libthor/thor_tar.c b/libthor/thor_tar.c
index b74e1269721c86fe41147700fc1f5acddf8b9a00..b91d102e94dcddca2f72904ba265742f67d9a567 100644 (file)
--- a/libthor/thor_tar.c
+++ b/libthor/thor_tar.c
@@ -28,10 +28,10 @@ struct tar_data_src {
        struct thor_data_src src;
        struct archive *ar;
        struct archive_entry *ae;
-       size_t total_size;
+       off_t total_size;
 };
 
-static size_t tar_get_file_length(struct thor_data_src *src)
+static off_t tar_get_file_length(struct thor_data_src *src)
 {
        struct tar_data_src *tardata =
                container_of(src, struct tar_data_src, src);
@@ -39,7 +39,7 @@ static size_t tar_get_file_length(struct thor_data_src *src)
        return archive_entry_size(tardata->ae);
 }
 
-static size_t tar_get_size(struct thor_data_src *src)
+static off_t tar_get_size(struct thor_data_src *src)
 {
        struct tar_data_src *tardata =
                container_of(src, struct tar_data_src, src);
@@ -47,8 +47,8 @@ static size_t tar_get_size(struct thor_data_src *src)
        return tardata->total_size;
 }
 
-static size_t tar_get_data_block(struct thor_data_src *src,
-                                void *data, size_t len)
+static off_t tar_get_data_block(struct thor_data_src *src,
+                                void *data, off_t len)
 {
        struct tar_data_src *tardata =
                container_of(src, struct tar_data_src, src);

diff --git a/libthor/thor_usb.c b/libthor/thor_usb.c
index 79d4567618a64385f9c836e86402d195c4fc2a89..9c93af2513f88dffd7bc29abdbd992916dfd6a6c 100644 (file)
--- a/libthor/thor_usb.c
+++ b/libthor/thor_usb.c
@@ -488,7 +488,7 @@ void t_usb_close_device(struct thor_device_handle *th)
 }
 
 int t_usb_send(struct thor_device_handle *th, unsigned char *buf,
-              size_t count, int timeout)
+              off_t count, int timeout)
 {
        int ret;
        int transferred = 0;
@@ -509,7 +509,7 @@ int t_usb_send(struct thor_device_handle *th, unsigned char *buf,
 }
 
 int t_usb_recv(struct thor_device_handle *th, unsigned char *buf,
-              size_t count, int timeout)
+              off_t count, int timeout)
 {
        int ret;
        int transferred = 0;
@@ -596,7 +596,7 @@ static void t_usb_transfer_finished(struct libusb_transfer *ltransfer)
 int t_usb_init_transfer(struct t_usb_transfer *t,
                        libusb_device_handle *devh,
                        unsigned char ep,
-                       unsigned char *buf, size_t size,
+                       unsigned char *buf, off_t size,
                        t_usb_transfer_cb transfer_finished,
                        unsigned int timeout)
 {

diff --git a/lthor.c b/lthor.c
index f17f78af1b5ecab8dc34d823eda75403aab4b2c3..4c5f99911c8b3b7b34154b07c4967ed8b98c3ac7 100644 (file)
--- a/lthor.c
+++ b/lthor.c
@@ -29,6 +29,7 @@
 
 #define KB                     (1024)
 #define MB                     (1024*KB)
+#define GB                     ((off_t)1024*MB)
 
 #define TERM_YELLOW      "\x1b[0;33;1m"
 #define TERM_LIGHT_GREEN "\x1b[0;32;1m"
@@ -203,7 +204,7 @@ static void report_progress(thor_device_handle *th, struct thor_data_src *data,
 }
 
 static int do_download(thor_device_handle *th, struct helper *data_parts,
-                      int entries, size_t total_size)
+                      int entries, off_t total_size)
 {
        struct time_data tdata;
        int i;
@@ -260,7 +261,7 @@ static int process_download(struct thor_device_id *dev_id, const char *pitfile,
                     char **tarfilelist)
 {
        thor_device_handle *th;
-       size_t total_size = 0;
+       off_t total_size = 0;
        struct helper *data_parts;
        int nfiles;
        int entries = 0;
@@ -289,12 +290,12 @@ static int process_download(struct thor_device_id *dev_id, const char *pitfile,
 
        /* Count the total size of data */
        for (i = 0; i < entries; ++i) {
-               size_t size = data_parts[i].data->get_size(data_parts[i].data);
+               off_t size = data_parts[i].data->get_size(data_parts[i].data);
 
                switch (data_parts[i].type) {
                case THOR_PIT_DATA:
-                       printf(TERM_YELLOW "%s :" TERM_NORMAL "%zuk\n",
-                              data_parts[i].name, size/KB);
+                       printf(TERM_YELLOW "%s :" TERM_NORMAL "%jdk\n",
+                              data_parts[i].name, (intmax_t)(size/KB));
                        break;
                case THOR_NORMAL_DATA:
                default:
@@ -307,8 +308,26 @@ static int process_download(struct thor_device_id *dev_id, const char *pitfile,
        printf("\t" TERM_YELLOW "total" TERM_NORMAL" :\t%.2fMB\n\n",
               (double)total_size/MB);
 
+       if (total_size > (4*GB - 1*KB)) {
+               fprintf(stderr,
+                       TERM_RED
+                       "[ERROR] Images over 4GB are not supported by thor protocol.\n"
+                       TERM_NORMAL);
+               ret = -EOVERFLOW;
+               goto release_data_srcs;
+       }
+
+       if (total_size > (2*GB - 1*KB)) {
+               fprintf(stderr,
+                       TERM_RED
+                       "[WARNING] Not all bootloaders support images over 2GB.\n"
+                       "          If your download will fail this may be a reason.\n"
+                       TERM_NORMAL);
+       }
+
        ret = do_download(th, data_parts, entries, total_size);
 
+release_data_srcs:
        for (i = 0; i < entries; ++i)
                thor_release_data_src(data_parts[i].data);