3 * Copyright (C) 2012 BMW AG
5 * This file is part of GENIVI Project Dlt - Diagnostic Log and Trace console apps.
7 * Contributions are licensed to the GENIVI Alliance under one or more
8 * Contribution License Agreements.
11 * This Source Code Form is subject to the terms of the
12 * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with
13 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
16 * \author Lassi Marttala <lassi.lm.marttala@partner.bmw.de> BMW 2012
18 * \file dlt-system-filetransfer.c
19 * For further information see http://www.genivi.org/.
23 /*******************************************************************************
25 ** SRC-MODULE: dlt-system-filetransfer.c **
31 ** AUTHOR : Lassi Marttala <lassi.lm.marttala@partner.bmw.de> **
32 ** Alexander Wenzel Alexander.AW.Wenzel@bmw.de **
38 ** PLATFORM DEPENDANT [yes/no]: yes **
40 ** TO BE CHANGED BY USER [yes/no]: no **
42 *******************************************************************************/
47 #include <sys/inotify.h>
55 #include "dlt-system.h"
57 #include "dlt_filetransfer.h"
59 #define INOTIFY_SZ (sizeof(struct inotify_event))
60 #define INOTIFY_LEN (INOTIFY_SZ + NAME_MAX + 1)
61 #define Z_CHUNK_SZ 1024*128
62 #define COMPRESS_EXTENSION ".gz"
63 #define SUBDIR_COMPRESS ".tocompress"
64 #define SUBDIR_TOSEND ".tosend"
68 extern DltSystemThreads threads;
69 // From dlt_filetransfer
70 extern unsigned long getFileSerialNumber(const char* file);
72 DLT_IMPORT_CONTEXT(dltsystem);
73 DLT_DECLARE_CONTEXT(filetransferContext)
77 int fd[DLT_SYSTEM_LOG_DIRS_MAX];
83 char *origin_name(char *src){
84 if (strlen( (char*) basename(src)) > 10 ){
85 return (char*)(basename(src)+10);
88 DLT_LOG(dltsystem, DLT_LOG_ERROR,
89 DLT_STRING("dlt-system-filetransfer, error in recreating origin name!"));
94 char *unique_name(char *src)
96 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
97 DLT_STRING("dlt-system-filetransfer, creating unique temporary file name."));
98 time_t t = time(NULL);
99 unsigned long l = getFileSerialNumber(src) ^ t;
100 // Length of ULONG_MAX + 1
101 //char *ret = malloc(11);
103 char *basename_f = basename(src);
104 // Length of ULONG_MAX + 1
105 int len = 11+strlen(basename_f);
107 DLT_LOG(dltsystem, DLT_LOG_WARN,
108 DLT_STRING("dlt-system-filetransfer, unique name creation needs to shorten the filename:"),DLT_STRING(basename_f));
112 char *ret = malloc(len);
115 snprintf(ret, len, "%lu%s", l,basename_f);
120 * Function which only calls the relevant part to transfer the payload
123 void send_dumped_file(FiletransferOptions const *opts,char *dst_tosend){
126 char *fn = origin_name(dst_tosend);
127 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
128 DLT_STRING("dlt-system-filetransfer, sending dumped file:"),DLT_STRING(fn));
129 if(dlt_user_log_file_header_alias(&filetransferContext, dst_tosend, fn) == 0)
131 int pkgcount = dlt_user_log_file_packagesCount(&filetransferContext, dst_tosend);
134 while(lastpkg < pkgcount)
138 dlt_user_check_buffer(&total, &used);
139 while((total-used) < (total/2))
143 t.tv_nsec = 1000000ul*opts->TimeoutBetweenLogs;
145 dlt_user_check_buffer(&total, &used);
148 if(dlt_user_log_file_data(&filetransferContext, dst_tosend, lastpkg, opts->TimeoutBetweenLogs) < 0)
155 dlt_user_log_file_end(&filetransferContext, dst_tosend, 1);
157 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
158 DLT_STRING("dlt-system-filetransfer, sent dumped file"));
162 * compress file, delete the source file
163 * modification: compress into subdirectory
164 * File whis is compress will be deleted afterwards
165 * @param src File to be sent
166 * @param dst destination where to compress the file
167 * @param level of compression
169 int compress_file_to(char *src, char *dst, int level)
171 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
172 DLT_STRING("dlt-system-filetransfer, compressing file from:"),DLT_STRING(src),DLT_STRING("to:"),DLT_STRING(dst));
177 snprintf(dst_mode,8, "wb%d", level);
182 dst_file = gzopen(dst, dst_mode);
183 if(dst_file == Z_NULL)
189 src_file = fopen(src, "r");
198 buf = malloc(Z_CHUNK_SZ);
201 while(!feof(src_file))
203 int read = fread(buf, 1, Z_CHUNK_SZ, src_file);
212 gzwrite(dst_file, buf, read);
216 DLT_LOG(dltsystem, DLT_LOG_WARN, DLT_STRING("Could not remove file"), DLT_STRING(src));
224 //!Sends one file over DLT.
226 * If configured in opts, compresses it, then sends it.
227 * uses subdirecties for compressing and before sending, to avoid that those files get changed in the meanwhile
230 int send_one(char *src, FiletransferOptions const *opts, int which)
232 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
233 DLT_STRING("dlt-system-filetransfer, sending a file."));
234 sleep(opts->TimeDelay);
236 // Prepare all needed file names
237 char *fn = basename(src);
238 char *fdir = malloc(strlen(src)+1);
240 strncpy(fdir,src,strlen(src));
241 *(fdir+strlen(fdir))='\0';
242 fdir = dirname(fdir);//dirname overwrites its argument anyway
243 char *dst_tosend;//file which is going to be sent
245 char *rn = unique_name(src);//new unique filename based on inode
251 // Compress if needed
252 if(opts->Compression[which] > 0)
254 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
255 DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory for compressing it."));
257 char *dst_tocompress;//file which is going to be compressed, the compressed one is named dst_tosend
260 int len = strlen(fdir)+strlen(SUBDIR_COMPRESS)+strlen(rn)+3;//the filename in .tocompress +2 for 2*"/", +1 for \0
261 dst_tocompress = malloc(len);
262 MALLOC_ASSERT(dst_tocompress);
264 snprintf(dst_tocompress,len,"%s/%s/%s",fdir,SUBDIR_COMPRESS,rn);
269 //moving in subdir, from where it can be compressed
270 if(rename(src, dst_tocompress) < 0)
272 DLT_LOG(dltsystem, DLT_LOG_ERROR,
273 DLT_STRING("Could not move file"),
275 DLT_STRING(dst_tocompress));
277 free(dst_tocompress);
281 len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+strlen(COMPRESS_EXTENSION)+3;//the resulting filename in .tosend +2 for 2*"/", +1 for \0
282 dst_tosend = malloc(len);
283 MALLOC_ASSERT(dst_tosend);
284 snprintf(dst_tosend,len,"%s/%s/%s%s",fdir,SUBDIR_TOSEND,rn,COMPRESS_EXTENSION);
288 if (compress_file_to(dst_tocompress,dst_tosend, opts->CompressionLevel[which]) != 0){
291 free(dst_tocompress);
295 free(dst_tocompress);
299 //move it directly into "tosend"
300 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
301 DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory."));
302 int len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+3;
303 dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 2*"/", +1 for \0
305 snprintf(dst_tosend,len,"%s/%s/%s",fdir,SUBDIR_TOSEND,rn);
307 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
308 DLT_STRING("dlt-system-filetransfer, Rename:"),DLT_STRING(src),DLT_STRING("to: "),DLT_STRING(dst_tosend));
309 //moving in subdir, from where it can be compressed
310 if(rename(src, dst_tosend) < 0)
312 DLT_LOG(dltsystem, DLT_LOG_ERROR,
313 DLT_STRING("Could not move file"),
315 DLT_STRING(dst_tosend));
324 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
325 DLT_STRING("dlt-system-filetransfer, File ready to send"));
327 send_dumped_file(opts,dst_tosend);
338 int flush_dir_send(FiletransferOptions const *opts, const char *compress_dir, const char *send_dir){
342 dir = opendir(send_dir);
346 while((dp = readdir(dir)) != NULL)
348 if(dp->d_type != DT_REG)
351 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
352 DLT_STRING("dlt-system-filetransfer, old compressed file found in send directory:"),DLT_STRING(dp->d_name));
353 int len = strlen(send_dir)+strlen(dp->d_name)+2;
356 snprintf(fn,len, "%s/%s", send_dir, dp->d_name);
359 //if we have a file here and in the to_compress dir, we delete the to_send file: we can not be sure, that it has been properly compressed!
360 if (!strncmp( dp->d_name+strlen(dp->d_name)-strlen(COMPRESS_EXTENSION),COMPRESS_EXTENSION,strlen(COMPRESS_EXTENSION)))
364 //old file name (not: path) would have been:
365 char tmp[strlen(dp->d_name)-strlen(COMPRESS_EXTENSION)+1];
366 strncpy(tmp,dp->d_name,strlen(dp->d_name)-strlen(COMPRESS_EXTENSION));
367 tmp[strlen(dp->d_name)-3]='\0';
369 int len = strlen(tmp)+strlen(compress_dir)+1+1;//2 sizes + 1*"/" + \0
370 char *path_uncompressed = malloc(len);
371 MALLOC_ASSERT(path_uncompressed);
372 snprintf(path_uncompressed,len,"%s/%s",compress_dir,tmp);
375 if (stat(path_uncompressed,&sb)==-1)
377 //uncompressed equivalent does not exist. We can send it out.
378 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
379 DLT_STRING("dlt-system-filetransfer, sending file."));
381 send_dumped_file(opts,fn);
385 //There is an uncompressed file. Compression seems to have been interrupted -> delete the compressed file instead of sending it!
386 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
387 DLT_STRING("dlt-system-filetransfer, uncompressed version exists. Deleting partially compressed version."));
388 if (sb.st_mode & S_IFREG)
392 if( remove(fn ) != 0 )
394 //"Error deleting file". Continue? If we would cancel, maybe the dump is never sent! Deletion would again be tried in next LC.
395 DLT_LOG(dltsystem, DLT_LOG_ERROR,
396 DLT_STRING("Error deleting file:"),DLT_STRING(fn));
401 //"Oldfile is a not reg file. Is this possible? Can we compress a directory?: %s\n",path_uncompressed);
402 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
403 DLT_STRING("dlt-system-filetransfer, Oldfile is a not regular file! Do we have a problem?"),DLT_STRING(fn));
407 free(path_uncompressed);//it is no more used. It would be transferred in next step.
410 //uncompressed file. We can just resend it, the action to put it here was a move action.
411 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
412 DLT_STRING("dlt-system-filetransfer, Sending uncompressed file from previous LC."),DLT_STRING(fn));
413 send_dumped_file(opts,fn);
420 DLT_LOG(dltsystem, DLT_LOG_ERROR,
421 DLT_STRING("Could not open directory"),
422 DLT_STRING(send_dir));
425 closedir(dir);//end: send_dir
430 int flush_dir_compress(FiletransferOptions const *opts, int which, const char *compress_dir, const char *send_dir){
432 //check for files in compress_dir. Assumption: a file which lies here, should have been compressed, but that action was interrupted.
433 //As it can arrive here only by a rename, it is most likely to be a complete file
436 dir = opendir(compress_dir);
439 while((dp = readdir(dir)) != NULL)
441 if(dp->d_type != DT_REG)
443 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
444 DLT_STRING("dlt-system-filetransfer, old file found in compress-directory."));
447 //compress file into to_send dir
448 int len = strlen(compress_dir)+strlen(dp->d_name)+2;
449 char *cd_filename = malloc(len);
450 MALLOC_ASSERT(cd_filename);
451 snprintf(cd_filename,len,"%s/%s",compress_dir,dp->d_name);
454 len = strlen(send_dir)+strlen(dp->d_name)+strlen(COMPRESS_EXTENSION)+2;
455 char *dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 1*"/", +1 for \0 + .gz
456 MALLOC_ASSERT(dst_tosend);
457 snprintf(dst_tosend,len,"%s/%s%s",send_dir,dp->d_name,COMPRESS_EXTENSION);
459 if (compress_file_to(cd_filename,dst_tosend, opts->CompressionLevel[which]) != 0){
467 send_dumped_file(opts,dst_tosend);
474 DLT_LOG(dltsystem, DLT_LOG_ERROR,
475 DLT_STRING("Could not open directory"),
476 DLT_STRING(compress_dir));
479 closedir(dir);//end: compress_dir
484 int flush_dir_original(FiletransferOptions const *opts, int which){
487 const char *sdir = opts->Directory[which];
491 while((dp = readdir(dir)) != NULL)
493 if(dp->d_type != DT_REG){
494 //we don't send directories
497 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
498 DLT_STRING("dlt-system-filetransfer, old file found in directory."));
499 int len = strlen(sdir)+strlen(dp->d_name)+2;
500 char *fn = malloc(len);
502 snprintf(fn,len, "%s/%s", sdir, dp->d_name);
503 if(send_one(fn, opts, which) < 0)
514 DLT_LOG(dltsystem, DLT_LOG_ERROR,
515 DLT_STRING("Could not open directory"),
523 //!Cleans the surveyed directories and subdirectories. Sends residing files into trace
525 * @param opts FiletransferOptions
526 * @param which which directory is affected -> position in list of opts->Directory
527 * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
529 int flush_dir(FiletransferOptions const *opts, int which)
533 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
534 DLT_STRING("dlt-system-filetransfer, flush directory of old files."));
538 int len = strlen(opts->Directory[which])+strlen(SUBDIR_COMPRESS)+2;
539 compress_dir = malloc (len);
540 MALLOC_ASSERT(compress_dir);
541 snprintf(compress_dir,len,"%s/%s",opts->Directory[which],SUBDIR_COMPRESS);
543 len = strlen(opts->Directory[which])+strlen(SUBDIR_TOSEND)+2;
544 send_dir = malloc (len);
545 MALLOC_ASSERT(send_dir);
546 snprintf(send_dir,len,"%s/%s",opts->Directory[which],SUBDIR_TOSEND);
548 //1st: scan the tosend directory.
549 if ( 0 != flush_dir_send(opts, compress_dir, send_dir) ){
555 //1nd: scan the tocompress directory.
556 if (0 != flush_dir_compress(opts, which, compress_dir, send_dir)){
562 free(send_dir);//no more used
565 //last step: scan the original directory - we can reuse the send_one function
566 if ( 0 != flush_dir_original(opts,which)){
573 //!Initializes the surveyed directories
574 /**On startup, the inotifiy handlers are created, and existing files shall be sent into DLT stream
575 * @param opts FiletransferOptions
576 * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
578 int init_filetransfer_dirs(FiletransferOptions const *opts)
580 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
581 DLT_STRING("dlt-system-filetransfer, initializing inotify on directories."));
582 ino.handle = inotify_init();
587 DLT_LOG(filetransferContext, DLT_LOG_FATAL,
588 DLT_STRING("Failed to initialize inotify in dlt-system file transfer."));
592 for(i = 0;i < opts->Count;i++)
594 //create subdirectories for processing the files
597 int len = strlen(opts->Directory[i])+strlen(SUBDIR_COMPRESS)+2;
598 subdirpath= malloc (len);
599 MALLOC_ASSERT(subdirpath);
600 snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_COMPRESS);
601 int ret = mkdir(subdirpath,0777);
603 if (0 != ret && EEXIST != errno){
604 DLT_LOG(dltsystem, DLT_LOG_ERROR,
605 DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
611 len = strlen(opts->Directory[i])+strlen(SUBDIR_TOSEND)+2;
612 subdirpath= malloc (len);
613 MALLOC_ASSERT(subdirpath);
614 snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_TOSEND);
615 ret = mkdir(subdirpath,0777);
616 if (0 != ret && EEXIST != errno){
617 DLT_LOG(dltsystem, DLT_LOG_ERROR,
618 DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
625 ino.fd[i] = inotify_add_watch(ino.handle, opts->Directory[i],
626 IN_CLOSE_WRITE|IN_MOVED_TO);
630 snprintf(buf, 1024, "Failed to add inotify watch to directory %s in dlt-system file transfer.",
632 DLT_LOG(filetransferContext, DLT_LOG_FATAL,
644 int wait_for_files(FiletransferOptions const *opts)
646 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, waiting for files."));
647 static char buf[INOTIFY_LEN];
648 ssize_t len = read(ino.handle, buf, INOTIFY_LEN);
651 DLT_LOG(filetransferContext, DLT_LOG_ERROR,
652 DLT_STRING("Error while waiting for files in dlt-system file transfer."));
657 while(i < (len-INOTIFY_SZ))
659 struct inotify_event *ie = (struct inotify_event *)&buf[i];
662 if((ie->mask & IN_CLOSE_WRITE) || (ie->mask & IN_MOVED_TO))
665 for(j = 0;j < opts->Count;j++)
667 if(ie->wd == ino.fd[j])
669 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, found new file."));
670 int length = strlen(opts->Directory[j])+ie->len+1;
671 char *tosend = malloc(length);
672 snprintf(tosend,length, "%s/%s", opts->Directory[j], ie->name);
673 send_one(tosend, opts, j);
679 i += INOTIFY_SZ + ie->len;
684 void filetransfer_thread(void *v_conf)
686 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, in thread."));
687 DltSystemConfiguration *conf = (DltSystemConfiguration *) v_conf;
688 DLT_REGISTER_CONTEXT(filetransferContext, conf->Filetransfer.ContextId,
689 "File transfer manager.");
691 sleep(conf->Filetransfer.TimeStartup);
693 if(init_filetransfer_dirs(&(conf->Filetransfer)) < 0)
696 while(!threads.shutdown)
698 if(wait_for_files(&(conf->Filetransfer)) < 0)
700 DLT_LOG(dltsystem, DLT_LOG_ERROR, DLT_STRING("Error while waiting files. File transfer shutdown."));
703 sleep(conf->Filetransfer.TimeDelay);
707 void start_filetransfer(DltSystemConfiguration *conf)
709 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, start."));
710 static pthread_attr_t t_attr;
712 pthread_create(&pt, &t_attr, (void *)filetransfer_thread, conf);
713 threads.threads[threads.count++] = pt;