3 * Copyright (C) 2012 BMW AG
5 * This file is part of GENIVI Project Dlt - Diagnostic Log and Trace console apps.
7 * Contributions are licensed to the GENIVI Alliance under one or more
8 * Contribution License Agreements.
11 * This Source Code Form is subject to the terms of the
12 * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with
13 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
16 * \author Lassi Marttala <lassi.lm.marttala@partner.bmw.de> BMW 2012
18 * \file dlt-system-filetransfer.c
19 * For further information see http://www.genivi.org/.
23 /*******************************************************************************
25 ** SRC-MODULE: dlt-system-filetransfer.c **
31 ** AUTHOR : Lassi Marttala <lassi.lm.marttala@partner.bmw.de> **
32 ** Alexander Wenzel Alexander.AW.Wenzel@bmw.de **
38 ** PLATFORM DEPENDANT [yes/no]: yes **
40 ** TO BE CHANGED BY USER [yes/no]: no **
42 *******************************************************************************/
47 #include <sys/inotify.h>
55 #include "dlt-system.h"
57 #include "dlt_filetransfer.h"
59 #define INOTIFY_SZ (sizeof(struct inotify_event))
60 #define INOTIFY_LEN (INOTIFY_SZ + NAME_MAX + 1)
61 #define Z_CHUNK_SZ 1024*128
62 #define COMPRESS_EXTENSION ".gz"
63 #define SUBDIR_COMPRESS ".tocompress"
64 #define SUBDIR_TOSEND ".tosend"
68 extern DltSystemThreads threads;
69 // From dlt_filetransfer
70 extern unsigned long getFileSerialNumber(const char* file);
72 DLT_IMPORT_CONTEXT(dltsystem);
73 DLT_DECLARE_CONTEXT(filetransferContext)
77 int fd[DLT_SYSTEM_LOG_DIRS_MAX];
83 char *origin_name(char *src){
84 if (strlen( (char*) basename(src)) > 10 ){
85 return (char*)(basename(src)+10);
88 DLT_LOG(dltsystem, DLT_LOG_ERROR,
89 DLT_STRING("dlt-system-filetransfer, error in recreating origin name!"));
94 char *unique_name(char *src)
96 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
97 DLT_STRING("dlt-system-filetransfer, creating unique temporary file name."));
98 time_t t = time(NULL);
99 unsigned long l = getFileSerialNumber(src) ^ t;
100 // Length of ULONG_MAX + 1
101 //char *ret = malloc(11);
103 char *basename_f = basename(src);
104 // Length of ULONG_MAX + 1
105 int len = 11+strlen(basename_f);
107 DLT_LOG(dltsystem, DLT_LOG_WARN,
108 DLT_STRING("dlt-system-filetransfer, unique name creation needs to shorten the filename:"),DLT_STRING(basename_f));
112 char *ret = malloc(len);
115 snprintf(ret, len, "%lu%s", l,basename_f);
120 * Function which only calls the relevant part to transfer the payload
123 void send_dumped_file(FiletransferOptions const *opts,char *dst_tosend){
126 char *fn = origin_name(dst_tosend);
127 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
128 DLT_STRING("dlt-system-filetransfer, sending dumped file:"),DLT_STRING(fn));
129 if(dlt_user_log_file_header_alias(&filetransferContext, dst_tosend, fn) == 0)
131 int pkgcount = dlt_user_log_file_packagesCount(&filetransferContext, dst_tosend);
134 while(lastpkg < pkgcount)
138 dlt_user_check_buffer(&total, &used);
139 while((total-used) < (total/2))
143 t.tv_nsec = 1000000ul*opts->TimeoutBetweenLogs;
145 dlt_user_check_buffer(&total, &used);
148 if(dlt_user_log_file_data(&filetransferContext, dst_tosend, lastpkg, opts->TimeoutBetweenLogs) < 0)
155 dlt_user_log_file_end(&filetransferContext, dst_tosend, 1);
157 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
158 DLT_STRING("dlt-system-filetransfer, sent dumped file"));
162 * compress file, delete the source file
163 * modification: compress into subdirectory
164 * File whis is compress will be deleted afterwards
165 * @param src File to be sent
166 * @param dst destination where to compress the file
167 * @param level of compression
169 int compress_file_to(char *src, char *dst, int level)
171 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
172 DLT_STRING("dlt-system-filetransfer, compressing file from:"),DLT_STRING(src),DLT_STRING("to:"),DLT_STRING(dst));
177 snprintf(dst_mode,8, "wb%d", level);
182 dst_file = gzopen(dst, dst_mode);
183 if(dst_file == Z_NULL)
189 src_file = fopen(src, "r");
198 buf = malloc(Z_CHUNK_SZ);
201 while(!feof(src_file))
203 int read = fread(buf, 1, Z_CHUNK_SZ, src_file);
212 gzwrite(dst_file, buf, read);
216 DLT_LOG(dltsystem, DLT_LOG_WARN, DLT_STRING("Could not remove file"), DLT_STRING(src));
224 //!Sends one file over DLT.
226 * If configured in opts, compresses it, then sends it.
227 * uses subdirecties for compressing and before sending, to avoid that those files get changed in the meanwhile
230 int send_one(char *src, FiletransferOptions const *opts, int which)
232 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
233 DLT_STRING("dlt-system-filetransfer, sending a file."));
234 sleep(opts->TimeDelay);
236 // Prepare all needed file names
237 char *fn = basename(src);
239 char *fdir = strndup(src,PATH_MAX);
241 fdir = dirname(fdir);//dirname overwrites its argument anyway
242 char *dst_tosend;//file which is going to be sent
244 char *rn = unique_name(src);//new unique filename based on inode
249 // Compress if needed
250 if(opts->Compression[which] > 0)
252 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
253 DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory for compressing it."));
255 char *dst_tocompress;//file which is going to be compressed, the compressed one is named dst_tosend
258 int len = strlen(fdir)+strlen(SUBDIR_COMPRESS)+strlen(rn)+3;//the filename in .tocompress +2 for 2*"/", +1 for \0
259 dst_tocompress = malloc(len);
260 MALLOC_ASSERT(dst_tocompress);
262 snprintf(dst_tocompress,len,"%s/%s/%s",fdir,SUBDIR_COMPRESS,rn);
264 //moving in subdir, from where it can be compressed
265 if(rename(src, dst_tocompress) < 0)
267 DLT_LOG(dltsystem, DLT_LOG_ERROR,
268 DLT_STRING("Could not move file"),
270 DLT_STRING(dst_tocompress));
272 free(dst_tocompress);
276 len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+strlen(COMPRESS_EXTENSION)+3;//the resulting filename in .tosend +2 for 2*"/", +1 for \0
277 dst_tosend = malloc(len);
278 MALLOC_ASSERT(dst_tosend);
279 snprintf(dst_tosend,len,"%s/%s/%s%s",fdir,SUBDIR_TOSEND,rn,COMPRESS_EXTENSION);
283 if (compress_file_to(dst_tocompress,dst_tosend, opts->CompressionLevel[which]) != 0){
286 free(dst_tocompress);
290 free(dst_tocompress);
294 //move it directly into "tosend"
295 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
296 DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory."));
297 int len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+3;
298 dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 2*"/", +1 for \0
300 snprintf(dst_tosend,len,"%s/%s/%s",fdir,SUBDIR_TOSEND,rn);
302 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
303 DLT_STRING("dlt-system-filetransfer, Rename:"),DLT_STRING(src),DLT_STRING("to: "),DLT_STRING(dst_tosend));
304 //moving in subdir, from where it can be compressed
305 if(rename(src, dst_tosend) < 0)
307 DLT_LOG(dltsystem, DLT_LOG_ERROR,
308 DLT_STRING("Could not move file"),
310 DLT_STRING(dst_tosend));
319 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
320 DLT_STRING("dlt-system-filetransfer, File ready to send"));
322 send_dumped_file(opts,dst_tosend);
333 int flush_dir_send(FiletransferOptions const *opts, const char *compress_dir, const char *send_dir){
337 dir = opendir(send_dir);
341 while((dp = readdir(dir)) != NULL)
343 if(dp->d_type != DT_REG)
346 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
347 DLT_STRING("dlt-system-filetransfer, old compressed file found in send directory:"),DLT_STRING(dp->d_name));
348 int len = strlen(send_dir)+strlen(dp->d_name)+2;
351 snprintf(fn,len, "%s/%s", send_dir, dp->d_name);
354 //if we have a file here and in the to_compress dir, we delete the to_send file: we can not be sure, that it has been properly compressed!
355 if (!strncmp( dp->d_name+strlen(dp->d_name)-strlen(COMPRESS_EXTENSION),COMPRESS_EXTENSION,strlen(COMPRESS_EXTENSION)))
359 //old file name (not: path) would have been:
360 char tmp[strlen(dp->d_name)-strlen(COMPRESS_EXTENSION)+1];
361 strncpy(tmp,dp->d_name,strlen(dp->d_name)-strlen(COMPRESS_EXTENSION));
362 tmp[strlen(dp->d_name)-3]='\0';
364 int len = strlen(tmp)+strlen(compress_dir)+1+1;//2 sizes + 1*"/" + \0
365 char *path_uncompressed = malloc(len);
366 MALLOC_ASSERT(path_uncompressed);
367 snprintf(path_uncompressed,len,"%s/%s",compress_dir,tmp);
370 if (stat(path_uncompressed,&sb)==-1)
372 //uncompressed equivalent does not exist. We can send it out.
373 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
374 DLT_STRING("dlt-system-filetransfer, sending file."));
376 send_dumped_file(opts,fn);
380 //There is an uncompressed file. Compression seems to have been interrupted -> delete the compressed file instead of sending it!
381 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
382 DLT_STRING("dlt-system-filetransfer, uncompressed version exists. Deleting partially compressed version."));
383 if (sb.st_mode & S_IFREG)
387 if( remove(fn ) != 0 )
389 //"Error deleting file". Continue? If we would cancel, maybe the dump is never sent! Deletion would again be tried in next LC.
390 DLT_LOG(dltsystem, DLT_LOG_ERROR,
391 DLT_STRING("Error deleting file:"),DLT_STRING(fn));
396 //"Oldfile is a not reg file. Is this possible? Can we compress a directory?: %s\n",path_uncompressed);
397 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
398 DLT_STRING("dlt-system-filetransfer, Oldfile is a not regular file! Do we have a problem?"),DLT_STRING(fn));
402 free(path_uncompressed);//it is no more used. It would be transferred in next step.
405 //uncompressed file. We can just resend it, the action to put it here was a move action.
406 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
407 DLT_STRING("dlt-system-filetransfer, Sending uncompressed file from previous LC."),DLT_STRING(fn));
408 send_dumped_file(opts,fn);
415 DLT_LOG(dltsystem, DLT_LOG_ERROR,
416 DLT_STRING("Could not open directory"),
417 DLT_STRING(send_dir));
420 closedir(dir);//end: send_dir
425 int flush_dir_compress(FiletransferOptions const *opts, int which, const char *compress_dir, const char *send_dir){
427 //check for files in compress_dir. Assumption: a file which lies here, should have been compressed, but that action was interrupted.
428 //As it can arrive here only by a rename, it is most likely to be a complete file
431 dir = opendir(compress_dir);
434 while((dp = readdir(dir)) != NULL)
436 if(dp->d_type != DT_REG)
438 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
439 DLT_STRING("dlt-system-filetransfer, old file found in compress-directory."));
442 //compress file into to_send dir
443 int len = strlen(compress_dir)+strlen(dp->d_name)+2;
444 char *cd_filename = malloc(len);
445 MALLOC_ASSERT(cd_filename);
446 snprintf(cd_filename,len,"%s/%s",compress_dir,dp->d_name);
449 len = strlen(send_dir)+strlen(dp->d_name)+strlen(COMPRESS_EXTENSION)+2;
450 char *dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 1*"/", +1 for \0 + .gz
451 MALLOC_ASSERT(dst_tosend);
452 snprintf(dst_tosend,len,"%s/%s%s",send_dir,dp->d_name,COMPRESS_EXTENSION);
454 if (compress_file_to(cd_filename,dst_tosend, opts->CompressionLevel[which]) != 0){
462 send_dumped_file(opts,dst_tosend);
469 DLT_LOG(dltsystem, DLT_LOG_ERROR,
470 DLT_STRING("Could not open directory"),
471 DLT_STRING(compress_dir));
474 closedir(dir);//end: compress_dir
479 int flush_dir_original(FiletransferOptions const *opts, int which){
482 const char *sdir = opts->Directory[which];
486 while((dp = readdir(dir)) != NULL)
488 if(dp->d_type != DT_REG){
489 //we don't send directories
492 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
493 DLT_STRING("dlt-system-filetransfer, old file found in directory."));
494 int len = strlen(sdir)+strlen(dp->d_name)+2;
495 char *fn = malloc(len);
497 snprintf(fn,len, "%s/%s", sdir, dp->d_name);
498 if(send_one(fn, opts, which) < 0)
509 DLT_LOG(dltsystem, DLT_LOG_ERROR,
510 DLT_STRING("Could not open directory"),
518 //!Cleans the surveyed directories and subdirectories. Sends residing files into trace
520 * @param opts FiletransferOptions
521 * @param which which directory is affected -> position in list of opts->Directory
522 * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
524 int flush_dir(FiletransferOptions const *opts, int which)
528 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
529 DLT_STRING("dlt-system-filetransfer, flush directory of old files."));
533 int len = strlen(opts->Directory[which])+strlen(SUBDIR_COMPRESS)+2;
534 compress_dir = malloc (len);
535 MALLOC_ASSERT(compress_dir);
536 snprintf(compress_dir,len,"%s/%s",opts->Directory[which],SUBDIR_COMPRESS);
538 len = strlen(opts->Directory[which])+strlen(SUBDIR_TOSEND)+2;
539 send_dir = malloc (len);
540 MALLOC_ASSERT(send_dir);
541 snprintf(send_dir,len,"%s/%s",opts->Directory[which],SUBDIR_TOSEND);
543 //1st: scan the tosend directory.
544 if ( 0 != flush_dir_send(opts, compress_dir, send_dir) ){
550 //1nd: scan the tocompress directory.
551 if (0 != flush_dir_compress(opts, which, compress_dir, send_dir)){
557 free(send_dir);//no more used
560 //last step: scan the original directory - we can reuse the send_one function
561 if ( 0 != flush_dir_original(opts,which)){
568 //!Initializes the surveyed directories
569 /**On startup, the inotifiy handlers are created, and existing files shall be sent into DLT stream
570 * @param opts FiletransferOptions
571 * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
573 int init_filetransfer_dirs(FiletransferOptions const *opts)
575 DLT_LOG(dltsystem, DLT_LOG_DEBUG,
576 DLT_STRING("dlt-system-filetransfer, initializing inotify on directories."));
577 ino.handle = inotify_init();
582 DLT_LOG(filetransferContext, DLT_LOG_FATAL,
583 DLT_STRING("Failed to initialize inotify in dlt-system file transfer."));
587 for(i = 0;i < opts->Count;i++)
589 //create subdirectories for processing the files
592 int len = strlen(opts->Directory[i])+strlen(SUBDIR_COMPRESS)+2;
593 subdirpath= malloc (len);
594 MALLOC_ASSERT(subdirpath);
595 snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_COMPRESS);
596 int ret = mkdir(subdirpath,0777);
598 if (0 != ret && EEXIST != errno){
599 DLT_LOG(dltsystem, DLT_LOG_ERROR,
600 DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
606 len = strlen(opts->Directory[i])+strlen(SUBDIR_TOSEND)+2;
607 subdirpath= malloc (len);
608 MALLOC_ASSERT(subdirpath);
609 snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_TOSEND);
610 ret = mkdir(subdirpath,0777);
611 if (0 != ret && EEXIST != errno){
612 DLT_LOG(dltsystem, DLT_LOG_ERROR,
613 DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
620 ino.fd[i] = inotify_add_watch(ino.handle, opts->Directory[i],
621 IN_CLOSE_WRITE|IN_MOVED_TO);
625 snprintf(buf, 1024, "Failed to add inotify watch to directory %s in dlt-system file transfer.",
627 DLT_LOG(filetransferContext, DLT_LOG_FATAL,
639 int wait_for_files(FiletransferOptions const *opts)
641 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, waiting for files."));
642 static char buf[INOTIFY_LEN];
643 ssize_t len = read(ino.handle, buf, INOTIFY_LEN);
646 DLT_LOG(filetransferContext, DLT_LOG_ERROR,
647 DLT_STRING("Error while waiting for files in dlt-system file transfer."));
652 while(i < (len-INOTIFY_SZ))
654 struct inotify_event *ie = (struct inotify_event *)&buf[i];
657 if((ie->mask & IN_CLOSE_WRITE) || (ie->mask & IN_MOVED_TO))
660 for(j = 0;j < opts->Count;j++)
662 if(ie->wd == ino.fd[j])
664 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, found new file."));
665 int length = strlen(opts->Directory[j])+ie->len+1;
666 char *tosend = malloc(length);
667 snprintf(tosend,length, "%s/%s", opts->Directory[j], ie->name);
668 send_one(tosend, opts, j);
674 i += INOTIFY_SZ + ie->len;
679 void filetransfer_thread(void *v_conf)
681 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, in thread."));
682 DltSystemConfiguration *conf = (DltSystemConfiguration *) v_conf;
683 DLT_REGISTER_CONTEXT(filetransferContext, conf->Filetransfer.ContextId,
684 "File transfer manager.");
686 sleep(conf->Filetransfer.TimeStartup);
688 if(init_filetransfer_dirs(&(conf->Filetransfer)) < 0)
691 while(!threads.shutdown)
693 if(wait_for_files(&(conf->Filetransfer)) < 0)
695 DLT_LOG(dltsystem, DLT_LOG_ERROR, DLT_STRING("Error while waiting files. File transfer shutdown."));
698 sleep(conf->Filetransfer.TimeDelay);
702 void start_filetransfer(DltSystemConfiguration *conf)
704 DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, start."));
705 static pthread_attr_t t_attr;
707 pthread_create(&pt, &t_attr, (void *)filetransfer_thread, conf);
708 threads.threads[threads.count++] = pt;