From eca1ee3488d9cced0f5dbfa35a233fae7d317e8e Mon Sep 17 00:00:00 2001 From: martin-s Date: Fri, 24 Dec 2010 22:26:36 +0000 Subject: [PATCH] Add:map_binfile:Better download on demand support git-svn-id: https://navit.svn.sourceforge.net/svnroot/navit/trunk@3828 ffa7fe5e-494d-0410-b361-a75ebd5db220 --- navit/navit/attr_def.h | 1 + navit/navit/file.c | 60 +++++--- navit/navit/file.h | 2 + navit/navit/map/binfile/binfile.c | 317 +++++++++++++++++++++++++++++--------- 4 files changed, 293 insertions(+), 87 deletions(-) diff --git a/navit/navit/attr_def.h b/navit/navit/attr_def.h index 7a44fd0..1a3afeb 100644 --- a/navit/navit/attr_def.h +++ b/navit/navit/attr_def.h @@ -226,6 +226,7 @@ ATTR(disable_reset) ATTR(autostart) ATTR(readwrite) ATTR(cache) +ATTR(create) ATTR2(0x0002ffff,type_int_end) ATTR2(0x00030000,type_string_begin) ATTR(type) diff --git a/navit/navit/file.c b/navit/navit/file.c index 6d00cfe..35bae39 100644 --- a/navit/navit/file.c +++ b/navit/navit/file.c @@ -113,6 +113,7 @@ file_http_request(struct file *file, char *method, char *host, char *path, char { char *request=g_strdup_printf("%s %s HTTP/1.0\r\nUser-Agent: navit %s\r\nHost: %s%s%s%s\r\n\r\n",method,path,version,host,header?"\r\n":"",header?header:"",header?"\r\n":""); write(file->fd, request, strlen(request)); + dbg(1,"%s\n",request); file->requests++; } @@ -162,7 +163,7 @@ file_create(char *name, struct attr **options) host[path-name-7]='\0'; if (port) *port++='\0'; - dbg(0,"host=%s path=%s\n",host,path); + dbg(1,"host=%s path=%s\n",host,path); file->fd=file_socket_connect(host,port?port:"80"); file_http_request(file,method,host,path,header); file->special=1; @@ -170,12 +171,14 @@ file_create(char *name, struct attr **options) } #endif } else { - if (options && (attr=attr_search(options, NULL, attr_readwrite)) && attr->u.num) + if (options && (attr=attr_search(options, NULL, attr_readwrite)) && attr->u.num) { open_flags |= O_RDWR; - else + if ((attr=attr_search(options, NULL, attr_create)) && attr->u.num) + open_flags |= O_CREAT; + } else open_flags |= O_RDONLY; file->name = g_strdup(name); - file->fd=open(name, open_flags); + file->fd=open(name, open_flags, 0666); if (file->fd == -1) { g_free(file); return NULL; @@ -311,31 +314,51 @@ file_process_headers(struct file *file, char *headers) } } +static void +file_shift_buffer(struct file *file, int amount) +{ + memmove(file->buffer, file->buffer+amount, file->buffer_len-amount); + file->buffer_len-=amount; +} + unsigned char * file_data_read_special(struct file *file, int size, int *size_ret) { - char *ret,*hdr; + unsigned char *ret,*hdr; int rets=0,rd; + int buffer_size=8192; + int eof=0; if (!file->special) return NULL; + if (!file->buffer) + file->buffer=g_malloc(buffer_size); ret=g_malloc(size); - while (size > 0) { - rd=read(file->fd, ret+rets, size); - if (rd <= 0) - break; - rets+=rd; - size-=rd; + while (size > 0 && (!eof || file->buffer_len)) { + int toread=buffer_size-file->buffer_len; + if (toread >= 4096 && !eof) { + rd=read(file->fd, file->buffer+file->buffer_len, toread); + if (rd > 0) { + file->buffer_len+=rd; + } else + eof=1; + } if (file->requests) { - if ((hdr=file_http_header_end(ret, rets))) { + dbg(1,"checking header\n"); + if ((hdr=file_http_header_end(file->buffer, file->buffer_len))) { hdr[-1]='\0'; - file_process_headers(file, ret); - rets-=hdr-ret; - memmove(ret, hdr, rets); + dbg(1,"found %s\n",file->buffer); + file_process_headers(file, file->buffer); + file_shift_buffer(file, hdr-file->buffer); file->requests--; - } else { - rets=0; - break; } + } else { + rd=file->buffer_len; + if (rd > size) + rd=size; + memcpy(ret+rets, file->buffer, rd); + file_shift_buffer(file, rd); + rets+=rd; + size-=rd; } } *size_ret=rets; @@ -637,6 +660,7 @@ file_destroy(struct file *f) file_unmap( f ); } + g_free(f->buffer); g_free(f->name); g_free(f); } diff --git a/navit/navit/file.h b/navit/navit/file.h index 2ee8bca..be5278a 100644 --- a/navit/navit/file.h +++ b/navit/navit/file.h @@ -54,6 +54,8 @@ struct file { int special; int cache; int requests; + unsigned char *buffer; + int buffer_len; }; /* prototypes */ diff --git a/navit/navit/map/binfile/binfile.c b/navit/navit/map/binfile/binfile.c index 69c7295..bdd11bf 100644 --- a/navit/navit/map/binfile/binfile.c +++ b/navit/navit/map/binfile/binfile.c @@ -225,6 +225,12 @@ binfile_read_eoc64(struct file *fi) return eoc; } +static int +binfile_cd_extra(struct zip_cd *cd) +{ + return cd->zipcfnl+cd->zipcxtl; +} + static struct zip_cd * binfile_read_cd(struct map_priv *m, int offset, int len) { @@ -233,7 +239,7 @@ binfile_read_cd(struct map_priv *m, int offset, int len) if (len == -1) { cd=(struct zip_cd *)file_data_read(m->fi,cdoffset+offset, sizeof(*cd)); cd_to_cpu(cd); - len=cd->zipcfnl+cd->zipcxtl; + len=binfile_cd_extra(cd); file_data_free(m->fi,(unsigned char *)cd); } cd=(struct zip_cd *)file_data_read(m->fi,cdoffset+offset, sizeof(*cd)+len); @@ -248,16 +254,28 @@ binfile_read_cd(struct map_priv *m, int offset, int len) return cd; } -static long long -binfile_cd_offset(struct zip_cd *cd) +static struct zip_cd_ext * +binfile_cd_ext(struct zip_cd *cd) { struct zip_cd_ext *ext; - if (cd->zipofst != 0xffffffff || cd->zipcxtl != sizeof(*ext)) - return cd->zipofst; + if (cd->zipofst != 0xffffffff) + return NULL; + if (cd->zipcxtl != sizeof(*ext)) + return NULL; ext=(struct zip_cd_ext *)((unsigned char *)cd+sizeof(*cd)+cd->zipcfnl); if (ext->tag != 0x0001 || ext->size != 8) + return NULL; + return ext; +} + +static long long +binfile_cd_offset(struct zip_cd *cd) +{ + struct zip_cd_ext *ext=binfile_cd_ext(cd); + if (ext) + return ext->zipofst; + else return cd->zipofst; - return ext->zipofst; } static struct zip_lfh * @@ -848,22 +866,116 @@ zipfile_to_tile(struct map_priv *m, struct zip_cd *cd, struct tile *t) return t->start != NULL; } +static long long +map_binfile_download_size(struct map_priv *m) +{ + struct attr url={attr_url}; + struct attr http_method={attr_http_method}; + struct attr *attrs[3]; + int size_ret; + long long ret; + struct file *http; + void *data; + + attrs[0]=&url; + url.u.str=m->url; + attrs[1]=&http_method; + http_method.u.str="HEAD"; + attrs[2]=NULL; + + http=file_create(NULL, attrs); + if (!http) + return 0; + data=file_data_read_special(http, 4096, &size_ret); + g_free(data); + if (size_ret < 0) + return 0; + ret=file_size(http); + file_destroy(http); + return ret; +} + +static struct file * +map_binfile_http_range(struct map_priv *m, long long offset, int size) +{ + struct file *http; + struct attr *attrs[3]; + struct attr url={attr_url}; + struct attr http_header={attr_http_header}; + + attrs[0]=&url; + attrs[1]=&http_header; + attrs[2]=NULL; + + url.u.str=m->url; + http_header.u.str=g_strdup_printf("Range: bytes=%Ld-%Ld",offset, offset+size-1); + + http=file_create(NULL, attrs); + g_free(http_header.u.str); + return http; +} + +static unsigned char * +map_binfile_download_range(struct map_priv *m, long long offset, int size) +{ + unsigned char *ret; + int size_ret; + struct file *http=map_binfile_http_range(m, offset, size); + + ret=file_data_read_special(http, size, &size_ret); + if (size_ret != size) { + g_free(ret); + return NULL; + } + file_destroy(http); + return ret; +} + +static struct zip_cd * +download_cd(struct map_download *download) +{ + struct map_priv *m=download->m; + struct zip64_eoc *zip64_eoc=(struct zip64_eoc *)file_data_read(m->fi, 0, sizeof(*zip64_eoc)); + struct zip_cd *cd=(struct zip_cd *)map_binfile_download_range(m, zip64_eoc->zip64eofst+download->zipfile*m->cde_size,m->cde_size); + file_data_free(m->fi, (unsigned char *)zip64_eoc); + dbg(0,"needed cd, result %p\n",cd); + return cd; +} + + static int download_start(struct map_download *download) { char tilename[download->cd->zipcfnl+1]; long long offset; struct zip_eoc *eoc; + struct zip_cd_ext *ext; struct attr url={attr_url}; - struct attr *attrs[]={&url, NULL}; + struct attr http_header={attr_http_header}; + struct attr *attrs[3]; - download->cd_copy=g_malloc(download->m->cde_size); - memcpy(download->cd_copy, download->cd, download->m->cde_size); - file_data_free(download->file, (unsigned char *)download->cd); + if (!download->cd->zipcensig) { + download->cd_copy=download_cd(download); + } else { + download->cd_copy=g_malloc(download->m->cde_size); + memcpy(download->cd_copy, download->cd, download->m->cde_size); + file_data_free(download->file, (unsigned char *)download->cd); + } download->cd=NULL; strncpy(tilename,(char *)(download->cd_copy+1),download->cd_copy->zipcfnl); tilename[download->cd_copy->zipcfnl]='\0'; - url.u.str=g_strdup_printf("%smemberid=%d",download->m->url,download->zipfile); + attrs[0]=&url; + attrs[1]=NULL; + if (strchr(download->m->url,'?')) + url.u.str=g_strdup_printf("%smemberid=%d",download->m->url,download->zipfile); + else { + long long offset=binfile_cd_offset(download->cd_copy); + int size=download->cd_copy->zipcsiz+sizeof(struct zip_lfh)+download->cd_copy->zipcfnl; + url.u.str=g_strdup(download->m->url); + http_header.u.str=g_strdup_printf("Range: bytes=%Ld-%Ld",offset,offset+size-1); + attrs[1]=&http_header; + attrs[2]=NULL; + } offset=file_size(download->file); offset-=sizeof(struct zip_eoc); eoc=(struct zip_eoc *)file_data_read(download->file, offset, sizeof(struct zip_eoc)); @@ -871,7 +983,11 @@ download_start(struct map_download *download) memcpy(download->eoc_copy, eoc, sizeof(struct zip_eoc)); file_data_free(download->file, (unsigned char *)eoc); dbg(0,"encountered missing tile %s(%s), downloading at %Ld\n",url.u.str,tilename,offset); - download->cd_copy->zipofst=offset; + ext=binfile_cd_ext(download->cd_copy); + if (ext) + ext->zipofst=offset; + else + download->cd_copy->zipofst=offset; download->tile=file_create(NULL, attrs); download->offset=offset; g_free(url.u.str); @@ -884,7 +1000,7 @@ download_download(struct map_download *download) int size=64*1024,size_ret,tile_size; unsigned char *data; data=file_data_read_special(download->tile, size, &size_ret); - dbg(0,"got %d bytes writing at offset %Ld\n",size_ret,download->offset); + dbg(1,"got %d bytes writing at offset %Ld\n",size_ret,download->offset); if (size_ret <= 0) { g_free(data); return 1; @@ -903,21 +1019,21 @@ download_finish(struct map_download *download) { struct zip_lfh *lfh; char *lfh_filename; - + file_data_write(download->file, download->offset, sizeof(struct zip_eoc), (void *)download->eoc_copy); - lfh=(struct zip_lfh *)(file_data_read(download->file,download->cd_copy->zipofst, sizeof(struct zip_lfh))); + lfh=(struct zip_lfh *)(file_data_read(download->file,binfile_cd_offset(download->cd_copy), sizeof(struct zip_lfh))); download->cd_copy->zipcsiz=lfh->zipsize; download->cd_copy->zipcunc=lfh->zipuncmp; download->cd_copy->zipccrc=lfh->zipcrc; - lfh_filename=(char *)file_data_read(download->file,download->cd_copy->zipofst+sizeof(struct zip_lfh),lfh->zipfnln); + lfh_filename=(char *)file_data_read(download->file,binfile_cd_offset(download->cd_copy)+sizeof(struct zip_lfh),lfh->zipfnln); memcpy(download->cd_copy+1,lfh_filename,lfh->zipfnln); file_data_free(download->file,(void *)lfh_filename); file_data_free(download->file,(void *)lfh); - file_data_write(download->file, download->m->eoc->zipeofst + download->zipfile*download->m->cde_size, download->m->cde_size, (void *)download->cd_copy); + file_data_write(download->file, download->m->eoc->zipeofst + download->zipfile*download->m->cde_size, binfile_cd_extra(download->cd_copy)+sizeof(struct zip_cd), (void *)download->cd_copy); g_free(download->cd_copy); download->cd=(struct zip_cd *)(file_data_read(download->file, download->m->eoc->zipeofst + download->zipfile*download->m->cde_size, download->m->cde_size)); cd_to_cpu(download->cd); - dbg(0,"Offset %d\n",download->cd->zipofst); + dbg(1,"Offset %d\n",download->cd->zipofst); return 1; } @@ -1141,7 +1257,8 @@ map_rect_destroy_binfile(struct map_rect_priv *mr) #ifdef DEBUG_SIZE dbg(0,"size=%d kb\n",mr->size/1024); #endif - file_data_free(mr->tiles[0].fi, (unsigned char *)(mr->tiles[0].start)); + if (mr->tiles[0].fi && mr->tiles[0].start) + file_data_free(mr->tiles[0].fi, (unsigned char *)(mr->tiles[0].start)); g_free(mr->url); g_free(mr); } @@ -1785,64 +1902,124 @@ map_binfile_zip_setup(struct map_priv *m, char *filename, int mmap) static int map_binfile_download_initial(struct map_priv *m) { - struct attr url={attr_url}; - struct attr http_method={attr_http_method}; - struct attr http_header={attr_http_header}; + struct attr readwrite={attr_readwrite,{(void *)1}}; + struct attr create={attr_create,{(void *)1}}; struct attr *attrs[4]; - struct file *planet; - long long offset=0,planet_size; - int size=64*1024,size_ret; - unsigned char *data; + struct file *out; + long long woffset=0,planet_size; + int size_ret; + int cd1size,cdisize; + long long cd1offset,cdioffset; struct zip64_eoc *zip64_eoc; + struct zip64_eocl *zip64_eocl; + struct zip_eoc *zip_eoc; + struct zip_cd *cd1,*cdn,*cdi; + int count,chunk,cdoffset=0; + int mode=1; + + planet_size=map_binfile_download_size(m); + if (!planet_size) + return 0; - attrs[0]=&url; - url.u.str=m->url; - attrs[1]=&http_method; - http_method.u.str="HEAD"; + attrs[0]=&readwrite; + attrs[1]=&create; attrs[2]=NULL; - - planet=file_create(NULL, attrs); - if (!planet) - return 0; - data=file_data_read_special(planet, size, &size_ret); - dbg(0,"size_ret=%d\n",size_ret); - if (size_ret < 0) - return 0; - g_free(data); - planet_size=file_size(planet); - file_destroy(planet); + out=file_create(m->filename,attrs); dbg(0,"planet_size %Ld\n",planet_size); - attrs[1]=&http_header; - http_header.u.str=g_strdup_printf("Range: bytes=%Ld-%Ld",planet_size-98,planet_size-1); - planet=file_create(NULL, attrs); - g_free(http_header.u.str); - if (!planet) - return 0; - data=file_data_read_special(planet, size, &size_ret); - dbg(0,"size_ret=%d\n",size_ret); - if (size_ret < 0) - return 0; - zip64_eoc=data; - if (zip64_eoc->zip64esig != zip64_eoc_sig) + zip64_eoc=(struct zip64_eoc *)map_binfile_download_range(m, planet_size-98, 98); + zip64_eocl=(struct zip64_eocl *)(zip64_eoc+1); + zip_eoc=(struct zip_eoc *)(zip64_eocl+1); + if (zip64_eoc->zip64esig != zip64_eoc_sig || zip64_eocl->zip64lsig != zip64_eocl_sig || zip_eoc->zipesig != zip_eoc_sig) { - g_free(data); - file_destroy(planet); + g_free(zip64_eoc); return 0; } - g_free(data); - file_destroy(planet); - http_header.u.str=g_strdup_printf("Range: bytes=%Ld-%Ld",zip64_eoc->zip64eofst,zip64_eoc->zip64eofst+zip64_eoc->zip64ecsz); - planet=file_create(NULL, attrs); - g_free(http_header.u.str); - for (;;) { - data=file_data_read_special(planet, size, &size_ret); - if (size_ret <= 0) - break; - g_free(data); - fprintf(stderr,"."); + if (mode == 1) { + struct file *http=map_binfile_http_range(m, zip64_eoc->zip64eofst, zip64_eoc->zip64ecsz); + for (;;) { + int cd_xlen; + unsigned char *cd_data; + cd1=(struct zip_cd *)file_data_read_special(http, sizeof(*cd1), &size_ret); + cd1->zipcunc=0; + dbg(1,"size_ret=%d\n",size_ret); + if (!size_ret) + break; + if (size_ret != sizeof(*cd1) || cd1->zipcensig != zip_cd_sig) { + dbg(0,"error1 size=%d vs %d\n",size_ret, sizeof(*cd1)); + break; + } + file_data_write(out, woffset, sizeof(*cd1), (unsigned char *)cd1); + woffset+=sizeof(*cd1); + cd_xlen=cd1->zipcfnl+cd1->zipcxtl; + cd_data=file_data_read_special(http, cd_xlen, &size_ret); + if (size_ret != cd_xlen) { + dbg(0,"error2 size=%d vs %d\n",size_ret,cd_xlen); + break; + } + file_data_write(out, woffset, cd_xlen, cd_data); + woffset+=cd_xlen; + g_free(cd1); + g_free(cd_data); + } + file_destroy(http); + } else { + cd1size=sizeof(*cd1); + cd1offset=zip64_eoc->zip64eofst; + cd1=(struct zip_cd *)map_binfile_download_range(m, cd1offset, cd1size); + if (!cd1) + return 0; + cd1size=sizeof(*cd1)+binfile_cd_extra(cd1); + g_free(cd1); + cd1=(struct zip_cd *)map_binfile_download_range(m, cd1offset, cd1size); + if (!cd1) + return 0; + cd1->zipcunc=0; + cdisize=sizeof(*cdi)+strlen("index")+cd1->zipcxtl; + cdioffset=zip64_eoc->zip64eofst+zip64_eoc->zip64ecsz-cdisize; + cdi=(struct zip_cd *)map_binfile_download_range(m, cdioffset, cdisize); + if (!cdi) { + g_free(cd1); + return 0; + } + cdi->zipcunc=0; + cdn=g_malloc0(cd1size*256); + + file_data_write(out, woffset, sizeof(*zip64_eoc), (unsigned char *)zip64_eoc); + woffset+=sizeof(*zip64_eoc); + cdoffset=woffset; + + file_data_write(out, woffset, cd1size, (unsigned char *)cd1); + woffset+=cd1size; + count=(cdioffset-cd1offset)/cd1size-1; + while (count > 0) { + if (count > 256) + chunk=256; + else + chunk=count; + file_data_write(out, woffset, cd1size*chunk, (unsigned char *)cdn); + woffset+=cd1size*chunk; + count-=chunk; + } + g_free(cdn); + g_free(cd1); + file_data_write(out, woffset, cdisize, (unsigned char *)cdi); + woffset+=cdisize; } - file_destroy(planet); + + zip64_eoc->zip64eofst=cdoffset; + zip64_eocl->zip64lofst=woffset; + zip_eoc->zipeofst=cdoffset; +#if 0 + file_data_write(out, woffset, sizeof(*zip64_eoc), (unsigned char *)zip64_eoc); + woffset+=sizeof(*zip64_eoc); + file_data_write(out, woffset, sizeof(*zip64_eocl), (unsigned char *)zip64_eocl); + woffset+=sizeof(*zip64_eocl); +#endif + file_data_write(out, woffset, sizeof(*zip_eoc), (unsigned char *)zip_eoc); + woffset+=sizeof(*zip_eoc); + g_free(zip64_eoc); + m->fi=out; return 1; } @@ -1858,6 +2035,8 @@ map_binfile_open(struct map_priv *m) dbg(1,"file_create %s\n", m->filename); m->fi=file_create(m->filename, m->url?attrs:NULL); + if (! m->fi && m->url) + map_binfile_download_initial(m); if (! m->fi) { dbg(0,"Failed to load '%s'\n", m->filename); return 0; @@ -1871,7 +2050,7 @@ map_binfile_open(struct map_priv *m) return 0; } *magic = le32_to_cpu(*magic); - if (*magic == zip_lfh_sig || *magic == zip_split_sig) { + if (*magic == zip_lfh_sig || *magic == zip_split_sig || *magic == zip_cd_sig || *magic == zip64_eoc_sig) { if (!map_binfile_zip_setup(m, m->filename, m->flags & 1)) { dbg(0,"invalid file format for '%s'\n", m->filename); file_destroy(m->fi); @@ -1885,7 +2064,7 @@ map_binfile_open(struct map_priv *m) m->map_version=0; mr=map_rect_new_binfile(m, NULL); if (mr) { - item=map_rect_get_item_binfile(mr); + while ((item=map_rect_get_item_binfile(mr)) == &busy_item); if (item && item->type == type_map_information) { if (binfile_attr_get(item->priv_data, attr_version, &attr)) m->map_version=attr.u.num; -- 2.7.4