/*
- * Copyright (c) 2007-2011, Novell Inc.
+ * Copyright (c) 2007-2012, Novell Inc.
*
* This program is licensed under the BSD license, read LICENSE.BSD
* for further information
#include <fcntl.h>
#include <time.h>
+#ifdef _WIN32
+ #include <windows.h>
+ #include <fileapi.h>
+ #include <io.h>
+#endif
+
#include "repo.h"
#include "repopage.h"
/*
The format is tailored for fast decompression (i.e. only byte based),
and skewed to ASCII content (highest bit often not set):
-
+
a 0LLLLLLL
- self-describing ASCII character hex L
b 100lllll <l+1 bytes>
litlen -= 32;
}
}
- litofs = 0;
}
return oo;
}
void repopagestore_free(Repopagestore *store)
{
- solv_free(store->blob_store);
- solv_free(store->pages);
- solv_free(store->mapped);
+ store->blob_store = solv_free(store->blob_store);
+ store->file_pages = solv_free(store->file_pages);
+ store->mapped_at = solv_free(store->mapped_at);
+ store->mapped = solv_free(store->mapped);
if (store->pagefd != -1)
close(store->pagefd);
+ store->pagefd = -1;
}
/* Make sure all pages from PSTART to PEND (inclusive) are loaded,
and are consecutive. Return a pointer to the mapping of PSTART. */
unsigned char buf[REPOPAGE_BLOBSIZE];
- unsigned int i, best;
+ unsigned int i, best, pnum;
- /* Quick check in case all pages are there already and consecutive. */
- for (i = pstart; i <= pend; i++)
- if (store->pages[i].mapped_at == -1
- || (i > pstart
- && store->pages[i].mapped_at
- != store->pages[i-1].mapped_at + REPOPAGE_BLOBSIZE))
- break;
- if (i > pend)
- return store->blob_store + store->pages[pstart].mapped_at;
+ if (pstart == pend)
+ {
+ /* Quick check in case the requested page is already mapped */
+ if (store->mapped_at[pstart] != -1)
+ return store->blob_store + store->mapped_at[pstart];
+ }
+ else
+ {
+ /* Quick check in case all pages are already mapped and consecutive. */
+ for (pnum = pstart; pnum <= pend; pnum++)
+ if (store->mapped_at[pnum] == -1
+ || (pnum > pstart
+ && store->mapped_at[pnum]
+ != store->mapped_at[pnum-1] + REPOPAGE_BLOBSIZE))
+ break;
+ if (pnum > pend)
+ return store->blob_store + store->mapped_at[pstart];
+ }
- if (store->pagefd == -1)
- return 0;
+ if (store->pagefd == -1 || !store->file_pages)
+ return 0; /* no backing file */
#ifdef DEBUG_PAGING
fprintf(stderr, "PAGE: want %d pages starting at %d\n", pend - pstart + 1, pstart);
#endif
/* Ensure that we can map the numbers of pages we need at all. */
- if (pend - pstart + 1 > store->ncanmap)
+ if (pend - pstart + 1 > store->nmapped)
{
- unsigned int oldcan = store->ncanmap;
- store->ncanmap = pend - pstart + 1;
- if (store->ncanmap < 4)
- store->ncanmap = 4;
- store->mapped = solv_realloc2(store->mapped, store->ncanmap, sizeof(store->mapped[0]));
- memset(store->mapped + oldcan, 0, (store->ncanmap - oldcan) * sizeof (store->mapped[0]));
- store->blob_store = solv_realloc2(store->blob_store, store->ncanmap, REPOPAGE_BLOBSIZE);
+ unsigned int oldcan = store->nmapped;
+ store->nmapped = pend - pstart + 1;
+ if (store->nmapped < 4)
+ store->nmapped = 4;
+ store->mapped = solv_realloc2(store->mapped, store->nmapped, sizeof(store->mapped[0]));
+ for (i = oldcan; i < store->nmapped; i++)
+ store->mapped[i] = -1;
+ store->blob_store = solv_realloc2(store->blob_store, store->nmapped, REPOPAGE_BLOBSIZE);
#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGE: can map %d pages\n", store->ncanmap);
+ fprintf(stderr, "PAGE: can map %d pages\n", store->nmapped);
#endif
}
- /* Now search for "cheap" space in our store. Space is cheap if it's either
- free (very cheap) or contains pages we search for anyway. */
-
- {
- unsigned int best_cost = -1;
- unsigned int same_cost = 0;
- unsigned int cost = 0;
- int ii;
-
- best = 0;
- for (i = 0, ii = -(pend - pstart); i < store->ncanmap; i++, ii++)
- {
- unsigned int pnum = store->mapped[i];
- if (pnum)
- {
- Attrblobpage *p = store->pages + --pnum;
- assert(p->mapped_at != -1);
- cost += pnum >= pstart && pnum <= pend ? 1 : 3;
- }
- if (ii < 0)
- continue; /* still need to accummulate cost */
- if (cost < best_cost)
- best_cost = cost, best = ii;
- else if (cost == best_cost)
- same_cost++;
- if (!cost)
- break; /* it won't get any better */
- /* now remove the cost of page ii again */
- if (i == ii)
- {
- cost = 0;
- continue;
- }
- pnum = store->mapped[ii];
- if (pnum)
- {
- Attrblobpage *p = store->pages + --pnum;
- assert(p->mapped_at != -1);
- cost -= pnum >= pstart && pnum <= pend ? 1 : 3;
- }
- }
-
-#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGE: best %d at cost %d, same %d\n", best, best_cost, same_cost);
-#endif
-
- /* If all places have the same cost we would thrash on slot 0. Avoid
- this by doing a round-robin strategy in this case. */
- if (same_cost == store->ncanmap - pend + pstart - 1)
- best = store->rr_counter++ % (store->ncanmap - pend + pstart);
- }
+ if (store->mapped_at[pstart] != -1)
+ {
+ /* assume forward search */
+ best = store->mapped_at[pstart] / REPOPAGE_BLOBSIZE;
+ if (best + (pend - pstart) >= store->nmapped)
+ best = 0;
+ }
+ else if (store->mapped_at[pend] != -1)
+ {
+ /* assume backward search */
+ best = store->mapped_at[pend] / REPOPAGE_BLOBSIZE;
+ if (best < pend - pstart)
+ best = store->nmapped - 1;
+ best -= pend - pstart;
+ }
+ else
+ {
+ /* choose some "random" location to avoid thrashing */
+ best = (pstart + store->rr_counter++) % (store->nmapped - pend + pstart);
+ }
/* So we want to map our pages from [best] to [best+pend-pstart].
Use a very simple strategy, which doesn't make the best use of
our resources, but works. Throw away all pages in that range
(even ours) then copy around ours or read them in. */
- for (i = best; i < best + pend - pstart + 1; i++)
+ for (i = best, pnum = pstart; pnum <= pend; i++, pnum++)
{
- unsigned int pnum = store->mapped[i];
- if (pnum--
- /* If this page is exactly at the right place already,
- no need to evict it. */
- && pnum != pstart + i - best)
+ unsigned int pnum_mapped_at;
+ unsigned int oldpnum = store->mapped[i];
+ if (oldpnum != -1)
{
- Attrblobpage *p;
+ if (oldpnum == pnum)
+ continue; /* already have the correct page */
/* Evict this page. */
#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGE: evict page %d from %d\n", pnum, i);
+ fprintf(stderr, "PAGE: evict page %d from %d\n", oldpnum, i);
#endif
- store->mapped[i] = 0;
- store->pages[pnum].mapped_at = -1;
- /* check if we can copy the correct content */
- p = store->pages + (pstart + i - best);
- if (p->mapped_at != -1 && p->mapped_at != i * REPOPAGE_BLOBSIZE)
- {
- void *dest = store->blob_store + i * REPOPAGE_BLOBSIZE;
+ store->mapped[i] = -1;
+ store->mapped_at[oldpnum] = -1;
+ }
+ /* check if we can copy the correct content (before it gets evicted) */
+ pnum_mapped_at = store->mapped_at[pnum];
+ if (pnum_mapped_at != -1 && pnum_mapped_at != i * REPOPAGE_BLOBSIZE)
+ {
+ void *dest = store->blob_store + i * REPOPAGE_BLOBSIZE;
#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGECOPY: %d from %d to %d\n", pstart + i - best, p->mapped_at / REPOPAGE_BLOBSIZE, i);
+ fprintf(stderr, "PAGECOPY: %d from %d to %d\n", pnum, pnum_mapped_at / REPOPAGE_BLOBSIZE, i);
#endif
- memcpy(dest, store->blob_store + p->mapped_at, REPOPAGE_BLOBSIZE);
- store->mapped[p->mapped_at / REPOPAGE_BLOBSIZE] = 0;
- p->mapped_at = i * REPOPAGE_BLOBSIZE;
- store->mapped[i] = pstart + i - best + 1;
- }
+ memcpy(dest, store->blob_store + pnum_mapped_at, REPOPAGE_BLOBSIZE);
+ store->mapped[pnum_mapped_at / REPOPAGE_BLOBSIZE] = -1;
+ store->mapped[i] = pnum;
+ store->mapped_at[pnum] = i * REPOPAGE_BLOBSIZE;
}
}
- /* Everything is free now. Read in the pages we want. */
- for (i = pstart; i <= pend; i++)
+ /* Everything is free now. Read in or copy the pages we want. */
+ for (i = best, pnum = pstart; pnum <= pend; i++, pnum++)
{
- Attrblobpage *p = store->pages + i;
- unsigned int pnum = i - pstart + best;
- void *dest = store->blob_store + pnum * REPOPAGE_BLOBSIZE;
- if (p->mapped_at != -1)
+ void *dest = store->blob_store + i * REPOPAGE_BLOBSIZE;
+ if (store->mapped_at[pnum] != -1)
{
- if (p->mapped_at != pnum * REPOPAGE_BLOBSIZE)
+ unsigned int pnum_mapped_at = store->mapped_at[pnum];
+ if (pnum_mapped_at != i * REPOPAGE_BLOBSIZE)
{
#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGECOPY: %d from %d to %d\n", i, p->mapped_at / REPOPAGE_BLOBSIZE, pnum);
+ fprintf(stderr, "PAGECOPY: %d from %d to %d\n", pnum, pnum_mapped_at / REPOPAGE_BLOBSIZE, i);
#endif
/* Still mapped somewhere else, so just copy it from there. */
- memcpy(dest, store->blob_store + p->mapped_at, REPOPAGE_BLOBSIZE);
- store->mapped[p->mapped_at / REPOPAGE_BLOBSIZE] = 0;
+ memcpy(dest, store->blob_store + pnum_mapped_at, REPOPAGE_BLOBSIZE);
+ store->mapped[pnum_mapped_at / REPOPAGE_BLOBSIZE] = -1;
}
}
else
{
- unsigned int in_len = p->file_size;
+ Attrblobpage *p = store->file_pages + pnum;
+ unsigned int in_len = p->page_size;
unsigned int compressed = in_len & 1;
in_len >>= 1;
#ifdef DEBUG_PAGING
- fprintf(stderr, "PAGEIN: %d to %d", i, pnum);
+ fprintf(stderr, "PAGEIN: %d to %d", pnum, i);
#endif
- if (pread(store->pagefd, compressed ? buf : dest, in_len, p->file_offset) != in_len)
+#ifndef _WIN32
+ if (pread(store->pagefd, compressed ? buf : dest, in_len, store->file_offset + p->page_offset) != in_len)
{
perror("mapping pread");
return 0;
}
+#else
+ DWORD read_len;
+ OVERLAPPED ovlp = {0};
+ ovlp.Offset = store->file_offset + p->page_offset;
+ if (!ReadFile((HANDLE) _get_osfhandle(store->pagefd), compressed ? buf : dest, in_len, &read_len, &ovlp) || read_len != in_len)
+ {
+ perror("mapping ReadFile");
+ return 0;
+ }
+#endif
if (compressed)
{
unsigned int out_len;
- out_len = unchecked_decompress_buf(buf, in_len,
- dest, REPOPAGE_BLOBSIZE);
- if (out_len != REPOPAGE_BLOBSIZE && i < store->num_pages - 1)
+ out_len = unchecked_decompress_buf(buf, in_len, dest, REPOPAGE_BLOBSIZE);
+ if (out_len != REPOPAGE_BLOBSIZE && pnum < store->num_pages - 1)
{
#ifdef DEBUG_PAGING
fprintf(stderr, "can't decompress\n");
fprintf(stderr, "\n");
#endif
}
- p->mapped_at = pnum * REPOPAGE_BLOBSIZE;
- store->mapped[pnum] = i + 1;
+ store->mapped_at[pnum] = i * REPOPAGE_BLOBSIZE;
+ store->mapped[i] = pnum;
}
return store->blob_store + best * REPOPAGE_BLOBSIZE;
}
read_u32(FILE *fp)
{
int c, i;
- unsigned int x = 0;
+ unsigned int x = 0;
- for (i = 0; i < 4; i++)
- {
+ for (i = 0; i < 4; i++)
+ {
c = getc(fp);
- if (c == EOF)
+ if (c == EOF)
return 0;
- x = (x << 8) | c;
- }
+ x = (x << 8) | c;
+ }
return x;
}
unsigned int npages;
unsigned int i;
unsigned int can_seek;
- long cur_file_ofs;
+ unsigned int cur_page_ofs;
unsigned char buf[REPOPAGE_BLOBSIZE];
if (pagesz != REPOPAGE_BLOBSIZE)
return SOLV_ERROR_CORRUPT;
}
can_seek = 1;
- if ((cur_file_ofs = ftell(fp)) < 0)
+ if ((store->file_offset = ftell(fp)) < 0)
can_seek = 0;
clearerr(fp);
if (can_seek)
if (store->pagefd == -1)
can_seek = 0;
else
- fcntl(store->pagefd, F_SETFD, FD_CLOEXEC);
+ solv_setcloexec(store->pagefd, 1);
#ifdef DEBUG_PAGING
fprintf(stderr, "can %sseek\n", can_seek ? "" : "NOT ");
npages = (blobsz + REPOPAGE_BLOBSIZE - 1) / REPOPAGE_BLOBSIZE;
store->num_pages = npages;
- store->pages = solv_malloc2(npages, sizeof(store->pages[0]));
+ store->mapped_at = solv_malloc2(npages, sizeof(*store->mapped_at));
- /* If we can't seek on our input we have to slurp in everything. */
- if (!can_seek)
+ /* If we can't seek on our input we have to slurp in everything.
+ * Otherwise set up file_pages containing offest/length of the
+ * pages */
+ if (can_seek)
+ store->file_pages = solv_malloc2(npages, sizeof(*store->file_pages));
+ else
store->blob_store = solv_malloc2(npages, REPOPAGE_BLOBSIZE);
+ cur_page_ofs = 0;
for (i = 0; i < npages; i++)
{
unsigned int in_len = read_u32(fp);
unsigned int compressed = in_len & 1;
- Attrblobpage *p = store->pages + i;
in_len >>= 1;
#ifdef DEBUG_PAGING
fprintf(stderr, "page %d: len %d (%scompressed)\n",
#endif
if (can_seek)
{
- cur_file_ofs += 4;
- p->mapped_at = -1;
- p->file_offset = cur_file_ofs;
- p->file_size = in_len * 2 + compressed;
+ Attrblobpage *p = store->file_pages + i;
+ cur_page_ofs += 4;
+ store->mapped_at[i] = -1; /* not mapped yet */
+ p->page_offset = cur_page_ofs;
+ p->page_size = in_len * 2 + compressed;
if (fseek(fp, in_len, SEEK_CUR) < 0)
{
/* We can't fall back to non-seeking behaviour as we already
store->pagefd = -1;
return SOLV_ERROR_EOF;
}
- cur_file_ofs += in_len;
+ cur_page_ofs += in_len;
}
else
{
unsigned int out_len;
void *dest = store->blob_store + i * REPOPAGE_BLOBSIZE;
- p->mapped_at = i * REPOPAGE_BLOBSIZE;
- p->file_offset = 0;
- p->file_size = 0;
+ store->mapped_at[i] = i * REPOPAGE_BLOBSIZE;
/* We can't seek, so suck everything in. */
if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
{
projects / platform / upstream / libsolv.git / blobdiff