2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
25 #include "poolid_private.h"
31 data_read_id(unsigned char *dp, Id *idp)
43 x = (x << 7) ^ c ^ 128;
47 static unsigned char *
48 data_read_ideof(unsigned char *dp, Id *idp, int *eof)
67 x = (x << 7) ^ c ^ 128;
71 static unsigned char *
72 data_skip(unsigned char *dp, int type)
83 while ((*dp & 0x80) != 0)
87 while ((*dp & 0xc0) != 0)
94 case TYPE_DIRSTRARRAY:
97 while ((*dp & 0x80) != 0)
106 case TYPE_DIRNUMNUMARRAY:
109 while ((*dp & 0x80) != 0)
112 while ((*dp & 0x80) != 0)
115 while ((*dp & 0x80) != 0)
122 fprintf(stderr, "unknown type in data_skip\n");
127 static unsigned char *
128 data_fetch(unsigned char *dp, KeyValue *kv, Repokey *key)
141 kv->str = (const char *)dp;
142 return dp + strlen(kv->str) + 1;
144 return data_read_id(dp, &kv->id);
146 return data_read_id(dp, &kv->num);
148 return data_read_ideof(dp, &kv->id, &kv->eof);
150 return data_read_id(dp, &kv->id);
151 case TYPE_DIRSTRARRAY:
152 dp = data_read_ideof(dp, &kv->id, &kv->eof);
153 kv->str = (const char *)dp;
154 return dp + strlen(kv->str) + 1;
155 case TYPE_DIRNUMNUMARRAY:
156 dp = data_read_id(dp, &kv->id);
157 dp = data_read_id(dp, &kv->num);
158 return data_read_ideof(dp, &kv->num2, &kv->eof);
164 static unsigned char *
165 forward_to_key(Repodata *data, Id key, Id schemaid, unsigned char *dp)
169 keyp = data->schemadata + data->schemata[schemaid];
170 while ((k = *keyp++) != 0)
174 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
176 dp = data_skip(dp, TYPE_ID); /* skip that offset */
177 dp = data_skip(dp, TYPE_ID); /* skip that length */
180 if (data->keys[k].storage != KEY_STORAGE_INCORE)
182 dp = data_skip(dp, data->keys[k].type);
187 #define BLOB_PAGEBITS 15
188 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
190 static unsigned char *
191 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
193 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
194 and are consecutive. Return a pointer to the mapping of PSTART. */
195 unsigned char buf[BLOB_PAGESIZE];
198 /* Quick check in case all pages are there already and consecutive. */
199 for (i = pstart; i <= pend; i++)
200 if (data->pages[i].mapped_at == -1
202 && data->pages[i].mapped_at
203 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
206 return data->blob_store + data->pages[pstart].mapped_at;
208 /* Ensure that we can map the numbers of pages we need at all. */
209 if (pend - pstart + 1 > data->ncanmap)
211 unsigned int oldcan = data->ncanmap;
212 data->ncanmap = pend - pstart + 1;
213 if (data->ncanmap < 4)
215 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
216 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
217 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
219 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
223 /* Now search for "cheap" space in our store. Space is cheap if it's either
224 free (very cheap) or contains pages we search for anyway. */
226 /* Setup cost array. */
227 unsigned int cost[data->ncanmap];
228 for (i = 0; i < data->ncanmap; i++)
230 unsigned int pnum = data->mapped[i];
236 Attrblobpage *p = data->pages + pnum;
237 assert (p->mapped_at != -1);
238 if (pnum >= pstart && pnum <= pend)
245 /* And search for cheapest space. */
246 unsigned int best_cost = -1;
247 unsigned int best = 0;
248 unsigned int same_cost = 0;
249 for (i = 0; i + pend - pstart < data->ncanmap; i++)
251 unsigned int c = cost[i];
253 for (j = 0; j < pend - pstart + 1; j++)
256 best_cost = c, best = i;
257 else if (c == best_cost)
259 /* A null cost won't become better. */
263 /* If all places have the same cost we would thrash on slot 0. Avoid
264 this by doing a round-robin strategy in this case. */
265 if (same_cost == data->ncanmap - pend + pstart - 1)
266 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
268 /* So we want to map our pages from [best] to [best+pend-pstart].
269 Use a very simple strategy, which doesn't make the best use of
270 our resources, but works. Throw away all pages in that range
271 (even ours) then copy around ours (in case they were outside the
272 range) or read them in. */
273 for (i = best; i < best + pend - pstart + 1; i++)
275 unsigned int pnum = data->mapped[i];
277 /* If this page is exactly at the right place already,
278 no need to evict it. */
279 && pnum != pstart + i - best)
281 /* Evict this page. */
283 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
287 data->pages[pnum].mapped_at = -1;
291 /* Everything is free now. Read in the pages we want. */
292 for (i = pstart; i <= pend; i++)
294 Attrblobpage *p = data->pages + i;
295 unsigned int pnum = i - pstart + best;
296 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
297 if (p->mapped_at != -1)
299 if (p->mapped_at != pnum * BLOB_PAGESIZE)
302 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
304 /* Still mapped somewhere else, so just copy it from there. */
305 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
306 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
311 unsigned int in_len = p->file_size;
312 unsigned int compressed = in_len & 1;
315 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
317 /* Not mapped, so read in this page. */
318 if (fseek(data->fp, p->file_offset, SEEK_SET) < 0)
320 perror ("mapping fseek");
323 if (fread(compressed ? buf : dest, in_len, 1, data->fp) != 1)
325 perror ("mapping fread");
330 unsigned int out_len;
331 out_len = unchecked_decompress_buf(buf, in_len,
332 dest, BLOB_PAGESIZE);
333 if (out_len != BLOB_PAGESIZE
334 && i < data->num_pages - 1)
336 fprintf (stderr, "can't decompress\n");
340 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
344 fprintf (stderr, "\n");
347 p->mapped_at = pnum * BLOB_PAGESIZE;
348 data->mapped[pnum] = i + 1;
350 return data->blob_store + best * BLOB_PAGESIZE;
353 static unsigned char *
354 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
357 if (key->type == TYPE_VOID)
359 if (off >= data->lastverticaloffset)
361 off -= data->lastverticaloffset;
362 if (off + len > data->vincorelen)
364 return data->vincore + off;
368 if (off + len > key->size)
370 /* we now have the offset, go into vertical */
371 off += data->verticaloffset[key - data->keys];
372 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
374 dp += off % BLOB_PAGESIZE;
378 static inline unsigned char *
379 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
381 unsigned char *dp = *dpp;
385 if (key->storage == KEY_STORAGE_INCORE)
387 /* hmm, this is a bit expensive */
388 *dpp = data_skip(dp, key->type);
391 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
394 dp = data_read_id(dp, &off);
395 dp = data_read_id(dp, &len);
397 return make_vertical_available(data, key, off, len);
404 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
411 dp = data->incoredata + data->incoreoffset[entry];
412 dp = data_read_id(dp, &schema);
413 /* make sure the schema of this solvable contains the key */
414 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
417 dp = forward_to_key(data, keyid, schema, dp);
418 key = data->keys + keyid;
419 dp = get_data(data, key, &dp);
422 if (key->type == TYPE_STR)
423 return (const char *)dp;
424 if (key->type != TYPE_ID)
426 /* id type, must either use global or local string strore*/
427 dp = data_read_id(dp, &id);
429 return data->spool.stringspace + data->spool.strings[id];
430 return id2str(data->repo->pool, id);
434 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
438 Id k, keyid, *kp, *keyp;
439 unsigned char *dp, *ddp;
444 dp = data->incoredata + data->incoreoffset[entry];
445 dp = data_read_id(dp, &schema);
446 keyp = data->schemadata + data->schemata[schema];
449 /* search in a specific key */
450 for (kp = keyp; (k = *kp++) != 0; )
451 if (data->keys[k].name == keyname)
455 dp = forward_to_key(data, k, schema, dp);
461 while ((keyid = *keyp++) != 0)
464 key = data->keys + keyid;
465 ddp = get_data(data, key, &dp);
468 ddp = data_fetch(ddp, &kv, key);
471 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
473 while (!kv.eof && !stop);
474 if (onekey || stop > SEARCH_NEXT_KEY)
480 /* extend repodata so that it includes solvables p */
482 repodata_extend(Repodata *data, Id p)
484 if (data->start == data->end)
485 data->start = data->end = p;
488 int old = data->end - data->start;
489 int new = p - data->end + 1;
492 data->attrs = sat_realloc2(data->attrs, old + new, sizeof(Id *));
493 memset(data->attrs + old, 0, new * sizeof(Id *));
495 data->incoreoffset = sat_realloc2(data->incoreoffset, old + new, sizeof(Id));
496 memset(data->incoreoffset + old, 0, new * sizeof(Id));
501 int old = data->end - data->start;
502 int new = data->start - p;
505 data->attrs = sat_realloc2(data->attrs, old + new, sizeof(Id *));
506 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
507 memset(data->attrs, 0, new * sizeof(Id *));
509 data->incoreoffset = sat_realloc2(data->incoreoffset, old + new, sizeof(Id));
510 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
511 memset(data->incoreoffset, 0, new * sizeof(Id));
517 repodata_insert_keyid(Repodata *data, Id entry, Id keyid, Id val, int overwrite)
522 data->attrs = sat_calloc(data->end - data->start + 1, sizeof(Id *));
524 if (data->attrs[entry])
526 for (pp = data->attrs[entry]; *pp; pp += 2)
535 i = pp - data->attrs[entry];
537 data->attrs[entry] = sat_realloc2(data->attrs[entry], i + 3, sizeof(Id));
538 pp = data->attrs[entry] + i;
545 repodata_set(Repodata *data, Id entry, Repokey *key, Id val)
549 /* find key in keys */
550 for (keyid = 1; keyid < data->nkeys; keyid++)
551 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
553 if (key->type == TYPE_CONSTANT && key->size != data->keys[keyid].size)
557 if (keyid == data->nkeys)
559 /* allocate new key */
560 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
561 data->keys[data->nkeys++] = *key;
562 if (data->verticaloffset)
564 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
565 data->verticaloffset[data->nkeys - 1] = 0;
568 repodata_insert_keyid(data, entry, keyid, val, 1);
572 repodata_set_id(Repodata *data, Id entry, Id keyname, Id id)
578 key.storage = KEY_STORAGE_INCORE;
579 repodata_set(data, entry, &key, id);
583 repodata_set_num(Repodata *data, Id entry, Id keyname, Id num)
589 key.storage = KEY_STORAGE_INCORE;
590 repodata_set(data, entry, &key, num);
594 repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str)
599 id = stringpool_str2id(&data->spool, str, 1);
601 id = str2id(data->repo->pool, str, 1);
605 key.storage = KEY_STORAGE_INCORE;
606 repodata_set(data, entry, &key, id);
610 repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant)
614 key.type = TYPE_CONSTANT;
616 key.storage = KEY_STORAGE_INCORE;
617 repodata_set(data, entry, &key, 0);
621 repodata_set_void(Repodata *data, Id entry, Id keyname)
625 key.type = TYPE_VOID;
627 key.storage = KEY_STORAGE_INCORE;
628 repodata_set(data, entry, &key, 0);
632 repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str)
641 key.storage = KEY_STORAGE_INCORE;
642 data->attrdata = sat_realloc(data->attrdata, data->attrdatalen + l);
643 memcpy(data->attrdata + data->attrdatalen, str, l);
644 repodata_set(data, entry, &key, data->attrdatalen);
645 data->attrdatalen += l;
649 repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2)
655 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", entry, dir, num, num2, data->attriddatalen);
657 if (data->attrs && data->attrs[entry])
659 for (pp = data->attrs[entry]; *pp; pp += 2)
660 if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRNUMNUMARRAY)
665 for (ida = data->attriddata + pp[1]; *ida; ida += 3)
667 if (ida + 1 == data->attriddata + data->attriddatalen)
669 /* this was the last entry, just append it */
670 data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + 3, sizeof(Id));
671 data->attriddatalen--; /* overwrite terminating 0 */
675 /* too bad. move to back. */
676 data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + oldsize + 4, sizeof(Id));
677 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
678 pp[1] = data->attriddatalen;
679 data->attriddatalen += oldsize;
681 data->attriddata[data->attriddatalen++] = dir;
682 data->attriddata[data->attriddatalen++] = num;
683 data->attriddata[data->attriddatalen++] = num2;
684 data->attriddata[data->attriddatalen++] = 0;
689 key.type = TYPE_DIRNUMNUMARRAY;
691 key.storage = KEY_STORAGE_INCORE;
692 data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + 4, sizeof(Id));
693 repodata_set(data, entry, &key, data->attriddatalen);
694 data->attriddata[data->attriddatalen++] = dir;
695 data->attriddata[data->attriddatalen++] = num;
696 data->attriddata[data->attriddatalen++] = num2;
697 data->attriddata[data->attriddatalen++] = 0;
701 repodata_merge_attrs (Repodata *data, Id dest, Id src)
704 for (keyp = data->attrs[src]; *keyp; keyp += 2)
705 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
708 /*********************************/
710 /* unify with repo_write! */
712 #define EXTDATA_BLOCK 1023
713 #define SCHEMATA_BLOCK 31
714 #define SCHEMATADATA_BLOCK 255
722 data_addid(struct extdata *xd, Id x)
725 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
726 dp = xd->buf + xd->len;
731 *dp++ = (x >> 28) | 128;
733 *dp++ = (x >> 21) | 128;
734 *dp++ = (x >> 14) | 128;
737 *dp++ = (x >> 7) | 128;
739 xd->len = dp - xd->buf;
743 data_addideof(struct extdata *xd, Id x, int eof)
746 x = (x & 63) | ((x & ~63) << 1);
747 data_addid(xd, (eof ? x: x | 64));
751 data_addblob(struct extdata *xd, unsigned char *blob, int len)
753 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
754 memcpy(xd->buf + xd->len, blob, len);
758 /*********************************/
761 addschema_prepare(Repodata *data, Id *schematacache)
766 memset(schematacache, 0, 256 * sizeof(Id));
767 for (i = 0; i < data->nschemata; i++)
769 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
772 schematacache[h] = i + 1;
774 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
775 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
779 addschema(Repodata *data, Id *schema, Id *schematacache)
784 for (sp = schema, len = 0, h = 0; *sp; len++)
789 cid = schematacache[h];
793 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
796 for (cid = 0; cid < data->nschemata; cid++)
797 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
800 /* a new one. make room. */
801 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
802 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
804 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
805 data->schemata[data->nschemata] = data->schemadatalen;
806 data->schemadatalen += len;
807 schematacache[h] = data->nschemata + 1;
809 fprintf(stderr, "addschema: new schema\n");
811 return data->nschemata++;
816 repodata_internalize(Repodata *data)
819 Id id, entry, nentry, *ida;
820 Id schematacache[256];
821 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
822 unsigned char *dp, *ndp;
823 int newschema, oldcount;
824 struct extdata newincore;
825 struct extdata newvincore;
830 newvincore.buf = data->vincore;
831 newvincore.len = data->vincorelen;
833 schema = sat_malloc2(data->nkeys, sizeof(Id));
834 seen = sat_malloc2(data->nkeys, sizeof(Id));
836 /* Merge the data already existing (in data->schemata, ->incoredata and
837 friends) with the new attributes in data->attrs[]. */
838 nentry = data->end - data->start;
839 addschema_prepare(data, schematacache);
840 memset(&newincore, 0, sizeof(newincore));
841 for (entry = 0; entry < nentry; entry++)
843 memset(seen, 0, data->nkeys * sizeof(Id));
845 dp = data->incoredata + data->incoreoffset[entry];
846 if (data->incoredata)
847 dp = data_read_id(dp, &oldschema);
851 fprintf(stderr, "oldschema %d\n", oldschema);
852 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
853 fprintf(stderr, "schemadata %p\n", data->schemadata);
855 /* seen: -1: old data 0: skipped >0: id + 1 */
858 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
862 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
869 if (data->attrs[entry])
870 for (keyp = data->attrs[entry]; *keyp; keyp += 2)
877 seen[*keyp] = keyp[1] + 1;
881 /* Ideally we'd like to sort the new schema here, to ensure
882 schema equality independend of the ordering. We can't do that
883 yet. For once see below (old ids need to come before new ids).
884 An additional difficulty is that we also need to move
885 the values with the keys. */
886 schemaid = addschema(data, schema, schematacache);
888 schemaid = oldschema;
891 /* Now create data blob. We walk through the (possibly new) schema
892 and either copy over old data, or insert the new. */
893 /* XXX Here we rely on the fact that the (new) schema has the form
894 o1 o2 o3 o4 ... | n1 n2 n3 ...
895 (oX being the old keyids (possibly overwritten), and nX being
896 the new keyids). This rules out sorting the keyids in order
897 to ensure a small schema count. */
898 data->incoreoffset[entry] = newincore.len;
899 data_addid(&newincore, schemaid);
900 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
902 key = data->keys + *keyp;
906 /* Skip the data associated with this old key. */
907 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
909 ndp = data_skip(dp, TYPE_ID);
910 ndp = data_skip(ndp, TYPE_ID);
912 else if (key->storage == KEY_STORAGE_INCORE)
913 ndp = data_skip(dp, key->type);
916 if (seen[*keyp] == -1)
918 /* If this key was an old one _and_ was not overwritten with
919 a different value copy over the old value (we skipped it
922 data_addblob(&newincore, dp, ndp - dp);
925 else if (seen[*keyp])
927 /* Otherwise we have a new value. Parse it into the internal
930 unsigned int oldvincorelen = 0;
933 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
936 oldvincorelen = xd->len;
938 id = seen[*keyp] - 1;
945 data_addblob(xd, data->attrdata + id, strlen((char *)(data->attrdata + id)) + 1);
952 case TYPE_DIRNUMNUMARRAY:
953 for (ida = data->attriddata + id; *ida; ida += 3)
955 data_addid(xd, ida[0]);
956 data_addid(xd, ida[1]);
957 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
961 fprintf(stderr, "don't know how to handle type %d\n", key->type);
964 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
966 /* put offset/len in incore */
967 data_addid(&newincore, data->lastverticaloffset + oldvincorelen);
968 oldvincorelen = xd->len - oldvincorelen;
969 data_addid(&newincore, oldvincorelen);
975 data->incoredata = newincore.buf;
976 data->incoredatalen = newincore.len;
977 data->incoredatafree = 0;
979 data->vincore = newvincore.buf;
980 data->vincorelen = newvincore.len;
982 data->attrs = sat_free(data->attrs);
983 data->attrdata = sat_free(data->attrdata);
984 data->attrdatalen = 0;
988 repodata_str2dir(Repodata *data, const char *dir, int create)
994 while (*dir == '/' && dir[1] == '/')
998 dire = strchrnul(dir, '/');
1000 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
1002 id = strn2id(data->repo->pool, dir, dire - dir, create);
1005 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
1018 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
1020 return compress_buf(page, len, cpage, max);
1023 #define SOLV_ERROR_EOF 3
1025 static inline unsigned int
1031 for (i = 0; i < 4; i++)
1041 /* Try to either setup on-demand paging (using FP as backing
1042 file), or in case that doesn't work (FP not seekable) slurps in
1043 all pages and deactivates paging. */
1046 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
1048 FILE *fp = data->fp;
1049 unsigned int npages;
1051 unsigned int can_seek;
1053 unsigned char buf[BLOB_PAGESIZE];
1054 if (pagesz != BLOB_PAGESIZE)
1056 /* We could handle this by slurping in everything. */
1057 fprintf (stderr, "non matching page size\n");
1061 if ((cur_file_ofs = ftell(fp)) < 0)
1065 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
1067 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
1069 data->num_pages = npages;
1070 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
1072 /* If we can't seek on our input we have to slurp in everything. */
1074 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
1075 for (i = 0; i < npages; i++)
1077 unsigned int in_len = read_u32(fp);
1078 unsigned int compressed = in_len & 1;
1079 Attrblobpage *p = data->pages + i;
1082 fprintf (stderr, "page %d: len %d (%scompressed)\n",
1083 i, in_len, compressed ? "" : "not ");
1089 p->file_offset = cur_file_ofs;
1090 p->file_size = in_len * 2 + compressed;
1091 if (fseek(fp, in_len, SEEK_CUR) < 0)
1094 fprintf (stderr, "can't seek after we thought we can\n");
1095 /* We can't fall back to non-seeking behaviour as we already
1096 read over some data pages without storing them away. */
1099 cur_file_ofs += in_len;
1103 unsigned int out_len;
1104 void *dest = data->blob_store + i * BLOB_PAGESIZE;
1105 p->mapped_at = i * BLOB_PAGESIZE;
1108 /* We can't seek, so suck everything in. */
1109 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
1116 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
1117 if (out_len != BLOB_PAGESIZE
1120 fprintf (stderr, "can't decompress\n");
1129 /* If we are here we were able to seek to all page
1130 positions, so activate paging by copying FP into our structure.
1131 We dup() the file, so that our callers can fclose() it and we
1132 still have it open. But this means that we share file positions
1133 with the input filedesc. So in case our caller reads it after us,
1134 and calls back into us we might change the file position unexpectedly
1136 int fd = dup (fileno (fp));
1139 /* Jeez! What a bloody system, we can't dup() anymore. */
1143 /* XXX we don't close this yet anywhere. */
1144 data->fp = fdopen (fd, "r");
1147 /* My God! What happened now? */