2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
29 #include "repo_write.h"
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
35 typedef struct needid {
41 #define NEEDIDOFF(id) (ISRELDEP(id) ? (needid[0].map + GETRELID(id)) : id)
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
48 * return size of array (including trailing zero)
53 incneedid(Id id, NeedId *needid)
55 needid[NEEDIDOFF(id)].need++;
59 incneedidarray(Id *idarray, NeedId *needid)
64 while ((id = *idarray++) != 0)
67 needid[NEEDIDOFF(id)].need++;
78 needid_cmp_need(const void *ap, const void *bp, void *dp)
83 r = b->need - a->need;
86 return a->map - b->map;
90 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
94 Stringpool *spool = dp;
99 r = b->need - a->need;
102 as = spool->stringspace + spool->strings[a->map];
103 bs = spool->stringspace + spool->strings[b->map];
104 return strcmp(as, bs);
108 /*------------------------------------------------------------------*/
109 /* output helper routines, used for writing the header */
110 /* (the data itself is accumulated in memory and written with
118 write_u32(Repodata *data, unsigned int x)
123 if (putc(x >> 24, fp) == EOF ||
124 putc(x >> 16, fp) == EOF ||
125 putc(x >> 8, fp) == EOF ||
128 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
138 write_u8(Repodata *data, unsigned int x)
142 if (putc(x, data->fp) == EOF)
144 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
153 write_blob(Repodata *data, void *blob, int len)
157 if (len && fwrite(blob, len, 1, data->fp) != 1)
159 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
168 write_id(Repodata *data, Id x)
176 putc((x >> 28) | 128, fp);
178 putc((x >> 21) | 128, fp);
179 putc((x >> 14) | 128, fp);
182 putc((x >> 7) | 128, fp);
183 if (putc(x & 127, fp) == EOF)
185 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
190 write_str(Repodata *data, const char *str)
194 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
196 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
205 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
219 id = needid[NEEDIDOFF(id)].need;
221 id = (id & 63) | ((id & ~63) << 1);
227 write_id(data, id | 64);
241 Stringpool *ownspool;
243 int clonepool; /* are the pool ids cloned into ownspool? */
245 Id *keymap; /* keymap for this repodata */
249 Id *schema; /* schema construction space */
250 Id *sp; /* pointer in above */
256 struct extdata *extdata;
260 Id vstart; /* offset of key in vertical data */
265 int doingsolvables; /* working on solvables data */
268 Id lastdirid; /* last dir id seen in this repodata */
269 Id lastdirid_own; /* last dir id put in own pool */
272 #define NEEDID_BLOCK 1023
273 #define SCHEMATA_BLOCK 31
274 #define EXTDATA_BLOCK 4095
277 data_addid(struct extdata *xd, Id sx)
279 unsigned int x = (unsigned int)sx;
282 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
283 dp = xd->buf + xd->len;
288 *dp++ = (x >> 28) | 128;
290 *dp++ = (x >> 21) | 128;
291 *dp++ = (x >> 14) | 128;
294 *dp++ = (x >> 7) | 128;
296 xd->len = dp - xd->buf;
300 data_addideof(struct extdata *xd, Id sx, int eof)
302 unsigned int x = (unsigned int)sx;
305 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
306 dp = xd->buf + xd->len;
311 *dp++ = (x >> 27) | 128;
313 *dp++ = (x >> 20) | 128;
314 *dp++ = (x >> 13) | 128;
317 *dp++ = (x >> 6) | 128;
318 *dp++ = eof ? (x & 63) : (x & 63) | 64;
319 xd->len = dp - xd->buf;
323 data_addideof_len(Id sx)
325 unsigned int x = (unsigned int)sx;
330 return x >= (1 << 20) ? 4 : 3;
332 return x >= (1 << 6) ? 2 : 1;
336 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
342 data_addid(xd, (Id)(hx >> 3));
343 xd->buf[xd->len - 1] |= 128;
346 data_addid(xd, (Id)(x | 0x80000000));
347 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
350 data_addid(xd, (Id)x);
353 #define USE_REL_IDARRAY
354 #ifdef USE_REL_IDARRAY
357 cmp_ids(const void *pa, const void *pb, void *dp)
365 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
378 for (len = 0; len < 64 && ids[len]; len++)
382 id = needid[NEEDIDOFF(id)].need;
387 for (i = len + 1; ids[i]; i++)
389 sids = solv_malloc2(i, sizeof(Id));
390 memcpy(sids, lids, 64 * sizeof(Id));
391 for (; ids[len]; len++)
395 id = needid[NEEDIDOFF(id)].need;
402 /* That bloody solvable:prereqmarker needs to stay in position :-( */
404 marker = needid[marker].need;
405 for (i = 0; i < len; i++)
406 if (sids[i] == marker)
409 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
411 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
415 /* The differencing above produces many runs of ones and twos. I tried
416 fairly elaborate schemes to RLE those, but they give only very mediocre
417 improvements in compression, as coding the escapes costs quite some
418 space. Even if they are coded only as bits in IDs. The best improvement
419 was about 2.7% for the whole .solv file. It's probably better to
420 invest some complexity into sharing idarrays, than RLEing. */
421 for (i = 0; i < len - 1; i++)
424 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
425 hence all real differences are offsetted by 1. Otherwise we would
426 have to handle negative differences, which would cost code space for
427 the encoding of the sign. We loose the exact mapping of prereq here,
428 but we know the result, so we can recover from that in the reader. */
436 /* XXX If difference is zero we have multiple equal elements,
437 we might want to skip writing them out. */
438 data_addideof(xd, id, 0);
445 data_addideof(xd, id, 1);
453 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
461 while ((id = *ids++) != 0)
464 id = needid[NEEDIDOFF(id)].need;
465 data_addideof(xd, id, *ids ? 0 : 1);
472 data_addblob(struct extdata *xd, unsigned char *blob, int len)
474 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
475 memcpy(xd->buf + xd->len, blob, len);
479 /* grow needid array so that it contains the specified id */
481 grow_needid(struct cbdata *cbdata, Id id)
483 int oldoff = cbdata->needid[0].map;
484 int newoff = (id + 1 + NEEDID_BLOCK) & ~NEEDID_BLOCK;
485 int nrels = cbdata->pool->nrels;
486 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
488 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
489 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
490 cbdata->needid[0].map = newoff;
494 putinownpool(struct cbdata *cbdata, Repodata *data, Id id)
496 Stringpool *ss = data->localpool ? &data->spool : &cbdata->pool->ss;
497 const char *str = stringpool_id2str(ss, id);
498 id = stringpool_str2id(cbdata->ownspool, str, 1);
499 if (id >= cbdata->needid[0].map)
500 grow_needid(cbdata, id);
505 putinowndirpool_slow(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
509 parent = dirpool_parent(dp, dir);
511 parent = putinowndirpool_slow(cbdata, data, dp, parent);
512 compid = dirpool_compid(dp, dir);
513 if (cbdata->ownspool && compid > 1 && (!cbdata->clonepool || data->localpool))
514 compid = putinownpool(cbdata, data, compid);
515 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
519 putinowndirpool(struct cbdata *cbdata, Repodata *data, Id dir)
521 if (dir && dir == cbdata->lastdirid)
522 return cbdata->lastdirid_own;
523 cbdata->lastdirid = dir;
524 cbdata->lastdirid_own = putinowndirpool_slow(cbdata, data, &data->dirpool, dir);
525 return cbdata->lastdirid_own;
530 * collect key/id/dirid usage information, create needed schemas
533 collect_needed_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
535 struct cbdata *cbdata = vcbdata;
540 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? (int)(s - cbdata->pool->solvables) : 0, s ? pool_id2str(cbdata->pool, s->name) : "", key->name, pool_id2str(cbdata->pool, key->name), key->type);
542 if (key->name == REPOSITORY_SOLVABLES)
543 return SEARCH_NEXT_KEY; /* we do not want this one */
545 rm = cbdata->keymap[key - data->keys];
547 return SEARCH_NEXT_KEY; /* we do not want this one */
549 /* record key in schema */
550 if (cbdata->sp[-1] != rm)
555 case REPOKEY_TYPE_ID:
556 case REPOKEY_TYPE_IDARRAY:
558 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
559 id = putinownpool(cbdata, data, id);
560 incneedid(id, cbdata->needid);
562 case REPOKEY_TYPE_DIR:
563 case REPOKEY_TYPE_DIRNUMNUMARRAY:
564 case REPOKEY_TYPE_DIRSTRARRAY:
566 if (cbdata->owndirpool)
567 putinowndirpool(cbdata, data, id);
569 cbdata->dirused[id] = 1;
571 case REPOKEY_TYPE_FIXARRAY:
572 case REPOKEY_TYPE_FLEXARRAY:
576 *cbdata->sp++ = 0; /* mark start */
580 /* just finished a schema, rewind to start */
581 Id *sp = cbdata->sp - 1;
585 if (kv->entry == 1 || key->type == REPOKEY_TYPE_FLEXARRAY)
587 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
588 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
590 cbdata->sp = kv->eof == 2 ? sp - 1: sp;
600 collect_needed_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
602 /* set schema info, keep in sync with collect_data_solvable */
603 Repo *repo = s->repo;
605 NeedId *needid = cbdata->needid;
606 Repodata *target = cbdata->target;
607 Id *idarraydata = repo->idarraydata;
609 if (keymap[SOLVABLE_NAME])
611 *sp++ = keymap[SOLVABLE_NAME];
612 needid[s->name].need++;
614 if (keymap[SOLVABLE_ARCH])
616 *sp++ = keymap[SOLVABLE_ARCH];
617 needid[s->arch].need++;
619 if (keymap[SOLVABLE_EVR])
621 *sp++ = keymap[SOLVABLE_EVR];
622 needid[s->evr].need++;
624 if (s->vendor && keymap[SOLVABLE_VENDOR])
626 *sp++ = keymap[SOLVABLE_VENDOR];
627 needid[s->vendor].need++;
629 if (s->provides && keymap[SOLVABLE_PROVIDES])
631 *sp++ = keymap[SOLVABLE_PROVIDES];
632 target->keys[keymap[SOLVABLE_PROVIDES]].size += incneedidarray(idarraydata + s->provides, needid);
634 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
636 *sp++ = keymap[SOLVABLE_OBSOLETES];
637 target->keys[keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(idarraydata + s->obsoletes, needid);
639 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
641 *sp++ = keymap[SOLVABLE_CONFLICTS];
642 target->keys[keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(idarraydata + s->conflicts, needid);
644 if (s->requires && keymap[SOLVABLE_REQUIRES])
646 *sp++ = keymap[SOLVABLE_REQUIRES];
647 target->keys[keymap[SOLVABLE_REQUIRES]].size += incneedidarray(idarraydata + s->requires, needid);
649 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
651 *sp++ = keymap[SOLVABLE_RECOMMENDS];
652 target->keys[keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(idarraydata + s->recommends, needid);
654 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
656 *sp++ = keymap[SOLVABLE_SUGGESTS];
657 target->keys[keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(idarraydata + s->suggests, needid);
659 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
661 *sp++ = keymap[SOLVABLE_SUPPLEMENTS];
662 target->keys[keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(idarraydata + s->supplements, needid);
664 if (s->enhances && keymap[SOLVABLE_ENHANCES])
666 *sp++ = keymap[SOLVABLE_ENHANCES];
667 target->keys[keymap[SOLVABLE_ENHANCES]].size += incneedidarray(idarraydata + s->enhances, needid);
669 if (repo->rpmdbid && keymap[RPM_RPMDBID])
671 *sp++ = keymap[RPM_RPMDBID];
672 target->keys[keymap[RPM_RPMDBID]].size++;
680 * encode all of the data into the correct buffers
683 collect_data_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
685 struct cbdata *cbdata = vcbdata;
691 if (key->name == REPOSITORY_SOLVABLES)
692 return SEARCH_NEXT_KEY;
694 rm = cbdata->keymap[key - data->keys];
696 return SEARCH_NEXT_KEY; /* we do not want this one */
697 storage = cbdata->target->keys[rm].storage;
699 xd = cbdata->extdata + 0; /* incore buffer */
700 if (storage == KEY_STORAGE_VERTICAL_OFFSET)
702 xd += rm; /* vertical buffer */
703 if (cbdata->vstart == -1)
704 cbdata->vstart = xd->len;
708 case REPOKEY_TYPE_DELETED:
709 case REPOKEY_TYPE_VOID:
710 case REPOKEY_TYPE_CONSTANT:
711 case REPOKEY_TYPE_CONSTANTID:
713 case REPOKEY_TYPE_ID:
715 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
716 id = putinownpool(cbdata, data, id);
717 needid = cbdata->needid;
718 id = needid[NEEDIDOFF(id)].need;
721 case REPOKEY_TYPE_IDARRAY:
723 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
724 id = putinownpool(cbdata, data, id);
725 needid = cbdata->needid;
726 id = needid[NEEDIDOFF(id)].need;
727 data_addideof(xd, id, kv->eof);
729 case REPOKEY_TYPE_STR:
730 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
732 case REPOKEY_TYPE_MD5:
733 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
735 case REPOKEY_TYPE_SHA1:
736 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
738 case REPOKEY_TYPE_SHA224:
739 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
741 case REPOKEY_TYPE_SHA256:
742 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
744 case REPOKEY_TYPE_SHA384:
745 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
747 case REPOKEY_TYPE_SHA512:
748 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
751 case REPOKEY_TYPE_NUM:
752 data_addid64(xd, kv->num, kv->num2);
754 case REPOKEY_TYPE_DIR:
756 if (cbdata->owndirpool)
757 id = putinowndirpool(cbdata, data, id);
758 id = cbdata->dirused[id];
761 case REPOKEY_TYPE_BINARY:
762 data_addid(xd, kv->num);
764 data_addblob(xd, (unsigned char *)kv->str, kv->num);
766 case REPOKEY_TYPE_DIRNUMNUMARRAY:
768 if (cbdata->owndirpool)
769 id = putinowndirpool(cbdata, data, id);
770 id = cbdata->dirused[id];
772 data_addid(xd, kv->num);
773 data_addideof(xd, kv->num2, kv->eof);
775 case REPOKEY_TYPE_DIRSTRARRAY:
777 if (cbdata->owndirpool)
778 id = putinowndirpool(cbdata, data, id);
779 id = cbdata->dirused[id];
780 if (rm == cbdata->filelistmode)
782 /* postpone adding to xd, just update len to get the correct offsets into the incore data*/
783 xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
786 data_addideof(xd, id, kv->eof);
787 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
789 case REPOKEY_TYPE_FIXARRAY:
790 case REPOKEY_TYPE_FLEXARRAY:
792 data_addid(xd, kv->num);
793 if (kv->eof != 2 && (!kv->entry || key->type == REPOKEY_TYPE_FLEXARRAY))
794 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
795 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
797 if (xd->len - cbdata->lastlen > cbdata->maxdata)
798 cbdata->maxdata = xd->len - cbdata->lastlen;
799 cbdata->lastlen = xd->len;
803 cbdata->target->error = pool_error(cbdata->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
806 if (storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
808 /* we can re-use old data in the blob here! */
809 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
810 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
816 /* special version of collect_data_cb that collects just one single REPOKEY_TYPE_DIRSTRARRAY vertical data */
818 collect_filelist_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
820 struct cbdata *cbdata = vcbdata;
825 rm = cbdata->keymap[key - data->keys];
826 if (rm != cbdata->filelistmode)
827 return SEARCH_NEXT_KEY; /* we do not want this one */
829 if (cbdata->owndirpool)
830 id = putinowndirpool(cbdata, data, id);
831 id = cbdata->dirused[id];
832 xd = cbdata->extdata + rm; /* vertical buffer */
833 data_addideof(xd, id, kv->eof);
834 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
839 collect_data_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
841 Repo *repo = s->repo;
842 Pool *pool = repo->pool;
843 struct extdata *xd = cbdata->extdata;
844 NeedId *needid = cbdata->needid;
845 Id *idarraydata = repo->idarraydata;
847 if (keymap[SOLVABLE_NAME])
848 data_addid(xd, needid[s->name].need);
849 if (keymap[SOLVABLE_ARCH])
850 data_addid(xd, needid[s->arch].need);
851 if (keymap[SOLVABLE_EVR])
852 data_addid(xd, needid[s->evr].need);
853 if (s->vendor && keymap[SOLVABLE_VENDOR])
854 data_addid(xd, needid[s->vendor].need);
855 if (s->provides && keymap[SOLVABLE_PROVIDES])
856 data_adddepids(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
857 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
858 data_adddepids(xd, pool, needid, idarraydata + s->obsoletes, 0);
859 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
860 data_adddepids(xd, pool, needid, idarraydata + s->conflicts, 0);
861 if (s->requires && keymap[SOLVABLE_REQUIRES])
862 data_adddepids(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
863 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
864 data_adddepids(xd, pool, needid, idarraydata + s->recommends, 0);
865 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
866 data_adddepids(xd, pool, needid, idarraydata + s->suggests, 0);
867 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
868 data_adddepids(xd, pool, needid, idarraydata + s->supplements, 0);
869 if (s->enhances && keymap[SOLVABLE_ENHANCES])
870 data_adddepids(xd, pool, needid, idarraydata + s->enhances, 0);
871 if (repo->rpmdbid && keymap[RPM_RPMDBID])
872 data_addid(xd, repo->rpmdbid[(s - pool->solvables) - repo->start]);
875 /* traverse through directory with first child "dir" */
877 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
883 /* special case for '/', which has to come first */
886 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
888 if (used && !used[sib])
890 if (sib == 1 && parent == 1)
891 continue; /* already did that one above */
895 /* check if our block has some content */
897 return n - 1; /* nope, drop parent id again */
899 /* now go through all the siblings we just added and
900 * do recursive calls on them */
902 for (; parent < lastn; parent++)
904 sib = dirmap[parent];
905 if (used && used[sib] != 2) /* 2: used as parent */
907 child = dirpool_child(dp, sib);
910 dirmap[n++] = -parent; /* start new block */
911 n = traverse_dirs(dp, dirmap, n, child, used);
918 write_compressed_page(Repodata *data, unsigned char *page, int len)
921 unsigned char cpage[REPOPAGE_BLOBSIZE];
923 clen = repopagestore_compress_page(page, len, cpage, len - 1);
926 write_u32(data, len * 2);
927 write_blob(data, page, len);
931 write_u32(data, clen * 2 + 1);
932 write_blob(data, cpage, clen);
936 static Id verticals[] = {
938 SOLVABLE_DESCRIPTION,
950 SOLVABLE_CHANGELOG_AUTHOR,
951 SOLVABLE_CHANGELOG_TEXT,
955 static char *languagetags[] = {
957 "solvable:description:",
958 "solvable:messageins:",
959 "solvable:messagedel:",
965 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
970 for (i = 0; verticals[i]; i++)
971 if (key->name == verticals[i])
972 return KEY_STORAGE_VERTICAL_OFFSET;
973 keyname = pool_id2str(repo->pool, key->name);
974 for (i = 0; languagetags[i] != 0; i++)
975 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
976 return KEY_STORAGE_VERTICAL_OFFSET;
977 return KEY_STORAGE_INCORE;
981 write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
983 unsigned char *dp = xd->buf;
987 int ll = REPOPAGE_BLOBSIZE - lpage;
990 memcpy(vpage + lpage, dp, ll);
994 if (lpage == REPOPAGE_BLOBSIZE)
996 write_compressed_page(target, vpage, lpage);
1005 create_keyskip(Repo *repo, Id entry, unsigned char *repodataused, Id **oldkeyskip)
1007 Repodata *data, *last = 0;
1011 if (repo->nrepodata <= 2)
1013 keyskip = *oldkeyskip;
1016 if (keyskip[1] >= 0x10000000)
1017 keyskip = solv_free(keyskip);
1019 keyskip[1] = keyskip[2];
1021 FOR_REPODATAS(repo, rdid, data)
1023 if (!repodataused[rdid])
1025 if (entry != SOLVID_META)
1027 if (entry < data->start || entry >= data->end)
1029 /* if repodataused is set we know that the state is AVAILABLE */
1030 if (!data->incoreoffset[entry - data->start])
1034 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1038 if (cnt <= 1) /* just one repodata means we don't need a keyskip */
1040 *oldkeyskip = keyskip;
1043 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1045 keyskip[2] = keyskip[1] + repo->nrepodata;
1046 *oldkeyskip = keyskip;
1055 repowriter_create(Repo *repo)
1057 Repowriter *writer = solv_calloc(1, sizeof(*writer));
1058 writer->repo = repo;
1059 writer->keyfilter = repo_write_stdkeyfilter;
1060 writer->repodatastart = 1;
1061 writer->repodataend = repo->nrepodata;
1062 writer->solvablestart = repo->start;
1063 writer->solvableend = repo->end;
1068 repowriter_free(Repowriter *writer)
1070 return solv_free(writer);
1074 repowriter_set_flags(Repowriter *writer, int flags)
1076 writer->flags = flags;
1080 repowriter_set_keyfilter(Repowriter *writer, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata)
1082 writer->keyfilter = keyfilter;
1083 writer->kfdata = kfdata;
1087 repowriter_set_keyqueue(Repowriter *writer, Queue *keyq)
1089 writer->keyq = keyq;
1093 repowriter_set_repodatarange(Repowriter *writer, int repodatastart, int repodataend)
1095 writer->repodatastart = repodatastart;
1096 writer->repodataend = repodataend;
1100 repowriter_set_solvablerange(Repowriter *writer, int solvablestart, int solvableend)
1102 writer->solvablestart = solvablestart;
1103 writer->solvableend = solvableend;
1107 * the code works the following way:
1109 * 1) find which keys should be written
1110 * 2) collect usage information for keys/ids/dirids, create schema
1112 * 3) use usage information to create mapping tables, so that often
1113 * used ids get a lower number
1114 * 4) encode data into buffers using the mapping tables
1115 * 5) write everything to disk
1118 repowriter_write(Repowriter *writer, FILE *fp)
1120 Repo *repo = writer->repo;
1121 Pool *pool = repo->pool;
1124 NeedId *needid, *needidp;
1125 int nstrings, nrels;
1126 unsigned int sizeid;
1127 unsigned int solv_flags;
1130 int searchflags = 0;
1134 Id *keymap; /* maps repo key to my key, 0 -> not used */
1136 int *keymapstart; /* maps repo number to keymap offset */
1142 unsigned char *repodataused;
1143 int anyrepodataused = 0;
1145 int solvablestart, solvableend;
1147 int anysolvableused = 0;
1150 struct cbdata cbdata;
1154 int poolusage, dirpoolusage;
1157 Repodata *data, *dirpooldata;
1164 Id mainschema, *mainschemakeys;
1168 Id type_constantid = 0;
1171 memset(&cbdata, 0, sizeof(cbdata));
1174 cbdata.target = ⌖
1176 repodata_initdata(&target, repo, 1);
1178 /* go through all repodata and find the keys we need */
1179 /* also unify keys */
1181 /* start with all KEY_STORAGE_SOLVABLE ids */
1183 n = ID_NUM_INTERNAL;
1184 FOR_REPODATAS(repo, i, data)
1187 keymap = solv_calloc(nkeymap, sizeof(Id));
1188 keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1189 repodataused = solv_calloc(repo->nrepodata, 1);
1194 if (!(writer->flags & REPOWRITER_NO_STORAGE_SOLVABLE))
1196 /* add keys for STORAGE_SOLVABLE */
1197 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1201 if (i < SOLVABLE_PROVIDES)
1202 keyd.type = REPOKEY_TYPE_ID;
1203 else if (i < RPM_RPMDBID)
1204 #ifdef USE_REL_IDARRAY
1205 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1207 keyd.type = REPOKEY_TYPE_IDARRAY;
1210 keyd.type = REPOKEY_TYPE_NUM;
1212 keyd.storage = KEY_STORAGE_SOLVABLE;
1213 if (writer->keyfilter)
1215 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1216 if (keyd.storage == KEY_STORAGE_DROPPED)
1218 keyd.storage = KEY_STORAGE_SOLVABLE;
1222 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1226 if (repo->nsolvables)
1229 keyd.name = REPOSITORY_SOLVABLES;
1230 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1232 keyd.storage = KEY_STORAGE_INCORE;
1233 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1241 n = ID_NUM_INTERNAL;
1242 FOR_REPODATAS(repo, i, data)
1244 int idused, dirused;
1245 if (i < writer->repodatastart || i >= writer->repodataend)
1247 if (writer->keyfilter && (writer->flags & REPOWRITER_LEGACY) != 0)
1249 /* ask keyfilter if we want this repodata */
1251 /* check if we want this repodata */
1252 memset(&keyd, 0, sizeof(keyd));
1256 if (writer->keyfilter(repo, &keyd, writer->kfdata) == -1)
1260 keymap[n++] = 0; /* key 0 */
1261 idused = dirused = 0;
1262 for (j = 1; j < data->nkeys; j++, n++)
1264 key = data->keys + j;
1265 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1267 keymap[n] = keymap[key->name];
1270 if (key->type == REPOKEY_TYPE_DELETED && (writer->flags & REPOWRITER_KEEP_TYPE_DELETED) == 0)
1275 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1277 Repokey keyd = *key;
1278 keyd.size = repodata_globalize_id(data, key->size, 1);
1279 id = repodata_key2id(&target, &keyd, 0);
1282 id = repodata_key2id(&target, key, 0);
1285 /* a new key. ask keyfilter if we want it before creating it */
1286 Repokey keyd = *key;
1287 keyd.storage = KEY_STORAGE_INCORE;
1288 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1289 keyd.size = repodata_globalize_id(data, key->size, 1);
1290 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1292 if (writer->keyfilter)
1294 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1295 if (keyd.storage == KEY_STORAGE_DROPPED)
1301 if (data->state != REPODATA_STUB)
1302 id = repodata_key2id(&target, &keyd, 1);
1305 /* load repodata if not already loaded */
1306 if (data->state == REPODATA_STUB)
1308 int oldnkeys = data->nkeys;
1309 repodata_load(data);
1310 if (oldnkeys != data->nkeys)
1312 nkeymap += data->nkeys - oldnkeys; /* grow/shrink keymap */
1313 keymap = solv_realloc2(keymap, nkeymap, sizeof(Id));
1315 if (data->state == REPODATA_AVAILABLE)
1317 /* redo this repodata! */
1323 if (data->state != REPODATA_AVAILABLE && data->state != REPODATA_LOADING)
1330 repodataused[i] = 1;
1331 anyrepodataused = 1;
1332 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1333 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1335 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1337 idused = 1; /* dirs also use ids */
1343 if (data->localpool)
1346 poolusage = 3; /* need own pool */
1350 spool = &data->spool;
1357 else if (poolusage != 1)
1358 poolusage = 3; /* need own pool */
1364 dirpoolusage = 3; /* need own dirpool */
1368 dirpool = &data->dirpool;
1373 nkeymap = n; /* update */
1375 /* 0: no pool needed at all */
1376 /* 1: use global pool */
1377 /* 2: use repodata local pool */
1378 /* 3: need own pool */
1383 spool = &target.spool;
1384 target.localpool = 1; /* so we can use repodata_translate */
1385 /* hack: reuse global pool data so we don't have to map pool ids */
1388 stringpool_free(spool);
1389 stringpool_clone(spool, &pool->ss);
1390 cbdata.clonepool = 1;
1392 cbdata.ownspool = spool;
1394 else if (poolusage == 0 || poolusage == 1)
1400 if (dirpoolusage == 3)
1402 /* dirpoolusage == 3 means that at least two repodata
1403 * areas have dir keys. This means that two areas have
1404 * idused set to 1, which results in poolusage being
1405 * either 1 (global pool) or 3 (own pool) */
1406 dirpool = &target.dirpool;
1408 cbdata.owndirpool = dirpool;
1411 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1414 /********************************************************************/
1416 fprintf(stderr, "poolusage: %d\n", poolusage);
1417 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1418 fprintf(stderr, "clonepool: %d\n", clonepool);
1419 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1420 for (i = 1; i < target.nkeys; i++)
1421 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1424 /********************************************************************/
1426 searchflags = SEARCH_SUB|SEARCH_ARRAYSENTINEL;
1427 if ((writer->flags & REPOWRITER_KEEP_TYPE_DELETED) != 0)
1428 searchflags |= SEARCH_KEEP_TYPE_DELETED;
1430 /* set needed count of all strings and rels,
1431 * find which keys are used in the solvables
1432 * put all strings in own spool
1435 reloff = spool->nstrings;
1436 if (cbdata.ownspool)
1437 reloff = (reloff + NEEDID_BLOCK) & ~NEEDID_BLOCK;
1438 else if (poolusage == 2)
1440 /* we'll need to put the key data into the spool,
1441 * so leave some room. 3 * nkeys is an upper bound */
1442 reloff += 3 * target.nkeys;
1445 needid = calloc(reloff + pool->nrels, sizeof(*needid));
1446 needid[0].map = reloff; /* remember size in case we need to grow */
1448 cbdata.needid = needid;
1449 cbdata.schema = solv_calloc(target.nkeys + 2, sizeof(Id));
1451 /* create main schema */
1452 cbdata.sp = cbdata.schema + 1;
1454 /* collect meta data from all repodatas */
1455 /* XXX: merge arrays of equal keys? */
1456 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1457 FOR_REPODATAS(repo, j, data)
1459 if (!repodataused[j])
1461 cbdata.keymap = keymap + keymapstart[j];
1462 cbdata.lastdirid = 0; /* clear dir mapping cache */
1463 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1465 needid = cbdata.needid; /* maybe relocated */
1467 /* add solvables if needed (may revert later) */
1468 if (repo->nsolvables)
1470 *sp++ = keymap[REPOSITORY_SOLVABLES];
1471 target.keys[keymap[REPOSITORY_SOLVABLES]].size++;
1474 /* stash away main schema (including terminating zero) */
1475 mainschemakeys = solv_memdup2(cbdata.schema + 1, sp - cbdata.schema, sizeof(Id));
1477 /* collect data for all solvables */
1478 solvschemata = solv_calloc(repo->nsolvables, sizeof(Id)); /* allocate upper bound */
1479 solvablestart = writer->solvablestart < repo->start ? repo->start : writer->solvablestart;
1480 solvableend = writer->solvableend > repo->end ? repo->end : writer->solvableend;
1481 anysolvableused = 0;
1482 nsolvables = 0; /* solvables we are going to write, will be <= repo->nsolvables */
1483 cbdata.doingsolvables = 1;
1484 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
1486 if (s->repo != repo)
1489 cbdata.sp = cbdata.schema + 1;
1490 collect_needed_solvable(&cbdata, s, keymap);
1492 if (anyrepodataused)
1494 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1495 FOR_REPODATAS(repo, j, data)
1497 if (!repodataused[j] || i < data->start || i >= data->end)
1499 cbdata.keymap = keymap + keymapstart[j];
1500 cbdata.lastdirid = 0;
1501 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1503 needid = cbdata.needid; /* maybe relocated */
1506 solvschemata[nsolvables] = repodata_schema2id(cbdata.target, cbdata.schema + 1, 1);
1507 if (solvschemata[nsolvables])
1508 anysolvableused = 1;
1511 cbdata.doingsolvables = 0;
1513 if (repo->nsolvables && !anysolvableused)
1515 /* strip off REPOSITORY_SOLVABLES from the main schema */
1516 for (sp = mainschemakeys; *sp; sp++)
1518 sp[-1] = 0; /* strip last entry */
1520 mainschema = repodata_schema2id(cbdata.target, mainschemakeys, 1);
1521 mainschemakeys = solv_free(mainschemakeys);
1523 /********************************************************************/
1525 /* remove unused keys */
1526 keyused = solv_calloc(target.nkeys, sizeof(Id));
1527 for (i = 1; i < (int)target.schemadatalen; i++)
1528 keyused[target.schemadata[i]] = 1;
1530 for (n = i = 1; i < target.nkeys; i++)
1535 target.keys[n] = target.keys[i];
1540 /* update schema data to the new key ids */
1541 for (i = 1; i < (int)target.schemadatalen; i++)
1542 target.schemadata[i] = keyused[target.schemadata[i]];
1543 /* update keymap to the new key ids */
1544 for (i = 0; i < nkeymap; i++)
1545 keymap[i] = keyused[keymap[i]];
1546 keyused = solv_free(keyused);
1548 /* copy keys if requested */
1551 queue_empty(writer->keyq);
1552 for (i = 1; i < target.nkeys; i++)
1553 queue_push2(writer->keyq, target.keys[i].name, target.keys[i].type);
1556 /********************************************************************/
1558 /* check if we can do the special filelist memory optimization
1559 * we do the check before the keys are mapped.
1560 * The optimization is done if there is just one vertical key and
1561 * it is of type REPOKEY_TYPE_DIRSTRARRAY */
1562 if (anysolvableused && anyrepodataused)
1564 for (i = 1; i < target.nkeys; i++)
1566 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1568 if (target.keys[i].type != REPOKEY_TYPE_DIRSTRARRAY || cbdata.filelistmode != 0)
1570 cbdata.filelistmode = 0;
1573 cbdata.filelistmode = i;
1577 /********************************************************************/
1581 /* put all the keys in our string pool */
1582 /* put mapped ids right into target.keys */
1583 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1585 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1586 id = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1587 if (key->type == REPOKEY_TYPE_CONSTANTID)
1589 type_constantid = id;
1590 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1595 stringpool_freehash(spool); /* free some mem */
1596 if (cbdata.ownspool && spool->nstrings > needid[0].map)
1598 grow_needid(&cbdata, spool->nstrings - 1);
1599 needid = cbdata.needid; /* we relocated */
1603 type_constantid = REPOKEY_TYPE_CONSTANTID;
1605 /* increment needid of the keys */
1606 for (i = 1; i < target.nkeys; i++)
1608 if (target.keys[i].type == type_constantid)
1609 needid[target.keys[i].size].need++;
1610 needid[target.keys[i].name].need++;
1611 needid[target.keys[i].type].need++;
1614 /********************************************************************/
1616 /* increment need id of all relations
1617 * if we refer to another relation, make sure that the
1618 * need value is it is bigger than our value so that
1621 reloff = needid[0].map;
1622 for (i = pool->nrels - 1, needidp = needid + (reloff + i); i > 0; i--, needidp--)
1627 /* we have some relations with a non-zero need */
1630 for (rd = pool->rels + i; i > 0; i--, rd--)
1632 int need = needid[reloff + i].need;
1639 if (needid[reloff + id].need < need + 1)
1640 needid[reloff + id].need = need + 1;
1644 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1646 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1647 if (id >= cbdata.needid[0].map)
1649 grow_needid(&cbdata, id);
1650 needid = cbdata.needid; /* we relocated */
1651 reloff = needid[0].map; /* we have a new offset */
1661 if (needid[reloff + id].need < need + 1)
1662 needid[reloff + id].need = need + 1;
1666 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1668 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1669 if (id >= cbdata.needid[0].map)
1671 grow_needid(&cbdata, id);
1672 needid = cbdata.needid; /* we relocated */
1673 reloff = needid[0].map; /* we have a new offset */
1681 /********************************************************************/
1683 /* increment need id for used dir components */
1684 if (cbdata.owndirpool)
1686 /* if we have own dirpool, all entries in it are used.
1687 also, all comp ids are already mapped by putinowndirpool(),
1688 so we can simply increment needid.
1689 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1690 for (i = 1; i < dirpool->ndirs; i++)
1692 id = dirpool->dirs[i];
1701 /* else we re-use a dirpool of repodata "dirpooldata".
1702 dirused tells us which of the ids are used.
1703 we need to map comp ids if we generate a new pool.
1704 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1705 for (i = dirpool->ndirs - 1; i > 0; i--)
1707 if (!cbdata.dirused[i])
1709 parent = dirpool_parent(dirpool, i); /* always < i */
1710 cbdata.dirused[parent] = 2; /* 2: used as parent */
1711 id = dirpool->dirs[i];
1714 if (cbdata.ownspool && id > 1 && (!cbdata.clonepool || dirpooldata->localpool))
1716 id = putinownpool(&cbdata, dirpooldata, id);
1717 needid = cbdata.needid;
1721 if (!cbdata.dirused[0])
1723 cbdata.dirused = solv_free(cbdata.dirused);
1729 /********************************************************************/
1732 * create mapping table, new keys are sorted by needid[].need
1734 * needid[key].need : old key -> new key
1735 * needid[key].map : new key -> old key
1738 /* zero out id 0 and rel 0 just in case */
1739 reloff = needid[0].map;
1741 needid[reloff].need = 0;
1743 for (i = 1; i < reloff + pool->nrels; i++)
1746 /* make first entry '' */
1748 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1749 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1750 /* now needid is in new order, needid[newid].map -> oldid */
1752 /* calculate string space size, also zero out needid[].need */
1754 for (i = 1; i < reloff; i++)
1756 if (!needid[i].need)
1757 break; /* as we have sorted, every entry after this also has need == 0 */
1759 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1761 nstrings = i; /* our new string id end */
1763 /* make needid[oldid].need point to newid */
1764 for (i = 1; i < nstrings; i++)
1765 needid[needid[i].map].need = i;
1767 /* same as above for relations */
1768 for (i = 0; i < pool->nrels; i++)
1770 if (!needid[reloff + i].need)
1772 needid[reloff + i].need = 0;
1774 nrels = i; /* our new rel id end */
1776 for (i = 0; i < nrels; i++)
1777 needid[needid[reloff + i].map].need = nstrings + i;
1779 /* now we have: needid[oldid].need -> newid
1780 needid[newid].map -> oldid
1781 both for strings and relations */
1784 /********************************************************************/
1788 if (dirpool && dirpool->ndirs)
1790 /* create our new target directory structure by traversing through all
1791 * used dirs. This will concatenate blocks with the same parent
1792 * directory into single blocks.
1793 * Instead of components, traverse_dirs stores the old dirids,
1794 * we will change this in the second step below */
1795 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1796 if (cbdata.dirused && !cbdata.dirused[1])
1798 cbdata.dirused[1] = 1; /* always want / entry */
1799 cbdata.dirused[0] = 2; /* always want / entry */
1801 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1803 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1805 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1806 /* change dirmap so that it maps from "new dirid" to "new compid" */
1807 if (!cbdata.dirused)
1808 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1809 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1810 for (i = 1; i < ndirmap; i++)
1814 cbdata.dirused[dirmap[i]] = i;
1815 id = dirpool->dirs[dirmap[i]];
1816 if (dirpooldata && cbdata.ownspool && id > 1)
1817 id = putinownpool(&cbdata, dirpooldata, id);
1818 dirmap[i] = needid[id].need;
1820 /* now the new target directory structure is complete (dirmap), and we have
1821 * dirused[olddirid] -> newdirid */
1824 /********************************************************************/
1827 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1829 * this must match the code above that creates the schema data!
1832 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1834 xd = cbdata.extdata;
1835 cbdata.current_sub = 0;
1836 /* add main schema */
1838 data_addid(xd, mainschema);
1840 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1841 FOR_REPODATAS(repo, j, data)
1843 if (!repodataused[j])
1845 cbdata.keymap = keymap + keymapstart[j];
1846 cbdata.lastdirid = 0;
1847 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1849 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1850 cbdata.maxdata = xd->len - cbdata.lastlen;
1851 cbdata.lastlen = xd->len;
1853 if (anysolvableused)
1855 data_addid(xd, nsolvables); /* FLEXARRAY nentries */
1856 cbdata.doingsolvables = 1;
1858 for (i = solvablestart, s = pool->solvables + i, n = 0; i < solvableend; i++, s++)
1860 if (s->repo != repo)
1862 data_addid(xd, solvschemata[n]);
1863 collect_data_solvable(&cbdata, s, keymap);
1864 if (anyrepodataused)
1866 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1868 FOR_REPODATAS(repo, j, data)
1870 if (!repodataused[j] || i < data->start || i >= data->end)
1872 cbdata.keymap = keymap + keymapstart[j];
1873 cbdata.lastdirid = 0;
1874 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1877 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1878 cbdata.maxdata = xd->len - cbdata.lastlen;
1879 cbdata.lastlen = xd->len;
1882 cbdata.doingsolvables = 0;
1885 assert(cbdata.current_sub == cbdata.nsubschemata);
1886 cbdata.subschemata = solv_free(cbdata.subschemata);
1887 cbdata.nsubschemata = 0;
1889 /********************************************************************/
1895 /* write file header */
1896 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1897 write_u32(&target, SOLV_VERSION_8);
1901 write_u32(&target, nstrings);
1902 write_u32(&target, nrels);
1903 write_u32(&target, ndirmap);
1904 write_u32(&target, anysolvableused ? nsolvables : 0);
1905 write_u32(&target, target.nkeys);
1906 write_u32(&target, target.nschemata);
1908 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1909 solv_flags |= SOLV_FLAG_SIZE_BYTES;
1910 write_u32(&target, solv_flags);
1915 * calculate prefix encoding of the strings
1917 unsigned char *prefixcomp = solv_malloc(nstrings);
1918 unsigned int compsum = 0;
1922 for (i = 1; i < nstrings; i++)
1924 char *str = spool->stringspace + spool->strings[needid[i].map];
1926 for (same = 0; same < 255; same++)
1927 if (!old_str[same] || old_str[same] != str[same])
1929 prefixcomp[i] = same;
1937 write_u32(&target, sizeid);
1938 /* we save compsum bytes but need 1 extra byte for every string */
1939 write_u32(&target, sizeid + nstrings - 1 - compsum);
1940 for (i = 1; i < nstrings; i++)
1942 char *str = spool->stringspace + spool->strings[needid[i].map];
1943 write_u8(&target, prefixcomp[i]);
1944 write_str(&target, str + prefixcomp[i]);
1946 solv_free(prefixcomp);
1950 write_u32(&target, 0);
1951 write_u32(&target, 0);
1957 for (i = 0; i < nrels; i++)
1959 Reldep *ran = pool->rels + (needid[reloff + i].map - reloff);
1960 write_id(&target, needid[NEEDIDOFF(ran->name)].need);
1961 write_id(&target, needid[NEEDIDOFF(ran->evr)].need);
1962 write_u8(&target, ran->flags);
1966 * write dirs (skip both root and / entry)
1968 for (i = 2; i < ndirmap; i++)
1971 write_id(&target, dirmap[i]);
1973 write_id(&target, nstrings - dirmap[i]);
1980 for (i = 1; i < target.nkeys; i++)
1982 write_id(&target, needid[target.keys[i].name].need);
1983 write_id(&target, needid[target.keys[i].type].need);
1984 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1986 if (target.keys[i].type == type_constantid)
1987 write_id(&target, needid[target.keys[i].size].need);
1989 write_id(&target, target.keys[i].size);
1992 write_id(&target, cbdata.extdata[i].len);
1993 write_id(&target, target.keys[i].storage);
1999 write_id(&target, target.schemadatalen); /* XXX -1? */
2000 for (i = 1; i < target.nschemata; i++)
2001 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
2006 write_id(&target, cbdata.maxdata);
2007 write_id(&target, cbdata.extdata[0].len);
2008 if (cbdata.extdata[0].len)
2009 write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
2010 solv_free(cbdata.extdata[0].buf);
2013 * write vertical data if we have any
2015 for (i = 1; i < target.nkeys; i++)
2016 if (cbdata.extdata[i].len)
2018 if (i < target.nkeys)
2020 /* have vertical data, write it in pages */
2021 unsigned char vpage[REPOPAGE_BLOBSIZE];
2024 write_u32(&target, REPOPAGE_BLOBSIZE);
2025 if (!cbdata.filelistmode)
2027 for (i = 1; i < target.nkeys; i++)
2028 if (cbdata.extdata[i].len)
2029 lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
2033 /* ok, just one single extdata which is of type REPOKEY_TYPE_DIRSTRARRAY */
2034 xd = cbdata.extdata + i;
2036 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
2037 FOR_REPODATAS(repo, j, data)
2039 if (!repodataused[j])
2041 cbdata.keymap = keymap + keymapstart[j];
2042 cbdata.lastdirid = 0;
2043 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2045 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
2047 if (s->repo != repo)
2049 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
2050 FOR_REPODATAS(repo, j, data)
2052 if (!repodataused[j] || i < data->start || i >= data->end)
2054 cbdata.keymap = keymap + keymapstart[j];
2055 cbdata.lastdirid = 0;
2056 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2058 if (xd->len > 1024 * 1024)
2060 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2065 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2068 write_compressed_page(&target, vpage, lpage);
2071 for (i = 1; i < target.nkeys; i++)
2072 solv_free(cbdata.extdata[i].buf);
2073 solv_free(cbdata.extdata);
2076 repodata_freedata(&target);
2079 solv_free(solvschemata);
2080 solv_free(cbdata.schema);
2083 solv_free(keymapstart);
2084 solv_free(cbdata.dirused);
2085 solv_free(repodataused);
2086 solv_free(oldkeyskip);
2087 return target.error;
2091 repo_write(Repo *repo, FILE *fp)
2094 Repowriter *writer = repowriter_create(repo);
2095 res = repowriter_write(writer, fp);
2096 repowriter_free(writer);
2101 repodata_write(Repodata *data, FILE *fp)
2104 Repowriter *writer = repowriter_create(data->repo);
2105 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2106 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
2107 res = repowriter_write(writer, fp);
2108 repowriter_free(writer);
2112 /* deprecated functions, do not use in new code! */
2114 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2117 Repowriter *writer = repowriter_create(repo);
2118 repowriter_set_flags(writer, REPOWRITER_LEGACY);
2119 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2120 repowriter_set_keyqueue(writer, keyq);
2121 res = repowriter_write(writer, fp);
2122 repowriter_free(writer);
2127 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2130 Repowriter *writer = repowriter_create(data->repo);
2131 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2132 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE | REPOWRITER_LEGACY);
2133 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2134 repowriter_set_keyqueue(writer, keyq);
2135 res = repowriter_write(writer, fp);
2136 repowriter_free(writer);