2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
29 #include "repo_write.h"
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
35 typedef struct needid {
41 #define RELOFF(id) (needid[0].map + GETRELID(id))
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
48 * return size of array (including trailing zero)
53 incneedid(Pool *pool, Id id, NeedId *needid)
57 Reldep *rd = GETRELDEP(pool, id);
58 needid[RELOFF(id)].need++;
59 if (ISRELDEP(rd->evr))
60 incneedid(pool, rd->evr, needid);
62 needid[rd->evr].need++;
69 incneedidarray(Pool *pool, Id *idarray, NeedId *needid)
76 while ((id = *idarray++) != 0)
81 Reldep *rd = GETRELDEP(pool, id);
82 needid[RELOFF(id)].need++;
83 if (ISRELDEP(rd->evr))
84 incneedid(pool, rd->evr, needid);
86 needid[rd->evr].need++;
100 needid_cmp_need(const void *ap, const void *bp, void *dp)
102 const NeedId *a = ap;
103 const NeedId *b = bp;
105 r = b->need - a->need;
108 return a->map - b->map;
112 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
114 const NeedId *a = ap;
115 const NeedId *b = bp;
116 Stringpool *spool = dp;
121 r = b->need - a->need;
124 as = spool->stringspace + spool->strings[a->map];
125 bs = spool->stringspace + spool->strings[b->map];
126 return strcmp(as, bs);
130 /*------------------------------------------------------------------*/
131 /* output helper routines, used for writing the header */
132 /* (the data itself is accumulated in memory and written with
140 write_u32(Repodata *data, unsigned int x)
145 if (putc(x >> 24, fp) == EOF ||
146 putc(x >> 16, fp) == EOF ||
147 putc(x >> 8, fp) == EOF ||
150 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
160 write_u8(Repodata *data, unsigned int x)
164 if (putc(x, data->fp) == EOF)
166 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
175 write_blob(Repodata *data, void *blob, int len)
179 if (len && fwrite(blob, len, 1, data->fp) != 1)
181 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
190 write_id(Repodata *data, Id x)
198 putc((x >> 28) | 128, fp);
200 putc((x >> 21) | 128, fp);
201 putc((x >> 14) | 128, fp);
204 putc((x >> 7) | 128, fp);
205 if (putc(x & 127, fp) == EOF)
207 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
212 write_id_eof(Repodata *data, Id x, int eof)
215 x = (x & 63) | ((x & ~63) << 1);
216 write_id(data, x | (eof ? 0 : 64));
222 write_str(Repodata *data, const char *str)
226 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
228 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
237 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
251 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
253 id = (id & 63) | ((id & ~63) << 1);
259 write_id(data, id | 64);
264 cmp_ids(const void *pa, const void *pb, void *dp)
273 write_idarray_sort(Repodata *data, Pool *pool, NeedId *needid, Id *ids, Id marker)
285 for (len = 0; len < 64 && ids[len]; len++)
289 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
294 for (i = len + 1; ids[i]; i++)
296 sids = solv_malloc2(i, sizeof(Id));
297 memcpy(sids, lids, 64 * sizeof(Id));
298 for (; ids[len]; len++)
302 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
309 /* That bloody solvable:prereqmarker needs to stay in position :-( */
311 marker = needid[marker].need;
312 for (i = 0; i < len; i++)
313 if (sids[i] == marker)
316 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
318 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
322 /* The differencing above produces many runs of ones and twos. I tried
323 fairly elaborate schemes to RLE those, but they give only very mediocre
324 improvements in compression, as coding the escapes costs quite some
325 space. Even if they are coded only as bits in IDs. The best improvement
326 was about 2.7% for the whole .solv file. It's probably better to
327 invest some complexity into sharing idarrays, than RLEing. */
328 for (i = 0; i < len - 1; i++)
331 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
332 hence all real differences are offsetted by 1. Otherwise we would
333 have to handle negative differences, which would cost code space for
334 the encoding of the sign. We loose the exact mapping of prereq here,
335 but we know the result, so we can recover from that in the reader. */
343 /* XXX If difference is zero we have multiple equal elements,
344 we might want to skip writing them out. */
346 id = (id & 63) | ((id & ~63) << 1);
347 write_id(data, id | 64);
355 id = (id & 63) | ((id & ~63) << 1);
372 Stringpool *ownspool;
381 Id *schema; /* schema construction space */
382 Id *sp; /* pointer in above */
383 Id *oldschema, *oldsp;
390 struct extdata *extdata;
399 int doingsolvables; /* working on solvables data */
402 #define NEEDED_BLOCK 1023
403 #define SCHEMATA_BLOCK 31
404 #define SCHEMATADATA_BLOCK 255
405 #define EXTDATA_BLOCK 4095
408 data_addid(struct extdata *xd, Id sx)
410 unsigned int x = (unsigned int)sx;
413 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
414 dp = xd->buf + xd->len;
419 *dp++ = (x >> 28) | 128;
421 *dp++ = (x >> 21) | 128;
422 *dp++ = (x >> 14) | 128;
425 *dp++ = (x >> 7) | 128;
427 xd->len = dp - xd->buf;
431 data_addideof(struct extdata *xd, Id sx, int eof)
433 unsigned int x = (unsigned int)sx;
436 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
437 dp = xd->buf + xd->len;
442 *dp++ = (x >> 27) | 128;
444 *dp++ = (x >> 20) | 128;
445 *dp++ = (x >> 13) | 128;
448 *dp++ = (x >> 6) | 128;
449 *dp++ = eof ? (x & 63) : (x & 63) | 64;
450 xd->len = dp - xd->buf;
454 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
460 data_addid(xd, (Id)(hx >> 3));
461 xd->buf[xd->len - 1] |= 128;
464 data_addid(xd, (Id)(x | 0x80000000));
465 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
468 data_addid(xd, (Id)x);
472 data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
485 for (len = 0; len < 64 && ids[len]; len++)
489 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
494 for (i = len + 1; ids[i]; i++)
496 sids = solv_malloc2(i, sizeof(Id));
497 memcpy(sids, lids, 64 * sizeof(Id));
498 for (; ids[len]; len++)
502 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
509 /* That bloody solvable:prereqmarker needs to stay in position :-( */
511 marker = needid[marker].need;
512 for (i = 0; i < len; i++)
513 if (sids[i] == marker)
516 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
518 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
522 /* The differencing above produces many runs of ones and twos. I tried
523 fairly elaborate schemes to RLE those, but they give only very mediocre
524 improvements in compression, as coding the escapes costs quite some
525 space. Even if they are coded only as bits in IDs. The best improvement
526 was about 2.7% for the whole .solv file. It's probably better to
527 invest some complexity into sharing idarrays, than RLEing. */
528 for (i = 0; i < len - 1; i++)
531 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
532 hence all real differences are offsetted by 1. Otherwise we would
533 have to handle negative differences, which would cost code space for
534 the encoding of the sign. We loose the exact mapping of prereq here,
535 but we know the result, so we can recover from that in the reader. */
543 /* XXX If difference is zero we have multiple equal elements,
544 we might want to skip writing them out. */
545 data_addideof(xd, id, 0);
552 data_addideof(xd, id, 1);
558 data_addblob(struct extdata *xd, unsigned char *blob, int len)
560 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
561 memcpy(xd->buf + xd->len, blob, len);
566 data_addu32(struct extdata *xd, unsigned int num)
573 data_addblob(xd, d, 4);
577 putinownpool(struct cbdata *cbdata, Stringpool *ss, Id id)
579 const char *str = stringpool_id2str(ss, id);
580 id = stringpool_str2id(cbdata->ownspool, str, 1);
581 if (id >= cbdata->needid[0].map)
583 int oldoff = cbdata->needid[0].map;
584 int newoff = (id + 1 + NEEDED_BLOCK) & ~NEEDED_BLOCK;
585 int nrels = cbdata->repo->pool->nrels;
586 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
588 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
589 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
590 cbdata->needid[0].map = newoff;
596 putinowndirpool(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
600 parent = dirpool_parent(dp, dir);
602 parent = putinowndirpool(cbdata, data, dp, parent);
603 compid = dp->dirs[dir];
604 if (cbdata->ownspool && compid > 1)
605 compid = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, compid);
606 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
610 * collect usage information about the dirs
611 * 1: dir used, no child of dir used
612 * 2: dir used as parent of another used dir
615 setdirused(struct cbdata *cbdata, Dirpool *dp, Id dir)
617 if (cbdata->dirused[dir])
619 cbdata->dirused[dir] = 1;
620 while ((dir = dirpool_parent(dp, dir)) != 0)
622 if (cbdata->dirused[dir] == 2)
624 if (cbdata->dirused[dir])
626 cbdata->dirused[dir] = 2;
629 cbdata->dirused[dir] = 2;
631 cbdata->dirused[0] = 2;
636 * collect key/id/dirid usage information, create needed schemas
639 repo_write_collect_needed(struct cbdata *cbdata, Repo *repo, Repodata *data, Repokey *key, KeyValue *kv)
644 if (key->name == REPOSITORY_SOLVABLES)
645 return SEARCH_NEXT_KEY; /* we do not want this one */
647 /* hack: ignore some keys, see BUGS */
648 if (data->repodataid != data->repo->nrepodata - 1)
649 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
650 return SEARCH_NEXT_KEY;
652 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
654 return SEARCH_NEXT_KEY; /* we do not want this one */
656 /* record key in schema */
657 if ((key->type != REPOKEY_TYPE_FIXARRAY || kv->eof == 0)
658 && (cbdata->sp == cbdata->schema || cbdata->sp[-1] != rm))
663 case REPOKEY_TYPE_ID:
664 case REPOKEY_TYPE_IDARRAY:
666 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
667 id = putinownpool(cbdata, data->localpool ? &data->spool : &repo->pool->ss, id);
668 incneedid(repo->pool, id, cbdata->needid);
670 case REPOKEY_TYPE_DIR:
671 case REPOKEY_TYPE_DIRNUMNUMARRAY:
672 case REPOKEY_TYPE_DIRSTRARRAY:
674 if (cbdata->owndirpool)
675 putinowndirpool(cbdata, data, &data->dirpool, id);
677 setdirused(cbdata, &data->dirpool, id);
679 case REPOKEY_TYPE_FIXARRAY:
682 if (cbdata->oldschema)
684 cbdata->target->error = pool_error(cbdata->repo->pool, -1, "nested fixarray structs not yet implemented");
685 return SEARCH_NEXT_KEY;
687 cbdata->oldschema = cbdata->schema;
688 cbdata->oldsp = cbdata->sp;
689 cbdata->schema = solv_calloc(cbdata->target->nkeys, sizeof(Id));
690 cbdata->sp = cbdata->schema;
692 else if (kv->eof == 1)
694 cbdata->current_sub++;
696 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
697 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, cbdata->schema, 1);
699 fprintf(stderr, "Have schema %d\n", cbdata->subschemata[cbdata->nsubschemata-1]);
701 cbdata->sp = cbdata->schema;
705 solv_free(cbdata->schema);
706 cbdata->schema = cbdata->oldschema;
707 cbdata->sp = cbdata->oldsp;
708 cbdata->oldsp = cbdata->oldschema = 0;
711 case REPOKEY_TYPE_FLEXARRAY:
715 *cbdata->sp++ = 0; /* mark start */
719 /* just finished a schema, rewind */
720 Id *sp = cbdata->sp - 1;
724 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
725 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
726 cbdata->sp = kv->eof == 2 ? sp - 1: sp;
736 repo_write_cb_needed(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
738 struct cbdata *cbdata = vcbdata;
739 Repo *repo = data->repo;
743 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? s - repo->pool->solvables : 0, s ? pool_id2str(repo->pool, s->name) : "", key->name, pool_id2str(repo->pool, key->name), key->type);
745 return repo_write_collect_needed(cbdata, repo, data, key, kv);
751 * encode all of the data into the correct buffers
755 repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue *kv)
764 if (key->name == REPOSITORY_SOLVABLES)
765 return SEARCH_NEXT_KEY;
767 /* hack: ignore some keys, see BUGS */
768 if (data->repodataid != data->repo->nrepodata - 1)
769 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
770 return SEARCH_NEXT_KEY;
772 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
774 return SEARCH_NEXT_KEY; /* we do not want this one */
776 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET)
778 xd = cbdata->extdata + rm; /* vertical buffer */
779 if (cbdata->vstart == -1)
780 cbdata->vstart = xd->len;
783 xd = cbdata->extdata + 0; /* incore buffer */
786 case REPOKEY_TYPE_VOID:
787 case REPOKEY_TYPE_CONSTANT:
788 case REPOKEY_TYPE_CONSTANTID:
790 case REPOKEY_TYPE_ID:
792 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
793 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
794 needid = cbdata->needid;
795 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
798 case REPOKEY_TYPE_IDARRAY:
800 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
801 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
802 needid = cbdata->needid;
803 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
804 data_addideof(xd, id, kv->eof);
806 case REPOKEY_TYPE_STR:
807 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
809 case REPOKEY_TYPE_MD5:
810 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
812 case REPOKEY_TYPE_SHA1:
813 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
815 case REPOKEY_TYPE_SHA256:
816 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
818 case REPOKEY_TYPE_U32:
824 data_addblob(xd, v, 4);
826 case REPOKEY_TYPE_NUM:
827 data_addid64(xd, kv->num, kv->num2);
829 case REPOKEY_TYPE_DIR:
831 if (cbdata->owndirpool)
832 id = putinowndirpool(cbdata, data, &data->dirpool, id);
833 id = cbdata->dirused[id];
836 case REPOKEY_TYPE_BINARY:
837 data_addid(xd, kv->num);
839 data_addblob(xd, (unsigned char *)kv->str, kv->num);
841 case REPOKEY_TYPE_DIRNUMNUMARRAY:
843 if (cbdata->owndirpool)
844 id = putinowndirpool(cbdata, data, &data->dirpool, id);
845 id = cbdata->dirused[id];
847 data_addid(xd, kv->num);
848 data_addideof(xd, kv->num2, kv->eof);
850 case REPOKEY_TYPE_DIRSTRARRAY:
852 if (cbdata->owndirpool)
853 id = putinowndirpool(cbdata, data, &data->dirpool, id);
854 id = cbdata->dirused[id];
855 data_addideof(xd, id, kv->eof);
856 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
858 case REPOKEY_TYPE_FIXARRAY:
863 data_addid(xd, kv->num);
864 data_addid(xd, cbdata->subschemata[cbdata->current_sub]);
866 fprintf(stderr, "writing %d %d\n", kv->num, cbdata->subschemata[cbdata->current_sub]);
870 else if (kv->eof == 1)
872 cbdata->current_sub++;
875 case REPOKEY_TYPE_FLEXARRAY:
877 data_addid(xd, kv->num);
879 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
880 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
882 if (xd->len - cbdata->lastlen > cbdata->maxdata)
883 cbdata->maxdata = xd->len - cbdata->lastlen;
884 cbdata->lastlen = xd->len;
888 cbdata->target->error = pool_error(cbdata->repo->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
891 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
893 /* we can re-use old data in the blob here! */
894 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
895 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
902 repo_write_cb_adddata(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
904 struct cbdata *cbdata = vcbdata;
905 return repo_write_adddata(cbdata, data, key, kv);
908 /* traverse through directory with first child "dir" */
910 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
916 /* special case for '/', which has to come first */
919 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
921 if (used && !used[sib])
923 if (sib == 1 && parent == 1)
924 continue; /* already did that one above */
928 /* now go through all the siblings we just added and
929 * do recursive calls on them */
931 for (; parent < lastn; parent++)
933 sib = dirmap[parent];
934 if (used && used[sib] != 2) /* 2: used as parent */
936 child = dirpool_child(dp, sib);
939 dirmap[n++] = -parent; /* start new block */
940 n = traverse_dirs(dp, dirmap, n, child, used);
947 write_compressed_page(Repodata *data, unsigned char *page, int len)
950 unsigned char cpage[REPOPAGE_BLOBSIZE];
952 clen = repopagestore_compress_page(page, len, cpage, len - 1);
955 write_u32(data, len * 2);
956 write_blob(data, page, len);
960 write_u32(data, clen * 2 + 1);
961 write_blob(data, cpage, clen);
965 static Id verticals[] = {
967 SOLVABLE_DESCRIPTION,
982 static char *languagetags[] = {
984 "solvable:description:",
985 "solvable:messageins:",
986 "solvable:messagedel:",
992 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
997 for (i = 0; verticals[i]; i++)
998 if (key->name == verticals[i])
999 return KEY_STORAGE_VERTICAL_OFFSET;
1000 keyname = pool_id2str(repo->pool, key->name);
1001 for (i = 0; languagetags[i] != 0; i++)
1002 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
1003 return KEY_STORAGE_VERTICAL_OFFSET;
1004 return KEY_STORAGE_INCORE;
1012 * the code works the following way:
1014 * 1) find which keys should be written
1015 * 2) collect usage information for keys/ids/dirids, create schema
1017 * 3) use usage information to create mapping tables, so that often
1018 * used ids get a lower number
1019 * 4) encode data into buffers using the mapping tables
1020 * 5) write everything to disk
1023 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
1025 Pool *pool = repo->pool;
1029 int nstrings, nrels;
1030 unsigned int sizeid;
1031 unsigned int solv_flags;
1040 unsigned char *repodataused;
1041 int anyrepodataused = 0;
1042 int anysolvableused = 0;
1044 struct cbdata cbdata;
1047 int poolusage, dirpoolusage, idused, dirused;
1050 Repodata *data, *dirpooldata;
1061 Id type_constantid = REPOKEY_TYPE_CONSTANTID;
1064 memset(&cbdata, 0, sizeof(cbdata));
1066 cbdata.target = ⌖
1068 repodata_initdata(&target, repo, 1);
1070 /* go through all repodata and find the keys we need */
1071 /* also unify keys */
1072 /* keymapstart - maps repo number to keymap offset */
1073 /* keymap - maps repo key to my key, 0 -> not used */
1075 /* start with all KEY_STORAGE_SOLVABLE ids */
1077 n = ID_NUM_INTERNAL;
1078 FOR_REPODATAS(repo, i, data)
1080 cbdata.keymap = solv_calloc(n, sizeof(Id));
1081 cbdata.keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1082 repodataused = solv_calloc(repo->nrepodata, 1);
1087 /* add keys for STORAGE_SOLVABLE */
1088 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1092 if (i < SOLVABLE_PROVIDES)
1093 keyd.type = REPOKEY_TYPE_ID;
1094 else if (i < RPM_RPMDBID)
1095 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1097 keyd.type = REPOKEY_TYPE_NUM;
1099 keyd.storage = KEY_STORAGE_SOLVABLE;
1102 keyd.storage = keyfilter(repo, &keyd, kfdata);
1103 if (keyd.storage == KEY_STORAGE_DROPPED)
1105 keyd.storage = KEY_STORAGE_SOLVABLE;
1109 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1112 if (repo->nsolvables)
1115 keyd.name = REPOSITORY_SOLVABLES;
1116 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1118 keyd.storage = KEY_STORAGE_INCORE;
1119 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1127 n = ID_NUM_INTERNAL;
1128 FOR_REPODATAS(repo, i, data)
1130 cbdata.keymapstart[i] = n;
1131 cbdata.keymap[n++] = 0; /* key 0 */
1137 /* check if we want this repodata */
1138 memset(&keyd, 0, sizeof(keyd));
1142 if (keyfilter(repo, &keyd, kfdata) == -1)
1145 for (j = 1; j < data->nkeys; j++, n++)
1147 key = data->keys + j;
1148 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1150 cbdata.keymap[n] = cbdata.keymap[key->name];
1153 if (key->type == REPOKEY_TYPE_DELETED)
1155 cbdata.keymap[n] = 0;
1158 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1160 Repokey keyd = *key;
1161 keyd.size = repodata_globalize_id(data, key->size, 1);
1162 id = repodata_key2id(&target, &keyd, 0);
1165 id = repodata_key2id(&target, key, 0);
1168 Repokey keyd = *key;
1169 keyd.storage = KEY_STORAGE_INCORE;
1170 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1171 keyd.size = repodata_globalize_id(data, key->size, 1);
1172 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1176 keyd.storage = keyfilter(repo, &keyd, kfdata);
1177 if (keyd.storage == KEY_STORAGE_DROPPED)
1179 cbdata.keymap[n] = 0;
1183 id = repodata_key2id(&target, &keyd, 1);
1185 cbdata.keymap[n] = id;
1186 /* load repodata if not already loaded */
1187 if (data->state == REPODATA_STUB)
1189 if (data->loadcallback)
1190 data->loadcallback(data);
1192 data->state = REPODATA_ERROR;
1193 if (data->state != REPODATA_ERROR)
1195 /* redo this repodata! */
1197 n = cbdata.keymapstart[i];
1201 if (data->state == REPODATA_ERROR)
1204 cbdata.keymap[n] = 0;
1208 repodataused[i] = 1;
1209 anyrepodataused = 1;
1210 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1211 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1213 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1215 idused = 1; /* dirs also use ids */
1221 if (data->localpool)
1224 poolusage = 3; /* need own pool */
1228 spool = &data->spool;
1235 else if (poolusage != 1)
1236 poolusage = 3; /* need own pool */
1242 dirpoolusage = 3; /* need own dirpool */
1246 dirpool = &data->dirpool;
1253 /* 0: no pool needed at all */
1254 /* 1: use global pool */
1255 /* 2: use repodata local pool */
1256 /* 3: need own pool */
1259 spool = &target.spool;
1260 /* hack: reuse global pool data so we don't have to map pool ids */
1263 stringpool_free(spool);
1264 stringpool_clone(spool, &pool->ss);
1266 cbdata.ownspool = spool;
1268 else if (poolusage == 0 || poolusage == 1)
1274 if (dirpoolusage == 3)
1276 dirpool = &target.dirpool;
1278 cbdata.owndirpool = dirpool;
1281 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1284 /********************************************************************/
1286 fprintf(stderr, "poolusage: %d\n", poolusage);
1287 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1288 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1289 for (i = 1; i < target.nkeys; i++)
1290 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1293 /* copy keys if requested */
1297 for (i = 1; i < target.nkeys; i++)
1298 queue_push2(keyq, target.keys[i].name, target.keys[i].type);
1303 /* put all the keys we need in our string pool */
1304 /* put mapped ids right into target.keys */
1305 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1307 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1308 if (key->type == REPOKEY_TYPE_CONSTANTID)
1310 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1311 type_constantid = key->type;
1312 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1315 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1318 stringpool_freehash(spool); /* free some mem */
1322 /********************************************************************/
1324 /* set needed count of all strings and rels,
1325 * find which keys are used in the solvables
1326 * put all strings in own spool
1329 reloff = spool->nstrings;
1331 reloff = (reloff + NEEDED_BLOCK) & ~NEEDED_BLOCK;
1333 needid = calloc(reloff + pool->nrels, sizeof(*needid));
1334 needid[0].map = reloff;
1336 cbdata.needid = needid;
1337 cbdata.schema = solv_calloc(target.nkeys, sizeof(Id));
1338 cbdata.sp = cbdata.schema;
1339 cbdata.solvschemata = solv_calloc(repo->nsolvables, sizeof(Id));
1341 /* create main schema */
1342 cbdata.sp = cbdata.schema;
1343 /* collect all other data from all repodatas */
1344 /* XXX: merge arrays of equal keys? */
1345 FOR_REPODATAS(repo, j, data)
1347 if (!repodataused[j])
1349 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1352 /* add solvables if needed (may revert later) */
1353 if (repo->nsolvables)
1355 *sp++ = cbdata.keymap[REPOSITORY_SOLVABLES];
1356 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size++;
1359 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1361 idarraydata = repo->idarraydata;
1363 anysolvableused = 0;
1364 cbdata.doingsolvables = 1;
1365 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1367 if (s->repo != repo)
1370 /* set schema info, keep in sync with further down */
1372 if (cbdata.keymap[SOLVABLE_NAME])
1374 *sp++ = cbdata.keymap[SOLVABLE_NAME];
1375 needid[s->name].need++;
1377 if (cbdata.keymap[SOLVABLE_ARCH])
1379 *sp++ = cbdata.keymap[SOLVABLE_ARCH];
1380 needid[s->arch].need++;
1382 if (cbdata.keymap[SOLVABLE_EVR])
1384 *sp++ = cbdata.keymap[SOLVABLE_EVR];
1385 needid[s->evr].need++;
1387 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1389 *sp++ = cbdata.keymap[SOLVABLE_VENDOR];
1390 needid[s->vendor].need++;
1392 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1394 *sp++ = cbdata.keymap[SOLVABLE_PROVIDES];
1395 target.keys[cbdata.keymap[SOLVABLE_PROVIDES]].size += incneedidarray(pool, idarraydata + s->provides, needid);
1397 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1399 *sp++ = cbdata.keymap[SOLVABLE_OBSOLETES];
1400 target.keys[cbdata.keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(pool, idarraydata + s->obsoletes, needid);
1402 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1404 *sp++ = cbdata.keymap[SOLVABLE_CONFLICTS];
1405 target.keys[cbdata.keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(pool, idarraydata + s->conflicts, needid);
1407 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1409 *sp++ = cbdata.keymap[SOLVABLE_REQUIRES];
1410 target.keys[cbdata.keymap[SOLVABLE_REQUIRES]].size += incneedidarray(pool, idarraydata + s->requires, needid);
1412 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1414 *sp++ = cbdata.keymap[SOLVABLE_RECOMMENDS];
1415 target.keys[cbdata.keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(pool, idarraydata + s->recommends, needid);
1417 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1419 *sp++ = cbdata.keymap[SOLVABLE_SUGGESTS];
1420 target.keys[cbdata.keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(pool, idarraydata + s->suggests, needid);
1422 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1424 *sp++ = cbdata.keymap[SOLVABLE_SUPPLEMENTS];
1425 target.keys[cbdata.keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(pool, idarraydata + s->supplements, needid);
1427 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1429 *sp++ = cbdata.keymap[SOLVABLE_ENHANCES];
1430 target.keys[cbdata.keymap[SOLVABLE_ENHANCES]].size += incneedidarray(pool, idarraydata + s->enhances, needid);
1432 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1434 *sp++ = cbdata.keymap[RPM_RPMDBID];
1435 target.keys[cbdata.keymap[RPM_RPMDBID]].size++;
1439 if (anyrepodataused)
1441 FOR_REPODATAS(repo, j, data)
1443 if (!repodataused[j])
1445 if (i < data->start || i >= data->end)
1447 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1448 needid = cbdata.needid;
1452 cbdata.solvschemata[n] = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1453 if (cbdata.solvschemata[n])
1454 anysolvableused = 1;
1457 cbdata.doingsolvables = 0;
1458 assert(n == repo->nsolvables);
1460 if (repo->nsolvables && !anysolvableused)
1462 /* strip off solvable from the main schema */
1463 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size = 0;
1465 for (i = 0; target.schemadata[target.schemata[mainschema] + i]; i++)
1467 *sp = target.schemadata[target.schemata[mainschema] + i];
1468 if (*sp != cbdata.keymap[REPOSITORY_SOLVABLES])
1471 assert(target.schemadatalen == target.schemata[mainschema] + i + 1);
1473 target.schemadatalen = target.schemata[mainschema];
1475 repodata_free_schemahash(&target);
1476 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1479 /********************************************************************/
1481 /* remove unused keys */
1482 keyused = solv_calloc(target.nkeys, sizeof(Id));
1483 for (i = 1; i < target.schemadatalen; i++)
1484 keyused[target.schemadata[i]] = 1;
1486 for (n = i = 1; i < target.nkeys; i++)
1493 target.keys[n] = target.keys[i];
1496 keyq->elements[2 * n - 2] = keyq->elements[2 * i - 2];
1497 keyq->elements[2 * n - 1] = keyq->elements[2 * i - 1];
1504 queue_truncate(keyq, 2 * n - 2);
1506 /* update schema data to the new key ids */
1507 for (i = 1; i < target.schemadatalen; i++)
1508 target.schemadata[i] = keyused[target.schemadata[i]];
1509 /* update keymap to the new key ids */
1510 for (i = 0; i < cbdata.nkeymap; i++)
1511 cbdata.keymap[i] = keyused[cbdata.keymap[i]];
1512 keyused = solv_free(keyused);
1514 /* increment needid of the used keys, they are already mapped to
1515 * the correct string pool */
1516 for (i = 1; i < target.nkeys; i++)
1518 if (target.keys[i].type == type_constantid)
1519 needid[target.keys[i].size].need++;
1520 needid[target.keys[i].name].need++;
1521 needid[target.keys[i].type].need++;
1524 /********************************************************************/
1526 if (dirpool && cbdata.dirused && !cbdata.dirused[0])
1528 /* no dirs used at all */
1529 cbdata.dirused = solv_free(cbdata.dirused);
1533 /* increment need id for used dir components */
1536 /* if we have own dirpool, all entries in it are used.
1537 also, all comp ids are already mapped by putinowndirpool(),
1538 so we can simply increment needid.
1539 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1540 /* else we re-use a dirpool of repodata "dirpooldata".
1541 dirused tells us which of the ids are used.
1542 we need to map comp ids if we generate a new pool.
1543 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1544 for (i = 1; i < dirpool->ndirs; i++)
1547 fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
1549 if (cbdata.dirused && !cbdata.dirused[i])
1551 id = dirpool->dirs[i];
1554 if (dirpooldata && cbdata.ownspool && id > 1)
1556 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1557 needid = cbdata.needid;
1564 /********************************************************************/
1567 * create mapping table, new keys are sorted by needid[].need
1569 * needid[key].need : old key -> new key
1570 * needid[key].map : new key -> old key
1573 /* zero out id 0 and rel 0 just in case */
1574 reloff = needid[0].map;
1576 needid[reloff].need = 0;
1578 for (i = 1; i < reloff + pool->nrels; i++)
1582 solv_sort(needid + 1, spool->nstrings - 1, sizeof(*needid), needid_cmp_need_s, spool);
1584 /* make first entry '' */
1586 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1588 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1589 /* now needid is in new order, needid[newid].map -> oldid */
1591 /* calculate string space size, also zero out needid[].need */
1593 for (i = 1; i < reloff; i++)
1595 if (!needid[i].need)
1596 break; /* as we have sorted, every entry after this also has need == 0 */
1598 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1600 nstrings = i; /* our new string id end */
1602 /* make needid[oldid].need point to newid */
1603 for (i = 1; i < nstrings; i++)
1604 needid[needid[i].map].need = i;
1606 /* same as above for relations */
1607 for (i = 0; i < pool->nrels; i++)
1609 if (!needid[reloff + i].need)
1611 needid[reloff + i].need = 0;
1613 nrels = i; /* our new rel id end */
1615 for (i = 0; i < nrels; i++)
1616 needid[needid[reloff + i].map].need = nstrings + i;
1618 /* now we have: needid[oldid].need -> newid
1619 needid[newid].map -> oldid
1620 both for strings and relations */
1623 /********************************************************************/
1629 /* create our new target directory structure by traversing through all
1630 * used dirs. This will concatenate blocks with the same parent
1631 * directory into single blocks.
1632 * Instead of components, traverse_dirs stores the old dirids,
1633 * we will change this in the second step below */
1634 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1635 if (cbdata.dirused && !cbdata.dirused[1])
1636 cbdata.dirused[1] = 1; /* always want / entry */
1637 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1639 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1641 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1642 /* change dirmap so that it maps from "new dirid" to "new compid" */
1643 if (!cbdata.dirused)
1644 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1645 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1646 for (i = 1; i < ndirmap; i++)
1650 cbdata.dirused[dirmap[i]] = i;
1651 id = dirpool->dirs[dirmap[i]];
1652 if (dirpooldata && cbdata.ownspool && id > 1)
1653 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1654 dirmap[i] = needid[id].need;
1656 /* now the new target directory structure is complete (dirmap), and we have
1657 * dirused[olddirid] -> newdirid */
1660 /********************************************************************/
1663 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1666 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1668 xd = cbdata.extdata;
1669 cbdata.current_sub = 0;
1670 /* add main schema */
1672 data_addid(xd, mainschema);
1675 FOR_REPODATAS(repo, j, data)
1677 if (!repodataused[j])
1679 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1683 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1684 cbdata.maxdata = xd->len - cbdata.lastlen;
1685 cbdata.lastlen = xd->len;
1687 if (anysolvableused)
1689 data_addid(xd, repo->nsolvables); /* FLEXARRAY nentries */
1690 cbdata.doingsolvables = 1;
1691 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1693 if (s->repo != repo)
1695 data_addid(xd, cbdata.solvschemata[n]);
1696 if (cbdata.keymap[SOLVABLE_NAME])
1697 data_addid(xd, needid[s->name].need);
1698 if (cbdata.keymap[SOLVABLE_ARCH])
1699 data_addid(xd, needid[s->arch].need);
1700 if (cbdata.keymap[SOLVABLE_EVR])
1701 data_addid(xd, needid[s->evr].need);
1702 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1703 data_addid(xd, needid[s->vendor].need);
1704 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1705 data_addidarray_sort(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
1706 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1707 data_addidarray_sort(xd, pool, needid, idarraydata + s->obsoletes, 0);
1708 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1709 data_addidarray_sort(xd, pool, needid, idarraydata + s->conflicts, 0);
1710 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1711 data_addidarray_sort(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
1712 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1713 data_addidarray_sort(xd, pool, needid, idarraydata + s->recommends, 0);
1714 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1715 data_addidarray_sort(xd, pool, needid, idarraydata + s->suggests, 0);
1716 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1717 data_addidarray_sort(xd, pool, needid, idarraydata + s->supplements, 0);
1718 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1719 data_addidarray_sort(xd, pool, needid, idarraydata + s->enhances, 0);
1720 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1721 data_addid(xd, repo->rpmdbid[i - repo->start]);
1722 if (anyrepodataused)
1725 FOR_REPODATAS(repo, j, data)
1727 if (!repodataused[j])
1729 if (i < data->start || i >= data->end)
1731 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1734 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1735 cbdata.maxdata = xd->len - cbdata.lastlen;
1736 cbdata.lastlen = xd->len;
1739 cbdata.doingsolvables = 0;
1742 assert(cbdata.current_sub == cbdata.nsubschemata);
1743 if (cbdata.subschemata)
1745 cbdata.subschemata = solv_free(cbdata.subschemata);
1746 cbdata.nsubschemata = 0;
1749 /********************************************************************/
1755 /* write file header */
1756 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1757 write_u32(&target, SOLV_VERSION_8);
1761 write_u32(&target, nstrings);
1762 write_u32(&target, nrels);
1763 write_u32(&target, ndirmap);
1764 write_u32(&target, anysolvableused ? repo->nsolvables : 0);
1765 write_u32(&target, target.nkeys);
1766 write_u32(&target, target.nschemata);
1768 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1769 solv_flags |= SOLV_FLAG_SIZE_BYTES;
1770 write_u32(&target, solv_flags);
1775 * calculate prefix encoding of the strings
1777 unsigned char *prefixcomp = solv_malloc(nstrings);
1778 unsigned int compsum = 0;
1782 for (i = 1; i < nstrings; i++)
1784 char *str = spool->stringspace + spool->strings[needid[i].map];
1786 for (same = 0; same < 255; same++)
1787 if (!old_str[same] || old_str[same] != str[same])
1789 prefixcomp[i] = same;
1797 write_u32(&target, sizeid);
1798 /* we save compsum bytes but need 1 extra byte for every string */
1799 write_u32(&target, sizeid + nstrings - 1 - compsum);
1800 for (i = 1; i < nstrings; i++)
1802 char *str = spool->stringspace + spool->strings[needid[i].map];
1803 write_u8(&target, prefixcomp[i]);
1804 write_str(&target, str + prefixcomp[i]);
1806 solv_free(prefixcomp);
1810 write_u32(&target, 0);
1811 write_u32(&target, 0);
1817 for (i = 0; i < nrels; i++)
1819 ran = pool->rels + (needid[reloff + i].map - reloff);
1820 write_id(&target, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
1821 write_id(&target, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
1822 write_u8(&target, ran->flags);
1826 * write dirs (skip both root and / entry)
1828 for (i = 2; i < ndirmap; i++)
1831 write_id(&target, dirmap[i]);
1833 write_id(&target, nstrings - dirmap[i]);
1840 for (i = 1; i < target.nkeys; i++)
1842 write_id(&target, needid[target.keys[i].name].need);
1843 write_id(&target, needid[target.keys[i].type].need);
1844 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1846 if (target.keys[i].type == type_constantid)
1847 write_id(&target, needid[target.keys[i].size].need);
1849 write_id(&target, target.keys[i].size);
1852 write_id(&target, cbdata.extdata[i].len);
1853 write_id(&target, target.keys[i].storage);
1859 write_id(&target, target.schemadatalen); /* XXX -1? */
1860 for (i = 1; i < target.nschemata; i++)
1861 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
1863 /********************************************************************/
1865 write_id(&target, cbdata.maxdata);
1866 write_id(&target, cbdata.extdata[0].len);
1867 if (cbdata.extdata[0].len)
1868 write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
1869 solv_free(cbdata.extdata[0].buf);
1871 /* do we have vertical data? */
1872 for (i = 1; i < target.nkeys; i++)
1873 if (cbdata.extdata[i].len)
1875 if (i < target.nkeys)
1877 /* yes, write it in pages */
1878 unsigned char *dp, vpage[REPOPAGE_BLOBSIZE];
1879 int l, ll, lpage = 0;
1881 write_u32(&target, REPOPAGE_BLOBSIZE);
1882 for (i = 1; i < target.nkeys; i++)
1884 if (!cbdata.extdata[i].len)
1886 l = cbdata.extdata[i].len;
1887 dp = cbdata.extdata[i].buf;
1890 ll = REPOPAGE_BLOBSIZE - lpage;
1893 memcpy(vpage + lpage, dp, ll);
1897 if (lpage == REPOPAGE_BLOBSIZE)
1899 write_compressed_page(&target, vpage, lpage);
1905 write_compressed_page(&target, vpage, lpage);
1908 for (i = 1; i < target.nkeys; i++)
1909 solv_free(cbdata.extdata[i].buf);
1910 solv_free(cbdata.extdata);
1913 repodata_freedata(&target);
1916 solv_free(cbdata.solvschemata);
1917 solv_free(cbdata.schema);
1919 solv_free(cbdata.keymap);
1920 solv_free(cbdata.keymapstart);
1921 solv_free(cbdata.dirused);
1922 solv_free(repodataused);
1923 return target.error;
1926 struct repodata_write_data {
1927 int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata);
1933 repodata_write_keyfilter(Repo *repo, Repokey *key, void *kfdata)
1935 struct repodata_write_data *wd = kfdata;
1937 /* XXX: special repodata selection hack */
1938 if (key->name == 1 && key->size != wd->repodataid)
1940 if (key->storage == KEY_STORAGE_SOLVABLE)
1941 return KEY_STORAGE_DROPPED; /* not part of this repodata */
1943 return (*wd->keyfilter)(repo, key, wd->kfdata);
1944 return key->storage;
1948 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
1950 struct repodata_write_data wd;
1952 wd.keyfilter = keyfilter;
1954 wd.repodataid = data->repodataid;
1955 return repo_write_filtered(data->repo, fp, repodata_write_keyfilter, &wd, keyq);
1959 repodata_write(Repodata *data, FILE *fp)
1961 return repodata_write_filtered(data, fp, repo_write_stdkeyfilter, 0, 0);
1965 repo_write(Repo *repo, FILE *fp)
1967 return repo_write_filtered(repo, fp, repo_write_stdkeyfilter, 0, 0);