2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
28 #include "repo_write.h"
31 /*------------------------------------------------------------------*/
32 /* Id map optimizations */
34 typedef struct needid {
40 #define RELOFF(id) (needid[0].map + GETRELID(id))
44 * idarray: array of Ids, ID_NULL terminated
45 * needid: array of Id->NeedId
47 * return size of array (including trailing zero)
52 incneedid(Pool *pool, Id id, NeedId *needid)
56 Reldep *rd = GETRELDEP(pool, id);
57 needid[RELOFF(id)].need++;
58 if (ISRELDEP(rd->evr))
59 incneedid(pool, rd->evr, needid);
61 needid[rd->evr].need++;
68 incneedidarray(Pool *pool, Id *idarray, NeedId *needid)
75 while ((id = *idarray++) != 0)
80 Reldep *rd = GETRELDEP(pool, id);
81 needid[RELOFF(id)].need++;
82 if (ISRELDEP(rd->evr))
83 incneedid(pool, rd->evr, needid);
85 needid[rd->evr].need++;
99 needid_cmp_need(const void *ap, const void *bp, void *dp)
101 const NeedId *a = ap;
102 const NeedId *b = bp;
104 r = b->need - a->need;
107 return a->map - b->map;
111 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
113 const NeedId *a = ap;
114 const NeedId *b = bp;
115 Stringpool *spool = dp;
120 r = b->need - a->need;
123 as = spool->stringspace + spool->strings[a->map];
124 bs = spool->stringspace + spool->strings[b->map];
125 return strcmp(as, bs);
129 /*------------------------------------------------------------------*/
130 /* output helper routines, used for writing the header */
131 /* (the data itself is accumulated in memory and written with
139 write_u32(FILE *fp, unsigned int x)
141 if (putc(x >> 24, fp) == EOF ||
142 putc(x >> 16, fp) == EOF ||
143 putc(x >> 8, fp) == EOF ||
146 perror("write error u32");
157 write_u8(FILE *fp, unsigned int x)
159 if (putc(x, fp) == EOF)
161 perror("write error u8");
171 write_blob(FILE *fp, void *data, int len)
173 if (len && fwrite(data, len, 1, fp) != 1)
175 perror("write error blob");
185 write_id(FILE *fp, Id x)
190 putc((x >> 28) | 128, fp);
192 putc((x >> 21) | 128, fp);
193 putc((x >> 14) | 128, fp);
196 putc((x >> 7) | 128, fp);
197 if (putc(x & 127, fp) == EOF)
199 perror("write error id");
205 write_id_eof(FILE *fp, Id x, int eof)
208 x = (x & 63) | ((x & ~63) << 1);
209 write_id(fp, x | (eof ? 0 : 64));
215 write_str(FILE *fp, const char *str)
217 if (fputs(str, fp) == EOF || putc(0, fp) == EOF)
219 perror("write error str");
229 write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
243 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
245 id = (id & 63) | ((id & ~63) << 1);
251 write_id(fp, id | 64);
256 cmp_ids(const void *pa, const void *pb, void *dp)
265 write_idarray_sort(FILE *fp, Pool *pool, NeedId *needid, Id *ids, Id marker)
277 for (len = 0; len < 64 && ids[len]; len++)
281 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
286 for (i = len + 1; ids[i]; i++)
288 sids = solv_malloc2(i, sizeof(Id));
289 memcpy(sids, lids, 64 * sizeof(Id));
290 for (; ids[len]; len++)
294 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
301 /* That bloody solvable:prereqmarker needs to stay in position :-( */
303 marker = needid[marker].need;
304 for (i = 0; i < len; i++)
305 if (sids[i] == marker)
308 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
310 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
314 /* The differencing above produces many runs of ones and twos. I tried
315 fairly elaborate schemes to RLE those, but they give only very mediocre
316 improvements in compression, as coding the escapes costs quite some
317 space. Even if they are coded only as bits in IDs. The best improvement
318 was about 2.7% for the whole .solv file. It's probably better to
319 invest some complexity into sharing idarrays, than RLEing. */
320 for (i = 0; i < len - 1; i++)
323 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
324 hence all real differences are offsetted by 1. Otherwise we would
325 have to handle negative differences, which would cost code space for
326 the encoding of the sign. We loose the exact mapping of prereq here,
327 but we know the result, so we can recover from that in the reader. */
335 /* XXX If difference is zero we have multiple equal elements,
336 we might want to skip writing them out. */
338 id = (id & 63) | ((id & ~63) << 1);
339 write_id(fp, id | 64);
347 id = (id & 63) | ((id & ~63) << 1);
364 Stringpool *ownspool;
373 Id *schema; /* schema construction space */
374 Id *sp; /* pointer in above */
375 Id *oldschema, *oldsp;
382 struct extdata *extdata;
391 int doingsolvables; /* working on solvables data */
394 #define NEEDED_BLOCK 1023
395 #define SCHEMATA_BLOCK 31
396 #define SCHEMATADATA_BLOCK 255
397 #define EXTDATA_BLOCK 4095
400 data_addid(struct extdata *xd, Id x)
403 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
404 dp = xd->buf + xd->len;
409 *dp++ = (x >> 28) | 128;
411 *dp++ = (x >> 21) | 128;
412 *dp++ = (x >> 14) | 128;
415 *dp++ = (x >> 7) | 128;
417 xd->len = dp - xd->buf;
421 data_addideof(struct extdata *xd, Id x, int eof)
424 x = (x & 63) | ((x & ~63) << 1);
425 data_addid(xd, (eof ? x: x | 64));
429 data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
442 for (len = 0; len < 64 && ids[len]; len++)
446 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
451 for (i = len + 1; ids[i]; i++)
453 sids = solv_malloc2(i, sizeof(Id));
454 memcpy(sids, lids, 64 * sizeof(Id));
455 for (; ids[len]; len++)
459 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
466 /* That bloody solvable:prereqmarker needs to stay in position :-( */
468 marker = needid[marker].need;
469 for (i = 0; i < len; i++)
470 if (sids[i] == marker)
473 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
475 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
479 /* The differencing above produces many runs of ones and twos. I tried
480 fairly elaborate schemes to RLE those, but they give only very mediocre
481 improvements in compression, as coding the escapes costs quite some
482 space. Even if they are coded only as bits in IDs. The best improvement
483 was about 2.7% for the whole .solv file. It's probably better to
484 invest some complexity into sharing idarrays, than RLEing. */
485 for (i = 0; i < len - 1; i++)
488 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
489 hence all real differences are offsetted by 1. Otherwise we would
490 have to handle negative differences, which would cost code space for
491 the encoding of the sign. We loose the exact mapping of prereq here,
492 but we know the result, so we can recover from that in the reader. */
500 /* XXX If difference is zero we have multiple equal elements,
501 we might want to skip writing them out. */
503 id = (id & 63) | ((id & ~63) << 1);
504 data_addid(xd, id | 64);
512 id = (id & 63) | ((id & ~63) << 1);
519 data_addblob(struct extdata *xd, unsigned char *blob, int len)
521 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
522 memcpy(xd->buf + xd->len, blob, len);
527 data_addu32(struct extdata *xd, unsigned int num)
534 data_addblob(xd, d, 4);
538 putinownpool(struct cbdata *cbdata, Stringpool *ss, Id id)
540 const char *str = stringpool_id2str(ss, id);
541 id = stringpool_str2id(cbdata->ownspool, str, 1);
542 if (id >= cbdata->needid[0].map)
544 int oldoff = cbdata->needid[0].map;
545 int newoff = (id + 1 + NEEDED_BLOCK) & ~NEEDED_BLOCK;
546 int nrels = cbdata->repo->pool->nrels;
547 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
549 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
550 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
551 cbdata->needid[0].map = newoff;
557 putinowndirpool(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
561 parent = dirpool_parent(dp, dir);
563 parent = putinowndirpool(cbdata, data, dp, parent);
564 compid = dp->dirs[dir];
565 if (cbdata->ownspool && compid > 1)
566 compid = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, compid);
567 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
571 * collect usage information about the dirs
572 * 1: dir used, no child of dir used
573 * 2: dir used as parent of another used dir
576 setdirused(struct cbdata *cbdata, Dirpool *dp, Id dir)
578 if (cbdata->dirused[dir])
580 cbdata->dirused[dir] = 1;
581 while ((dir = dirpool_parent(dp, dir)) != 0)
583 if (cbdata->dirused[dir] == 2)
585 if (cbdata->dirused[dir])
587 cbdata->dirused[dir] = 2;
590 cbdata->dirused[dir] = 2;
592 cbdata->dirused[0] = 2;
597 * collect key/id/dirid usage information, create needed schemas
600 repo_write_collect_needed(struct cbdata *cbdata, Repo *repo, Repodata *data, Repokey *key, KeyValue *kv)
605 if (key->name == REPOSITORY_SOLVABLES)
606 return SEARCH_NEXT_KEY; /* we do not want this one */
608 /* hack: ignore some keys, see BUGS */
609 if (data->repodataid != data->repo->nrepodata - 1)
610 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
611 return SEARCH_NEXT_KEY;
613 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
615 return SEARCH_NEXT_KEY; /* we do not want this one */
617 /* record key in schema */
618 if ((key->type != REPOKEY_TYPE_FIXARRAY || kv->eof == 0)
619 && (cbdata->sp == cbdata->schema || cbdata->sp[-1] != rm))
624 case REPOKEY_TYPE_ID:
625 case REPOKEY_TYPE_IDARRAY:
627 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
628 id = putinownpool(cbdata, data->localpool ? &data->spool : &repo->pool->ss, id);
629 incneedid(repo->pool, id, cbdata->needid);
631 case REPOKEY_TYPE_DIR:
632 case REPOKEY_TYPE_DIRNUMNUMARRAY:
633 case REPOKEY_TYPE_DIRSTRARRAY:
635 if (cbdata->owndirpool)
636 putinowndirpool(cbdata, data, &data->dirpool, id);
638 setdirused(cbdata, &data->dirpool, id);
640 case REPOKEY_TYPE_FIXARRAY:
643 if (cbdata->oldschema)
645 fprintf(stderr, "nested structs not yet implemented\n");
648 cbdata->oldschema = cbdata->schema;
649 cbdata->oldsp = cbdata->sp;
650 cbdata->schema = solv_calloc(cbdata->target->nkeys, sizeof(Id));
651 cbdata->sp = cbdata->schema;
653 else if (kv->eof == 1)
655 cbdata->current_sub++;
657 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
658 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, cbdata->schema, 1);
660 fprintf(stderr, "Have schema %d\n", cbdata->subschemata[cbdata->nsubschemata-1]);
662 cbdata->sp = cbdata->schema;
666 solv_free(cbdata->schema);
667 cbdata->schema = cbdata->oldschema;
668 cbdata->sp = cbdata->oldsp;
669 cbdata->oldsp = cbdata->oldschema = 0;
672 case REPOKEY_TYPE_FLEXARRAY:
676 *cbdata->sp++ = 0; /* mark start */
680 /* just finished a schema, rewind */
681 Id *sp = cbdata->sp - 1;
685 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
686 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
687 cbdata->sp = kv->eof == 2 ? sp - 1: sp;
697 repo_write_cb_needed(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
699 struct cbdata *cbdata = vcbdata;
700 Repo *repo = data->repo;
704 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? s - repo->pool->solvables : 0, s ? pool_id2str(repo->pool, s->name) : "", key->name, pool_id2str(repo->pool, key->name), key->type);
706 return repo_write_collect_needed(cbdata, repo, data, key, kv);
712 * encode all of the data into the correct buffers
716 repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue *kv)
725 if (key->name == REPOSITORY_SOLVABLES)
726 return SEARCH_NEXT_KEY;
728 /* hack: ignore some keys, see BUGS */
729 if (data->repodataid != data->repo->nrepodata - 1)
730 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
731 return SEARCH_NEXT_KEY;
733 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
735 return SEARCH_NEXT_KEY; /* we do not want this one */
737 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET)
739 xd = cbdata->extdata + rm; /* vertical buffer */
740 if (cbdata->vstart == -1)
741 cbdata->vstart = xd->len;
744 xd = cbdata->extdata + 0; /* incore buffer */
747 case REPOKEY_TYPE_VOID:
748 case REPOKEY_TYPE_CONSTANT:
749 case REPOKEY_TYPE_CONSTANTID:
751 case REPOKEY_TYPE_ID:
753 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
754 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
755 needid = cbdata->needid;
756 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
759 case REPOKEY_TYPE_IDARRAY:
761 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
762 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
763 needid = cbdata->needid;
764 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
765 data_addideof(xd, id, kv->eof);
767 case REPOKEY_TYPE_STR:
768 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
770 case REPOKEY_TYPE_MD5:
771 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
773 case REPOKEY_TYPE_SHA1:
774 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
776 case REPOKEY_TYPE_SHA256:
777 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
779 case REPOKEY_TYPE_U32:
785 data_addblob(xd, v, 4);
787 case REPOKEY_TYPE_NUM:
788 data_addid(xd, kv->num);
790 case REPOKEY_TYPE_DIR:
792 if (cbdata->owndirpool)
793 id = putinowndirpool(cbdata, data, &data->dirpool, id);
794 id = cbdata->dirused[id];
797 case REPOKEY_TYPE_BINARY:
798 data_addid(xd, kv->num);
800 data_addblob(xd, (unsigned char *)kv->str, kv->num);
802 case REPOKEY_TYPE_DIRNUMNUMARRAY:
804 if (cbdata->owndirpool)
805 id = putinowndirpool(cbdata, data, &data->dirpool, id);
806 id = cbdata->dirused[id];
808 data_addid(xd, kv->num);
809 data_addideof(xd, kv->num2, kv->eof);
811 case REPOKEY_TYPE_DIRSTRARRAY:
813 if (cbdata->owndirpool)
814 id = putinowndirpool(cbdata, data, &data->dirpool, id);
815 id = cbdata->dirused[id];
816 data_addideof(xd, id, kv->eof);
817 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
819 case REPOKEY_TYPE_FIXARRAY:
824 data_addid(xd, kv->num);
825 data_addid(xd, cbdata->subschemata[cbdata->current_sub]);
827 fprintf(stderr, "writing %d %d\n", kv->num, cbdata->subschemata[cbdata->current_sub]);
831 else if (kv->eof == 1)
833 cbdata->current_sub++;
836 case REPOKEY_TYPE_FLEXARRAY:
838 data_addid(xd, kv->num);
840 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
841 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
843 if (xd->len - cbdata->lastlen > cbdata->maxdata)
844 cbdata->maxdata = xd->len - cbdata->lastlen;
845 cbdata->lastlen = xd->len;
849 fprintf(stderr, "unknown type for %d: %d\n", key->name, key->type);
852 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
854 /* we can re-use old data in the blob here! */
855 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
856 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
863 repo_write_cb_adddata(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
865 struct cbdata *cbdata = vcbdata;
866 return repo_write_adddata(cbdata, data, key, kv);
869 /* traverse through directory with first child "dir" */
871 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
877 /* special case for '/', which has to come first */
880 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
882 if (used && !used[sib])
884 if (sib == 1 && parent == 1)
885 continue; /* already did that one above */
889 /* now go through all the siblings we just added and
890 * do recursive calls on them */
892 for (; parent < lastn; parent++)
894 sib = dirmap[parent];
895 if (used && used[sib] != 2) /* 2: used as parent */
897 child = dirpool_child(dp, sib);
900 dirmap[n++] = -parent; /* start new block */
901 n = traverse_dirs(dp, dirmap, n, child, used);
908 write_compressed_page(FILE *fp, unsigned char *page, int len)
911 unsigned char cpage[REPOPAGE_BLOBSIZE];
913 clen = repopagestore_compress_page(page, len, cpage, len - 1);
916 write_u32(fp, len * 2);
917 write_blob(fp, page, len);
921 write_u32(fp, clen * 2 + 1);
922 write_blob(fp, cpage, clen);
926 static Id verticals[] = {
928 SOLVABLE_DESCRIPTION,
937 static char *languagetags[] = {
939 "solvable:description:",
940 "solvable:messageins:",
941 "solvable:messagedel:",
947 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
952 for (i = 0; verticals[i]; i++)
953 if (key->name == verticals[i])
954 return KEY_STORAGE_VERTICAL_OFFSET;
955 keyname = pool_id2str(repo->pool, key->name);
956 for (i = 0; languagetags[i] != 0; i++)
957 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
958 return KEY_STORAGE_VERTICAL_OFFSET;
959 return KEY_STORAGE_INCORE;
967 * the code works the following way:
969 * 1) find which keys should be written
970 * 2) collect usage information for keys/ids/dirids, create schema
972 * 3) use usage information to create mapping tables, so that often
973 * used ids get a lower number
974 * 4) encode data into buffers using the mapping tables
975 * 5) write everything to disk
978 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
980 Pool *pool = repo->pool;
986 unsigned int solv_flags;
995 unsigned char *repodataused;
996 int anyrepodataused = 0;
997 int anysolvableused = 0;
999 struct cbdata cbdata;
1002 int poolusage, dirpoolusage, idused, dirused;
1005 Repodata *data, *dirpooldata;
1016 Id type_constantid = REPOKEY_TYPE_CONSTANTID;
1018 unsigned char *prefixcomp;
1019 unsigned int compsum;
1023 memset(&cbdata, 0, sizeof(cbdata));
1025 cbdata.target = ⌖
1027 repodata_initdata(&target, repo, 1);
1029 /* go through all repodata and find the keys we need */
1030 /* also unify keys */
1031 /* keymapstart - maps repo number to keymap offset */
1032 /* keymap - maps repo key to my key, 0 -> not used */
1034 /* start with all KEY_STORAGE_SOLVABLE ids */
1036 n = ID_NUM_INTERNAL;
1037 FOR_REPODATAS(repo, i, data)
1039 cbdata.keymap = solv_calloc(n, sizeof(Id));
1040 cbdata.keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1041 repodataused = solv_calloc(repo->nrepodata, 1);
1046 /* add keys for STORAGE_SOLVABLE */
1047 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1051 if (i < SOLVABLE_PROVIDES)
1052 keyd.type = REPOKEY_TYPE_ID;
1053 else if (i < RPM_RPMDBID)
1054 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1056 keyd.type = REPOKEY_TYPE_U32;
1058 keyd.storage = KEY_STORAGE_SOLVABLE;
1061 keyd.storage = keyfilter(repo, &keyd, kfdata);
1062 if (keyd.storage == KEY_STORAGE_DROPPED)
1064 keyd.storage = KEY_STORAGE_SOLVABLE;
1068 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1071 if (repo->nsolvables)
1074 keyd.name = REPOSITORY_SOLVABLES;
1075 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1077 keyd.storage = KEY_STORAGE_INCORE;
1078 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1086 n = ID_NUM_INTERNAL;
1087 FOR_REPODATAS(repo, i, data)
1089 cbdata.keymapstart[i] = n;
1090 cbdata.keymap[n++] = 0; /* key 0 */
1096 /* check if we want this repodata */
1097 memset(&keyd, 0, sizeof(keyd));
1101 if (keyfilter(repo, &keyd, kfdata) == -1)
1104 for (j = 1; j < data->nkeys; j++, n++)
1106 key = data->keys + j;
1107 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1109 cbdata.keymap[n] = cbdata.keymap[key->name];
1112 if (key->type == REPOKEY_TYPE_DELETED)
1114 cbdata.keymap[n] = 0;
1117 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1119 Repokey keyd = *key;
1120 keyd.size = repodata_globalize_id(data, key->size, 1);
1121 id = repodata_key2id(&target, &keyd, 0);
1124 id = repodata_key2id(&target, key, 0);
1127 Repokey keyd = *key;
1128 keyd.storage = KEY_STORAGE_INCORE;
1129 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1130 keyd.size = repodata_globalize_id(data, key->size, 1);
1131 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1135 keyd.storage = keyfilter(repo, &keyd, kfdata);
1136 if (keyd.storage == KEY_STORAGE_DROPPED)
1138 cbdata.keymap[n] = 0;
1142 id = repodata_key2id(&target, &keyd, 1);
1144 cbdata.keymap[n] = id;
1145 /* load repodata if not already loaded */
1146 if (data->state == REPODATA_STUB)
1148 if (data->loadcallback)
1149 data->loadcallback(data);
1151 data->state = REPODATA_ERROR;
1152 if (data->state != REPODATA_ERROR)
1154 /* redo this repodata! */
1156 n = cbdata.keymapstart[i];
1160 if (data->state == REPODATA_ERROR)
1163 cbdata.keymap[n] = 0;
1167 repodataused[i] = 1;
1168 anyrepodataused = 1;
1169 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1170 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1172 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1174 idused = 1; /* dirs also use ids */
1180 if (data->localpool)
1183 poolusage = 3; /* need own pool */
1187 spool = &data->spool;
1194 else if (poolusage != 1)
1195 poolusage = 3; /* need own pool */
1201 dirpoolusage = 3; /* need own dirpool */
1205 dirpool = &data->dirpool;
1212 /* 0: no pool needed at all */
1213 /* 1: use global pool */
1214 /* 2: use repodata local pool */
1215 /* 3: need own pool */
1218 spool = &target.spool;
1219 /* hack: reuse global pool data so we don't have to map pool ids */
1222 stringpool_free(spool);
1223 stringpool_clone(spool, &pool->ss);
1225 cbdata.ownspool = spool;
1227 else if (poolusage == 0 || poolusage == 1)
1233 if (dirpoolusage == 3)
1235 dirpool = &target.dirpool;
1237 cbdata.owndirpool = dirpool;
1240 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1243 /********************************************************************/
1245 fprintf(stderr, "poolusage: %d\n", poolusage);
1246 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1247 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1248 for (i = 1; i < target.nkeys; i++)
1249 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1252 /* copy keys if requested */
1256 for (i = 1; i < target.nkeys; i++)
1257 queue_push2(keyq, target.keys[i].name, target.keys[i].type);
1262 /* put all the keys we need in our string pool */
1263 /* put mapped ids right into target.keys */
1264 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1266 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1267 if (key->type == REPOKEY_TYPE_CONSTANTID)
1269 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1270 type_constantid = key->type;
1271 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1274 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1277 stringpool_freehash(spool); /* free some mem */
1281 /********************************************************************/
1283 /* set needed count of all strings and rels,
1284 * find which keys are used in the solvables
1285 * put all strings in own spool
1288 reloff = spool->nstrings;
1290 reloff = (reloff + NEEDED_BLOCK) & ~NEEDED_BLOCK;
1292 needid = calloc(reloff + pool->nrels, sizeof(*needid));
1293 needid[0].map = reloff;
1295 cbdata.needid = needid;
1296 cbdata.schema = solv_calloc(target.nkeys, sizeof(Id));
1297 cbdata.sp = cbdata.schema;
1298 cbdata.solvschemata = solv_calloc(repo->nsolvables, sizeof(Id));
1300 /* create main schema */
1301 cbdata.sp = cbdata.schema;
1302 /* collect all other data from all repodatas */
1303 /* XXX: merge arrays of equal keys? */
1304 FOR_REPODATAS(repo, j, data)
1306 if (!repodataused[j])
1308 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1311 /* add solvables if needed (may revert later) */
1312 if (repo->nsolvables)
1314 *sp++ = cbdata.keymap[REPOSITORY_SOLVABLES];
1315 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size++;
1318 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1320 idarraydata = repo->idarraydata;
1322 anysolvableused = 0;
1323 cbdata.doingsolvables = 1;
1324 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1326 if (s->repo != repo)
1329 /* set schema info, keep in sync with further down */
1331 if (cbdata.keymap[SOLVABLE_NAME])
1333 *sp++ = cbdata.keymap[SOLVABLE_NAME];
1334 needid[s->name].need++;
1336 if (cbdata.keymap[SOLVABLE_ARCH])
1338 *sp++ = cbdata.keymap[SOLVABLE_ARCH];
1339 needid[s->arch].need++;
1341 if (cbdata.keymap[SOLVABLE_EVR])
1343 *sp++ = cbdata.keymap[SOLVABLE_EVR];
1344 needid[s->evr].need++;
1346 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1348 *sp++ = cbdata.keymap[SOLVABLE_VENDOR];
1349 needid[s->vendor].need++;
1351 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1353 *sp++ = cbdata.keymap[SOLVABLE_PROVIDES];
1354 target.keys[cbdata.keymap[SOLVABLE_PROVIDES]].size += incneedidarray(pool, idarraydata + s->provides, needid);
1356 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1358 *sp++ = cbdata.keymap[SOLVABLE_OBSOLETES];
1359 target.keys[cbdata.keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(pool, idarraydata + s->obsoletes, needid);
1361 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1363 *sp++ = cbdata.keymap[SOLVABLE_CONFLICTS];
1364 target.keys[cbdata.keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(pool, idarraydata + s->conflicts, needid);
1366 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1368 *sp++ = cbdata.keymap[SOLVABLE_REQUIRES];
1369 target.keys[cbdata.keymap[SOLVABLE_REQUIRES]].size += incneedidarray(pool, idarraydata + s->requires, needid);
1371 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1373 *sp++ = cbdata.keymap[SOLVABLE_RECOMMENDS];
1374 target.keys[cbdata.keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(pool, idarraydata + s->recommends, needid);
1376 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1378 *sp++ = cbdata.keymap[SOLVABLE_SUGGESTS];
1379 target.keys[cbdata.keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(pool, idarraydata + s->suggests, needid);
1381 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1383 *sp++ = cbdata.keymap[SOLVABLE_SUPPLEMENTS];
1384 target.keys[cbdata.keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(pool, idarraydata + s->supplements, needid);
1386 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1388 *sp++ = cbdata.keymap[SOLVABLE_ENHANCES];
1389 target.keys[cbdata.keymap[SOLVABLE_ENHANCES]].size += incneedidarray(pool, idarraydata + s->enhances, needid);
1391 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1393 *sp++ = cbdata.keymap[RPM_RPMDBID];
1394 target.keys[cbdata.keymap[RPM_RPMDBID]].size++;
1398 if (anyrepodataused)
1400 FOR_REPODATAS(repo, j, data)
1402 if (!repodataused[j])
1404 if (i < data->start || i >= data->end)
1406 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1407 needid = cbdata.needid;
1411 cbdata.solvschemata[n] = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1412 if (cbdata.solvschemata[n])
1413 anysolvableused = 1;
1416 cbdata.doingsolvables = 0;
1417 assert(n == repo->nsolvables);
1419 if (repo->nsolvables && !anysolvableused)
1421 /* strip off solvable from the main schema */
1422 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size = 0;
1424 for (i = 0; target.schemadata[target.schemata[mainschema] + i]; i++)
1426 *sp = target.schemadata[target.schemata[mainschema] + i];
1427 if (*sp != cbdata.keymap[REPOSITORY_SOLVABLES])
1430 assert(target.schemadatalen == target.schemata[mainschema] + i + 1);
1432 target.schemadatalen = target.schemata[mainschema];
1434 repodata_free_schemahash(&target);
1435 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1438 /********************************************************************/
1440 /* remove unused keys */
1441 keyused = solv_calloc(target.nkeys, sizeof(Id));
1442 for (i = 1; i < target.schemadatalen; i++)
1443 keyused[target.schemadata[i]] = 1;
1445 for (n = i = 1; i < target.nkeys; i++)
1452 target.keys[n] = target.keys[i];
1455 keyq->elements[2 * n - 2] = keyq->elements[2 * i - 2];
1456 keyq->elements[2 * n - 1] = keyq->elements[2 * i - 1];
1463 queue_truncate(keyq, 2 * n - 2);
1465 /* update schema data to the new key ids */
1466 for (i = 1; i < target.schemadatalen; i++)
1467 target.schemadata[i] = keyused[target.schemadata[i]];
1468 /* update keymap to the new key ids */
1469 for (i = 0; i < cbdata.nkeymap; i++)
1470 cbdata.keymap[i] = keyused[cbdata.keymap[i]];
1471 keyused = solv_free(keyused);
1473 /* increment needid of the used keys, they are already mapped to
1474 * the correct string pool */
1475 for (i = 1; i < target.nkeys; i++)
1477 if (target.keys[i].type == type_constantid)
1478 needid[target.keys[i].size].need++;
1479 needid[target.keys[i].name].need++;
1480 needid[target.keys[i].type].need++;
1483 /********************************************************************/
1485 if (dirpool && cbdata.dirused && !cbdata.dirused[0])
1487 /* no dirs used at all */
1488 cbdata.dirused = solv_free(cbdata.dirused);
1492 /* increment need id for used dir components */
1495 /* if we have own dirpool, all entries in it are used.
1496 also, all comp ids are already mapped by putinowndirpool(),
1497 so we can simply increment needid.
1498 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1499 /* else we re-use a dirpool of repodata "dirpooldata".
1500 dirused tells us which of the ids are used.
1501 we need to map comp ids if we generate a new pool.
1502 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1503 for (i = 1; i < dirpool->ndirs; i++)
1506 fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
1508 if (cbdata.dirused && !cbdata.dirused[i])
1510 id = dirpool->dirs[i];
1513 if (dirpooldata && cbdata.ownspool && id > 1)
1515 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1516 needid = cbdata.needid;
1523 /********************************************************************/
1526 * create mapping table, new keys are sorted by needid[].need
1528 * needid[key].need : old key -> new key
1529 * needid[key].map : new key -> old key
1532 /* zero out id 0 and rel 0 just in case */
1533 reloff = needid[0].map;
1535 needid[reloff].need = 0;
1537 for (i = 1; i < reloff + pool->nrels; i++)
1541 solv_sort(needid + 1, spool->nstrings - 1, sizeof(*needid), needid_cmp_need_s, spool);
1543 /* make first entry '' */
1545 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1547 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1548 /* now needid is in new order, needid[newid].map -> oldid */
1550 /* calculate string space size, also zero out needid[].need */
1552 for (i = 1; i < reloff; i++)
1554 if (!needid[i].need)
1555 break; /* as we have sorted, every entry after this also has need == 0 */
1557 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1559 nstrings = i; /* our new string id end */
1561 /* make needid[oldid].need point to newid */
1562 for (i = 1; i < nstrings; i++)
1563 needid[needid[i].map].need = i;
1565 /* same as above for relations */
1566 for (i = 0; i < pool->nrels; i++)
1568 if (!needid[reloff + i].need)
1570 needid[reloff + i].need = 0;
1572 nrels = i; /* our new rel id end */
1574 for (i = 0; i < nrels; i++)
1575 needid[needid[reloff + i].map].need = nstrings + i;
1577 /* now we have: needid[oldid].need -> newid
1578 needid[newid].map -> oldid
1579 both for strings and relations */
1582 /********************************************************************/
1588 /* create our new target directory structure by traversing through all
1589 * used dirs. This will concatenate blocks with the same parent
1590 * directory into single blocks.
1591 * Instead of components, traverse_dirs stores the old dirids,
1592 * we will change this in the second step below */
1593 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1594 if (cbdata.dirused && !cbdata.dirused[1])
1595 cbdata.dirused[1] = 1; /* always want / entry */
1596 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1598 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1600 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1601 /* change dirmap so that it maps from "new dirid" to "new compid" */
1602 if (!cbdata.dirused)
1603 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1604 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1605 for (i = 1; i < ndirmap; i++)
1609 cbdata.dirused[dirmap[i]] = i;
1610 id = dirpool->dirs[dirmap[i]];
1611 if (dirpooldata && cbdata.ownspool && id > 1)
1612 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1613 dirmap[i] = needid[id].need;
1615 /* now the new target directory structure is complete (dirmap), and we have
1616 * dirused[olddirid] -> newdirid */
1619 /********************************************************************/
1622 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1625 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1627 xd = cbdata.extdata;
1628 cbdata.current_sub = 0;
1629 /* add main schema */
1631 data_addid(xd, mainschema);
1634 FOR_REPODATAS(repo, j, data)
1636 if (!repodataused[j])
1638 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1642 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1643 cbdata.maxdata = xd->len - cbdata.lastlen;
1644 cbdata.lastlen = xd->len;
1646 if (anysolvableused)
1648 data_addid(xd, repo->nsolvables); /* FLEXARRAY nentries */
1649 cbdata.doingsolvables = 1;
1650 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1652 if (s->repo != repo)
1654 data_addid(xd, cbdata.solvschemata[n]);
1655 if (cbdata.keymap[SOLVABLE_NAME])
1656 data_addid(xd, needid[s->name].need);
1657 if (cbdata.keymap[SOLVABLE_ARCH])
1658 data_addid(xd, needid[s->arch].need);
1659 if (cbdata.keymap[SOLVABLE_EVR])
1660 data_addid(xd, needid[s->evr].need);
1661 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1662 data_addid(xd, needid[s->vendor].need);
1663 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1664 data_addidarray_sort(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
1665 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1666 data_addidarray_sort(xd, pool, needid, idarraydata + s->obsoletes, 0);
1667 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1668 data_addidarray_sort(xd, pool, needid, idarraydata + s->conflicts, 0);
1669 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1670 data_addidarray_sort(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
1671 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1672 data_addidarray_sort(xd, pool, needid, idarraydata + s->recommends, 0);
1673 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1674 data_addidarray_sort(xd, pool, needid, idarraydata + s->suggests, 0);
1675 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1676 data_addidarray_sort(xd, pool, needid, idarraydata + s->supplements, 0);
1677 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1678 data_addidarray_sort(xd, pool, needid, idarraydata + s->enhances, 0);
1679 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1680 data_addu32(xd, repo->rpmdbid[i - repo->start]);
1681 if (anyrepodataused)
1684 FOR_REPODATAS(repo, j, data)
1686 if (!repodataused[j])
1688 if (i < data->start || i >= data->end)
1690 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1693 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1694 cbdata.maxdata = xd->len - cbdata.lastlen;
1695 cbdata.lastlen = xd->len;
1698 cbdata.doingsolvables = 0;
1701 assert(cbdata.current_sub == cbdata.nsubschemata);
1702 if (cbdata.subschemata)
1704 cbdata.subschemata = solv_free(cbdata.subschemata);
1705 cbdata.nsubschemata = 0;
1708 /********************************************************************/
1712 /* write file header */
1713 write_u32(fp, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1714 write_u32(fp, SOLV_VERSION_8);
1718 write_u32(fp, nstrings);
1719 write_u32(fp, nrels);
1720 write_u32(fp, ndirmap);
1721 write_u32(fp, anysolvableused ? repo->nsolvables : 0);
1722 write_u32(fp, target.nkeys);
1723 write_u32(fp, target.nschemata);
1725 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1726 write_u32(fp, solv_flags);
1729 * calculate prefix encoding of the strings
1731 prefixcomp = solv_malloc(nstrings);
1736 for (i = 1; i < nstrings; i++)
1738 char *str = spool->stringspace + spool->strings[needid[i].map];
1740 for (same = 0; same < 255; same++)
1741 if (!old_str[same] || old_str[same] != str[same])
1743 prefixcomp[i] = same;
1751 write_u32(fp, sizeid);
1752 /* we save compsum bytes but need 1 extra byte for every string */
1753 write_u32(fp, sizeid + (nstrings ? nstrings - 1 : 0) - compsum);
1754 if (sizeid + (nstrings ? nstrings - 1 : 0) != compsum)
1756 for (i = 1; i < nstrings; i++)
1758 char *str = spool->stringspace + spool->strings[needid[i].map];
1759 write_u8(fp, prefixcomp[i]);
1760 write_str(fp, str + prefixcomp[i]);
1763 solv_free(prefixcomp);
1766 /* Build the prefix-encoding of the string pool. We need to know
1767 the size of that before writing it to the file, so we have to
1768 build a separate buffer for that. As it's temporarily possible
1769 that this actually is an expansion we can't easily reuse the
1770 stringspace for this. The max expansion per string is 1 byte,
1771 so it will fit into sizeid+nstrings bytes. */
1772 char *prefix = solv_malloc(sizeid + nstrings);
1775 for (i = 1; i < nstrings; i++)
1777 char *str = spool->stringspace + spool->strings[needid[i].map];
1780 for (same = 0; same < 255; same++)
1781 if (!old_str[same] || !str[same] || old_str[same] != str[same])
1784 len = strlen(str + same) + 1;
1785 memcpy(pp, str + same, len);
1793 write_u32(fp, sizeid);
1794 write_u32(fp, pp - prefix);
1797 if (fwrite(prefix, pp - prefix, 1, fp) != 1)
1799 perror("write error prefix");
1809 for (i = 0; i < nrels; i++)
1811 ran = pool->rels + (needid[reloff + i].map - reloff);
1812 write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
1813 write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
1814 write_u8(fp, ran->flags);
1818 * write dirs (skip both root and / entry)
1820 for (i = 2; i < ndirmap; i++)
1823 write_id(fp, dirmap[i]);
1825 write_id(fp, nstrings - dirmap[i]);
1832 for (i = 1; i < target.nkeys; i++)
1834 write_id(fp, needid[target.keys[i].name].need);
1835 write_id(fp, needid[target.keys[i].type].need);
1836 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1838 if (target.keys[i].type == type_constantid)
1839 write_id(fp, needid[target.keys[i].size].need);
1841 write_id(fp, target.keys[i].size);
1844 write_id(fp, cbdata.extdata[i].len);
1845 write_id(fp, target.keys[i].storage);
1851 write_id(fp, target.schemadatalen); /* XXX -1? */
1852 for (i = 1; i < target.nschemata; i++)
1853 write_idarray(fp, pool, 0, repodata_id2schema(&target, i));
1855 /********************************************************************/
1857 write_id(fp, cbdata.maxdata);
1858 write_id(fp, cbdata.extdata[0].len);
1859 if (cbdata.extdata[0].len)
1860 write_blob(fp, cbdata.extdata[0].buf, cbdata.extdata[0].len);
1861 solv_free(cbdata.extdata[0].buf);
1863 /* do we have vertical data? */
1864 for (i = 1; i < target.nkeys; i++)
1865 if (cbdata.extdata[i].len)
1867 if (i < target.nkeys)
1869 /* yes, write it in pages */
1870 unsigned char *dp, vpage[REPOPAGE_BLOBSIZE];
1871 int l, ll, lpage = 0;
1873 write_u32(fp, REPOPAGE_BLOBSIZE);
1874 for (i = 1; i < target.nkeys; i++)
1876 if (!cbdata.extdata[i].len)
1878 l = cbdata.extdata[i].len;
1879 dp = cbdata.extdata[i].buf;
1882 ll = REPOPAGE_BLOBSIZE - lpage;
1885 memcpy(vpage + lpage, dp, ll);
1889 if (lpage == REPOPAGE_BLOBSIZE)
1891 write_compressed_page(fp, vpage, lpage);
1897 write_compressed_page(fp, vpage, lpage);
1900 for (i = 1; i < target.nkeys; i++)
1901 solv_free(cbdata.extdata[i].buf);
1902 solv_free(cbdata.extdata);
1904 repodata_freedata(&target);
1907 solv_free(cbdata.solvschemata);
1908 solv_free(cbdata.schema);
1910 solv_free(cbdata.keymap);
1911 solv_free(cbdata.keymapstart);
1912 solv_free(cbdata.dirused);
1913 solv_free(repodataused);
1917 struct repodata_write_data {
1918 int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata);
1924 repodata_write_keyfilter(Repo *repo, Repokey *key, void *kfdata)
1926 struct repodata_write_data *wd = kfdata;
1928 /* XXX: special repodata selection hack */
1929 if (key->name == 1 && key->size != wd->repodataid)
1931 if (key->storage == KEY_STORAGE_SOLVABLE)
1932 return KEY_STORAGE_DROPPED; /* not part of this repodata */
1934 return (*wd->keyfilter)(repo, key, wd->kfdata);
1935 return key->storage;
1939 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
1941 struct repodata_write_data wd;
1943 wd.keyfilter = keyfilter;
1945 wd.repodataid = data->repodataid;
1946 return repo_write_filtered(data->repo, fp, repodata_write_keyfilter, &wd, keyq);
1950 repodata_write(Repodata *data, FILE *fp)
1952 return repodata_write_filtered(data, fp, repo_write_stdkeyfilter, 0, 0);
1956 repo_write(Repo *repo, FILE *fp)
1958 return repo_write_filtered(repo, fp, repo_write_stdkeyfilter, 0, 0);