2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
29 #include "repo_write.h"
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
35 typedef struct needid {
41 #define RELOFF(id) (needid[0].map + GETRELID(id))
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
48 * return size of array (including trailing zero)
53 incneedid(Pool *pool, Id id, NeedId *needid)
57 Reldep *rd = GETRELDEP(pool, id);
58 needid[RELOFF(id)].need++;
59 if (ISRELDEP(rd->evr))
60 incneedid(pool, rd->evr, needid);
62 needid[rd->evr].need++;
69 incneedidarray(Pool *pool, Id *idarray, NeedId *needid)
76 while ((id = *idarray++) != 0)
81 Reldep *rd = GETRELDEP(pool, id);
82 needid[RELOFF(id)].need++;
83 if (ISRELDEP(rd->evr))
84 incneedid(pool, rd->evr, needid);
86 needid[rd->evr].need++;
100 needid_cmp_need(const void *ap, const void *bp, void *dp)
102 const NeedId *a = ap;
103 const NeedId *b = bp;
105 r = b->need - a->need;
108 return a->map - b->map;
112 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
114 const NeedId *a = ap;
115 const NeedId *b = bp;
116 Stringpool *spool = dp;
121 r = b->need - a->need;
124 as = spool->stringspace + spool->strings[a->map];
125 bs = spool->stringspace + spool->strings[b->map];
126 return strcmp(as, bs);
130 /*------------------------------------------------------------------*/
131 /* output helper routines, used for writing the header */
132 /* (the data itself is accumulated in memory and written with
140 write_u32(Repodata *data, unsigned int x)
145 if (putc(x >> 24, fp) == EOF ||
146 putc(x >> 16, fp) == EOF ||
147 putc(x >> 8, fp) == EOF ||
150 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
160 write_u8(Repodata *data, unsigned int x)
164 if (putc(x, data->fp) == EOF)
166 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
175 write_blob(Repodata *data, void *blob, int len)
179 if (len && fwrite(blob, len, 1, data->fp) != 1)
181 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
190 write_id(Repodata *data, Id x)
198 putc((x >> 28) | 128, fp);
200 putc((x >> 21) | 128, fp);
201 putc((x >> 14) | 128, fp);
204 putc((x >> 7) | 128, fp);
205 if (putc(x & 127, fp) == EOF)
207 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
212 write_id_eof(Repodata *data, Id x, int eof)
215 x = (x & 63) | ((x & ~63) << 1);
216 write_id(data, x | (eof ? 0 : 64));
222 write_str(Repodata *data, const char *str)
226 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
228 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
237 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
251 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
253 id = (id & 63) | ((id & ~63) << 1);
259 write_id(data, id | 64);
264 cmp_ids(const void *pa, const void *pb, void *dp)
273 write_idarray_sort(Repodata *data, Pool *pool, NeedId *needid, Id *ids, Id marker)
285 for (len = 0; len < 64 && ids[len]; len++)
289 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
294 for (i = len + 1; ids[i]; i++)
296 sids = solv_malloc2(i, sizeof(Id));
297 memcpy(sids, lids, 64 * sizeof(Id));
298 for (; ids[len]; len++)
302 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
309 /* That bloody solvable:prereqmarker needs to stay in position :-( */
311 marker = needid[marker].need;
312 for (i = 0; i < len; i++)
313 if (sids[i] == marker)
316 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
318 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
322 /* The differencing above produces many runs of ones and twos. I tried
323 fairly elaborate schemes to RLE those, but they give only very mediocre
324 improvements in compression, as coding the escapes costs quite some
325 space. Even if they are coded only as bits in IDs. The best improvement
326 was about 2.7% for the whole .solv file. It's probably better to
327 invest some complexity into sharing idarrays, than RLEing. */
328 for (i = 0; i < len - 1; i++)
331 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
332 hence all real differences are offsetted by 1. Otherwise we would
333 have to handle negative differences, which would cost code space for
334 the encoding of the sign. We loose the exact mapping of prereq here,
335 but we know the result, so we can recover from that in the reader. */
343 /* XXX If difference is zero we have multiple equal elements,
344 we might want to skip writing them out. */
346 id = (id & 63) | ((id & ~63) << 1);
347 write_id(data, id | 64);
355 id = (id & 63) | ((id & ~63) << 1);
372 Stringpool *ownspool;
381 Id *schema; /* schema construction space */
382 Id *sp; /* pointer in above */
383 Id *oldschema, *oldsp;
390 struct extdata *extdata;
399 int doingsolvables; /* working on solvables data */
403 #define NEEDED_BLOCK 1023
404 #define SCHEMATA_BLOCK 31
405 #define SCHEMATADATA_BLOCK 255
406 #define EXTDATA_BLOCK 4095
409 data_addid(struct extdata *xd, Id sx)
411 unsigned int x = (unsigned int)sx;
414 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
415 dp = xd->buf + xd->len;
420 *dp++ = (x >> 28) | 128;
422 *dp++ = (x >> 21) | 128;
423 *dp++ = (x >> 14) | 128;
426 *dp++ = (x >> 7) | 128;
428 xd->len = dp - xd->buf;
432 data_addideof(struct extdata *xd, Id sx, int eof)
434 unsigned int x = (unsigned int)sx;
437 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
438 dp = xd->buf + xd->len;
443 *dp++ = (x >> 27) | 128;
445 *dp++ = (x >> 20) | 128;
446 *dp++ = (x >> 13) | 128;
449 *dp++ = (x >> 6) | 128;
450 *dp++ = eof ? (x & 63) : (x & 63) | 64;
451 xd->len = dp - xd->buf;
455 data_addideof_len(Id sx)
457 unsigned int x = (unsigned int)sx;
462 return x >= (1 << 20) ? 4 : 3;
464 return x >= (1 << 6) ? 2 : 1;
468 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
474 data_addid(xd, (Id)(hx >> 3));
475 xd->buf[xd->len - 1] |= 128;
478 data_addid(xd, (Id)(x | 0x80000000));
479 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
482 data_addid(xd, (Id)x);
486 data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
499 for (len = 0; len < 64 && ids[len]; len++)
503 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
508 for (i = len + 1; ids[i]; i++)
510 sids = solv_malloc2(i, sizeof(Id));
511 memcpy(sids, lids, 64 * sizeof(Id));
512 for (; ids[len]; len++)
516 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
523 /* That bloody solvable:prereqmarker needs to stay in position :-( */
525 marker = needid[marker].need;
526 for (i = 0; i < len; i++)
527 if (sids[i] == marker)
530 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
532 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
536 /* The differencing above produces many runs of ones and twos. I tried
537 fairly elaborate schemes to RLE those, but they give only very mediocre
538 improvements in compression, as coding the escapes costs quite some
539 space. Even if they are coded only as bits in IDs. The best improvement
540 was about 2.7% for the whole .solv file. It's probably better to
541 invest some complexity into sharing idarrays, than RLEing. */
542 for (i = 0; i < len - 1; i++)
545 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
546 hence all real differences are offsetted by 1. Otherwise we would
547 have to handle negative differences, which would cost code space for
548 the encoding of the sign. We loose the exact mapping of prereq here,
549 but we know the result, so we can recover from that in the reader. */
557 /* XXX If difference is zero we have multiple equal elements,
558 we might want to skip writing them out. */
559 data_addideof(xd, id, 0);
566 data_addideof(xd, id, 1);
572 data_addblob(struct extdata *xd, unsigned char *blob, int len)
574 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
575 memcpy(xd->buf + xd->len, blob, len);
580 data_addu32(struct extdata *xd, unsigned int num)
587 data_addblob(xd, d, 4);
591 putinownpool(struct cbdata *cbdata, Stringpool *ss, Id id)
593 const char *str = stringpool_id2str(ss, id);
594 id = stringpool_str2id(cbdata->ownspool, str, 1);
595 if (id >= cbdata->needid[0].map)
597 int oldoff = cbdata->needid[0].map;
598 int newoff = (id + 1 + NEEDED_BLOCK) & ~NEEDED_BLOCK;
599 int nrels = cbdata->repo->pool->nrels;
600 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
602 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
603 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
604 cbdata->needid[0].map = newoff;
610 putinowndirpool(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
614 parent = dirpool_parent(dp, dir);
616 parent = putinowndirpool(cbdata, data, dp, parent);
617 compid = dp->dirs[dir];
618 if (cbdata->ownspool && compid > 1)
619 compid = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, compid);
620 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
624 * collect usage information about the dirs
625 * 1: dir used, no child of dir used
626 * 2: dir used as parent of another used dir
629 setdirused(struct cbdata *cbdata, Dirpool *dp, Id dir)
631 if (cbdata->dirused[dir])
633 cbdata->dirused[dir] = 1;
634 while ((dir = dirpool_parent(dp, dir)) != 0)
636 if (cbdata->dirused[dir] == 2)
638 if (cbdata->dirused[dir])
640 cbdata->dirused[dir] = 2;
643 cbdata->dirused[dir] = 2;
645 cbdata->dirused[0] = 2;
650 * collect key/id/dirid usage information, create needed schemas
653 repo_write_collect_needed(struct cbdata *cbdata, Repo *repo, Repodata *data, Repokey *key, KeyValue *kv)
658 if (key->name == REPOSITORY_SOLVABLES)
659 return SEARCH_NEXT_KEY; /* we do not want this one */
661 /* hack: ignore some keys, see BUGS */
662 if (data->repodataid != data->repo->nrepodata - 1)
663 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
664 return SEARCH_NEXT_KEY;
666 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
668 return SEARCH_NEXT_KEY; /* we do not want this one */
670 /* record key in schema */
671 if ((key->type != REPOKEY_TYPE_FIXARRAY || kv->eof == 0)
672 && (cbdata->sp == cbdata->schema || cbdata->sp[-1] != rm))
677 case REPOKEY_TYPE_ID:
678 case REPOKEY_TYPE_IDARRAY:
680 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
681 id = putinownpool(cbdata, data->localpool ? &data->spool : &repo->pool->ss, id);
682 incneedid(repo->pool, id, cbdata->needid);
684 case REPOKEY_TYPE_DIR:
685 case REPOKEY_TYPE_DIRNUMNUMARRAY:
686 case REPOKEY_TYPE_DIRSTRARRAY:
688 if (cbdata->owndirpool)
689 putinowndirpool(cbdata, data, &data->dirpool, id);
691 setdirused(cbdata, &data->dirpool, id);
693 case REPOKEY_TYPE_FIXARRAY:
696 if (cbdata->oldschema)
698 cbdata->target->error = pool_error(cbdata->repo->pool, -1, "nested fixarray structs not yet implemented");
699 return SEARCH_NEXT_KEY;
701 cbdata->oldschema = cbdata->schema;
702 cbdata->oldsp = cbdata->sp;
703 cbdata->schema = solv_calloc(cbdata->target->nkeys, sizeof(Id));
704 cbdata->sp = cbdata->schema;
706 else if (kv->eof == 1)
708 cbdata->current_sub++;
710 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
711 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, cbdata->schema, 1);
713 fprintf(stderr, "Have schema %d\n", cbdata->subschemata[cbdata->nsubschemata-1]);
715 cbdata->sp = cbdata->schema;
719 solv_free(cbdata->schema);
720 cbdata->schema = cbdata->oldschema;
721 cbdata->sp = cbdata->oldsp;
722 cbdata->oldsp = cbdata->oldschema = 0;
725 case REPOKEY_TYPE_FLEXARRAY:
729 *cbdata->sp++ = 0; /* mark start */
733 /* just finished a schema, rewind */
734 Id *sp = cbdata->sp - 1;
738 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
739 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
740 cbdata->sp = kv->eof == 2 ? sp - 1: sp;
750 repo_write_cb_needed(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
752 struct cbdata *cbdata = vcbdata;
753 Repo *repo = data->repo;
757 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? s - repo->pool->solvables : 0, s ? pool_id2str(repo->pool, s->name) : "", key->name, pool_id2str(repo->pool, key->name), key->type);
759 return repo_write_collect_needed(cbdata, repo, data, key, kv);
765 * encode all of the data into the correct buffers
769 repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue *kv)
778 if (key->name == REPOSITORY_SOLVABLES)
779 return SEARCH_NEXT_KEY;
781 /* hack: ignore some keys, see BUGS */
782 if (data->repodataid != data->repo->nrepodata - 1)
783 if (key->name == REPOSITORY_ADDEDFILEPROVIDES || key->name == REPOSITORY_EXTERNAL || key->name == REPOSITORY_LOCATION || key->name == REPOSITORY_KEYS || key->name == REPOSITORY_TOOLVERSION)
784 return SEARCH_NEXT_KEY;
786 rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
788 return SEARCH_NEXT_KEY; /* we do not want this one */
790 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET)
792 xd = cbdata->extdata + rm; /* vertical buffer */
793 if (cbdata->vstart == -1)
794 cbdata->vstart = xd->len;
797 xd = cbdata->extdata + 0; /* incore buffer */
800 case REPOKEY_TYPE_VOID:
801 case REPOKEY_TYPE_CONSTANT:
802 case REPOKEY_TYPE_CONSTANTID:
804 case REPOKEY_TYPE_ID:
806 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
807 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
808 needid = cbdata->needid;
809 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
812 case REPOKEY_TYPE_IDARRAY:
814 if (!ISRELDEP(id) && cbdata->ownspool && id > 1)
815 id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id);
816 needid = cbdata->needid;
817 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
818 data_addideof(xd, id, kv->eof);
820 case REPOKEY_TYPE_STR:
821 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
823 case REPOKEY_TYPE_MD5:
824 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
826 case REPOKEY_TYPE_SHA1:
827 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
829 case REPOKEY_TYPE_SHA224:
830 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
832 case REPOKEY_TYPE_SHA256:
833 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
835 case REPOKEY_TYPE_SHA384:
836 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
838 case REPOKEY_TYPE_SHA512:
839 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
841 case REPOKEY_TYPE_U32:
847 data_addblob(xd, v, 4);
849 case REPOKEY_TYPE_NUM:
850 data_addid64(xd, kv->num, kv->num2);
852 case REPOKEY_TYPE_DIR:
854 if (cbdata->owndirpool)
855 id = putinowndirpool(cbdata, data, &data->dirpool, id);
856 id = cbdata->dirused[id];
859 case REPOKEY_TYPE_BINARY:
860 data_addid(xd, kv->num);
862 data_addblob(xd, (unsigned char *)kv->str, kv->num);
864 case REPOKEY_TYPE_DIRNUMNUMARRAY:
866 if (cbdata->owndirpool)
867 id = putinowndirpool(cbdata, data, &data->dirpool, id);
868 id = cbdata->dirused[id];
870 data_addid(xd, kv->num);
871 data_addideof(xd, kv->num2, kv->eof);
873 case REPOKEY_TYPE_DIRSTRARRAY:
875 if (cbdata->owndirpool)
876 id = putinowndirpool(cbdata, data, &data->dirpool, id);
877 id = cbdata->dirused[id];
878 if (cbdata->filelistmode > 0)
880 xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
883 data_addideof(xd, id, kv->eof);
884 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
885 if (cbdata->filelistmode < 0)
888 case REPOKEY_TYPE_FIXARRAY:
893 data_addid(xd, kv->num);
894 data_addid(xd, cbdata->subschemata[cbdata->current_sub]);
896 fprintf(stderr, "writing %d %d\n", kv->num, cbdata->subschemata[cbdata->current_sub]);
900 else if (kv->eof == 1)
902 cbdata->current_sub++;
905 case REPOKEY_TYPE_FLEXARRAY:
907 data_addid(xd, kv->num);
909 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
910 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
912 if (xd->len - cbdata->lastlen > cbdata->maxdata)
913 cbdata->maxdata = xd->len - cbdata->lastlen;
914 cbdata->lastlen = xd->len;
918 cbdata->target->error = pool_error(cbdata->repo->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
921 if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
923 /* we can re-use old data in the blob here! */
924 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
925 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
932 repo_write_cb_adddata(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
934 struct cbdata *cbdata = vcbdata;
935 return repo_write_adddata(cbdata, data, key, kv);
938 /* traverse through directory with first child "dir" */
940 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
946 /* special case for '/', which has to come first */
949 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
951 if (used && !used[sib])
953 if (sib == 1 && parent == 1)
954 continue; /* already did that one above */
958 /* now go through all the siblings we just added and
959 * do recursive calls on them */
961 for (; parent < lastn; parent++)
963 sib = dirmap[parent];
964 if (used && used[sib] != 2) /* 2: used as parent */
966 child = dirpool_child(dp, sib);
969 dirmap[n++] = -parent; /* start new block */
970 n = traverse_dirs(dp, dirmap, n, child, used);
977 write_compressed_page(Repodata *data, unsigned char *page, int len)
980 unsigned char cpage[REPOPAGE_BLOBSIZE];
982 clen = repopagestore_compress_page(page, len, cpage, len - 1);
985 write_u32(data, len * 2);
986 write_blob(data, page, len);
990 write_u32(data, clen * 2 + 1);
991 write_blob(data, cpage, clen);
995 static Id verticals[] = {
997 SOLVABLE_DESCRIPTION,
1009 SOLVABLE_CHANGELOG_AUTHOR,
1010 SOLVABLE_CHANGELOG_TEXT,
1014 static char *languagetags[] = {
1015 "solvable:summary:",
1016 "solvable:description:",
1017 "solvable:messageins:",
1018 "solvable:messagedel:",
1024 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
1026 const char *keyname;
1029 for (i = 0; verticals[i]; i++)
1030 if (key->name == verticals[i])
1031 return KEY_STORAGE_VERTICAL_OFFSET;
1032 keyname = pool_id2str(repo->pool, key->name);
1033 for (i = 0; languagetags[i] != 0; i++)
1034 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
1035 return KEY_STORAGE_VERTICAL_OFFSET;
1036 return KEY_STORAGE_INCORE;
1040 * return true if the repodata contains the filelist (and just
1041 * the filelist). The same code is used in the dataiterator. The way
1042 * it is used is completely wrong, of course, as having the filelist
1043 * key does not mean it is used for a specific solvable. Nevertheless
1044 * it is better to have it than to write broken solv files.
1047 is_filelist_extension(Repodata *data)
1050 for (j = 1; j < data->nkeys; j++)
1051 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1058 write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
1060 unsigned char *dp = xd->buf;
1064 int ll = REPOPAGE_BLOBSIZE - lpage;
1067 memcpy(vpage + lpage, dp, ll);
1071 if (lpage == REPOPAGE_BLOBSIZE)
1073 write_compressed_page(target, vpage, lpage);
1085 * the code works the following way:
1087 * 1) find which keys should be written
1088 * 2) collect usage information for keys/ids/dirids, create schema
1090 * 3) use usage information to create mapping tables, so that often
1091 * used ids get a lower number
1092 * 4) encode data into buffers using the mapping tables
1093 * 5) write everything to disk
1096 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
1098 Pool *pool = repo->pool;
1099 int i, j, n, lastfilelistn;
1102 int nstrings, nrels;
1103 unsigned int sizeid;
1104 unsigned int solv_flags;
1113 unsigned char *repodataused;
1114 int anyrepodataused = 0;
1115 int anysolvableused = 0;
1117 struct cbdata cbdata;
1120 int poolusage, dirpoolusage, idused, dirused;
1123 Repodata *data, *dirpooldata;
1134 Id type_constantid = REPOKEY_TYPE_CONSTANTID;
1137 memset(&cbdata, 0, sizeof(cbdata));
1139 cbdata.target = ⌖
1141 repodata_initdata(&target, repo, 1);
1143 /* go through all repodata and find the keys we need */
1144 /* also unify keys */
1145 /* keymapstart - maps repo number to keymap offset */
1146 /* keymap - maps repo key to my key, 0 -> not used */
1148 /* start with all KEY_STORAGE_SOLVABLE ids */
1150 n = ID_NUM_INTERNAL;
1151 FOR_REPODATAS(repo, i, data)
1153 cbdata.keymap = solv_calloc(n, sizeof(Id));
1154 cbdata.keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1155 repodataused = solv_calloc(repo->nrepodata, 1);
1160 /* add keys for STORAGE_SOLVABLE */
1161 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1165 if (i < SOLVABLE_PROVIDES)
1166 keyd.type = REPOKEY_TYPE_ID;
1167 else if (i < RPM_RPMDBID)
1168 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1170 keyd.type = REPOKEY_TYPE_NUM;
1172 keyd.storage = KEY_STORAGE_SOLVABLE;
1175 keyd.storage = keyfilter(repo, &keyd, kfdata);
1176 if (keyd.storage == KEY_STORAGE_DROPPED)
1178 keyd.storage = KEY_STORAGE_SOLVABLE;
1182 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1185 if (repo->nsolvables)
1188 keyd.name = REPOSITORY_SOLVABLES;
1189 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1191 keyd.storage = KEY_STORAGE_INCORE;
1192 cbdata.keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1200 n = ID_NUM_INTERNAL;
1202 FOR_REPODATAS(repo, i, data)
1204 cbdata.keymapstart[i] = n;
1205 cbdata.keymap[n++] = 0; /* key 0 */
1211 /* check if we want this repodata */
1212 memset(&keyd, 0, sizeof(keyd));
1216 if (keyfilter(repo, &keyd, kfdata) == -1)
1219 for (j = 1; j < data->nkeys; j++, n++)
1221 key = data->keys + j;
1222 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1224 cbdata.keymap[n] = cbdata.keymap[key->name];
1227 if (key->type == REPOKEY_TYPE_DELETED)
1229 cbdata.keymap[n] = 0;
1232 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1234 Repokey keyd = *key;
1235 keyd.size = repodata_globalize_id(data, key->size, 1);
1236 id = repodata_key2id(&target, &keyd, 0);
1239 id = repodata_key2id(&target, key, 0);
1242 Repokey keyd = *key;
1243 keyd.storage = KEY_STORAGE_INCORE;
1244 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1245 keyd.size = repodata_globalize_id(data, key->size, 1);
1246 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1250 keyd.storage = keyfilter(repo, &keyd, kfdata);
1251 if (keyd.storage == KEY_STORAGE_DROPPED)
1253 cbdata.keymap[n] = 0;
1257 id = repodata_key2id(&target, &keyd, 1);
1259 cbdata.keymap[n] = id;
1260 /* load repodata if not already loaded */
1261 if (data->state == REPODATA_STUB)
1263 if (data->loadcallback)
1264 data->loadcallback(data);
1266 data->state = REPODATA_ERROR;
1267 if (data->state != REPODATA_ERROR)
1269 /* redo this repodata! */
1271 n = cbdata.keymapstart[i];
1275 if (data->state == REPODATA_ERROR)
1278 cbdata.keymap[n] = 0;
1282 repodataused[i] = 1;
1283 anyrepodataused = 1;
1284 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1285 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1287 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1289 idused = 1; /* dirs also use ids */
1292 if (key->type == REPOKEY_TYPE_DIRSTRARRAY && key->name == SOLVABLE_FILELIST)
1294 /* is this a file list extension */
1295 if (is_filelist_extension(data))
1297 /* hmm, we have a file list extension. Kill filelist of other repodata.
1298 * XXX: this is wrong, as the extension does not need to cover all
1299 * solvables of the other repodata */
1301 cbdata.keymap[lastfilelistn] = 0;
1309 if (data->localpool)
1312 poolusage = 3; /* need own pool */
1316 spool = &data->spool;
1323 else if (poolusage != 1)
1324 poolusage = 3; /* need own pool */
1330 dirpoolusage = 3; /* need own dirpool */
1334 dirpool = &data->dirpool;
1341 /* 0: no pool needed at all */
1342 /* 1: use global pool */
1343 /* 2: use repodata local pool */
1344 /* 3: need own pool */
1347 spool = &target.spool;
1348 /* hack: reuse global pool data so we don't have to map pool ids */
1351 stringpool_free(spool);
1352 stringpool_clone(spool, &pool->ss);
1354 cbdata.ownspool = spool;
1356 else if (poolusage == 0 || poolusage == 1)
1362 if (dirpoolusage == 3)
1364 dirpool = &target.dirpool;
1366 cbdata.owndirpool = dirpool;
1369 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1372 /********************************************************************/
1374 fprintf(stderr, "poolusage: %d\n", poolusage);
1375 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1376 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1377 for (i = 1; i < target.nkeys; i++)
1378 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1381 /* copy keys if requested */
1385 for (i = 1; i < target.nkeys; i++)
1386 queue_push2(keyq, target.keys[i].name, target.keys[i].type);
1391 /* put all the keys we need in our string pool */
1392 /* put mapped ids right into target.keys */
1393 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1395 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1396 if (key->type == REPOKEY_TYPE_CONSTANTID)
1398 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1399 type_constantid = key->type;
1400 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1403 key->type = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1406 stringpool_freehash(spool); /* free some mem */
1410 /********************************************************************/
1412 /* set needed count of all strings and rels,
1413 * find which keys are used in the solvables
1414 * put all strings in own spool
1417 reloff = spool->nstrings;
1419 reloff = (reloff + NEEDED_BLOCK) & ~NEEDED_BLOCK;
1421 needid = calloc(reloff + pool->nrels, sizeof(*needid));
1422 needid[0].map = reloff;
1424 cbdata.needid = needid;
1425 cbdata.schema = solv_calloc(target.nkeys, sizeof(Id));
1426 cbdata.sp = cbdata.schema;
1427 cbdata.solvschemata = solv_calloc(repo->nsolvables, sizeof(Id));
1429 /* create main schema */
1430 cbdata.sp = cbdata.schema;
1431 /* collect all other data from all repodatas */
1432 /* XXX: merge arrays of equal keys? */
1433 FOR_REPODATAS(repo, j, data)
1435 if (!repodataused[j])
1437 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1440 /* add solvables if needed (may revert later) */
1441 if (repo->nsolvables)
1443 *sp++ = cbdata.keymap[REPOSITORY_SOLVABLES];
1444 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size++;
1447 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1449 idarraydata = repo->idarraydata;
1451 anysolvableused = 0;
1452 cbdata.doingsolvables = 1;
1453 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1455 if (s->repo != repo)
1458 /* set schema info, keep in sync with further down */
1460 if (cbdata.keymap[SOLVABLE_NAME])
1462 *sp++ = cbdata.keymap[SOLVABLE_NAME];
1463 needid[s->name].need++;
1465 if (cbdata.keymap[SOLVABLE_ARCH])
1467 *sp++ = cbdata.keymap[SOLVABLE_ARCH];
1468 needid[s->arch].need++;
1470 if (cbdata.keymap[SOLVABLE_EVR])
1472 *sp++ = cbdata.keymap[SOLVABLE_EVR];
1473 needid[s->evr].need++;
1475 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1477 *sp++ = cbdata.keymap[SOLVABLE_VENDOR];
1478 needid[s->vendor].need++;
1480 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1482 *sp++ = cbdata.keymap[SOLVABLE_PROVIDES];
1483 target.keys[cbdata.keymap[SOLVABLE_PROVIDES]].size += incneedidarray(pool, idarraydata + s->provides, needid);
1485 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1487 *sp++ = cbdata.keymap[SOLVABLE_OBSOLETES];
1488 target.keys[cbdata.keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(pool, idarraydata + s->obsoletes, needid);
1490 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1492 *sp++ = cbdata.keymap[SOLVABLE_CONFLICTS];
1493 target.keys[cbdata.keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(pool, idarraydata + s->conflicts, needid);
1495 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1497 *sp++ = cbdata.keymap[SOLVABLE_REQUIRES];
1498 target.keys[cbdata.keymap[SOLVABLE_REQUIRES]].size += incneedidarray(pool, idarraydata + s->requires, needid);
1500 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1502 *sp++ = cbdata.keymap[SOLVABLE_RECOMMENDS];
1503 target.keys[cbdata.keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(pool, idarraydata + s->recommends, needid);
1505 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1507 *sp++ = cbdata.keymap[SOLVABLE_SUGGESTS];
1508 target.keys[cbdata.keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(pool, idarraydata + s->suggests, needid);
1510 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1512 *sp++ = cbdata.keymap[SOLVABLE_SUPPLEMENTS];
1513 target.keys[cbdata.keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(pool, idarraydata + s->supplements, needid);
1515 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1517 *sp++ = cbdata.keymap[SOLVABLE_ENHANCES];
1518 target.keys[cbdata.keymap[SOLVABLE_ENHANCES]].size += incneedidarray(pool, idarraydata + s->enhances, needid);
1520 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1522 *sp++ = cbdata.keymap[RPM_RPMDBID];
1523 target.keys[cbdata.keymap[RPM_RPMDBID]].size++;
1527 if (anyrepodataused)
1529 FOR_REPODATAS(repo, j, data)
1531 if (!repodataused[j])
1533 if (i < data->start || i >= data->end)
1535 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_needed, &cbdata);
1536 needid = cbdata.needid;
1540 cbdata.solvschemata[n] = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1541 if (cbdata.solvschemata[n])
1542 anysolvableused = 1;
1545 cbdata.doingsolvables = 0;
1546 assert(n == repo->nsolvables);
1548 if (repo->nsolvables && !anysolvableused)
1550 /* strip off solvable from the main schema */
1551 target.keys[cbdata.keymap[REPOSITORY_SOLVABLES]].size = 0;
1553 for (i = 0; target.schemadata[target.schemata[mainschema] + i]; i++)
1555 *sp = target.schemadata[target.schemata[mainschema] + i];
1556 if (*sp != cbdata.keymap[REPOSITORY_SOLVABLES])
1559 assert(target.schemadatalen == target.schemata[mainschema] + i + 1);
1561 target.schemadatalen = target.schemata[mainschema];
1563 repodata_free_schemahash(&target);
1564 mainschema = repodata_schema2id(cbdata.target, cbdata.schema, 1);
1567 /********************************************************************/
1569 /* remove unused keys */
1570 keyused = solv_calloc(target.nkeys, sizeof(Id));
1571 for (i = 1; i < (int)target.schemadatalen; i++)
1572 keyused[target.schemadata[i]] = 1;
1574 for (n = i = 1; i < target.nkeys; i++)
1581 target.keys[n] = target.keys[i];
1584 keyq->elements[2 * n - 2] = keyq->elements[2 * i - 2];
1585 keyq->elements[2 * n - 1] = keyq->elements[2 * i - 1];
1592 queue_truncate(keyq, 2 * n - 2);
1594 /* update schema data to the new key ids */
1595 for (i = 1; i < (int)target.schemadatalen; i++)
1596 target.schemadata[i] = keyused[target.schemadata[i]];
1597 /* update keymap to the new key ids */
1598 for (i = 0; i < cbdata.nkeymap; i++)
1599 cbdata.keymap[i] = keyused[cbdata.keymap[i]];
1600 keyused = solv_free(keyused);
1602 /* increment needid of the used keys, they are already mapped to
1603 * the correct string pool */
1604 for (i = 1; i < target.nkeys; i++)
1606 if (target.keys[i].type == type_constantid)
1607 needid[target.keys[i].size].need++;
1608 needid[target.keys[i].name].need++;
1609 needid[target.keys[i].type].need++;
1612 /********************************************************************/
1614 if (dirpool && cbdata.dirused && !cbdata.dirused[0])
1616 /* no dirs used at all */
1617 cbdata.dirused = solv_free(cbdata.dirused);
1621 /* increment need id for used dir components */
1624 /* if we have own dirpool, all entries in it are used.
1625 also, all comp ids are already mapped by putinowndirpool(),
1626 so we can simply increment needid.
1627 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1628 /* else we re-use a dirpool of repodata "dirpooldata".
1629 dirused tells us which of the ids are used.
1630 we need to map comp ids if we generate a new pool.
1631 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1632 for (i = 1; i < dirpool->ndirs; i++)
1635 fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
1637 if (cbdata.dirused && !cbdata.dirused[i])
1639 id = dirpool->dirs[i];
1642 if (dirpooldata && cbdata.ownspool && id > 1)
1644 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1645 needid = cbdata.needid;
1652 /********************************************************************/
1655 * create mapping table, new keys are sorted by needid[].need
1657 * needid[key].need : old key -> new key
1658 * needid[key].map : new key -> old key
1661 /* zero out id 0 and rel 0 just in case */
1662 reloff = needid[0].map;
1664 needid[reloff].need = 0;
1666 for (i = 1; i < reloff + pool->nrels; i++)
1670 solv_sort(needid + 1, spool->nstrings - 1, sizeof(*needid), needid_cmp_need_s, spool);
1672 /* make first entry '' */
1674 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1676 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1677 /* now needid is in new order, needid[newid].map -> oldid */
1679 /* calculate string space size, also zero out needid[].need */
1681 for (i = 1; i < reloff; i++)
1683 if (!needid[i].need)
1684 break; /* as we have sorted, every entry after this also has need == 0 */
1686 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1688 nstrings = i; /* our new string id end */
1690 /* make needid[oldid].need point to newid */
1691 for (i = 1; i < nstrings; i++)
1692 needid[needid[i].map].need = i;
1694 /* same as above for relations */
1695 for (i = 0; i < pool->nrels; i++)
1697 if (!needid[reloff + i].need)
1699 needid[reloff + i].need = 0;
1701 nrels = i; /* our new rel id end */
1703 for (i = 0; i < nrels; i++)
1704 needid[needid[reloff + i].map].need = nstrings + i;
1706 /* now we have: needid[oldid].need -> newid
1707 needid[newid].map -> oldid
1708 both for strings and relations */
1711 /********************************************************************/
1717 /* create our new target directory structure by traversing through all
1718 * used dirs. This will concatenate blocks with the same parent
1719 * directory into single blocks.
1720 * Instead of components, traverse_dirs stores the old dirids,
1721 * we will change this in the second step below */
1722 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1723 if (cbdata.dirused && !cbdata.dirused[1])
1724 cbdata.dirused[1] = 1; /* always want / entry */
1725 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1727 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1729 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1730 /* change dirmap so that it maps from "new dirid" to "new compid" */
1731 if (!cbdata.dirused)
1732 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1733 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1734 for (i = 1; i < ndirmap; i++)
1738 cbdata.dirused[dirmap[i]] = i;
1739 id = dirpool->dirs[dirmap[i]];
1740 if (dirpooldata && cbdata.ownspool && id > 1)
1741 id = putinownpool(&cbdata, dirpooldata->localpool ? &dirpooldata->spool : &pool->ss, id);
1742 dirmap[i] = needid[id].need;
1744 /* now the new target directory structure is complete (dirmap), and we have
1745 * dirused[olddirid] -> newdirid */
1748 /********************************************************************/
1751 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1754 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1756 xd = cbdata.extdata;
1757 cbdata.current_sub = 0;
1758 /* add main schema */
1760 data_addid(xd, mainschema);
1763 FOR_REPODATAS(repo, j, data)
1765 if (!repodataused[j])
1767 repodata_search(data, SOLVID_META, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1771 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1772 cbdata.maxdata = xd->len - cbdata.lastlen;
1773 cbdata.lastlen = xd->len;
1775 if (anysolvableused)
1777 data_addid(xd, repo->nsolvables); /* FLEXARRAY nentries */
1778 cbdata.doingsolvables = 1;
1780 /* check if we can do the special filelist memory optimization */
1781 if (anyrepodataused)
1783 for (i = 1; i < target.nkeys; i++)
1784 if (target.keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1785 cbdata.filelistmode |= cbdata.filelistmode == 0 && target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY ? 1 : 2;
1786 else if (target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY)
1787 cbdata.filelistmode = 2;
1788 if (cbdata.filelistmode != 1)
1789 cbdata.filelistmode = 0;
1792 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
1794 if (s->repo != repo)
1796 data_addid(xd, cbdata.solvschemata[n]);
1797 if (cbdata.keymap[SOLVABLE_NAME])
1798 data_addid(xd, needid[s->name].need);
1799 if (cbdata.keymap[SOLVABLE_ARCH])
1800 data_addid(xd, needid[s->arch].need);
1801 if (cbdata.keymap[SOLVABLE_EVR])
1802 data_addid(xd, needid[s->evr].need);
1803 if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR])
1804 data_addid(xd, needid[s->vendor].need);
1805 if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES])
1806 data_addidarray_sort(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
1807 if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES])
1808 data_addidarray_sort(xd, pool, needid, idarraydata + s->obsoletes, 0);
1809 if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS])
1810 data_addidarray_sort(xd, pool, needid, idarraydata + s->conflicts, 0);
1811 if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES])
1812 data_addidarray_sort(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
1813 if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS])
1814 data_addidarray_sort(xd, pool, needid, idarraydata + s->recommends, 0);
1815 if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS])
1816 data_addidarray_sort(xd, pool, needid, idarraydata + s->suggests, 0);
1817 if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS])
1818 data_addidarray_sort(xd, pool, needid, idarraydata + s->supplements, 0);
1819 if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES])
1820 data_addidarray_sort(xd, pool, needid, idarraydata + s->enhances, 0);
1821 if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID])
1822 data_addid(xd, repo->rpmdbid[i - repo->start]);
1823 if (anyrepodataused)
1826 FOR_REPODATAS(repo, j, data)
1828 if (!repodataused[j])
1830 if (i < data->start || i >= data->end)
1832 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
1835 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1836 cbdata.maxdata = xd->len - cbdata.lastlen;
1837 cbdata.lastlen = xd->len;
1840 cbdata.doingsolvables = 0;
1843 assert(cbdata.current_sub == cbdata.nsubschemata);
1844 if (cbdata.subschemata)
1846 cbdata.subschemata = solv_free(cbdata.subschemata);
1847 cbdata.nsubschemata = 0;
1850 /********************************************************************/
1856 /* write file header */
1857 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1858 write_u32(&target, SOLV_VERSION_8);
1862 write_u32(&target, nstrings);
1863 write_u32(&target, nrels);
1864 write_u32(&target, ndirmap);
1865 write_u32(&target, anysolvableused ? repo->nsolvables : 0);
1866 write_u32(&target, target.nkeys);
1867 write_u32(&target, target.nschemata);
1869 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1870 solv_flags |= SOLV_FLAG_SIZE_BYTES;
1871 write_u32(&target, solv_flags);
1876 * calculate prefix encoding of the strings
1878 unsigned char *prefixcomp = solv_malloc(nstrings);
1879 unsigned int compsum = 0;
1883 for (i = 1; i < nstrings; i++)
1885 char *str = spool->stringspace + spool->strings[needid[i].map];
1887 for (same = 0; same < 255; same++)
1888 if (!old_str[same] || old_str[same] != str[same])
1890 prefixcomp[i] = same;
1898 write_u32(&target, sizeid);
1899 /* we save compsum bytes but need 1 extra byte for every string */
1900 write_u32(&target, sizeid + nstrings - 1 - compsum);
1901 for (i = 1; i < nstrings; i++)
1903 char *str = spool->stringspace + spool->strings[needid[i].map];
1904 write_u8(&target, prefixcomp[i]);
1905 write_str(&target, str + prefixcomp[i]);
1907 solv_free(prefixcomp);
1911 write_u32(&target, 0);
1912 write_u32(&target, 0);
1918 for (i = 0; i < nrels; i++)
1920 ran = pool->rels + (needid[reloff + i].map - reloff);
1921 write_id(&target, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
1922 write_id(&target, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
1923 write_u8(&target, ran->flags);
1927 * write dirs (skip both root and / entry)
1929 for (i = 2; i < ndirmap; i++)
1932 write_id(&target, dirmap[i]);
1934 write_id(&target, nstrings - dirmap[i]);
1941 for (i = 1; i < target.nkeys; i++)
1943 write_id(&target, needid[target.keys[i].name].need);
1944 write_id(&target, needid[target.keys[i].type].need);
1945 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1947 if (target.keys[i].type == type_constantid)
1948 write_id(&target, needid[target.keys[i].size].need);
1950 write_id(&target, target.keys[i].size);
1953 write_id(&target, cbdata.extdata[i].len);
1954 write_id(&target, target.keys[i].storage);
1960 write_id(&target, target.schemadatalen); /* XXX -1? */
1961 for (i = 1; i < target.nschemata; i++)
1962 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
1964 /********************************************************************/
1966 write_id(&target, cbdata.maxdata);
1967 write_id(&target, cbdata.extdata[0].len);
1968 if (cbdata.extdata[0].len)
1969 write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
1970 solv_free(cbdata.extdata[0].buf);
1972 /* do we have vertical data? */
1973 for (i = 1; i < target.nkeys; i++)
1974 if (cbdata.extdata[i].len)
1976 if (i < target.nkeys)
1978 /* yes, write it in pages */
1979 unsigned char vpage[REPOPAGE_BLOBSIZE];
1982 write_u32(&target, REPOPAGE_BLOBSIZE);
1983 for (i = 1; i < target.nkeys; i++)
1984 if (cbdata.extdata[i].len)
1986 if (cbdata.filelistmode)
1988 lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
1990 if (cbdata.filelistmode && i < target.nkeys)
1992 /* ok, just this single extdata, which is a filelist */
1993 xd = cbdata.extdata + i;
1995 cbdata.filelistmode = -1;
1996 for (j = 0; j < cbdata.nkeymap; j++)
1997 if (cbdata.keymap[j] != i)
1998 cbdata.keymap[j] = 0;
1999 for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
2001 if (s->repo != repo)
2003 FOR_REPODATAS(repo, j, data)
2005 if (!repodataused[j])
2007 if (i < data->start || i >= data->end)
2009 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
2011 if (xd->len > 1024 * 1024)
2013 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2018 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2021 write_compressed_page(&target, vpage, lpage);
2024 for (i = 1; i < target.nkeys; i++)
2025 solv_free(cbdata.extdata[i].buf);
2026 solv_free(cbdata.extdata);
2029 repodata_freedata(&target);
2032 solv_free(cbdata.solvschemata);
2033 solv_free(cbdata.schema);
2035 solv_free(cbdata.keymap);
2036 solv_free(cbdata.keymapstart);
2037 solv_free(cbdata.dirused);
2038 solv_free(repodataused);
2039 return target.error;
2042 struct repodata_write_data {
2043 int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata);
2049 repodata_write_keyfilter(Repo *repo, Repokey *key, void *kfdata)
2051 struct repodata_write_data *wd = kfdata;
2053 /* XXX: special repodata selection hack */
2054 if (key->name == 1 && key->size != wd->repodataid)
2056 if (key->storage == KEY_STORAGE_SOLVABLE)
2057 return KEY_STORAGE_DROPPED; /* not part of this repodata */
2059 return (*wd->keyfilter)(repo, key, wd->kfdata);
2060 return key->storage;
2064 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2066 struct repodata_write_data wd;
2068 wd.keyfilter = keyfilter;
2070 wd.repodataid = data->repodataid;
2071 return repo_write_filtered(data->repo, fp, repodata_write_keyfilter, &wd, keyq);
2075 repodata_write(Repodata *data, FILE *fp)
2077 return repodata_write_filtered(data, fp, repo_write_stdkeyfilter, 0, 0);
2081 repo_write(Repo *repo, FILE *fp)
2083 return repo_write_filtered(repo, fp, repo_write_stdkeyfilter, 0, 0);