2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
29 #include "repo_write.h"
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
35 typedef struct needid {
41 #define NEEDIDOFF(id) (ISRELDEP(id) ? (needid[0].map + GETRELID(id)) : id)
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
48 * return size of array (including trailing zero)
53 incneedid(Id id, NeedId *needid)
55 needid[NEEDIDOFF(id)].need++;
59 incneedidarray(Id *idarray, NeedId *needid)
64 while ((id = *idarray++) != 0)
67 needid[NEEDIDOFF(id)].need++;
78 needid_cmp_need(const void *ap, const void *bp, void *dp)
83 r = b->need - a->need;
86 return a->map - b->map;
90 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
94 Stringpool *spool = dp;
99 r = b->need - a->need;
102 as = spool->stringspace + spool->strings[a->map];
103 bs = spool->stringspace + spool->strings[b->map];
104 return strcmp(as, bs);
108 /*------------------------------------------------------------------*/
109 /* output helper routines, used for writing the header */
110 /* (the data itself is accumulated in memory and written with
118 write_u32(Repodata *data, unsigned int x)
123 if (putc(x >> 24, fp) == EOF ||
124 putc(x >> 16, fp) == EOF ||
125 putc(x >> 8, fp) == EOF ||
128 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
138 write_u8(Repodata *data, unsigned int x)
142 if (putc(x, data->fp) == EOF)
144 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
153 write_blob(Repodata *data, void *blob, int len)
157 if (len && fwrite(blob, len, 1, data->fp) != 1)
159 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
168 write_id(Repodata *data, Id x)
176 putc((x >> 28) | 128, fp);
178 putc((x >> 21) | 128, fp);
179 putc((x >> 14) | 128, fp);
182 putc((x >> 7) | 128, fp);
183 if (putc(x & 127, fp) == EOF)
185 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
190 write_id_eof(Repodata *data, Id x, int eof)
193 x = (x & 63) | ((x & ~63) << 1);
194 write_id(data, x | (eof ? 0 : 64));
200 write_str(Repodata *data, const char *str)
204 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
206 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
215 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
229 id = needid[NEEDIDOFF(id)].need;
231 id = (id & 63) | ((id & ~63) << 1);
237 write_id(data, id | 64);
251 Stringpool *ownspool;
253 int clonepool; /* are the pool ids cloned into ownspool? */
255 Id *keymap; /* keymap for this repodata */
259 Id *schema; /* schema construction space */
260 Id *sp; /* pointer in above */
266 struct extdata *extdata;
270 Id vstart; /* offset of key in vertical data */
275 int doingsolvables; /* working on solvables data */
278 Id lastdirid; /* last dir id seen in this repodata */
279 Id lastdirid_own; /* last dir id put in own pool */
282 #define NEEDID_BLOCK 1023
283 #define SCHEMATA_BLOCK 31
284 #define EXTDATA_BLOCK 4095
287 data_addid(struct extdata *xd, Id sx)
289 unsigned int x = (unsigned int)sx;
292 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
293 dp = xd->buf + xd->len;
298 *dp++ = (x >> 28) | 128;
300 *dp++ = (x >> 21) | 128;
301 *dp++ = (x >> 14) | 128;
304 *dp++ = (x >> 7) | 128;
306 xd->len = dp - xd->buf;
310 data_addideof(struct extdata *xd, Id sx, int eof)
312 unsigned int x = (unsigned int)sx;
315 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
316 dp = xd->buf + xd->len;
321 *dp++ = (x >> 27) | 128;
323 *dp++ = (x >> 20) | 128;
324 *dp++ = (x >> 13) | 128;
327 *dp++ = (x >> 6) | 128;
328 *dp++ = eof ? (x & 63) : (x & 63) | 64;
329 xd->len = dp - xd->buf;
333 data_addideof_len(Id sx)
335 unsigned int x = (unsigned int)sx;
340 return x >= (1 << 20) ? 4 : 3;
342 return x >= (1 << 6) ? 2 : 1;
346 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
352 data_addid(xd, (Id)(hx >> 3));
353 xd->buf[xd->len - 1] |= 128;
356 data_addid(xd, (Id)(x | 0x80000000));
357 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
360 data_addid(xd, (Id)x);
363 #define USE_REL_IDARRAY
364 #ifdef USE_REL_IDARRAY
367 cmp_ids(const void *pa, const void *pb, void *dp)
375 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
388 for (len = 0; len < 64 && ids[len]; len++)
392 id = needid[NEEDIDOFF(id)].need;
397 for (i = len + 1; ids[i]; i++)
399 sids = solv_malloc2(i, sizeof(Id));
400 memcpy(sids, lids, 64 * sizeof(Id));
401 for (; ids[len]; len++)
405 id = needid[NEEDIDOFF(id)].need;
412 /* That bloody solvable:prereqmarker needs to stay in position :-( */
414 marker = needid[marker].need;
415 for (i = 0; i < len; i++)
416 if (sids[i] == marker)
419 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
421 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
425 /* The differencing above produces many runs of ones and twos. I tried
426 fairly elaborate schemes to RLE those, but they give only very mediocre
427 improvements in compression, as coding the escapes costs quite some
428 space. Even if they are coded only as bits in IDs. The best improvement
429 was about 2.7% for the whole .solv file. It's probably better to
430 invest some complexity into sharing idarrays, than RLEing. */
431 for (i = 0; i < len - 1; i++)
434 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
435 hence all real differences are offsetted by 1. Otherwise we would
436 have to handle negative differences, which would cost code space for
437 the encoding of the sign. We loose the exact mapping of prereq here,
438 but we know the result, so we can recover from that in the reader. */
446 /* XXX If difference is zero we have multiple equal elements,
447 we might want to skip writing them out. */
448 data_addideof(xd, id, 0);
455 data_addideof(xd, id, 1);
463 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
471 while ((id = *ids++) != 0)
474 id = needid[NEEDIDOFF(id)].need;
475 data_addideof(xd, id, *ids ? 0 : 1);
482 data_addblob(struct extdata *xd, unsigned char *blob, int len)
484 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
485 memcpy(xd->buf + xd->len, blob, len);
489 /* grow needid array so that it contains the specified id */
491 grow_needid(struct cbdata *cbdata, Id id)
493 int oldoff = cbdata->needid[0].map;
494 int newoff = (id + 1 + NEEDID_BLOCK) & ~NEEDID_BLOCK;
495 int nrels = cbdata->pool->nrels;
496 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
498 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
499 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
500 cbdata->needid[0].map = newoff;
504 putinownpool(struct cbdata *cbdata, Repodata *data, Id id)
506 Stringpool *ss = data->localpool ? &data->spool : &cbdata->pool->ss;
507 const char *str = stringpool_id2str(ss, id);
508 id = stringpool_str2id(cbdata->ownspool, str, 1);
509 if (id >= cbdata->needid[0].map)
510 grow_needid(cbdata, id);
515 putinowndirpool_slow(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
519 parent = dirpool_parent(dp, dir);
521 parent = putinowndirpool_slow(cbdata, data, dp, parent);
522 compid = dirpool_compid(dp, dir);
523 if (cbdata->ownspool && compid > 1 && (!cbdata->clonepool || data->localpool))
524 compid = putinownpool(cbdata, data, compid);
525 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
529 putinowndirpool(struct cbdata *cbdata, Repodata *data, Id dir)
531 if (dir && dir == cbdata->lastdirid)
532 return cbdata->lastdirid_own;
533 cbdata->lastdirid = dir;
534 cbdata->lastdirid_own = putinowndirpool_slow(cbdata, data, &data->dirpool, dir);
535 return cbdata->lastdirid_own;
540 * collect key/id/dirid usage information, create needed schemas
543 collect_needed_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
545 struct cbdata *cbdata = vcbdata;
550 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? (int)(s - cbdata->pool->solvables) : 0, s ? pool_id2str(cbdata->pool, s->name) : "", key->name, pool_id2str(cbdata->pool, key->name), key->type);
552 if (key->name == REPOSITORY_SOLVABLES)
553 return SEARCH_NEXT_KEY; /* we do not want this one */
555 rm = cbdata->keymap[key - data->keys];
557 return SEARCH_NEXT_KEY; /* we do not want this one */
559 /* record key in schema */
560 if (cbdata->sp[-1] != rm)
565 case REPOKEY_TYPE_ID:
566 case REPOKEY_TYPE_IDARRAY:
568 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
569 id = putinownpool(cbdata, data, id);
570 incneedid(id, cbdata->needid);
572 case REPOKEY_TYPE_DIR:
573 case REPOKEY_TYPE_DIRNUMNUMARRAY:
574 case REPOKEY_TYPE_DIRSTRARRAY:
576 if (cbdata->owndirpool)
577 putinowndirpool(cbdata, data, id);
579 cbdata->dirused[id] = 1;
581 case REPOKEY_TYPE_FIXARRAY:
582 case REPOKEY_TYPE_FLEXARRAY:
586 *cbdata->sp++ = 0; /* mark start */
590 /* just finished a schema, rewind to start */
591 Id *sp = cbdata->sp - 1;
595 if (kv->entry == 1 || key->type == REPOKEY_TYPE_FLEXARRAY)
597 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
598 cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
600 cbdata->sp = kv->eof == 2 ? sp - 1: sp;
610 collect_needed_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
612 /* set schema info, keep in sync with collect_data_solvable */
613 Repo *repo = s->repo;
615 NeedId *needid = cbdata->needid;
616 Repodata *target = cbdata->target;
617 Id *idarraydata = repo->idarraydata;
619 if (keymap[SOLVABLE_NAME])
621 *sp++ = keymap[SOLVABLE_NAME];
622 needid[s->name].need++;
624 if (keymap[SOLVABLE_ARCH])
626 *sp++ = keymap[SOLVABLE_ARCH];
627 needid[s->arch].need++;
629 if (keymap[SOLVABLE_EVR])
631 *sp++ = keymap[SOLVABLE_EVR];
632 needid[s->evr].need++;
634 if (s->vendor && keymap[SOLVABLE_VENDOR])
636 *sp++ = keymap[SOLVABLE_VENDOR];
637 needid[s->vendor].need++;
639 if (s->provides && keymap[SOLVABLE_PROVIDES])
641 *sp++ = keymap[SOLVABLE_PROVIDES];
642 target->keys[keymap[SOLVABLE_PROVIDES]].size += incneedidarray(idarraydata + s->provides, needid);
644 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
646 *sp++ = keymap[SOLVABLE_OBSOLETES];
647 target->keys[keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(idarraydata + s->obsoletes, needid);
649 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
651 *sp++ = keymap[SOLVABLE_CONFLICTS];
652 target->keys[keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(idarraydata + s->conflicts, needid);
654 if (s->requires && keymap[SOLVABLE_REQUIRES])
656 *sp++ = keymap[SOLVABLE_REQUIRES];
657 target->keys[keymap[SOLVABLE_REQUIRES]].size += incneedidarray(idarraydata + s->requires, needid);
659 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
661 *sp++ = keymap[SOLVABLE_RECOMMENDS];
662 target->keys[keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(idarraydata + s->recommends, needid);
664 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
666 *sp++ = keymap[SOLVABLE_SUGGESTS];
667 target->keys[keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(idarraydata + s->suggests, needid);
669 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
671 *sp++ = keymap[SOLVABLE_SUPPLEMENTS];
672 target->keys[keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(idarraydata + s->supplements, needid);
674 if (s->enhances && keymap[SOLVABLE_ENHANCES])
676 *sp++ = keymap[SOLVABLE_ENHANCES];
677 target->keys[keymap[SOLVABLE_ENHANCES]].size += incneedidarray(idarraydata + s->enhances, needid);
679 if (repo->rpmdbid && keymap[RPM_RPMDBID])
681 *sp++ = keymap[RPM_RPMDBID];
682 target->keys[keymap[RPM_RPMDBID]].size++;
690 * encode all of the data into the correct buffers
693 collect_data_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
695 struct cbdata *cbdata = vcbdata;
701 if (key->name == REPOSITORY_SOLVABLES)
702 return SEARCH_NEXT_KEY;
704 rm = cbdata->keymap[key - data->keys];
706 return SEARCH_NEXT_KEY; /* we do not want this one */
707 storage = cbdata->target->keys[rm].storage;
709 xd = cbdata->extdata + 0; /* incore buffer */
710 if (storage == KEY_STORAGE_VERTICAL_OFFSET)
712 xd += rm; /* vertical buffer */
713 if (cbdata->vstart == -1)
714 cbdata->vstart = xd->len;
718 case REPOKEY_TYPE_DELETED:
719 case REPOKEY_TYPE_VOID:
720 case REPOKEY_TYPE_CONSTANT:
721 case REPOKEY_TYPE_CONSTANTID:
723 case REPOKEY_TYPE_ID:
725 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
726 id = putinownpool(cbdata, data, id);
727 needid = cbdata->needid;
728 id = needid[NEEDIDOFF(id)].need;
731 case REPOKEY_TYPE_IDARRAY:
733 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
734 id = putinownpool(cbdata, data, id);
735 needid = cbdata->needid;
736 id = needid[NEEDIDOFF(id)].need;
737 data_addideof(xd, id, kv->eof);
739 case REPOKEY_TYPE_STR:
740 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
742 case REPOKEY_TYPE_MD5:
743 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
745 case REPOKEY_TYPE_SHA1:
746 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
748 case REPOKEY_TYPE_SHA224:
749 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
751 case REPOKEY_TYPE_SHA256:
752 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
754 case REPOKEY_TYPE_SHA384:
755 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
757 case REPOKEY_TYPE_SHA512:
758 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
761 case REPOKEY_TYPE_NUM:
762 data_addid64(xd, kv->num, kv->num2);
764 case REPOKEY_TYPE_DIR:
766 if (cbdata->owndirpool)
767 id = putinowndirpool(cbdata, data, id);
768 id = cbdata->dirused[id];
771 case REPOKEY_TYPE_BINARY:
772 data_addid(xd, kv->num);
774 data_addblob(xd, (unsigned char *)kv->str, kv->num);
776 case REPOKEY_TYPE_DIRNUMNUMARRAY:
778 if (cbdata->owndirpool)
779 id = putinowndirpool(cbdata, data, id);
780 id = cbdata->dirused[id];
782 data_addid(xd, kv->num);
783 data_addideof(xd, kv->num2, kv->eof);
785 case REPOKEY_TYPE_DIRSTRARRAY:
787 if (cbdata->owndirpool)
788 id = putinowndirpool(cbdata, data, id);
789 id = cbdata->dirused[id];
790 if (rm == cbdata->filelistmode)
792 /* postpone adding to xd, just update len to get the correct offsets into the incore data*/
793 xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
796 data_addideof(xd, id, kv->eof);
797 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
799 case REPOKEY_TYPE_FIXARRAY:
800 case REPOKEY_TYPE_FLEXARRAY:
802 data_addid(xd, kv->num);
803 if (kv->eof != 2 && (!kv->entry || key->type == REPOKEY_TYPE_FLEXARRAY))
804 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
805 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
807 if (xd->len - cbdata->lastlen > cbdata->maxdata)
808 cbdata->maxdata = xd->len - cbdata->lastlen;
809 cbdata->lastlen = xd->len;
813 cbdata->target->error = pool_error(cbdata->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
816 if (storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
818 /* we can re-use old data in the blob here! */
819 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
820 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
826 /* special version of collect_data_cb that collects just one single REPOKEY_TYPE_DIRSTRARRAY vertical data */
828 collect_filelist_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
830 struct cbdata *cbdata = vcbdata;
835 rm = cbdata->keymap[key - data->keys];
836 if (rm != cbdata->filelistmode)
837 return SEARCH_NEXT_KEY; /* we do not want this one */
839 if (cbdata->owndirpool)
840 id = putinowndirpool(cbdata, data, id);
841 id = cbdata->dirused[id];
842 xd = cbdata->extdata + rm; /* vertical buffer */
843 data_addideof(xd, id, kv->eof);
844 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
849 collect_data_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
851 Repo *repo = s->repo;
852 Pool *pool = repo->pool;
853 struct extdata *xd = cbdata->extdata;
854 NeedId *needid = cbdata->needid;
855 Id *idarraydata = repo->idarraydata;
857 if (keymap[SOLVABLE_NAME])
858 data_addid(xd, needid[s->name].need);
859 if (keymap[SOLVABLE_ARCH])
860 data_addid(xd, needid[s->arch].need);
861 if (keymap[SOLVABLE_EVR])
862 data_addid(xd, needid[s->evr].need);
863 if (s->vendor && keymap[SOLVABLE_VENDOR])
864 data_addid(xd, needid[s->vendor].need);
865 if (s->provides && keymap[SOLVABLE_PROVIDES])
866 data_adddepids(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
867 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
868 data_adddepids(xd, pool, needid, idarraydata + s->obsoletes, 0);
869 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
870 data_adddepids(xd, pool, needid, idarraydata + s->conflicts, 0);
871 if (s->requires && keymap[SOLVABLE_REQUIRES])
872 data_adddepids(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
873 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
874 data_adddepids(xd, pool, needid, idarraydata + s->recommends, 0);
875 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
876 data_adddepids(xd, pool, needid, idarraydata + s->suggests, 0);
877 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
878 data_adddepids(xd, pool, needid, idarraydata + s->supplements, 0);
879 if (s->enhances && keymap[SOLVABLE_ENHANCES])
880 data_adddepids(xd, pool, needid, idarraydata + s->enhances, 0);
881 if (repo->rpmdbid && keymap[RPM_RPMDBID])
882 data_addid(xd, repo->rpmdbid[(s - pool->solvables) - repo->start]);
885 /* traverse through directory with first child "dir" */
887 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
893 /* special case for '/', which has to come first */
896 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
898 if (used && !used[sib])
900 if (sib == 1 && parent == 1)
901 continue; /* already did that one above */
905 /* check if our block has some content */
907 return n - 1; /* nope, drop parent id again */
909 /* now go through all the siblings we just added and
910 * do recursive calls on them */
912 for (; parent < lastn; parent++)
914 sib = dirmap[parent];
915 if (used && used[sib] != 2) /* 2: used as parent */
917 child = dirpool_child(dp, sib);
920 dirmap[n++] = -parent; /* start new block */
921 n = traverse_dirs(dp, dirmap, n, child, used);
928 write_compressed_page(Repodata *data, unsigned char *page, int len)
931 unsigned char cpage[REPOPAGE_BLOBSIZE];
933 clen = repopagestore_compress_page(page, len, cpage, len - 1);
936 write_u32(data, len * 2);
937 write_blob(data, page, len);
941 write_u32(data, clen * 2 + 1);
942 write_blob(data, cpage, clen);
946 static Id verticals[] = {
948 SOLVABLE_DESCRIPTION,
960 SOLVABLE_CHANGELOG_AUTHOR,
961 SOLVABLE_CHANGELOG_TEXT,
965 static char *languagetags[] = {
967 "solvable:description:",
968 "solvable:messageins:",
969 "solvable:messagedel:",
975 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
980 for (i = 0; verticals[i]; i++)
981 if (key->name == verticals[i])
982 return KEY_STORAGE_VERTICAL_OFFSET;
983 keyname = pool_id2str(repo->pool, key->name);
984 for (i = 0; languagetags[i] != 0; i++)
985 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
986 return KEY_STORAGE_VERTICAL_OFFSET;
987 return KEY_STORAGE_INCORE;
991 write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
993 unsigned char *dp = xd->buf;
997 int ll = REPOPAGE_BLOBSIZE - lpage;
1000 memcpy(vpage + lpage, dp, ll);
1004 if (lpage == REPOPAGE_BLOBSIZE)
1006 write_compressed_page(target, vpage, lpage);
1015 create_keyskip(Repo *repo, Id entry, unsigned char *repodataused, Id **oldkeyskip)
1017 Repodata *data, *last = 0;
1021 if (repo->nrepodata <= 2)
1023 keyskip = *oldkeyskip;
1026 if (keyskip[1] >= 0x10000000)
1027 keyskip = solv_free(keyskip);
1029 keyskip[1] = keyskip[2];
1031 FOR_REPODATAS(repo, rdid, data)
1033 if (!repodataused[rdid])
1035 if (entry != SOLVID_META)
1037 if (entry < data->start || entry >= data->end)
1039 /* if repodataused is set we know that the state is AVAILABLE */
1040 if (!data->incoreoffset[entry - data->start])
1044 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1048 if (cnt <= 1) /* just one repodata means we don't need a keyskip */
1050 *oldkeyskip = keyskip;
1053 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1055 keyskip[2] = keyskip[1] + repo->nrepodata;
1056 *oldkeyskip = keyskip;
1065 repowriter_create(Repo *repo)
1067 Repowriter *writer = solv_calloc(1, sizeof(*writer));
1068 writer->repo = repo;
1069 writer->keyfilter = repo_write_stdkeyfilter;
1070 writer->repodatastart = 1;
1071 writer->repodataend = repo->nrepodata;
1072 writer->solvablestart = repo->start;
1073 writer->solvableend = repo->end;
1078 repowriter_free(Repowriter *writer)
1080 return solv_free(writer);
1084 repowriter_set_flags(Repowriter *writer, int flags)
1086 writer->flags = flags;
1090 repowriter_set_keyfilter(Repowriter *writer, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata)
1092 writer->keyfilter = keyfilter;
1093 writer->kfdata = kfdata;
1097 repowriter_set_keyqueue(Repowriter *writer, Queue *keyq)
1099 writer->keyq = keyq;
1103 repowriter_set_repodatarange(Repowriter *writer, int repodatastart, int repodataend)
1105 writer->repodatastart = repodatastart;
1106 writer->repodataend = repodataend;
1110 repowriter_set_solvablerange(Repowriter *writer, int solvablestart, int solvableend)
1112 writer->solvablestart = solvablestart;
1113 writer->solvableend = solvableend;
1117 * the code works the following way:
1119 * 1) find which keys should be written
1120 * 2) collect usage information for keys/ids/dirids, create schema
1122 * 3) use usage information to create mapping tables, so that often
1123 * used ids get a lower number
1124 * 4) encode data into buffers using the mapping tables
1125 * 5) write everything to disk
1128 repowriter_write(Repowriter *writer, FILE *fp)
1130 Repo *repo = writer->repo;
1131 Pool *pool = repo->pool;
1134 NeedId *needid, *needidp;
1135 int nstrings, nrels;
1136 unsigned int sizeid;
1137 unsigned int solv_flags;
1140 int searchflags = 0;
1144 Id *keymap; /* maps repo key to my key, 0 -> not used */
1146 int *keymapstart; /* maps repo number to keymap offset */
1152 unsigned char *repodataused;
1153 int anyrepodataused = 0;
1155 int solvablestart, solvableend;
1157 int anysolvableused = 0;
1160 struct cbdata cbdata;
1164 int poolusage, dirpoolusage;
1167 Repodata *data, *dirpooldata;
1174 Id mainschema, *mainschemakeys;
1178 Id type_constantid = 0;
1181 memset(&cbdata, 0, sizeof(cbdata));
1184 cbdata.target = ⌖
1186 repodata_initdata(&target, repo, 1);
1188 /* go through all repodata and find the keys we need */
1189 /* also unify keys */
1191 /* start with all KEY_STORAGE_SOLVABLE ids */
1193 n = ID_NUM_INTERNAL;
1194 FOR_REPODATAS(repo, i, data)
1197 keymap = solv_calloc(nkeymap, sizeof(Id));
1198 keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1199 repodataused = solv_calloc(repo->nrepodata, 1);
1204 if (!(writer->flags & REPOWRITER_NO_STORAGE_SOLVABLE))
1206 /* add keys for STORAGE_SOLVABLE */
1207 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1211 if (i < SOLVABLE_PROVIDES)
1212 keyd.type = REPOKEY_TYPE_ID;
1213 else if (i < RPM_RPMDBID)
1214 #ifdef USE_REL_IDARRAY
1215 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1217 keyd.type = REPOKEY_TYPE_IDARRAY;
1220 keyd.type = REPOKEY_TYPE_NUM;
1222 keyd.storage = KEY_STORAGE_SOLVABLE;
1223 if (writer->keyfilter)
1225 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1226 if (keyd.storage == KEY_STORAGE_DROPPED)
1228 keyd.storage = KEY_STORAGE_SOLVABLE;
1232 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1236 if (repo->nsolvables)
1239 keyd.name = REPOSITORY_SOLVABLES;
1240 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1242 keyd.storage = KEY_STORAGE_INCORE;
1243 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1251 n = ID_NUM_INTERNAL;
1252 FOR_REPODATAS(repo, i, data)
1254 int idused, dirused;
1255 if (i < writer->repodatastart || i >= writer->repodataend)
1257 if (writer->keyfilter && (writer->flags & REPOWRITER_LEGACY) != 0)
1259 /* ask keyfilter if we want this repodata */
1261 /* check if we want this repodata */
1262 memset(&keyd, 0, sizeof(keyd));
1266 if (writer->keyfilter(repo, &keyd, writer->kfdata) == -1)
1270 keymap[n++] = 0; /* key 0 */
1271 idused = dirused = 0;
1272 for (j = 1; j < data->nkeys; j++, n++)
1274 key = data->keys + j;
1275 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1277 keymap[n] = keymap[key->name];
1280 if (key->type == REPOKEY_TYPE_DELETED && (writer->flags & REPOWRITER_KEEP_TYPE_DELETED) == 0)
1285 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1287 Repokey keyd = *key;
1288 keyd.size = repodata_globalize_id(data, key->size, 1);
1289 id = repodata_key2id(&target, &keyd, 0);
1292 id = repodata_key2id(&target, key, 0);
1295 /* a new key. ask keyfilter if we want it before creating it */
1296 Repokey keyd = *key;
1297 keyd.storage = KEY_STORAGE_INCORE;
1298 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1299 keyd.size = repodata_globalize_id(data, key->size, 1);
1300 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1302 if (writer->keyfilter)
1304 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1305 if (keyd.storage == KEY_STORAGE_DROPPED)
1311 if (data->state != REPODATA_STUB)
1312 id = repodata_key2id(&target, &keyd, 1);
1315 /* load repodata if not already loaded */
1316 if (data->state == REPODATA_STUB)
1318 int oldnkeys = data->nkeys;
1319 repodata_load(data);
1320 if (oldnkeys != data->nkeys)
1322 nkeymap += data->nkeys - oldnkeys; /* grow/shrink keymap */
1323 keymap = solv_realloc2(keymap, nkeymap, sizeof(Id));
1325 if (data->state == REPODATA_AVAILABLE)
1327 /* redo this repodata! */
1333 if (data->state != REPODATA_AVAILABLE && data->state != REPODATA_LOADING)
1340 repodataused[i] = 1;
1341 anyrepodataused = 1;
1342 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1343 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1345 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1347 idused = 1; /* dirs also use ids */
1353 if (data->localpool)
1356 poolusage = 3; /* need own pool */
1360 spool = &data->spool;
1367 else if (poolusage != 1)
1368 poolusage = 3; /* need own pool */
1374 dirpoolusage = 3; /* need own dirpool */
1378 dirpool = &data->dirpool;
1383 nkeymap = n; /* update */
1385 /* 0: no pool needed at all */
1386 /* 1: use global pool */
1387 /* 2: use repodata local pool */
1388 /* 3: need own pool */
1393 spool = &target.spool;
1394 target.localpool = 1; /* so we can use repodata_translate */
1395 /* hack: reuse global pool data so we don't have to map pool ids */
1398 stringpool_free(spool);
1399 stringpool_clone(spool, &pool->ss);
1400 cbdata.clonepool = 1;
1402 cbdata.ownspool = spool;
1404 else if (poolusage == 0 || poolusage == 1)
1410 if (dirpoolusage == 3)
1412 /* dirpoolusage == 3 means that at least two repodata
1413 * areas have dir keys. This means that two areas have
1414 * idused set to 1, which results in poolusage being
1415 * either 1 (global pool) or 3 (own pool) */
1416 dirpool = &target.dirpool;
1418 cbdata.owndirpool = dirpool;
1421 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1424 /********************************************************************/
1426 fprintf(stderr, "poolusage: %d\n", poolusage);
1427 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1428 fprintf(stderr, "clonepool: %d\n", clonepool);
1429 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1430 for (i = 1; i < target.nkeys; i++)
1431 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1434 /********************************************************************/
1436 searchflags = SEARCH_SUB|SEARCH_ARRAYSENTINEL;
1437 if ((writer->flags & REPOWRITER_KEEP_TYPE_DELETED) != 0)
1438 searchflags |= SEARCH_KEEP_TYPE_DELETED;
1440 /* set needed count of all strings and rels,
1441 * find which keys are used in the solvables
1442 * put all strings in own spool
1445 reloff = spool->nstrings;
1446 if (cbdata.ownspool)
1447 reloff = (reloff + NEEDID_BLOCK) & ~NEEDID_BLOCK;
1448 else if (poolusage == 2)
1450 /* we'll need to put the key data into the spool,
1451 * so leave some room. 3 * nkeys is an upper bound */
1452 reloff += 3 * target.nkeys;
1455 needid = calloc(reloff + pool->nrels, sizeof(*needid));
1456 needid[0].map = reloff; /* remember size in case we need to grow */
1458 cbdata.needid = needid;
1459 cbdata.schema = solv_calloc(target.nkeys + 2, sizeof(Id));
1461 /* create main schema */
1462 cbdata.sp = cbdata.schema + 1;
1464 /* collect meta data from all repodatas */
1465 /* XXX: merge arrays of equal keys? */
1466 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1467 FOR_REPODATAS(repo, j, data)
1469 if (!repodataused[j])
1471 cbdata.keymap = keymap + keymapstart[j];
1472 cbdata.lastdirid = 0; /* clear dir mapping cache */
1473 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1475 needid = cbdata.needid; /* maybe relocated */
1477 /* add solvables if needed (may revert later) */
1478 if (repo->nsolvables)
1480 *sp++ = keymap[REPOSITORY_SOLVABLES];
1481 target.keys[keymap[REPOSITORY_SOLVABLES]].size++;
1484 /* stash away main schema (including terminating zero) */
1485 mainschemakeys = solv_memdup2(cbdata.schema + 1, sp - cbdata.schema, sizeof(Id));
1487 /* collect data for all solvables */
1488 solvschemata = solv_calloc(repo->nsolvables, sizeof(Id)); /* allocate upper bound */
1489 solvablestart = writer->solvablestart < repo->start ? repo->start : writer->solvablestart;
1490 solvableend = writer->solvableend > repo->end ? repo->end : writer->solvableend;
1491 anysolvableused = 0;
1492 nsolvables = 0; /* solvables we are going to write, will be <= repo->nsolvables */
1493 cbdata.doingsolvables = 1;
1494 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
1496 if (s->repo != repo)
1499 cbdata.sp = cbdata.schema + 1;
1500 collect_needed_solvable(&cbdata, s, keymap);
1502 if (anyrepodataused)
1504 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1505 FOR_REPODATAS(repo, j, data)
1507 if (!repodataused[j] || i < data->start || i >= data->end)
1509 cbdata.keymap = keymap + keymapstart[j];
1510 cbdata.lastdirid = 0;
1511 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1513 needid = cbdata.needid; /* maybe relocated */
1516 solvschemata[nsolvables] = repodata_schema2id(cbdata.target, cbdata.schema + 1, 1);
1517 if (solvschemata[nsolvables])
1518 anysolvableused = 1;
1521 cbdata.doingsolvables = 0;
1523 if (repo->nsolvables && !anysolvableused)
1525 /* strip off REPOSITORY_SOLVABLES from the main schema */
1526 for (sp = mainschemakeys; *sp; sp++)
1528 sp[-1] = 0; /* strip last entry */
1530 mainschema = repodata_schema2id(cbdata.target, mainschemakeys, 1);
1531 mainschemakeys = solv_free(mainschemakeys);
1533 /********************************************************************/
1535 /* remove unused keys */
1536 keyused = solv_calloc(target.nkeys, sizeof(Id));
1537 for (i = 1; i < (int)target.schemadatalen; i++)
1538 keyused[target.schemadata[i]] = 1;
1540 for (n = i = 1; i < target.nkeys; i++)
1545 target.keys[n] = target.keys[i];
1550 /* update schema data to the new key ids */
1551 for (i = 1; i < (int)target.schemadatalen; i++)
1552 target.schemadata[i] = keyused[target.schemadata[i]];
1553 /* update keymap to the new key ids */
1554 for (i = 0; i < nkeymap; i++)
1555 keymap[i] = keyused[keymap[i]];
1556 keyused = solv_free(keyused);
1558 /* copy keys if requested */
1561 queue_empty(writer->keyq);
1562 for (i = 1; i < target.nkeys; i++)
1563 queue_push2(writer->keyq, target.keys[i].name, target.keys[i].type);
1566 /********************************************************************/
1568 /* check if we can do the special filelist memory optimization
1569 * we do the check before the keys are mapped.
1570 * The optimization is done if there is just one vertical key and
1571 * it is of type REPOKEY_TYPE_DIRSTRARRAY */
1572 if (anysolvableused && anyrepodataused)
1574 for (i = 1; i < target.nkeys; i++)
1576 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1578 if (target.keys[i].type != REPOKEY_TYPE_DIRSTRARRAY || cbdata.filelistmode != 0)
1580 cbdata.filelistmode = 0;
1583 cbdata.filelistmode = i;
1587 /********************************************************************/
1591 /* put all the keys in our string pool */
1592 /* put mapped ids right into target.keys */
1593 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1595 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1596 id = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1597 if (key->type == REPOKEY_TYPE_CONSTANTID)
1599 type_constantid = id;
1600 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1605 stringpool_freehash(spool); /* free some mem */
1606 if (cbdata.ownspool && spool->nstrings > needid[0].map)
1608 grow_needid(&cbdata, spool->nstrings - 1);
1609 needid = cbdata.needid; /* we relocated */
1613 type_constantid = REPOKEY_TYPE_CONSTANTID;
1615 /* increment needid of the keys */
1616 for (i = 1; i < target.nkeys; i++)
1618 if (target.keys[i].type == type_constantid)
1619 needid[target.keys[i].size].need++;
1620 needid[target.keys[i].name].need++;
1621 needid[target.keys[i].type].need++;
1624 /********************************************************************/
1626 /* increment need id of all relations
1627 * if we refer to another relation, make sure that the
1628 * need value is it is bigger than our value so that
1631 reloff = needid[0].map;
1632 for (i = pool->nrels - 1, needidp = needid + (reloff + i); i > 0; i--, needidp--)
1637 /* we have some relations with a non-zero need */
1640 for (rd = pool->rels + i; i > 0; i--, rd--)
1642 int need = needid[reloff + i].need;
1649 if (needid[reloff + id].need < need + 1)
1650 needid[reloff + id].need = need + 1;
1654 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1656 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1657 if (id >= cbdata.needid[0].map)
1659 grow_needid(&cbdata, id);
1660 needid = cbdata.needid; /* we relocated */
1661 reloff = needid[0].map; /* we have a new offset */
1671 if (needid[reloff + id].need < need + 1)
1672 needid[reloff + id].need = need + 1;
1676 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1678 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1679 if (id >= cbdata.needid[0].map)
1681 grow_needid(&cbdata, id);
1682 needid = cbdata.needid; /* we relocated */
1683 reloff = needid[0].map; /* we have a new offset */
1691 /********************************************************************/
1693 /* increment need id for used dir components */
1694 if (cbdata.owndirpool)
1696 /* if we have own dirpool, all entries in it are used.
1697 also, all comp ids are already mapped by putinowndirpool(),
1698 so we can simply increment needid.
1699 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1700 for (i = 1; i < dirpool->ndirs; i++)
1702 id = dirpool->dirs[i];
1711 /* else we re-use a dirpool of repodata "dirpooldata".
1712 dirused tells us which of the ids are used.
1713 we need to map comp ids if we generate a new pool.
1714 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1715 for (i = dirpool->ndirs - 1; i > 0; i--)
1717 if (!cbdata.dirused[i])
1719 parent = dirpool_parent(dirpool, i); /* always < i */
1720 cbdata.dirused[parent] = 2; /* 2: used as parent */
1721 id = dirpool->dirs[i];
1724 if (cbdata.ownspool && id > 1 && (!cbdata.clonepool || dirpooldata->localpool))
1726 id = putinownpool(&cbdata, dirpooldata, id);
1727 needid = cbdata.needid;
1731 if (!cbdata.dirused[0])
1733 cbdata.dirused = solv_free(cbdata.dirused);
1739 /********************************************************************/
1742 * create mapping table, new keys are sorted by needid[].need
1744 * needid[key].need : old key -> new key
1745 * needid[key].map : new key -> old key
1748 /* zero out id 0 and rel 0 just in case */
1749 reloff = needid[0].map;
1751 needid[reloff].need = 0;
1753 for (i = 1; i < reloff + pool->nrels; i++)
1756 /* make first entry '' */
1758 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1759 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1760 /* now needid is in new order, needid[newid].map -> oldid */
1762 /* calculate string space size, also zero out needid[].need */
1764 for (i = 1; i < reloff; i++)
1766 if (!needid[i].need)
1767 break; /* as we have sorted, every entry after this also has need == 0 */
1769 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1771 nstrings = i; /* our new string id end */
1773 /* make needid[oldid].need point to newid */
1774 for (i = 1; i < nstrings; i++)
1775 needid[needid[i].map].need = i;
1777 /* same as above for relations */
1778 for (i = 0; i < pool->nrels; i++)
1780 if (!needid[reloff + i].need)
1782 needid[reloff + i].need = 0;
1784 nrels = i; /* our new rel id end */
1786 for (i = 0; i < nrels; i++)
1787 needid[needid[reloff + i].map].need = nstrings + i;
1789 /* now we have: needid[oldid].need -> newid
1790 needid[newid].map -> oldid
1791 both for strings and relations */
1794 /********************************************************************/
1798 if (dirpool && dirpool->ndirs)
1800 /* create our new target directory structure by traversing through all
1801 * used dirs. This will concatenate blocks with the same parent
1802 * directory into single blocks.
1803 * Instead of components, traverse_dirs stores the old dirids,
1804 * we will change this in the second step below */
1805 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1806 if (cbdata.dirused && !cbdata.dirused[1])
1808 cbdata.dirused[1] = 1; /* always want / entry */
1809 cbdata.dirused[0] = 2; /* always want / entry */
1811 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1813 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1815 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1816 /* change dirmap so that it maps from "new dirid" to "new compid" */
1817 if (!cbdata.dirused)
1818 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1819 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1820 for (i = 1; i < ndirmap; i++)
1824 cbdata.dirused[dirmap[i]] = i;
1825 id = dirpool->dirs[dirmap[i]];
1826 if (dirpooldata && cbdata.ownspool && id > 1)
1827 id = putinownpool(&cbdata, dirpooldata, id);
1828 dirmap[i] = needid[id].need;
1830 /* now the new target directory structure is complete (dirmap), and we have
1831 * dirused[olddirid] -> newdirid */
1834 /********************************************************************/
1837 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1839 * this must match the code above that creates the schema data!
1842 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1844 xd = cbdata.extdata;
1845 cbdata.current_sub = 0;
1846 /* add main schema */
1848 data_addid(xd, mainschema);
1850 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1851 FOR_REPODATAS(repo, j, data)
1853 if (!repodataused[j])
1855 cbdata.keymap = keymap + keymapstart[j];
1856 cbdata.lastdirid = 0;
1857 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1859 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1860 cbdata.maxdata = xd->len - cbdata.lastlen;
1861 cbdata.lastlen = xd->len;
1863 if (anysolvableused)
1865 data_addid(xd, nsolvables); /* FLEXARRAY nentries */
1866 cbdata.doingsolvables = 1;
1868 for (i = solvablestart, s = pool->solvables + i, n = 0; i < solvableend; i++, s++)
1870 if (s->repo != repo)
1872 data_addid(xd, solvschemata[n]);
1873 collect_data_solvable(&cbdata, s, keymap);
1874 if (anyrepodataused)
1876 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1878 FOR_REPODATAS(repo, j, data)
1880 if (!repodataused[j] || i < data->start || i >= data->end)
1882 cbdata.keymap = keymap + keymapstart[j];
1883 cbdata.lastdirid = 0;
1884 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1887 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1888 cbdata.maxdata = xd->len - cbdata.lastlen;
1889 cbdata.lastlen = xd->len;
1892 cbdata.doingsolvables = 0;
1895 assert(cbdata.current_sub == cbdata.nsubschemata);
1896 cbdata.subschemata = solv_free(cbdata.subschemata);
1897 cbdata.nsubschemata = 0;
1899 /********************************************************************/
1905 /* write file header */
1906 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1907 write_u32(&target, SOLV_VERSION_8);
1911 write_u32(&target, nstrings);
1912 write_u32(&target, nrels);
1913 write_u32(&target, ndirmap);
1914 write_u32(&target, anysolvableused ? nsolvables : 0);
1915 write_u32(&target, target.nkeys);
1916 write_u32(&target, target.nschemata);
1918 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1919 solv_flags |= SOLV_FLAG_SIZE_BYTES;
1920 write_u32(&target, solv_flags);
1925 * calculate prefix encoding of the strings
1927 unsigned char *prefixcomp = solv_malloc(nstrings);
1928 unsigned int compsum = 0;
1932 for (i = 1; i < nstrings; i++)
1934 char *str = spool->stringspace + spool->strings[needid[i].map];
1936 for (same = 0; same < 255; same++)
1937 if (!old_str[same] || old_str[same] != str[same])
1939 prefixcomp[i] = same;
1947 write_u32(&target, sizeid);
1948 /* we save compsum bytes but need 1 extra byte for every string */
1949 write_u32(&target, sizeid + nstrings - 1 - compsum);
1950 for (i = 1; i < nstrings; i++)
1952 char *str = spool->stringspace + spool->strings[needid[i].map];
1953 write_u8(&target, prefixcomp[i]);
1954 write_str(&target, str + prefixcomp[i]);
1956 solv_free(prefixcomp);
1960 write_u32(&target, 0);
1961 write_u32(&target, 0);
1967 for (i = 0; i < nrels; i++)
1969 Reldep *ran = pool->rels + (needid[reloff + i].map - reloff);
1970 write_id(&target, needid[NEEDIDOFF(ran->name)].need);
1971 write_id(&target, needid[NEEDIDOFF(ran->evr)].need);
1972 write_u8(&target, ran->flags);
1976 * write dirs (skip both root and / entry)
1978 for (i = 2; i < ndirmap; i++)
1981 write_id(&target, dirmap[i]);
1983 write_id(&target, nstrings - dirmap[i]);
1990 for (i = 1; i < target.nkeys; i++)
1992 write_id(&target, needid[target.keys[i].name].need);
1993 write_id(&target, needid[target.keys[i].type].need);
1994 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1996 if (target.keys[i].type == type_constantid)
1997 write_id(&target, needid[target.keys[i].size].need);
1999 write_id(&target, target.keys[i].size);
2002 write_id(&target, cbdata.extdata[i].len);
2003 write_id(&target, target.keys[i].storage);
2009 write_id(&target, target.schemadatalen); /* XXX -1? */
2010 for (i = 1; i < target.nschemata; i++)
2011 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
2016 write_id(&target, cbdata.maxdata);
2017 write_id(&target, cbdata.extdata[0].len);
2018 if (cbdata.extdata[0].len)
2019 write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
2020 solv_free(cbdata.extdata[0].buf);
2023 * write vertical data if we have any
2025 for (i = 1; i < target.nkeys; i++)
2026 if (cbdata.extdata[i].len)
2028 if (i < target.nkeys)
2030 /* have vertical data, write it in pages */
2031 unsigned char vpage[REPOPAGE_BLOBSIZE];
2034 write_u32(&target, REPOPAGE_BLOBSIZE);
2035 if (!cbdata.filelistmode)
2037 for (i = 1; i < target.nkeys; i++)
2038 if (cbdata.extdata[i].len)
2039 lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
2043 /* ok, just one single extdata which is of type REPOKEY_TYPE_DIRSTRARRAY */
2044 xd = cbdata.extdata + i;
2046 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
2047 FOR_REPODATAS(repo, j, data)
2049 if (!repodataused[j])
2051 cbdata.keymap = keymap + keymapstart[j];
2052 cbdata.lastdirid = 0;
2053 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2055 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
2057 if (s->repo != repo)
2059 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
2060 FOR_REPODATAS(repo, j, data)
2062 if (!repodataused[j] || i < data->start || i >= data->end)
2064 cbdata.keymap = keymap + keymapstart[j];
2065 cbdata.lastdirid = 0;
2066 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2068 if (xd->len > 1024 * 1024)
2070 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2075 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2078 write_compressed_page(&target, vpage, lpage);
2081 for (i = 1; i < target.nkeys; i++)
2082 solv_free(cbdata.extdata[i].buf);
2083 solv_free(cbdata.extdata);
2086 repodata_freedata(&target);
2089 solv_free(solvschemata);
2090 solv_free(cbdata.schema);
2093 solv_free(keymapstart);
2094 solv_free(cbdata.dirused);
2095 solv_free(repodataused);
2096 solv_free(oldkeyskip);
2097 return target.error;
2101 repo_write(Repo *repo, FILE *fp)
2104 Repowriter *writer = repowriter_create(repo);
2105 res = repowriter_write(writer, fp);
2106 repowriter_free(writer);
2111 repodata_write(Repodata *data, FILE *fp)
2114 Repowriter *writer = repowriter_create(data->repo);
2115 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2116 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
2117 res = repowriter_write(writer, fp);
2118 repowriter_free(writer);
2122 /* deprecated functions, do not use in new code! */
2124 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2127 Repowriter *writer = repowriter_create(repo);
2128 repowriter_set_flags(writer, REPOWRITER_LEGACY);
2129 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2130 repowriter_set_keyqueue(writer, keyq);
2131 res = repowriter_write(writer, fp);
2132 repowriter_free(writer);
2137 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2140 Repowriter *writer = repowriter_create(data->repo);
2141 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2142 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE | REPOWRITER_LEGACY);
2143 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2144 repowriter_set_keyqueue(writer, keyq);
2145 res = repowriter_write(writer, fp);
2146 repowriter_free(writer);