2 * Copyright (c) 2007-2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to a file in solv format
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
29 #include "repo_write.h"
32 #undef USE_IDARRAYBLOCK
33 #define USE_REL_IDARRAY
35 /*------------------------------------------------------------------*/
36 /* Id map optimizations */
38 typedef struct needid {
44 #define NEEDIDOFF(id) (ISRELDEP(id) ? (needid[0].map + GETRELID(id)) : id)
48 * idarray: array of Ids, ID_NULL terminated
49 * needid: array of Id->NeedId
51 * return size of array (including trailing zero)
56 incneedid(Id id, NeedId *needid)
58 needid[NEEDIDOFF(id)].need++;
62 incneedidarray(Id *idarray, NeedId *needid)
67 while ((id = *idarray++) != 0)
70 needid[NEEDIDOFF(id)].need++;
81 needid_cmp_need(const void *ap, const void *bp, void *dp)
86 r = b->need - a->need;
89 return a->map - b->map;
93 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
97 Stringpool *spool = dp;
102 r = b->need - a->need;
105 as = spool->stringspace + spool->strings[a->map];
106 bs = spool->stringspace + spool->strings[b->map];
107 return strcmp(as, bs);
111 /*------------------------------------------------------------------*/
112 /* output helper routines, used for writing the header */
113 /* (the data itself is accumulated in memory and written with
121 write_u32(Repodata *data, unsigned int x)
126 if (putc(x >> 24, fp) == EOF ||
127 putc(x >> 16, fp) == EOF ||
128 putc(x >> 8, fp) == EOF ||
131 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
141 write_u8(Repodata *data, unsigned int x)
145 if (putc(x, data->fp) == EOF)
147 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
156 write_blob(Repodata *data, void *blob, int len)
160 if (len && fwrite(blob, len, 1, data->fp) != 1)
162 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
167 write_compressed_blob(Repodata *data, void *blob, int len)
169 unsigned char cpage[65536];
174 int chunk = len > sizeof(cpage) ? sizeof(cpage) : len;
175 int flag = (chunk == len ? 0x80 : 0x00);
176 int clen = repopagestore_compress_page(blob, chunk, cpage, sizeof(cpage) - 1);
179 write_u8(data, flag);
180 write_u8(data, chunk >> 8);
181 write_u8(data, chunk);
182 write_blob(data, blob, chunk);
186 write_u8(data, flag | 0x40);
187 write_u8(data, clen >> 8);
188 write_u8(data, clen);
189 write_blob(data, cpage, clen);
201 write_id(Repodata *data, Id x)
209 putc((x >> 28) | 128, fp);
211 putc((x >> 21) | 128, fp);
212 putc((x >> 14) | 128, fp);
215 putc((x >> 7) | 128, fp);
216 if (putc(x & 127, fp) == EOF)
218 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
223 write_str(Repodata *data, const char *str)
227 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
229 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
238 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
252 id = needid[NEEDIDOFF(id)].need;
254 id = (id & 63) | ((id & ~63) << 1);
260 write_id(data, id | 64);
269 #define DIRIDCACHE_SIZE 1024
276 Stringpool *ownspool;
278 int clonepool; /* are the pool ids cloned into ownspool? */
280 Id *keymap; /* keymap for this repodata */
284 Id *schema; /* schema construction space */
285 Id *sp; /* pointer in above */
291 struct extdata *extdata;
295 Id vstart; /* offset of key in vertical data */
300 int doingsolvables; /* working on solvables data */
303 Id lastdirid; /* last dir id seen in this repodata */
304 Id lastdirid_own; /* last dir id put in own pool */
306 Id diridcache[3 * DIRIDCACHE_SIZE];
309 #define NEEDID_BLOCK 1023
310 #define SCHEMATA_BLOCK 31
311 #define EXTDATA_BLOCK 4095
314 data_addid(struct extdata *xd, Id sx)
316 unsigned int x = (unsigned int)sx;
319 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
320 dp = xd->buf + xd->len;
325 *dp++ = (x >> 28) | 128;
327 *dp++ = (x >> 21) | 128;
328 *dp++ = (x >> 14) | 128;
331 *dp++ = (x >> 7) | 128;
333 xd->len = dp - xd->buf;
337 data_addideof(struct extdata *xd, Id sx, int eof)
339 unsigned int x = (unsigned int)sx;
342 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
343 dp = xd->buf + xd->len;
348 *dp++ = (x >> 27) | 128;
350 *dp++ = (x >> 20) | 128;
351 *dp++ = (x >> 13) | 128;
354 *dp++ = (x >> 6) | 128;
355 *dp++ = eof ? (x & 63) : (x & 63) | 64;
356 xd->len = dp - xd->buf;
360 data_addideof_len(Id sx)
362 unsigned int x = (unsigned int)sx;
367 return x >= (1 << 20) ? 4 : 3;
369 return x >= (1 << 6) ? 2 : 1;
373 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
379 data_addid(xd, (Id)(hx >> 3));
380 xd->buf[xd->len - 1] |= 128;
383 data_addid(xd, (Id)(x | 0x80000000));
384 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
387 data_addid(xd, (Id)x);
390 #ifdef USE_REL_IDARRAY
393 cmp_ids(const void *pa, const void *pb, void *dp)
401 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
409 data_addideof(xd, 0, 1);
412 for (len = 0; len < 64 && ids[len]; len++)
416 id = needid[NEEDIDOFF(id)].need;
421 for (i = len + 1; ids[i]; i++)
423 sids = solv_malloc2(i, sizeof(Id));
424 memcpy(sids, lids, 64 * sizeof(Id));
425 for (; ids[len]; len++)
429 id = needid[NEEDIDOFF(id)].need;
436 /* That bloody solvable:prereqmarker needs to stay in position :-( */
438 marker = needid[marker].need;
439 for (i = 0; i < len; i++)
440 if (sids[i] == marker)
443 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
445 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
449 /* The differencing above produces many runs of ones and twos. I tried
450 fairly elaborate schemes to RLE those, but they give only very mediocre
451 improvements in compression, as coding the escapes costs quite some
452 space. Even if they are coded only as bits in IDs. The best improvement
453 was about 2.7% for the whole .solv file. It's probably better to
454 invest some complexity into sharing idarrays, than RLEing. */
455 for (i = 0; i < len - 1; i++)
458 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
459 hence all real differences are offsetted by 1. Otherwise we would
460 have to handle negative differences, which would cost code space for
461 the encoding of the sign. We loose the exact mapping of prereq here,
462 but we know the result, so we can recover from that in the reader. */
470 /* XXX If difference is zero we have multiple equal elements,
471 we might want to skip writing them out. */
472 data_addideof(xd, id, 0);
479 data_addideof(xd, id, 1);
486 #ifdef USE_IDARRAYBLOCK
489 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
495 data_addideof(xd, 0, 1);
498 while ((id = *ids++) != 0)
501 id = needid[NEEDIDOFF(id)].need;
504 id = (last - id) * 2 - 1; /* [1, 2 * last - 1] odd */
505 else if (id < 2 * last)
506 id = (id - last) * 2; /* [0, 2 * last - 2] even */
508 data_addideof(xd, id, *ids ? 0 : 1);
515 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
520 data_addideof(xd, 0, 1);
523 while ((id = *ids++) != 0)
526 id = needid[NEEDIDOFF(id)].need;
527 data_addideof(xd, id, *ids ? 0 : 1);
536 data_addblob(struct extdata *xd, unsigned char *blob, int len)
538 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
539 memcpy(xd->buf + xd->len, blob, len);
543 /* grow needid array so that it contains the specified id */
545 grow_needid(struct cbdata *cbdata, Id id)
547 int oldoff = cbdata->needid[0].map;
548 int newoff = (id + 1 + NEEDID_BLOCK) & ~NEEDID_BLOCK;
549 int nrels = cbdata->pool->nrels;
550 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
552 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
553 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
554 cbdata->needid[0].map = newoff;
558 putinownpool(struct cbdata *cbdata, Repodata *data, Id id)
560 Stringpool *ss = data->localpool ? &data->spool : &cbdata->pool->ss;
561 const char *str = stringpool_id2str(ss, id);
562 id = stringpool_str2id(cbdata->ownspool, str, 1);
563 if (id >= cbdata->needid[0].map)
564 grow_needid(cbdata, id);
569 putinowndirpool_slow(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
571 Id compid, parent, id;
574 parent = dirpool_parent(dp, dir);
577 /* put parent in own pool first */
578 cacheent = cbdata->diridcache + (parent & (DIRIDCACHE_SIZE - 1));
579 if (cacheent[0] == parent && cacheent[DIRIDCACHE_SIZE] == data->repodataid)
580 parent = cacheent[2 * DIRIDCACHE_SIZE];
582 parent = putinowndirpool_slow(cbdata, data, dp, parent);
584 compid = dirpool_compid(dp, dir);
585 if (cbdata->ownspool && compid > 1 && (!cbdata->clonepool || data->localpool))
586 compid = putinownpool(cbdata, data, compid);
587 id = dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
589 cacheent = cbdata->diridcache + (dir & (DIRIDCACHE_SIZE - 1));
591 cacheent[DIRIDCACHE_SIZE] = data->repodataid;
592 cacheent[2 * DIRIDCACHE_SIZE] = id;
597 putinowndirpool(struct cbdata *cbdata, Repodata *data, Id dir)
600 if (dir && dir == cbdata->lastdirid)
601 return cbdata->lastdirid_own;
602 cacheent = cbdata->diridcache + (dir & (DIRIDCACHE_SIZE - 1));
603 if (dir && cacheent[0] == dir && cacheent[DIRIDCACHE_SIZE] == data->repodataid)
604 return cacheent[2 * DIRIDCACHE_SIZE];
605 cbdata->lastdirid = dir;
606 cbdata->lastdirid_own = putinowndirpool_slow(cbdata, data, &data->dirpool, dir);
607 return cbdata->lastdirid_own;
612 * collect key/id/dirid usage information, create needed schemas
615 collect_needed_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
617 struct cbdata *cbdata = vcbdata;
622 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? (int)(s - cbdata->pool->solvables) : 0, s ? pool_id2str(cbdata->pool, s->name) : "", key->name, pool_id2str(cbdata->pool, key->name), key->type);
624 if (key->name == REPOSITORY_SOLVABLES)
625 return SEARCH_NEXT_KEY; /* we do not want this one */
627 rm = cbdata->keymap[key - data->keys];
629 return SEARCH_NEXT_KEY; /* we do not want this one */
631 /* record key in schema */
632 if (cbdata->sp[-1] != rm)
637 case REPOKEY_TYPE_ID:
638 case REPOKEY_TYPE_IDARRAY:
640 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
641 id = putinownpool(cbdata, data, id);
642 incneedid(id, cbdata->needid);
644 case REPOKEY_TYPE_DIR:
645 case REPOKEY_TYPE_DIRNUMNUMARRAY:
646 case REPOKEY_TYPE_DIRSTRARRAY:
648 if (cbdata->owndirpool)
649 putinowndirpool(cbdata, data, id);
651 cbdata->dirused[id] = 1;
653 case REPOKEY_TYPE_FIXARRAY:
654 case REPOKEY_TYPE_FLEXARRAY:
657 /* finish schema, rewind to start */
658 Id *sp = cbdata->sp - 1;
663 cbdata->subschemata[sp[-2]] = repodata_schema2id(cbdata->target, sp, 1);
668 /* start new schema */
669 if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
671 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
672 *cbdata->sp++ = cbdata->nsubschemata++;
686 collect_needed_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
688 /* set schema info, keep in sync with collect_data_solvable */
689 Repo *repo = s->repo;
691 NeedId *needid = cbdata->needid;
692 Repodata *target = cbdata->target;
693 Id *idarraydata = repo->idarraydata;
695 if (keymap[SOLVABLE_NAME])
697 *sp++ = keymap[SOLVABLE_NAME];
698 needid[s->name].need++;
700 if (keymap[SOLVABLE_ARCH])
702 *sp++ = keymap[SOLVABLE_ARCH];
703 needid[s->arch].need++;
705 if (keymap[SOLVABLE_EVR])
707 *sp++ = keymap[SOLVABLE_EVR];
708 needid[s->evr].need++;
710 if (s->vendor && keymap[SOLVABLE_VENDOR])
712 *sp++ = keymap[SOLVABLE_VENDOR];
713 needid[s->vendor].need++;
715 if (s->provides && keymap[SOLVABLE_PROVIDES])
717 *sp++ = keymap[SOLVABLE_PROVIDES];
718 target->keys[keymap[SOLVABLE_PROVIDES]].size += incneedidarray(idarraydata + s->provides, needid);
720 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
722 *sp++ = keymap[SOLVABLE_OBSOLETES];
723 target->keys[keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(idarraydata + s->obsoletes, needid);
725 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
727 *sp++ = keymap[SOLVABLE_CONFLICTS];
728 target->keys[keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(idarraydata + s->conflicts, needid);
730 if (s->requires && keymap[SOLVABLE_REQUIRES])
732 *sp++ = keymap[SOLVABLE_REQUIRES];
733 target->keys[keymap[SOLVABLE_REQUIRES]].size += incneedidarray(idarraydata + s->requires, needid);
735 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
737 *sp++ = keymap[SOLVABLE_RECOMMENDS];
738 target->keys[keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(idarraydata + s->recommends, needid);
740 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
742 *sp++ = keymap[SOLVABLE_SUGGESTS];
743 target->keys[keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(idarraydata + s->suggests, needid);
745 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
747 *sp++ = keymap[SOLVABLE_SUPPLEMENTS];
748 target->keys[keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(idarraydata + s->supplements, needid);
750 if (s->enhances && keymap[SOLVABLE_ENHANCES])
752 *sp++ = keymap[SOLVABLE_ENHANCES];
753 target->keys[keymap[SOLVABLE_ENHANCES]].size += incneedidarray(idarraydata + s->enhances, needid);
755 if (repo->rpmdbid && keymap[RPM_RPMDBID])
757 *sp++ = keymap[RPM_RPMDBID];
758 target->keys[keymap[RPM_RPMDBID]].size++;
766 * encode all of the data into the correct buffers
769 collect_data_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
771 struct cbdata *cbdata = vcbdata;
777 if (key->name == REPOSITORY_SOLVABLES)
778 return SEARCH_NEXT_KEY;
780 rm = cbdata->keymap[key - data->keys];
782 return SEARCH_NEXT_KEY; /* we do not want this one */
783 storage = cbdata->target->keys[rm].storage;
785 xd = cbdata->extdata + 0; /* incore buffer */
786 if (storage == KEY_STORAGE_VERTICAL_OFFSET)
788 xd += rm; /* vertical buffer */
789 if (cbdata->vstart == -1)
790 cbdata->vstart = xd->len;
794 case REPOKEY_TYPE_DELETED:
795 case REPOKEY_TYPE_VOID:
796 case REPOKEY_TYPE_CONSTANT:
797 case REPOKEY_TYPE_CONSTANTID:
799 case REPOKEY_TYPE_ID:
801 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
802 id = putinownpool(cbdata, data, id);
803 needid = cbdata->needid;
804 id = needid[NEEDIDOFF(id)].need;
807 case REPOKEY_TYPE_IDARRAY:
809 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
810 id = putinownpool(cbdata, data, id);
811 needid = cbdata->needid;
812 id = needid[NEEDIDOFF(id)].need;
813 data_addideof(xd, id, kv->eof);
815 case REPOKEY_TYPE_STR:
816 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
818 case REPOKEY_TYPE_MD5:
819 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
821 case REPOKEY_TYPE_SHA1:
822 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
824 case REPOKEY_TYPE_SHA224:
825 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
827 case REPOKEY_TYPE_SHA256:
828 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
830 case REPOKEY_TYPE_SHA384:
831 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
833 case REPOKEY_TYPE_SHA512:
834 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
837 case REPOKEY_TYPE_NUM:
838 data_addid64(xd, kv->num, kv->num2);
840 case REPOKEY_TYPE_DIR:
842 if (cbdata->owndirpool)
843 id = putinowndirpool(cbdata, data, id);
844 id = cbdata->dirused[id];
847 case REPOKEY_TYPE_BINARY:
848 data_addid(xd, kv->num);
850 data_addblob(xd, (unsigned char *)kv->str, kv->num);
852 case REPOKEY_TYPE_DIRNUMNUMARRAY:
854 if (cbdata->owndirpool)
855 id = putinowndirpool(cbdata, data, id);
856 id = cbdata->dirused[id];
858 data_addid(xd, kv->num);
859 data_addideof(xd, kv->num2, kv->eof);
861 case REPOKEY_TYPE_DIRSTRARRAY:
863 if (cbdata->owndirpool)
864 id = putinowndirpool(cbdata, data, id);
865 id = cbdata->dirused[id];
866 if (rm == cbdata->filelistmode)
868 /* postpone adding to xd, just update len to get the correct offsets into the incore data*/
869 xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
872 data_addideof(xd, id, kv->eof);
873 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
875 case REPOKEY_TYPE_FIXARRAY:
876 case REPOKEY_TYPE_FLEXARRAY:
878 data_addid(xd, kv->num);
879 if (kv->eof != 2 && (!kv->entry || key->type == REPOKEY_TYPE_FLEXARRAY))
880 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
881 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
883 if (xd->len - cbdata->lastlen > cbdata->maxdata)
884 cbdata->maxdata = xd->len - cbdata->lastlen;
885 cbdata->lastlen = xd->len;
889 cbdata->target->error = pool_error(cbdata->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
892 if (storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
894 /* we can re-use old data in the blob here! */
895 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
896 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
902 /* special version of collect_data_cb that collects just one single REPOKEY_TYPE_DIRSTRARRAY vertical data */
904 collect_filelist_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
906 struct cbdata *cbdata = vcbdata;
911 rm = cbdata->keymap[key - data->keys];
912 if (rm != cbdata->filelistmode)
913 return SEARCH_NEXT_KEY; /* we do not want this one */
915 if (cbdata->owndirpool)
916 id = putinowndirpool(cbdata, data, id);
917 id = cbdata->dirused[id];
918 xd = cbdata->extdata + rm; /* vertical buffer */
919 data_addideof(xd, id, kv->eof);
920 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
925 collect_data_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
927 Repo *repo = s->repo;
928 Pool *pool = repo->pool;
929 struct extdata *xd = cbdata->extdata;
930 #ifdef USE_IDARRAYBLOCK
931 struct extdata *xda = xd + cbdata->target->nkeys; /* idarray block */
933 struct extdata *xda = xd;
936 NeedId *needid = cbdata->needid;
937 Id *idarraydata = repo->idarraydata;
939 if (keymap[SOLVABLE_NAME])
940 data_addid(xd, needid[s->name].need);
941 if (keymap[SOLVABLE_ARCH])
942 data_addid(xd, needid[s->arch].need);
943 if (keymap[SOLVABLE_EVR])
944 data_addid(xd, needid[s->evr].need);
945 if (s->vendor && keymap[SOLVABLE_VENDOR])
946 data_addid(xd, needid[s->vendor].need);
947 if (s->provides && keymap[SOLVABLE_PROVIDES])
948 data_adddepids(xda, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
949 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
950 data_adddepids(xda, pool, needid, idarraydata + s->obsoletes, 0);
951 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
952 data_adddepids(xda, pool, needid, idarraydata + s->conflicts, 0);
953 if (s->requires && keymap[SOLVABLE_REQUIRES])
954 data_adddepids(xda, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
955 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
956 data_adddepids(xda, pool, needid, idarraydata + s->recommends, 0);
957 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
958 data_adddepids(xda, pool, needid, idarraydata + s->suggests, 0);
959 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
960 data_adddepids(xda, pool, needid, idarraydata + s->supplements, 0);
961 if (s->enhances && keymap[SOLVABLE_ENHANCES])
962 data_adddepids(xda, pool, needid, idarraydata + s->enhances, 0);
963 if (repo->rpmdbid && keymap[RPM_RPMDBID])
964 data_addid(xd, repo->rpmdbid[(s - pool->solvables) - repo->start]);
967 /* traverse through directory with first child "dir" */
969 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
975 /* special case for '/', which has to come first */
978 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
980 if (used && !used[sib])
982 if (sib == 1 && parent == 1)
983 continue; /* already did that one above */
987 /* check if our block has some content */
989 return n - 1; /* nope, drop parent id again */
991 /* now go through all the siblings we just added and
992 * do recursive calls on them */
994 for (; parent < lastn; parent++)
996 sib = dirmap[parent];
997 if (used && used[sib] != 2) /* 2: used as parent */
999 child = dirpool_child(dp, sib);
1002 dirmap[n++] = -parent; /* start new block */
1003 n = traverse_dirs(dp, dirmap, n, child, used);
1010 write_compressed_page(Repodata *data, unsigned char *page, int len)
1013 unsigned char cpage[REPOPAGE_BLOBSIZE];
1015 clen = repopagestore_compress_page(page, len, cpage, len - 1);
1018 write_u32(data, len * 2);
1019 write_blob(data, page, len);
1023 write_u32(data, clen * 2 + 1);
1024 write_blob(data, cpage, clen);
1028 static Id verticals[] = {
1030 SOLVABLE_DESCRIPTION,
1031 SOLVABLE_MESSAGEDEL,
1032 SOLVABLE_MESSAGEINS,
1042 SOLVABLE_CHANGELOG_AUTHOR,
1043 SOLVABLE_CHANGELOG_TEXT,
1044 SOLVABLE_SIGNATUREDATA,
1048 static char *languagetags[] = {
1049 "solvable:summary:",
1050 "solvable:description:",
1051 "solvable:messageins:",
1052 "solvable:messagedel:",
1058 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
1060 const char *keyname;
1063 for (i = 0; verticals[i]; i++)
1064 if (key->name == verticals[i])
1065 return KEY_STORAGE_VERTICAL_OFFSET;
1066 keyname = pool_id2str(repo->pool, key->name);
1067 for (i = 0; languagetags[i] != 0; i++)
1068 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
1069 return KEY_STORAGE_VERTICAL_OFFSET;
1070 return KEY_STORAGE_INCORE;
1074 write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
1076 unsigned char *dp = xd->buf;
1080 int ll = REPOPAGE_BLOBSIZE - lpage;
1083 memcpy(vpage + lpage, dp, ll);
1087 if (lpage == REPOPAGE_BLOBSIZE)
1089 write_compressed_page(target, vpage, lpage);
1098 create_keyskip(Repo *repo, Id entry, unsigned char *repodataused, Id **oldkeyskip)
1100 Repodata *data, *last = 0;
1104 if (repo->nrepodata <= 2)
1106 keyskip = *oldkeyskip;
1109 if (keyskip[1] >= 0x10000000)
1110 keyskip = solv_free(keyskip);
1112 keyskip[1] = keyskip[2];
1114 FOR_REPODATAS(repo, rdid, data)
1116 if (!repodataused[rdid])
1118 if (entry != SOLVID_META)
1120 if (entry < data->start || entry >= data->end)
1122 /* if repodataused is set we know that the state is AVAILABLE */
1123 if (!data->incoreoffset[entry - data->start])
1127 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1131 if (cnt <= 1) /* just one repodata means we don't need a keyskip */
1133 *oldkeyskip = keyskip;
1136 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1138 keyskip[2] = keyskip[1] + repo->nrepodata;
1139 *oldkeyskip = keyskip;
1148 repowriter_create(Repo *repo)
1150 Repowriter *writer = solv_calloc(1, sizeof(*writer));
1151 writer->repo = repo;
1152 writer->keyfilter = repo_write_stdkeyfilter;
1153 writer->repodatastart = 1;
1154 writer->repodataend = repo->nrepodata;
1155 writer->solvablestart = repo->start;
1156 writer->solvableend = repo->end;
1161 repowriter_free(Repowriter *writer)
1163 solv_free(writer->userdata);
1164 return solv_free(writer);
1168 repowriter_set_flags(Repowriter *writer, int flags)
1170 writer->flags = flags;
1174 repowriter_set_keyfilter(Repowriter *writer, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata)
1176 writer->keyfilter = keyfilter;
1177 writer->kfdata = kfdata;
1181 repowriter_set_keyqueue(Repowriter *writer, Queue *keyq)
1183 writer->keyq = keyq;
1187 repowriter_set_repodatarange(Repowriter *writer, int repodatastart, int repodataend)
1189 writer->repodatastart = repodatastart;
1190 writer->repodataend = repodataend;
1194 repowriter_set_solvablerange(Repowriter *writer, int solvablestart, int solvableend)
1196 writer->solvablestart = solvablestart;
1197 writer->solvableend = solvableend;
1201 repowriter_set_userdata(Repowriter *writer, const void *data, int len)
1203 writer->userdata = solv_free(writer->userdata);
1204 writer->userdatalen = 0;
1207 writer->userdata = solv_memdup(data, len);
1208 writer->userdatalen = len;
1212 * the code works the following way:
1214 * 1) find which keys should be written
1215 * 2) collect usage information for keys/ids/dirids, create schema
1217 * 3) use usage information to create mapping tables, so that often
1218 * used ids get a lower number
1219 * 4) encode data into buffers using the mapping tables
1220 * 5) write everything to disk
1223 repowriter_write(Repowriter *writer, FILE *fp)
1225 Repo *repo = writer->repo;
1226 Pool *pool = repo->pool;
1229 NeedId *needid, *needidp;
1230 int nstrings, nrels;
1231 unsigned int sizeid;
1232 unsigned int solv_flags;
1235 int searchflags = 0;
1239 Id *keymap; /* maps repo key to my key, 0 -> not used */
1241 int *keymapstart; /* maps repo number to keymap offset */
1247 unsigned char *repodataused;
1248 int anyrepodataused = 0;
1250 int solvablestart, solvableend;
1252 int anysolvableused = 0;
1255 struct cbdata cbdata;
1259 int poolusage, dirpoolusage;
1262 Repodata *data, *dirpooldata;
1269 Id mainschema, *mainschemakeys;
1273 Id type_constantid = 0;
1276 if (writer->userdatalen < 0 || writer->userdatalen >= 65536)
1277 return pool_error(pool, -1, "illegal userdata length: %d", writer->userdatalen);
1279 memset(&cbdata, 0, sizeof(cbdata));
1282 cbdata.target = ⌖
1284 repodata_initdata(&target, repo, 1);
1286 /* go through all repodata and find the keys we need */
1287 /* also unify keys */
1289 /* start with all KEY_STORAGE_SOLVABLE ids */
1291 n = ID_NUM_INTERNAL;
1292 FOR_REPODATAS(repo, i, data)
1295 keymap = solv_calloc(nkeymap, sizeof(Id));
1296 keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1297 repodataused = solv_calloc(repo->nrepodata, 1);
1302 if (!(writer->flags & REPOWRITER_NO_STORAGE_SOLVABLE))
1304 /* add keys for STORAGE_SOLVABLE */
1305 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1309 if (i < SOLVABLE_PROVIDES)
1310 keyd.type = REPOKEY_TYPE_ID;
1311 else if (i < RPM_RPMDBID)
1312 keyd.type = REPOKEY_TYPE_IDARRAY;
1314 keyd.type = REPOKEY_TYPE_NUM;
1315 #ifdef USE_REL_IDARRAY
1316 if (keyd.type == REPOKEY_TYPE_IDARRAY)
1317 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1320 keyd.storage = KEY_STORAGE_SOLVABLE;
1321 if (writer->keyfilter)
1323 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1324 if (keyd.storage == KEY_STORAGE_DROPPED)
1326 keyd.storage = KEY_STORAGE_SOLVABLE;
1328 #ifdef USE_IDARRAYBLOCK
1329 if (keyd.type == REPOKEY_TYPE_IDARRAY)
1330 keyd.storage = KEY_STORAGE_IDARRAYBLOCK;
1334 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1338 if (repo->nsolvables)
1341 keyd.name = REPOSITORY_SOLVABLES;
1342 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1344 keyd.storage = KEY_STORAGE_INCORE;
1345 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1353 n = ID_NUM_INTERNAL;
1354 FOR_REPODATAS(repo, i, data)
1356 int idused, dirused;
1357 if (i < writer->repodatastart || i >= writer->repodataend)
1359 if (writer->keyfilter && (writer->flags & REPOWRITER_LEGACY) != 0)
1361 /* ask keyfilter if we want this repodata */
1363 /* check if we want this repodata */
1364 memset(&keyd, 0, sizeof(keyd));
1368 if (writer->keyfilter(repo, &keyd, writer->kfdata) == -1)
1372 keymap[n++] = 0; /* key 0 */
1373 idused = dirused = 0;
1374 for (j = 1; j < data->nkeys; j++, n++)
1376 key = data->keys + j;
1377 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1379 keymap[n] = keymap[key->name];
1382 if (key->type == REPOKEY_TYPE_DELETED && (writer->flags & REPOWRITER_KEEP_TYPE_DELETED) == 0)
1387 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1389 Repokey keyd = *key;
1390 keyd.size = repodata_globalize_id(data, key->size, 1);
1391 id = repodata_key2id(&target, &keyd, 0);
1394 id = repodata_key2id(&target, key, 0);
1397 /* a new key. ask keyfilter if we want it before creating it */
1398 Repokey keyd = *key;
1399 keyd.storage = KEY_STORAGE_INCORE;
1400 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1401 keyd.size = repodata_globalize_id(data, key->size, 1);
1402 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1404 if (writer->keyfilter)
1406 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1407 if (keyd.storage == KEY_STORAGE_DROPPED)
1412 if (keyd.storage != KEY_STORAGE_VERTICAL_OFFSET)
1413 keyd.storage = KEY_STORAGE_INCORE; /* do not mess with us */
1415 if (data->state != REPODATA_STUB)
1416 id = repodata_key2id(&target, &keyd, 1);
1419 /* load repodata if not already loaded */
1420 if (data->state == REPODATA_STUB)
1422 int oldnkeys = data->nkeys;
1423 repodata_load(data);
1424 if (oldnkeys != data->nkeys)
1426 nkeymap += data->nkeys - oldnkeys; /* grow/shrink keymap */
1427 keymap = solv_realloc2(keymap, nkeymap, sizeof(Id));
1429 if (data->state == REPODATA_AVAILABLE)
1431 /* redo this repodata! */
1437 if (data->state != REPODATA_AVAILABLE && data->state != REPODATA_LOADING)
1444 repodataused[i] = 1;
1445 anyrepodataused = 1;
1446 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1447 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1449 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1451 idused = 1; /* dirs also use ids */
1457 if (data->localpool)
1460 poolusage = 3; /* need own pool */
1464 spool = &data->spool;
1471 else if (poolusage != 1)
1472 poolusage = 3; /* need own pool */
1478 dirpoolusage = 3; /* need own dirpool */
1482 dirpool = &data->dirpool;
1487 nkeymap = n; /* update */
1489 /* 0: no pool needed at all */
1490 /* 1: use global pool */
1491 /* 2: use repodata local pool */
1492 /* 3: need own pool */
1497 spool = &target.spool;
1498 target.localpool = 1; /* so we can use repodata_translate */
1499 /* hack: reuse global pool data so we don't have to map pool ids */
1502 stringpool_free(spool);
1503 stringpool_clone(spool, &pool->ss);
1504 cbdata.clonepool = 1;
1506 cbdata.ownspool = spool;
1508 else if (poolusage == 0 || poolusage == 1)
1514 if (dirpoolusage == 3)
1516 /* dirpoolusage == 3 means that at least two repodata
1517 * areas have dir keys. This means that two areas have
1518 * idused set to 1, which results in poolusage being
1519 * either 1 (global pool) or 3 (own pool) */
1520 dirpool = &target.dirpool;
1522 cbdata.owndirpool = dirpool;
1525 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1528 /********************************************************************/
1530 fprintf(stderr, "poolusage: %d\n", poolusage);
1531 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1532 fprintf(stderr, "clonepool: %d\n", clonepool);
1533 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1534 for (i = 1; i < target.nkeys; i++)
1535 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1538 /********************************************************************/
1540 searchflags = SEARCH_SUB|SEARCH_ARRAYSENTINEL;
1541 if ((writer->flags & REPOWRITER_KEEP_TYPE_DELETED) != 0)
1542 searchflags |= SEARCH_KEEP_TYPE_DELETED;
1544 /* set needed count of all strings and rels,
1545 * find which keys are used in the solvables
1546 * put all strings in own spool
1549 reloff = spool->nstrings;
1550 if (cbdata.ownspool)
1551 reloff = (reloff + NEEDID_BLOCK) & ~NEEDID_BLOCK;
1552 else if (poolusage == 2)
1554 /* we'll need to put the key data into the spool,
1555 * so leave some room. 3 * nkeys is an upper bound */
1556 reloff += 3 * target.nkeys;
1559 needid = solv_calloc(reloff + pool->nrels, sizeof(*needid));
1560 needid[0].map = reloff; /* remember size in case we need to grow */
1562 cbdata.needid = needid;
1563 cbdata.schema = solv_calloc(target.nkeys + 2, sizeof(Id));
1565 /* create main schema */
1566 cbdata.sp = cbdata.schema + 1;
1568 /* collect meta data from all repodatas */
1569 /* XXX: merge arrays of equal keys? */
1570 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1571 FOR_REPODATAS(repo, j, data)
1573 if (!repodataused[j])
1575 cbdata.keymap = keymap + keymapstart[j];
1576 cbdata.lastdirid = 0; /* clear dir mapping cache */
1577 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1579 needid = cbdata.needid; /* maybe relocated */
1581 /* add solvables if needed (may revert later) */
1582 if (repo->nsolvables)
1584 *sp++ = keymap[REPOSITORY_SOLVABLES];
1585 target.keys[keymap[REPOSITORY_SOLVABLES]].size++;
1588 /* stash away main schema (including terminating zero) */
1589 mainschemakeys = solv_memdup2(cbdata.schema + 1, sp - cbdata.schema, sizeof(Id));
1591 /* collect data for all solvables */
1592 solvschemata = solv_calloc(repo->nsolvables, sizeof(Id)); /* allocate upper bound */
1593 solvablestart = writer->solvablestart < repo->start ? repo->start : writer->solvablestart;
1594 solvableend = writer->solvableend > repo->end ? repo->end : writer->solvableend;
1595 anysolvableused = 0;
1596 nsolvables = 0; /* solvables we are going to write, will be <= repo->nsolvables */
1597 cbdata.doingsolvables = 1;
1598 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
1600 if (s->repo != repo)
1603 cbdata.sp = cbdata.schema + 1;
1604 collect_needed_solvable(&cbdata, s, keymap);
1606 if (anyrepodataused)
1608 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1609 FOR_REPODATAS(repo, j, data)
1611 if (!repodataused[j] || i < data->start || i >= data->end)
1613 cbdata.keymap = keymap + keymapstart[j];
1614 cbdata.lastdirid = 0;
1615 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1617 needid = cbdata.needid; /* maybe relocated */
1620 solvschemata[nsolvables] = repodata_schema2id(cbdata.target, cbdata.schema + 1, 1);
1621 if (solvschemata[nsolvables])
1622 anysolvableused = 1;
1625 cbdata.doingsolvables = 0;
1627 if (repo->nsolvables && !anysolvableused)
1629 /* strip off REPOSITORY_SOLVABLES from the main schema */
1630 for (sp = mainschemakeys; *sp; sp++)
1632 sp[-1] = 0; /* strip last entry */
1634 mainschema = repodata_schema2id(cbdata.target, mainschemakeys, 1);
1635 mainschemakeys = solv_free(mainschemakeys);
1637 /********************************************************************/
1639 /* remove unused keys */
1640 keyused = solv_calloc(target.nkeys, sizeof(Id));
1641 for (i = 1; i < (int)target.schemadatalen; i++)
1642 keyused[target.schemadata[i]] = 1;
1644 for (n = i = 1; i < target.nkeys; i++)
1649 target.keys[n] = target.keys[i];
1654 /* update schema data to the new key ids */
1655 for (i = 1; i < (int)target.schemadatalen; i++)
1656 target.schemadata[i] = keyused[target.schemadata[i]];
1657 /* update keymap to the new key ids */
1658 for (i = 0; i < nkeymap; i++)
1659 keymap[i] = keyused[keymap[i]];
1660 keyused = solv_free(keyused);
1662 /* copy keys if requested */
1665 queue_empty(writer->keyq);
1666 for (i = 1; i < target.nkeys; i++)
1667 queue_push2(writer->keyq, target.keys[i].name, target.keys[i].type);
1670 /********************************************************************/
1672 /* check if we can do the special filelist memory optimization
1673 * we do the check before the keys are mapped.
1674 * The optimization is done if there is just one vertical key and
1675 * it is of type REPOKEY_TYPE_DIRSTRARRAY */
1676 if (anysolvableused && anyrepodataused)
1678 for (i = 1; i < target.nkeys; i++)
1680 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1682 if (target.keys[i].type != REPOKEY_TYPE_DIRSTRARRAY || cbdata.filelistmode != 0)
1684 cbdata.filelistmode = 0;
1687 cbdata.filelistmode = i;
1691 /********************************************************************/
1695 /* put all the keys in our string pool */
1696 /* put mapped ids right into target.keys */
1697 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1699 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1700 id = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1701 if (key->type == REPOKEY_TYPE_CONSTANTID)
1703 type_constantid = id;
1704 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1709 stringpool_freehash(spool); /* free some mem */
1710 if (cbdata.ownspool && spool->nstrings > needid[0].map)
1712 grow_needid(&cbdata, spool->nstrings - 1);
1713 needid = cbdata.needid; /* we relocated */
1717 type_constantid = REPOKEY_TYPE_CONSTANTID;
1719 /* increment needid of the keys */
1720 for (i = 1; i < target.nkeys; i++)
1722 if (target.keys[i].type == type_constantid)
1723 needid[target.keys[i].size].need++;
1724 needid[target.keys[i].name].need++;
1725 needid[target.keys[i].type].need++;
1728 /********************************************************************/
1730 /* increment need id of all relations
1731 * if we refer to another relation, make sure that the
1732 * need value is it is bigger than our value so that
1735 reloff = needid[0].map;
1736 for (i = pool->nrels - 1, needidp = needid + (reloff + i); i > 0; i--, needidp--)
1741 /* we have some relations with a non-zero need */
1744 for (rd = pool->rels + i; i > 0; i--, rd--)
1746 int need = needid[reloff + i].need;
1753 if (needid[reloff + id].need < need + 1)
1754 needid[reloff + id].need = need + 1;
1758 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1760 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1761 if (id >= cbdata.needid[0].map)
1763 grow_needid(&cbdata, id);
1764 needid = cbdata.needid; /* we relocated */
1765 reloff = needid[0].map; /* we have a new offset */
1775 if (needid[reloff + id].need < need + 1)
1776 needid[reloff + id].need = need + 1;
1780 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1782 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1783 if (id >= cbdata.needid[0].map)
1785 grow_needid(&cbdata, id);
1786 needid = cbdata.needid; /* we relocated */
1787 reloff = needid[0].map; /* we have a new offset */
1795 /********************************************************************/
1797 /* increment need id for used dir components */
1798 if (cbdata.owndirpool)
1800 /* if we have own dirpool, all entries in it are used.
1801 also, all comp ids are already mapped by putinowndirpool(),
1802 so we can simply increment needid.
1803 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1804 for (i = 1; i < dirpool->ndirs; i++)
1806 id = dirpool->dirs[i];
1815 /* else we re-use a dirpool of repodata "dirpooldata".
1816 dirused tells us which of the ids are used.
1817 we need to map comp ids if we generate a new pool.
1818 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1819 for (i = dirpool->ndirs - 1; i > 0; i--)
1821 if (!cbdata.dirused[i])
1823 parent = dirpool_parent(dirpool, i); /* always < i */
1824 cbdata.dirused[parent] = 2; /* 2: used as parent */
1825 id = dirpool->dirs[i];
1828 if (cbdata.ownspool && id > 1 && (!cbdata.clonepool || dirpooldata->localpool))
1830 id = putinownpool(&cbdata, dirpooldata, id);
1831 needid = cbdata.needid;
1835 if (!cbdata.dirused[0])
1837 cbdata.dirused = solv_free(cbdata.dirused);
1843 /********************************************************************/
1846 * create mapping table, new keys are sorted by needid[].need
1848 * needid[key].need : old key -> new key
1849 * needid[key].map : new key -> old key
1852 /* zero out id 0 and rel 0 just in case */
1853 reloff = needid[0].map;
1855 needid[reloff].need = 0;
1857 for (i = 1; i < reloff + pool->nrels; i++)
1860 /* make first entry '' */
1862 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1863 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1864 /* now needid is in new order, needid[newid].map -> oldid */
1866 /* calculate string space size, also zero out needid[].need */
1868 for (i = 1; i < reloff; i++)
1870 if (!needid[i].need)
1871 break; /* as we have sorted, every entry after this also has need == 0 */
1873 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1875 nstrings = i; /* our new string id end */
1877 /* make needid[oldid].need point to newid */
1878 for (i = 1; i < nstrings; i++)
1879 needid[needid[i].map].need = i;
1881 /* same as above for relations */
1882 for (i = 0; i < pool->nrels; i++)
1884 if (!needid[reloff + i].need)
1886 needid[reloff + i].need = 0;
1888 nrels = i; /* our new rel id end */
1890 for (i = 0; i < nrels; i++)
1891 needid[needid[reloff + i].map].need = nstrings + i;
1893 /* now we have: needid[oldid].need -> newid
1894 needid[newid].map -> oldid
1895 both for strings and relations */
1898 /********************************************************************/
1902 if (dirpool && dirpool->ndirs)
1904 /* create our new target directory structure by traversing through all
1905 * used dirs. This will concatenate blocks with the same parent
1906 * directory into single blocks.
1907 * Instead of components, traverse_dirs stores the old dirids,
1908 * we will change this in the second step below */
1909 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1910 if (cbdata.dirused && !cbdata.dirused[1])
1912 cbdata.dirused[1] = 1; /* always want / entry */
1913 cbdata.dirused[0] = 2; /* always want / entry */
1915 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1917 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1919 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1920 /* change dirmap so that it maps from "new dirid" to "new compid" */
1921 if (!cbdata.dirused)
1922 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1923 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1924 for (i = 1; i < ndirmap; i++)
1928 cbdata.dirused[dirmap[i]] = i;
1929 id = dirpool->dirs[dirmap[i]];
1930 if (dirpooldata && cbdata.ownspool && id > 1)
1931 id = putinownpool(&cbdata, dirpooldata, id);
1932 dirmap[i] = needid[id].need;
1934 /* now the new target directory structure is complete (dirmap), and we have
1935 * dirused[olddirid] -> newdirid */
1938 /********************************************************************/
1941 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1942 * we use extdata[nkeys] for the idarray_block data
1944 * this must match the code above that creates the schema data!
1947 cbdata.extdata = solv_calloc(target.nkeys + 1, sizeof(struct extdata));
1949 xd = cbdata.extdata;
1950 cbdata.current_sub = 0;
1951 /* add main schema */
1953 data_addid(xd, mainschema);
1955 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1956 FOR_REPODATAS(repo, j, data)
1958 if (!repodataused[j])
1960 cbdata.keymap = keymap + keymapstart[j];
1961 cbdata.lastdirid = 0;
1962 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1964 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1965 cbdata.maxdata = xd->len - cbdata.lastlen;
1966 cbdata.lastlen = xd->len;
1968 if (anysolvableused)
1970 data_addid(xd, nsolvables); /* FLEXARRAY nentries */
1971 cbdata.doingsolvables = 1;
1973 for (i = solvablestart, s = pool->solvables + i, n = 0; i < solvableend; i++, s++)
1975 if (s->repo != repo)
1977 data_addid(xd, solvschemata[n]);
1978 collect_data_solvable(&cbdata, s, keymap);
1979 if (anyrepodataused)
1981 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1983 FOR_REPODATAS(repo, j, data)
1985 if (!repodataused[j] || i < data->start || i >= data->end)
1987 cbdata.keymap = keymap + keymapstart[j];
1988 cbdata.lastdirid = 0;
1989 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1992 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1993 cbdata.maxdata = xd->len - cbdata.lastlen;
1994 cbdata.lastlen = xd->len;
1997 cbdata.doingsolvables = 0;
2000 assert(cbdata.current_sub == cbdata.nsubschemata);
2001 cbdata.subschemata = solv_free(cbdata.subschemata);
2002 cbdata.nsubschemata = 0;
2004 /********************************************************************/
2010 solv_flags |= SOLV_FLAG_PREFIX_POOL;
2011 solv_flags |= SOLV_FLAG_SIZE_BYTES;
2012 if (writer->userdatalen)
2013 solv_flags |= SOLV_FLAG_USERDATA;
2014 if (cbdata.extdata[target.nkeys].len)
2015 solv_flags |= SOLV_FLAG_IDARRAYBLOCK;
2017 /* write file header */
2018 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
2019 if ((solv_flags & (SOLV_FLAG_USERDATA | SOLV_FLAG_IDARRAYBLOCK)) != 0)
2020 write_u32(&target, SOLV_VERSION_9);
2022 write_u32(&target, SOLV_VERSION_8);
2025 write_u32(&target, nstrings);
2026 write_u32(&target, nrels);
2027 write_u32(&target, ndirmap);
2028 write_u32(&target, anysolvableused ? nsolvables : 0);
2029 write_u32(&target, target.nkeys);
2030 write_u32(&target, target.nschemata);
2031 write_u32(&target, solv_flags);
2033 /* write userdata */
2034 if ((solv_flags & SOLV_FLAG_USERDATA) != 0)
2036 write_u32(&target, writer->userdatalen);
2037 write_blob(&target, writer->userdata, writer->userdatalen);
2043 * calculate prefix encoding of the strings
2045 unsigned char *prefixcomp = solv_malloc(nstrings);
2046 unsigned int compsum = 0;
2050 for (i = 1; i < nstrings; i++)
2052 char *str = spool->stringspace + spool->strings[needid[i].map];
2054 for (same = 0; same < 255; same++)
2055 if (!old_str[same] || old_str[same] != str[same])
2057 prefixcomp[i] = same;
2065 write_u32(&target, sizeid);
2066 /* we save compsum bytes but need 1 extra byte for every string */
2067 write_u32(&target, sizeid + nstrings - 1 - compsum);
2068 for (i = 1; i < nstrings; i++)
2070 char *str = spool->stringspace + spool->strings[needid[i].map];
2071 write_u8(&target, prefixcomp[i]);
2072 write_str(&target, str + prefixcomp[i]);
2074 solv_free(prefixcomp);
2078 write_u32(&target, 0); /* unpacked size */
2079 write_u32(&target, 0); /* compressed size */
2085 for (i = 0; i < nrels; i++)
2087 Reldep *ran = pool->rels + (needid[reloff + i].map - reloff);
2088 write_id(&target, needid[NEEDIDOFF(ran->name)].need);
2089 write_id(&target, needid[NEEDIDOFF(ran->evr)].need);
2090 write_u8(&target, ran->flags);
2094 * write dirs (skip both root and / entry)
2096 for (i = 2; i < ndirmap; i++)
2099 write_id(&target, dirmap[i]);
2101 write_id(&target, nstrings - dirmap[i]);
2108 for (i = 1; i < target.nkeys; i++)
2110 write_id(&target, needid[target.keys[i].name].need);
2111 write_id(&target, needid[target.keys[i].type].need);
2112 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
2114 if (target.keys[i].type == type_constantid)
2115 write_id(&target, needid[target.keys[i].size].need);
2117 write_id(&target, target.keys[i].size);
2120 write_id(&target, cbdata.extdata[i].len);
2121 write_id(&target, target.keys[i].storage);
2127 write_id(&target, target.schemadatalen); /* XXX -1? */
2128 for (i = 1; i < target.nschemata; i++)
2129 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
2131 /* write idarray_block data if not empty */
2132 if (cbdata.extdata[target.nkeys].len)
2134 unsigned int cnt = 0;
2138 xd = cbdata.extdata + target.nkeys;
2139 /* calculate number of entries */
2140 for (l = xd->len, b = xd->buf; l--;)
2142 unsigned char x = *b++;
2143 if ((x & 0x80) == 0)
2144 cnt += (x & 0x40) ? 1 : 2;
2146 write_id(&target, cnt);
2148 write_compressed_blob(&target, xd->buf, xd->len);
2155 xd = cbdata.extdata;
2156 write_id(&target, cbdata.maxdata);
2157 write_id(&target, xd->len);
2159 write_blob(&target, xd->buf, xd->len);
2163 * write vertical data if we have any
2165 for (i = 1; i < target.nkeys; i++)
2166 if (cbdata.extdata[i].len)
2168 if (i < target.nkeys)
2170 /* have vertical data, write it in pages */
2171 unsigned char vpage[REPOPAGE_BLOBSIZE];
2174 write_u32(&target, REPOPAGE_BLOBSIZE);
2175 if (!cbdata.filelistmode)
2177 for (i = 1; i < target.nkeys; i++)
2178 if (cbdata.extdata[i].len)
2179 lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
2183 /* ok, just one single extdata which is of type REPOKEY_TYPE_DIRSTRARRAY */
2184 xd = cbdata.extdata + i;
2186 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
2187 FOR_REPODATAS(repo, j, data)
2189 if (!repodataused[j])
2191 cbdata.keymap = keymap + keymapstart[j];
2192 cbdata.lastdirid = 0;
2193 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2195 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
2197 if (s->repo != repo)
2199 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
2200 FOR_REPODATAS(repo, j, data)
2202 if (!repodataused[j] || i < data->start || i >= data->end)
2204 cbdata.keymap = keymap + keymapstart[j];
2205 cbdata.lastdirid = 0;
2206 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2208 if (xd->len > 1024 * 1024)
2210 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2215 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2218 write_compressed_page(&target, vpage, lpage);
2221 for (i = 1; i < target.nkeys; i++)
2222 solv_free(cbdata.extdata[i].buf);
2223 solv_free(cbdata.extdata);
2226 repodata_freedata(&target);
2229 solv_free(solvschemata);
2230 solv_free(cbdata.schema);
2233 solv_free(keymapstart);
2234 solv_free(cbdata.dirused);
2235 solv_free(repodataused);
2236 solv_free(oldkeyskip);
2237 return target.error;
2241 repo_write(Repo *repo, FILE *fp)
2244 Repowriter *writer = repowriter_create(repo);
2245 res = repowriter_write(writer, fp);
2246 repowriter_free(writer);
2251 repodata_write(Repodata *data, FILE *fp)
2254 Repowriter *writer = repowriter_create(data->repo);
2255 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2256 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
2257 res = repowriter_write(writer, fp);
2258 repowriter_free(writer);
2262 /* deprecated functions, do not use in new code! */
2264 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2267 Repowriter *writer = repowriter_create(repo);
2268 repowriter_set_flags(writer, REPOWRITER_LEGACY);
2269 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2270 repowriter_set_keyqueue(writer, keyq);
2271 res = repowriter_write(writer, fp);
2272 repowriter_free(writer);
2277 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2280 Repowriter *writer = repowriter_create(data->repo);
2281 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2282 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE | REPOWRITER_LEGACY);
2283 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2284 repowriter_set_keyqueue(writer, keyq);
2285 res = repowriter_write(writer, fp);
2286 repowriter_free(writer);