2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Write Repo data out to binary file
13 * See doc/README.format for a description
14 * of the binary file format
18 #include <sys/types.h>
28 #include "repo_write.h"
30 /* #define IGNORE_NEED_FREQUENCY */
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
35 typedef struct needid {
41 #define RELOFF(id) (pool->ss.nstrings + GETRELID(id))
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
53 incneedid(Pool *pool, Id *idarray, NeedId *needid)
61 while ((id = *idarray++) != 0)
66 Reldep *rd = GETRELDEP(pool, id);
67 needid[RELOFF(id)].need++;
68 if (ISRELDEP(rd->evr))
73 incneedid(pool, ida, needid);
76 needid[rd->evr].need++;
90 needid_cmp_need(const void *ap, const void *bp)
95 r = b->need - a->need;
98 return a->map - b->map;
101 static Pool *cmp_pool;
103 needid_cmp_need_s(const void *ap, const void *bp)
105 const NeedId *a = ap;
106 const NeedId *b = bp;
109 #ifdef IGNORE_NEED_FREQUENCY
112 else if (b->need == 0)
116 r = b->need - a->need;
120 char *as = cmp_pool->ss.stringspace + cmp_pool->ss.strings[a->map];
121 char *bs = cmp_pool->ss.stringspace + cmp_pool->ss.strings[b->map];
122 size_t alen = strlen (as);
123 size_t blen = strlen (bs);
124 return memcmp (as, bs, alen < blen ? alen + 1 : blen + 1);
128 /*------------------------------------------------------------------*/
129 /* output routines */
136 write_u32(FILE *fp, unsigned int x)
138 if (putc(x >> 24, fp) == EOF ||
139 putc(x >> 16, fp) == EOF ||
140 putc(x >> 8, fp) == EOF ||
143 perror("write error");
154 write_u8(FILE *fp, unsigned int x)
156 if (putc(x, fp) == EOF)
158 perror("write error");
169 write_id(FILE *fp, Id x)
174 putc((x >> 28) | 128, fp);
176 putc((x >> 21) | 128, fp);
177 putc((x >> 14) | 128, fp);
180 putc((x >> 7) | 128, fp);
181 if (putc(x & 127, fp) == EOF)
183 perror("write error");
189 write_str(FILE *fp, const char *str)
191 if (fputs (str, fp) == EOF
192 || putc (0, fp) == EOF)
194 perror("write error");
204 write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
218 id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
220 id = (id & 63) | ((id & ~63) << 1);
226 write_id(fp, id | 64);
231 cmp_ids (const void *pa, const void *pb)
239 write_idarray_sort(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
249 /* If we ever share idarrays we can't do this in-place. */
250 for (len = 0; ids[len]; len++)
254 ids[len] = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
257 /* That bloody solvable:prereqmarker needs to stay in position :-( */
258 Id prereq = SOLVABLE_PREREQMARKER;
260 prereq = needid[prereq].need;
261 for (i = 0; i < len; i++)
262 if (ids[i] == prereq)
265 qsort (ids, i, sizeof (Id), cmp_ids);
267 qsort (ids + i + 1, len - i - 1, sizeof (Id), cmp_ids);
270 for (i = 0; i < len; i++)
271 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
272 hence all real differences are offsetted by 1. Otherwise we would
273 have to handle negative differences, which would cost code space for
274 the encoding of the sign. We loose the exact mapping of prereq here,
275 but we know the result, so we can recover from that in the reader. */
276 if (ids[i] == prereq)
282 /* XXX If difference is zero we have multiple equal elements,
283 we might want to skip writing them out. */
287 /* The differencing above produces many runs of ones and twos. I tried
288 fairly elaborate schemes to RLE those, but they give only very mediocre
289 improvements in compression, as coding the escapes costs quite some
290 space. Even if they are coded only as bits in IDs. The best improvement
291 was about 2.7% for the whole .solv file. It's probably better to
292 invest some complexity into sharing idarrays, than RLEing. */
293 for (i = 0; i < len - 1; i++)
297 id = (id & 63) | ((id & ~63) << 1);
298 write_id(fp, id | 64);
302 old = (old & 63) | ((old & ~63) << 1);
307 write_id_value(FILE *fp, Id id, Id value, int eof)
310 id = (id & 63) | ((id & ~63) << 1);
311 write_id(fp, id | 64);
313 value = (value & 63) | ((value & ~63) << 1);
314 write_id(fp, value | (eof ? 0 : 64));
319 Id *schemadata, *schemadatap;
323 Id lastschemakey[256];
327 addschema(struct schemata *schemata, Id *schema)
332 for (sp = schema, len = 0, h = 0; *sp; len++)
336 if (schemata->lastschema[h] && !memcmp(schemata->schemadata + schemata->lastschema[h], schema, len * sizeof(Id)))
337 return schemata->lastschemakey[h];
340 for (sp = schemata->schemadata + 1; sp < schemata->schemadatap; )
342 if (!memcmp(sp, schemata->schemadatap, len * sizeof(Id)))
350 if (len > schemata->schemadatafree)
352 int l = schemata->schemadatap - schemata->schemadata;
353 schemata->schemadata = xrealloc(schemata->schemadata, (schemata->schemadatap - schemata->schemadata + len + 256) * sizeof(Id));
354 schemata->schemadatafree = len + 256;
355 schemata->schemadatap = schemata->schemadata + l;
358 /* leave first one free so that our lastschema test works */
359 *schemata->schemadatap++ = 0;
360 schemata->schemadatafree--;
363 if (schemaid != schemata->nschemata)
365 schemata->lastschema[h] = schemata->schemadatap - schemata->schemadata;
366 schemata->lastschemakey[h] = schemaid;
367 memcpy(schemata->schemadatap, schema, len * sizeof(Id));
368 schemata->schemadatafree -= len;
369 schemata->schemadatap += len;
370 schemata->nschemata++;
379 repo_write(Repo *repo, FILE *fp)
381 Pool *pool = repo->pool;
385 int nstrings, nrels, nkeys;
387 unsigned int solv_flags;
391 int idsizes[ID_NUM_INTERNAL];
392 int id2key[ID_NUM_INTERNAL];
395 Id schema[ID_NUM_INTERNAL], *sp;
396 struct schemata schemata;
397 Id *solvschema; /* schema of our solvables */
398 Id repodataschema, repodataschema_internal;
401 idarraydata = repo->idarraydata;
403 needid = (NeedId *)xcalloc(pool->ss.nstrings + pool->nrels, sizeof(*needid));
404 memset(idsizes, 0, sizeof(idsizes));
406 repodataschema = repodataschema_internal = 0;
407 for (i = 0; i < repo->nrepodata; i++)
410 idsizes[REPODATA_EXTERNAL] = 1;
411 idsizes[REPODATA_KEYS]++;
412 if (repo->repodata[i].location)
414 repodataschema = 1; /* mark that we need it */
415 idsizes[REPODATA_LOCATION] = 1;
418 repodataschema_internal = 1; /* mark that we need it */
419 for (j = 0; j < repo->repodata[i].nkeys; j++)
420 needid[repo->repodata[i].keys[j].name].need++;
421 idsizes[REPODATA_KEYS] += 2 * repo->repodata[i].nkeys;
424 for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
429 needid[s->name].need++;
430 needid[s->arch].need++;
431 needid[s->evr].need++;
434 needid[s->vendor].need++;
435 idsizes[SOLVABLE_VENDOR] = 1;
438 idsizes[SOLVABLE_PROVIDES] += incneedid(pool, idarraydata + s->provides, needid);
440 idsizes[SOLVABLE_REQUIRES] += incneedid(pool, idarraydata + s->requires, needid);
442 idsizes[SOLVABLE_CONFLICTS] += incneedid(pool, idarraydata + s->conflicts, needid);
444 idsizes[SOLVABLE_OBSOLETES] += incneedid(pool, idarraydata + s->obsoletes, needid);
446 idsizes[SOLVABLE_RECOMMENDS] += incneedid(pool, idarraydata + s->recommends, needid);
448 idsizes[SOLVABLE_SUGGESTS] += incneedid(pool, idarraydata + s->suggests, needid);
450 idsizes[SOLVABLE_SUPPLEMENTS] += incneedid(pool, idarraydata + s->supplements, needid);
452 idsizes[SOLVABLE_ENHANCES] += incneedid(pool, idarraydata + s->enhances, needid);
454 idsizes[SOLVABLE_FRESHENS] += incneedid(pool, idarraydata + s->freshens, needid);
456 if (nsolvables != repo->nsolvables)
459 idsizes[SOLVABLE_NAME] = 1;
460 idsizes[SOLVABLE_ARCH] = 1;
461 idsizes[SOLVABLE_EVR] = 1;
463 idsizes[RPM_RPMDBID] = 1;
465 for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++)
472 needid[pool->ss.nstrings].need = 0;
473 for (i = 0; i < pool->ss.nstrings + pool->nrels; i++)
477 qsort(needid + 1, pool->ss.nstrings - 1, sizeof(*needid), needid_cmp_need_s);
478 qsort(needid + pool->ss.nstrings, pool->nrels, sizeof(*needid), needid_cmp_need);
481 for (i = 1; i < pool->ss.nstrings; i++)
486 sizeid += strlen(pool->ss.stringspace + pool->ss.strings[needid[i].map]) + 1;
490 for (i = 0; i < nstrings; i++)
491 needid[needid[i].map].need = i;
493 for (i = 0; i < pool->nrels; i++)
495 if (!needid[pool->ss.nstrings + i].need)
498 needid[pool->ss.nstrings + i].need = 0;
502 for (i = 0; i < nrels; i++)
504 needid[needid[pool->ss.nstrings + i].map].need = nstrings + i;
507 /* find the keys we need */
509 memset(id2key, 0, sizeof(id2key));
510 for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++)
514 /* find the schemata we need */
515 memset(&schemata, 0, sizeof(schemata));
516 solvschema = xcalloc(repo->nsolvables, sizeof(Id));
518 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
523 *sp++ = SOLVABLE_NAME;
524 *sp++ = SOLVABLE_ARCH;
525 *sp++ = SOLVABLE_EVR;
527 *sp++ = SOLVABLE_VENDOR;
529 *sp++ = SOLVABLE_PROVIDES;
531 *sp++ = SOLVABLE_OBSOLETES;
533 *sp++ = SOLVABLE_CONFLICTS;
535 *sp++ = SOLVABLE_REQUIRES;
537 *sp++ = SOLVABLE_RECOMMENDS;
539 *sp++ = SOLVABLE_SUGGESTS;
541 *sp++ = SOLVABLE_SUPPLEMENTS;
543 *sp++ = SOLVABLE_ENHANCES;
545 *sp++ = SOLVABLE_FRESHENS;
549 solvschema[n++] = addschema(&schemata, schema);
554 /* add us a schema for our repodata */
556 *sp++ = REPODATA_EXTERNAL;
557 *sp++ = REPODATA_KEYS;
558 *sp++ = REPODATA_LOCATION;
560 repodataschema = addschema(&schemata, schema);
562 if (repodataschema_internal)
565 *sp++ = REPODATA_EXTERNAL;
566 *sp++ = REPODATA_KEYS;
568 repodataschema_internal = addschema(&schemata, schema);
571 /* convert all schemas to local keys */
572 if (schemata.nschemata)
573 for (sp = schemata.schemadata; sp < schemata.schemadatap; sp++)
576 /* write file header */
577 write_u32(fp, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
578 write_u32(fp, SOLV_VERSION_3);
581 write_u32(fp, nstrings);
582 write_u32(fp, nrels);
583 write_u32(fp, nsolvables);
584 write_u32(fp, nkeys);
585 write_u32(fp, schemata.nschemata);
586 write_u32(fp, repo->nrepodata); /* info blocks. */
588 solv_flags |= SOLV_FLAG_PREFIX_POOL;
590 solv_flags |= SOLV_FLAG_VERTICAL;
592 write_u32(fp, solv_flags);
594 /* Build the prefix-encoding of the string pool. We need to know
595 the size of that before writing it to the file, so we have to
596 build a separate buffer for that. As it's temporarily possible
597 that this actually is an expansion we can't easily reuse the
598 stringspace for this. The max expansion per string is 1 byte,
599 so it will fit into sizeid+nstrings bytes. */
600 char *prefix = xmalloc (sizeid + nstrings);
603 for (i = 1; i < nstrings; i++)
605 char *str = pool->ss.stringspace + pool->ss.strings[needid[i].map];
608 for (same = 0; same < 255; same++)
609 if (old_str[same] != str[same])
612 len = strlen (str + same) + 1;
613 memcpy (pp, str + same, len);
621 write_u32(fp, sizeid);
622 write_u32(fp, pp - prefix);
623 if (fwrite(prefix, pp - prefix, 1, fp) != 1)
625 perror("write error");
633 for (i = 0; i < nrels; i++)
635 ran = pool->rels + (needid[pool->ss.nstrings + i].map - pool->ss.nstrings);
636 write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
637 write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
638 write_u8(fp, ran->flags);
644 for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++)
648 write_id(fp, needid[i].need);
649 if (i >= SOLVABLE_PROVIDES && i <= SOLVABLE_FRESHENS)
650 write_id(fp, TYPE_REL_IDARRAY);
651 else if (i == RPM_RPMDBID)
652 write_id(fp, TYPE_U32);
653 else if (i == REPODATA_EXTERNAL)
654 write_id(fp, TYPE_VOID);
655 else if (i == REPODATA_KEYS)
656 write_id(fp, TYPE_IDVALUEARRAY);
657 else if (i == REPODATA_LOCATION)
658 write_id(fp, TYPE_STR);
660 write_id(fp, TYPE_ID);
661 write_id(fp, idsizes[i]);
667 if (schemata.nschemata)
669 write_id(fp, schemata.schemadatap - schemata.schemadata - 1);
670 for (sp = schemata.schemadata + 1; sp < schemata.schemadatap; )
672 write_idarray(fp, pool, 0, sp);
683 for (i = 0; i < repo->nrepodata; i++)
687 if (repo->repodata[i].location)
688 write_id(fp, repodataschema);
690 write_id(fp, repodataschema_internal);
691 /* keys + location, write idarray */
692 for (j = 0; j < repo->repodata[i].nkeys; j++)
694 Id id = needid[repo->repodata[i].keys[j].name].need;
695 write_id_value(fp, id, repo->repodata[i].keys[j].type, j == repo->repodata[i].nkeys - 1);
697 if (repo->repodata[i].location)
698 write_str(fp, repo->repodata[i].location);
706 for (i = 0; i < nsolvables; i++)
707 write_id(fp, solvschema[i]);
708 unsigned char *used = xmalloc(schemata.nschemata);
709 for (id = SOLVABLE_NAME; id <= RPM_RPMDBID; id++)
712 memset(used, 0, nschemata);
713 for (sp = schemata.schemadata + 1, i = 0; sp < schemata.schemadatap; sp++)
720 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
724 if (!used[solvschema[n++]])
729 write_id(fp, needid[s->name].need);
732 write_id(fp, needid[s->arch].need);
735 write_id(fp, needid[s->evr].need);
737 case SOLVABLE_VENDOR:
738 write_id(fp, needid[s->vendor].need);
741 write_u32(fp, repo->rpmdbid[i - repo->start]);
743 case SOLVABLE_PROVIDES:
744 write_idarray(fp, pool, needid, idarraydata + s->provides);
746 case SOLVABLE_OBSOLETES:
747 write_idarray(fp, pool, needid, idarraydata + s->obsoletes);
749 case SOLVABLE_CONFLICTS:
750 write_idarray(fp, pool, needid, idarraydata + s->conflicts);
752 case SOLVABLE_REQUIRES:
753 write_idarray(fp, pool, needid, idarraydata + s->requires);
755 case SOLVABLE_RECOMMENDS:
756 write_idarray(fp, pool, needid, idarraydata + s->recommends);
758 case SOLVABLE_SUPPLEMENTS:
759 write_idarray(fp, pool, needid, idarraydata + s->supplements);
761 case SOLVABLE_SUGGESTS:
762 write_idarray(fp, pool, needid, idarraydata + s->suggests);
764 case SOLVABLE_ENHANCES:
765 write_idarray(fp, pool, needid, idarraydata + s->enhances);
767 case SOLVABLE_FRESHENS:
768 write_idarray(fp, pool, needid, idarraydata + s->freshens);
776 xfree(schemata.schemadata);
785 for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
789 /* keep in sync with schema generation! */
790 write_id(fp, solvschema[n++]);
791 write_id(fp, needid[s->name].need);
792 write_id(fp, needid[s->arch].need);
793 write_id(fp, needid[s->evr].need);
795 write_id(fp, needid[s->vendor].need);
797 write_idarray_sort(fp, pool, needid, idarraydata + s->provides);
799 write_idarray_sort(fp, pool, needid, idarraydata + s->obsoletes);
801 write_idarray_sort(fp, pool, needid, idarraydata + s->conflicts);
803 write_idarray_sort(fp, pool, needid, idarraydata + s->requires);
805 write_idarray_sort(fp, pool, needid, idarraydata + s->recommends);
807 write_idarray_sort(fp, pool, needid, idarraydata + s->suggests);
809 write_idarray_sort(fp, pool, needid, idarraydata + s->supplements);
811 write_idarray_sort(fp, pool, needid, idarraydata + s->enhances);
813 write_idarray_sort(fp, pool, needid, idarraydata + s->freshens);
815 write_u32(fp, repo->rpmdbid[i - repo->start]);
820 xfree(schemata.schemadata);