tools/repo_write.c

   1 /*
   2  * Copyright (c) 2007, Novell Inc.
   3  *
   4  * This program is licensed under the BSD license, read LICENSE.BSD
   5  * for further information
   6  */
   7
   8 /*
   9  * repo_write.c
  10  *
  11  * Write Repo data out to binary file
  12  *
  13  * See doc/README.format for a description
  14  * of the binary file format
  15  *
  16  */
  17
  18 #include <sys/types.h>
  19 #include <limits.h>
  20 #include <fcntl.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24
  25 #include "pool.h"
  26 #include "util.h"
  27 #include "repo_write.h"
  28
  29 /*------------------------------------------------------------------*/
  30 /* Id map optimizations */
  31
  32 typedef struct needid {
  33   Id need;
  34   Id map;
  35 } NeedId;
  36
  37
  38 #define RELOFF(id) (pool->ss.nstrings + GETRELID(id))
  39
  40 /*
  41  * increment need Id
  42  * idarray: array of Ids, ID_NULL terminated
  43  * needid: array of Id->NeedId
  44  *
  45  * return count
  46  *
  47  */
  48
  49 static int
  50 incneedid(Pool *pool, Id *idarray, NeedId *needid)
  51 {
  52   if (!idarray)
  53     return 0;
  54
  55   Id id;
  56   int n = 0;
  57
  58   while ((id = *idarray++) != 0)
  59     {
  60       n++;
  61       while (ISRELDEP(id))
  62         {
  63           Reldep *rd = GETRELDEP(pool, id);
  64           needid[RELOFF(id)].need++;
  65           if (ISRELDEP(rd->evr))
  66             {
  67               Id ida[2];
  68               ida[0] = rd->evr;
  69               ida[1] = 0;
  70               incneedid(pool, ida, needid);
  71             }
  72           else
  73             needid[rd->evr].need++;
  74           id = rd->name;
  75         }
  76       needid[id].need++;
  77     }
  78   return n + 1;
  79 }
  80
  81
  82 /*
  83  *
  84  */
  85
  86 static int
  87 needid_cmp_need(const void *ap, const void *bp)
  88 {
  89   const NeedId *a = ap;
  90   const NeedId *b = bp;
  91   int r;
  92   r = b->need - a->need;
  93   if (r)
  94     return r;
  95   return a->map - b->map;
  96 }
  97
  98
  99 /*------------------------------------------------------------------*/
 100 /* output routines */
 101
 102 /*
 103  * unsigned 32-bit
 104  */
 105
 106 static void
 107 write_u32(FILE *fp, unsigned int x)
 108 {
 109   if (putc(x >> 24, fp) == EOF ||
 110       putc(x >> 16, fp) == EOF ||
 111       putc(x >> 8, fp) == EOF ||
 112       putc(x, fp) == EOF)
 113     {
 114       perror("write error");
 115       exit(1);
 116     }
 117 }
 118
 119
 120 /*
 121  * unsigned 8-bit
 122  */
 123
 124 static void
 125 write_u8(FILE *fp, unsigned int x)
 126 {
 127   if (putc(x, fp) == EOF)
 128     {
 129       perror("write error");
 130       exit(1);
 131     }
 132 }
 133
 134
 135 /*
 136  * Id
 137  */
 138
 139 static void
 140 write_id(FILE *fp, Id x)
 141 {
 142   if (x >= (1 << 14))
 143     {
 144       if (x >= (1 << 28))
 145         putc((x >> 28) | 128, fp);
 146       if (x >= (1 << 21))
 147         putc((x >> 21) | 128, fp);
 148       putc((x >> 14) | 128, fp);
 149     }
 150   if (x >= (1 << 7))
 151     putc((x >> 7) | 128, fp);
 152   if (putc(x & 127, fp) == EOF)
 153     {
 154       perror("write error");
 155       exit(1);
 156     }
 157 }
 158
 159
 160 /*
 161  * Array of Ids
 162  */
 163
 164 static void
 165 write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
 166 {
 167   Id id;
 168   if (!ids)
 169     return;
 170   if (!*ids)
 171     {
 172       write_u8(fp, 0);
 173       return;
 174     }
 175   for (;;)
 176     {
 177       id = *ids++;
 178       if (needid)
 179         id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
 180       if (id >= 64)
 181         id = (id & 63) | ((id & ~63) << 1);
 182       if (!*ids)
 183         {
 184           write_id(fp, id);
 185           return;
 186         }
 187       write_id(fp, id | 64);
 188     }
 189 }
 190
 191
 192 /*
 193  * Repo
 194  */
 195
 196 void
 197 repo_write(Repo *repo, FILE *fp)
 198 {
 199   Pool *pool = repo->pool;
 200   int i, n;
 201   Solvable *s;
 202   NeedId *needid;
 203   int nstrings, nrels, nkeys, nschemata;
 204   unsigned int sizeid;
 205   Reldep *ran;
 206   Id *idarraydata;
 207
 208   int idsizes[RPM_RPMDBID + 1];
 209   int id2key[RPM_RPMDBID + 1];
 210   int nsolvables;
 211
 212   Id *schemadata, *schemadatap, *schema, *sp;
 213   Id schemaid;
 214   int schemadatalen;
 215   Id *solvschema;       /* schema of our solvables */
 216   Id lastschema[256];
 217   Id lastschemakey[256];
 218
 219   nsolvables = 0;
 220   idarraydata = repo->idarraydata;
 221
 222   needid = (NeedId *)xcalloc(pool->ss.nstrings + pool->nrels, sizeof(*needid));
 223
 224   memset(idsizes, 0, sizeof(idsizes));
 225
 226   for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
 227     {
 228       if (s->repo != repo)
 229         continue;
 230       nsolvables++;
 231       needid[s->name].need++;
 232       needid[s->arch].need++;
 233       needid[s->evr].need++;
 234       if (s->vendor)
 235         {
 236           needid[s->vendor].need++;
 237           idsizes[SOLVABLE_VENDOR] = 1;
 238         }
 239       if (s->provides)
 240         idsizes[SOLVABLE_PROVIDES]    += incneedid(pool, idarraydata + s->provides, needid);
 241       if (s->requires)
 242         idsizes[SOLVABLE_REQUIRES]    += incneedid(pool, idarraydata + s->requires, needid);
 243       if (s->conflicts)
 244         idsizes[SOLVABLE_CONFLICTS]   += incneedid(pool, idarraydata + s->conflicts, needid);
 245       if (s->obsoletes)
 246         idsizes[SOLVABLE_OBSOLETES]   += incneedid(pool, idarraydata + s->obsoletes, needid);
 247       if (s->recommends)
 248         idsizes[SOLVABLE_RECOMMENDS]  += incneedid(pool, idarraydata + s->recommends, needid);
 249       if (s->suggests)
 250         idsizes[SOLVABLE_SUGGESTS]    += incneedid(pool, idarraydata + s->suggests, needid);
 251       if (s->supplements)
 252         idsizes[SOLVABLE_SUPPLEMENTS] += incneedid(pool, idarraydata + s->supplements, needid);
 253       if (s->enhances)
 254         idsizes[SOLVABLE_ENHANCES]    += incneedid(pool, idarraydata + s->enhances, needid);
 255       if (s->freshens)
 256         idsizes[SOLVABLE_FRESHENS]    += incneedid(pool, idarraydata + s->freshens, needid);
 257     }
 258   if (nsolvables != repo->nsolvables)
 259     abort();
 260
 261   idsizes[SOLVABLE_NAME] = 1;
 262   idsizes[SOLVABLE_ARCH] = 1;
 263   idsizes[SOLVABLE_EVR] = 1;
 264   if (repo->rpmdbid)
 265     idsizes[RPM_RPMDBID] = 1;
 266
 267   for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
 268     {
 269       if (idsizes[i])
 270         needid[i].need++;
 271     }
 272
 273   needid[0].need = 0;
 274   needid[pool->ss.nstrings].need = 0;
 275   for (i = 0; i < pool->ss.nstrings + pool->nrels; i++)
 276     needid[i].map = i;
 277
 278   qsort(needid + 1, pool->ss.nstrings - 1, sizeof(*needid), needid_cmp_need);
 279   qsort(needid + pool->ss.nstrings, pool->nrels, sizeof(*needid), needid_cmp_need);
 280
 281   sizeid = 0;
 282   for (i = 1; i < pool->ss.nstrings; i++)
 283     {
 284       if (!needid[i].need)
 285         break;
 286       needid[i].need = 0;
 287       sizeid += strlen(pool->ss.stringspace + pool->ss.strings[needid[i].map]) + 1;
 288     }
 289
 290   nstrings = i;
 291   for (i = 0; i < nstrings; i++)
 292     needid[needid[i].map].need = i;
 293
 294   for (i = 0; i < pool->nrels; i++)
 295     {
 296       if (!needid[pool->ss.nstrings + i].need)
 297         break;
 298       else
 299         needid[pool->ss.nstrings + i].need = 0;
 300     }
 301
 302   nrels = i;
 303   for (i = 0; i < nrels; i++)
 304     {
 305       needid[needid[pool->ss.nstrings + i].map].need = nstrings + i;
 306     }
 307
 308   /* find the keys we need */
 309   nkeys = 1;
 310   memset(id2key, 0, sizeof(id2key));
 311   for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
 312     if (idsizes[i])
 313       id2key[i] = nkeys++;
 314
 315   /* find the schemata we need */
 316   solvschema = xcalloc(repo->nsolvables, sizeof(Id));
 317
 318   memset(lastschema, 0, sizeof(lastschema));
 319   memset(lastschemakey, 0, sizeof(lastschemakey));
 320   schemadata = xmalloc(256 * sizeof(Id));
 321   schemadatalen = 256;
 322   schemadatap = schemadata;
 323   *schemadatap++ = 0;
 324   schemadatalen--;
 325   nschemata = 0;
 326   for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
 327     {
 328       unsigned int h;
 329       Id *sp;
 330
 331       if (s->repo != repo)
 332         continue;
 333       if (schemadatalen < 32)
 334         {
 335           int l = schemadatap - schemadata;
 336           fprintf(stderr, "growing schemadata\n");
 337           schemadata = xrealloc(schemadata, (schemadatap - schemadata + 256) * sizeof(Id));
 338           schemadatalen = 256;
 339           schemadatap = schemadata + l;
 340         }
 341       schema = schemadatap;
 342       *schema++ = SOLVABLE_NAME;
 343       *schema++ = SOLVABLE_ARCH;
 344       *schema++ = SOLVABLE_EVR;
 345       if (s->vendor)
 346         *schema++ = SOLVABLE_VENDOR;
 347       if (s->provides)
 348         *schema++ = SOLVABLE_PROVIDES;
 349       if (s->obsoletes)
 350         *schema++ = SOLVABLE_OBSOLETES;
 351       if (s->conflicts)
 352         *schema++ = SOLVABLE_CONFLICTS;
 353       if (s->requires)
 354         *schema++ = SOLVABLE_REQUIRES;
 355       if (s->recommends)
 356         *schema++ = SOLVABLE_RECOMMENDS;
 357       if (s->suggests)
 358         *schema++ = SOLVABLE_SUGGESTS;
 359       if (s->supplements)
 360         *schema++ = SOLVABLE_SUPPLEMENTS;
 361       if (s->enhances)
 362         *schema++ = SOLVABLE_ENHANCES;
 363       if (s->freshens)
 364         *schema++ = SOLVABLE_FRESHENS;
 365       if (repo->rpmdbid)
 366         *schema++ = RPM_RPMDBID;
 367       *schema++ = 0;
 368       for (sp = schemadatap, h = 0; *sp; )
 369         h = h * 7 + *sp++;
 370       h &= 255;
 371       if (lastschema[h] && !memcmp(schemadata + lastschema[h], schemadatap, (schema - schemadatap) * sizeof(Id)))
 372         {
 373           solvschema[n++] = lastschemakey[h];
 374           continue;
 375         }
 376       schemaid = 0;
 377       for (sp = schemadata + 1; sp < schemadatap; )
 378         {
 379           if (!memcmp(sp, schemadatap, (schema - schemadatap) * sizeof(Id)))
 380             break;
 381           while (*sp++)
 382             ;
 383           schemaid++;
 384         }
 385       if (sp >= schemadatap)
 386         {
 387           if (schemaid != nschemata)
 388             abort();
 389           lastschema[h] = schemadatap - schemadata;
 390           lastschemakey[h] = schemaid;
 391           schemadatalen -= schema - schemadatap;
 392           schemadatap = schema;
 393           nschemata++;
 394         }
 395       solvschema[n++] = schemaid;
 396     }
 397   /* convert all schemas to keys */
 398   for (sp = schemadata; sp < schemadatap; sp++)
 399     *sp = id2key[*sp];
 400
 401   /* write file header */
 402   write_u32(fp, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
 403   write_u32(fp, SOLV_VERSION_1);
 404
 405   /* write counts */
 406   write_u32(fp, nstrings);
 407   write_u32(fp, nrels);
 408   write_u32(fp, nsolvables);
 409   write_u32(fp, nkeys);
 410   write_u32(fp, nschemata);
 411   write_u32(fp, 0);     /* no info block */
 412 #if 0
 413   write_u32(fp, SOLV_FLAG_VERTICAL);    /* no flags */
 414 #else
 415   write_u32(fp, 0);     /* no flags */
 416 #endif
 417
 418   /*
 419    * write strings
 420    */
 421   write_u32(fp, sizeid);
 422   for (i = 1; i < nstrings; i++)
 423     {
 424       char *str = pool->ss.stringspace + pool->ss.strings[needid[i].map];
 425       if (fwrite(str, strlen(str) + 1, 1, fp) != 1)
 426         {
 427           perror("write error");
 428           exit(1);
 429         }
 430     }
 431
 432   /*
 433    * write RelDeps
 434    */
 435   for (i = 0; i < nrels; i++)
 436     {
 437       ran = pool->rels + (needid[pool->ss.nstrings + i].map - pool->ss.nstrings);
 438       write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
 439       write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
 440       write_u8( fp, ran->flags);
 441     }
 442
 443   /*
 444    * write keys
 445    */
 446   for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
 447     {
 448       if (!idsizes[i])
 449         continue;
 450       write_id(fp, needid[i].need);
 451       if (i >= SOLVABLE_PROVIDES && i <= SOLVABLE_FRESHENS)
 452         write_id(fp, TYPE_IDARRAY);
 453       else if (i == RPM_RPMDBID)
 454         write_id(fp, TYPE_U32);
 455       else
 456         write_id(fp, TYPE_ID);
 457       write_id(fp, idsizes[i]);
 458     }
 459
 460   /*
 461    * write schemata
 462    */
 463   write_id(fp, schemadatap - schemadata - 1);
 464   for (sp = schemadata + 1; sp < schemadatap; )
 465     {
 466       write_idarray(fp, pool, 0, sp);
 467       while (*sp++)
 468         ;
 469     }
 470
 471 #if 0
 472   if (1)
 473     {
 474       Id id, key;
 475
 476       for (i = 0; i < nsolvables; i++)
 477         write_id(fp, solvschema[i]);
 478       unsigned char *used = xmalloc(nschemata);
 479       for (id = SOLVABLE_NAME; id <= RPM_RPMDBID; id++)
 480         {
 481           key = id2key[id];
 482           memset(used, 0, nschemata);
 483           for (sp = schemadata + 1, i = 0; sp < schemadatap; sp++)
 484             {
 485               if (*sp == 0)
 486                 i++;
 487               else if (*sp == key)
 488                 used[i] = 1;
 489             }
 490           for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
 491             {
 492               if (s->repo != repo)
 493                 continue;
 494               if (!used[solvschema[n++]])
 495                 continue;
 496               switch(id)
 497                 {
 498                 case SOLVABLE_NAME:
 499                   write_id(fp, needid[s->name].need);
 500                   break;
 501                 case SOLVABLE_ARCH:
 502                   write_id(fp, needid[s->arch].need);
 503                   break;
 504                 case SOLVABLE_EVR:
 505                   write_id(fp, needid[s->evr].need);
 506                   break;
 507                 case SOLVABLE_VENDOR:
 508                   write_id(fp, needid[s->vendor].need);
 509                   break;
 510                 case RPM_RPMDBID:
 511                   write_u32(fp, repo->rpmdbid[i - repo->start]);
 512                   break;
 513                 case SOLVABLE_PROVIDES:
 514                   write_idarray(fp, pool, needid, idarraydata + s->provides);
 515                   break;
 516                 case SOLVABLE_OBSOLETES:
 517                   write_idarray(fp, pool, needid, idarraydata + s->obsoletes);
 518                   break;
 519                 case SOLVABLE_CONFLICTS:
 520                   write_idarray(fp, pool, needid, idarraydata + s->conflicts);
 521                   break;
 522                 case SOLVABLE_REQUIRES:
 523                   write_idarray(fp, pool, needid, idarraydata + s->requires);
 524                   break;
 525                 case SOLVABLE_RECOMMENDS:
 526                   write_idarray(fp, pool, needid, idarraydata + s->recommends);
 527                   break;
 528                 case SOLVABLE_SUPPLEMENTS:
 529                   write_idarray(fp, pool, needid, idarraydata + s->supplements);
 530                   break;
 531                 case SOLVABLE_SUGGESTS:
 532                   write_idarray(fp, pool, needid, idarraydata + s->suggests);
 533                   break;
 534                 case SOLVABLE_ENHANCES:
 535                   write_idarray(fp, pool, needid, idarraydata + s->enhances);
 536                   break;
 537                 case SOLVABLE_FRESHENS:
 538                   write_idarray(fp, pool, needid, idarraydata + s->freshens);
 539                   break;
 540                 }
 541             }
 542         }
 543       xfree(used);
 544       xfree(needid);
 545       xfree(solvschema);
 546       xfree(schemadata);
 547       return;
 548     }
 549
 550 #endif
 551
 552   /*
 553    * write Solvables
 554    */
 555   for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
 556     {
 557       if (s->repo != repo)
 558         continue;
 559       /* keep in sync with schema generation! */
 560       write_id(fp, solvschema[n++]);
 561       write_id(fp, needid[s->name].need);
 562       write_id(fp, needid[s->arch].need);
 563       write_id(fp, needid[s->evr].need);
 564       if (s->vendor)
 565         write_id(fp, needid[s->vendor].need);
 566       if (s->provides)
 567         write_idarray(fp, pool, needid, idarraydata + s->provides);
 568       if (s->obsoletes)
 569         write_idarray(fp, pool, needid, idarraydata + s->obsoletes);
 570       if (s->conflicts)
 571         write_idarray(fp, pool, needid, idarraydata + s->conflicts);
 572       if (s->requires)
 573         write_idarray(fp, pool, needid, idarraydata + s->requires);
 574       if (s->recommends)
 575         write_idarray(fp, pool, needid, idarraydata + s->recommends);
 576       if (s->suggests)
 577         write_idarray(fp, pool, needid, idarraydata + s->suggests);
 578       if (s->supplements)
 579         write_idarray(fp, pool, needid, idarraydata + s->supplements);
 580       if (s->enhances)
 581         write_idarray(fp, pool, needid, idarraydata + s->enhances);
 582       if (s->freshens)
 583         write_idarray(fp, pool, needid, idarraydata + s->freshens);
 584       if (repo->rpmdbid)
 585         write_u32(fp, repo->rpmdbid[i - repo->start]);
 586     }
 587
 588   xfree(needid);
 589   xfree(solvschema);
 590   xfree(schemadata);
 591 }
 592
 593 // EOF