X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Frepodata.c;h=b611afce0e192932bc431ef553cdaeade83c86da;hb=a52345de1f81f92a1bd69365def64a91cfd782ff;hp=df120ac5a1be21497452e612fec7609bbf7d49e9;hpb=89331e25f8970a00c380c71f57a3cfcea14017a6;p=platform%2Fupstream%2Flibsolv.git diff --git a/src/repodata.c b/src/repodata.c index df120ac..b611afc 100644 --- a/src/repodata.c +++ b/src/repodata.c @@ -34,29 +34,24 @@ #include "repopack.h" #include "repopage.h" -extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len, - unsigned char *out, unsigned int out_len); -extern unsigned int unchecked_decompress_buf (const unsigned char *in, - unsigned int in_len, - unsigned char *out, - unsigned int out_len); - #define REPODATA_BLOCK 255 +static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key); void repodata_initdata(Repodata *data, Repo *repo, int localpool) { memset(data, 0, sizeof (*data)); + data->repodataid = data - repo->repodata; data->repo = repo; data->localpool = localpool; if (localpool) stringpool_init_empty(&data->spool); /* dirpool_init(&data->dirpool); just zeros out again */ - data->keys = sat_calloc(1, sizeof(Repokey)); + data->keys = solv_calloc(1, sizeof(Repokey)); data->nkeys = 1; - data->schemata = sat_calloc(1, sizeof(Id)); - data->schemadata = sat_calloc(1, sizeof(Id)); + data->schemata = solv_calloc(1, sizeof(Id)); + data->schemadata = solv_calloc(1, sizeof(Id)); data->nschemata = 1; data->schemadatalen = 1; repopagestore_init(&data->store); @@ -67,47 +62,38 @@ repodata_freedata(Repodata *data) { int i; - sat_free(data->keys); + solv_free(data->keys); - sat_free(data->schemata); - sat_free(data->schemadata); - sat_free(data->schematahash); + solv_free(data->schemata); + solv_free(data->schemadata); + solv_free(data->schematahash); stringpool_free(&data->spool); dirpool_free(&data->dirpool); - sat_free(data->mainschemaoffsets); - sat_free(data->incoredata); - sat_free(data->incoreoffset); - sat_free(data->verticaloffset); + solv_free(data->mainschemaoffsets); + solv_free(data->incoredata); + solv_free(data->incoreoffset); + solv_free(data->verticaloffset); repopagestore_free(&data->store); - sat_free(data->vincore); + solv_free(data->vincore); if (data->attrs) for (i = 0; i < data->end - data->start; i++) - sat_free(data->attrs[i]); - sat_free(data->attrs); + solv_free(data->attrs[i]); + solv_free(data->attrs); if (data->xattrs) for (i = 0; i < data->nxattrs; i++) - sat_free(data->xattrs[i]); - sat_free(data->xattrs); - - sat_free(data->attrdata); - sat_free(data->attriddata); -} + solv_free(data->xattrs[i]); + solv_free(data->xattrs); -Repodata * -repodata_create(Repo *repo, int localpool) -{ - Repodata *data; + solv_free(data->attrdata); + solv_free(data->attriddata); + solv_free(data->attrnum64data); - repo->nrepodata++; - repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data)); - data = repo->repodata + repo->nrepodata - 1; - repodata_initdata(data, repo, localpool); - return data; + solv_free(data->dircache); } void @@ -115,10 +101,22 @@ repodata_free(Repodata *data) { Repo *repo = data->repo; int i = data - repo->repodata; + if (i == 0) + return; repodata_freedata(data); if (i < repo->nrepodata - 1) - memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata)); + { + /* whoa! this changes the repodataids! */ + memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata)); + for (; i < repo->nrepodata - 1; i++) + repo->repodata[i].repodataid = i; + } repo->nrepodata--; + if (repo->nrepodata == 1) + { + repo->repodata = solv_free(repo->repodata); + repo->nrepodata = 0; + } } void @@ -156,11 +154,11 @@ repodata_key2id(Repodata *data, Repokey *key, int create) if (!create) return 0; /* allocate new key */ - data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey)); + data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey)); data->keys[data->nkeys++] = *key; if (data->verticaloffset) { - data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id)); + data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id)); data->verticaloffset[data->nkeys - 1] = 0; } data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7); @@ -187,16 +185,16 @@ repodata_schema2id(Repodata *data, Id *schema, int create) return 0; /* XXX: allow empty schema? */ if ((schematahash = data->schematahash) == 0) { - data->schematahash = schematahash = sat_calloc(256, sizeof(Id)); + data->schematahash = schematahash = solv_calloc(256, sizeof(Id)); for (i = 1; i < data->nschemata; i++) { - for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++) + for (sp = data->schemadata + data->schemata[i], h = 0; *sp;) h = h * 7 + *sp++; h &= 255; schematahash[h] = i; } - data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK); - data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK); + data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK); + data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK); } for (sp = schema, len = 0, h = 0; *sp; len++) @@ -217,8 +215,8 @@ repodata_schema2id(Repodata *data, Id *schema, int create) /* a new one */ if (!create) return 0; - data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK); - data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK); + data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK); + data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK); /* add schema */ memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id)); data->schemata[data->nschemata] = data->schemadatalen; @@ -233,10 +231,10 @@ fprintf(stderr, "schema2id: new schema\n"); void repodata_free_schemahash(Repodata *data) { - data->schematahash = sat_free(data->schematahash); + data->schematahash = solv_free(data->schematahash); /* shrink arrays */ - data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id)); - data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id)); + data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id)); + data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id)); } @@ -252,13 +250,27 @@ static inline const char *strchrnul(const char *str, char x) } #endif +#define DIRCACHE_SIZE 41 /* < 1k */ + +#ifdef DIRCACHE_SIZE +struct dircache { + Id ids[DIRCACHE_SIZE]; + char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2]; +}; +#endif + Id repodata_str2dir(Repodata *data, const char *dir, int create) { Id id, parent; +#ifdef DIRCACHE_SIZE + const char *dirs; +#endif const char *dire; parent = 0; + if (!*dir) + return 0; while (*dir == '/' && dir[1] == '/') dir++; if (*dir == '/' && !dir[1]) @@ -267,18 +279,56 @@ repodata_str2dir(Repodata *data, const char *dir, int create) return 1; return dirpool_add_dir(&data->dirpool, 0, 1, create); } +#ifdef DIRCACHE_SIZE + dirs = dir; + if (data->dircache) + { + int l; + struct dircache *dircache = data->dircache; + l = strlen(dir); + while (l > 0) + { + if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l)) + { + parent = dircache->ids[l]; + dir += l; + if (!*dir) + return parent; + while (*dir == '/') + dir++; + break; + } + while (--l) + if (dir[l] == '/') + break; + } + } +#endif while (*dir) { dire = strchrnul(dir, '/'); if (data->localpool) id = stringpool_strn2id(&data->spool, dir, dire - dir, create); else - id = strn2id(data->repo->pool, dir, dire - dir, create); + id = pool_strn2id(data->repo->pool, dir, dire - dir, create); if (!id) return 0; parent = dirpool_add_dir(&data->dirpool, parent, id, create); if (!parent) return 0; +#ifdef DIRCACHE_SIZE + if (!data->dircache) + data->dircache = solv_calloc(1, sizeof(struct dircache)); + if (data->dircache) + { + int l = dire - dirs; + if (l < DIRCACHE_SIZE) + { + data->dircache->ids[l] = parent; + memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l); + } + } +#endif if (!*dire) break; dir = dire + 1; @@ -288,6 +338,12 @@ repodata_str2dir(Repodata *data, const char *dir, int create) return parent; } +void +repodata_free_dircache(Repodata *data) +{ + data->dircache = solv_free(data->dircache); +} + const char * repodata_dir2str(Repodata *data, Id did, const char *suf) { @@ -348,7 +404,7 @@ data_skip_schema(Repodata *data, unsigned char *dp, Id schema) return dp; } -unsigned char * +static unsigned char * data_skip_key(Repodata *data, unsigned char *dp, Repokey *key) { int nentries, schema; @@ -418,23 +474,24 @@ static unsigned char * get_vertical_data(Repodata *data, Repokey *key, Id off, Id len) { unsigned char *dp; - if (!len) + if (len <= 0) return 0; if (off >= data->lastverticaloffset) { off -= data->lastverticaloffset; - if (off + len > data->vincorelen) + if ((unsigned int)off + len > data->vincorelen) return 0; return data->vincore + off; } - if (off + len > key->size) + if ((unsigned int)off + len > key->size) return 0; /* we now have the offset, go into vertical */ off += data->verticaloffset[key - data->keys]; - /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */ - dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE); + /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */ + dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE); + data->storestate++; if (dp) - dp += off % BLOB_PAGESIZE; + dp += off % REPOPAGE_BLOBSIZE; return dp; } @@ -487,7 +544,7 @@ maybe_load_repodata(Repodata *data, Id keyname) if (keyname) { int i; - for (i = 0; i < data->nkeys; i++) + for (i = 1; i < data->nkeys; i++) if (keyname == data->keys[i].name) break; if (i == data->nkeys) @@ -511,17 +568,21 @@ solvid2data(Repodata *data, Id solvid, Id *schemap) unsigned char *dp = data->incoredata; if (!dp) return 0; - if (solvid == SOLVID_META) /* META */ - dp += 1; - else if (solvid == SOLVID_POS) /* META */ + if (solvid == SOLVID_META) + dp += 1; /* offset of "meta" solvable */ + else if (solvid == SOLVID_POS) { Pool *pool = data->repo->pool; if (data->repo != pool->pos.repo) return 0; if (data != data->repo->repodata + pool->pos.repodataid) return 0; - *schemap = pool->pos.schema; - return data->incoredata + pool->pos.dp; + dp += pool->pos.dp; + if (pool->pos.dp != 1) + { + *schemap = pool->pos.schema; + return dp; + } } else { @@ -536,7 +597,7 @@ solvid2data(Repodata *data, Id solvid, Id *schemap) * data lookup */ -static inline unsigned char * +static unsigned char * find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp) { unsigned char *dp; @@ -559,6 +620,8 @@ find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp) return 0; if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) return dp; /* no need to forward... */ + if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET) + return 0; /* get_data will not work, no need to forward */ dp = forward_to_key(data, *kp, keyp, dp); if (!dp) return 0; @@ -611,36 +674,43 @@ repodata_lookup_str(Repodata *data, Id solvid, Id keyname) if (key->type == REPOKEY_TYPE_STR) return (const char *)dp; if (key->type == REPOKEY_TYPE_CONSTANTID) - return id2str(data->repo->pool, key->size); - if (key->type == REPOKEY_TYPE_ID) + id = key->size; + else if (key->type == REPOKEY_TYPE_ID) dp = data_read_id(dp, &id); else return 0; if (data->localpool) - return data->spool.stringspace + data->spool.strings[id]; - return id2str(data->repo->pool, id); + return stringpool_id2str(&data->spool, id); + return pool_id2str(data->repo->pool, id); } int -repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value) +repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value) { unsigned char *dp; Repokey *key; - KeyValue kv; + unsigned int high, low; *value = 0; dp = find_key_data(data, solvid, keyname, &key); if (!dp) return 0; - if (key->type == REPOKEY_TYPE_NUM - || key->type == REPOKEY_TYPE_U32 - || key->type == REPOKEY_TYPE_CONSTANT) + switch (key->type) { - dp = data_fetch(dp, &kv, key); - *value = kv.num; + case REPOKEY_TYPE_NUM: + data_read_num64(dp, &low, &high); + *value = (unsigned long long)high << 32 | low; + return 1; + case REPOKEY_TYPE_U32: + data_read_u32(dp, &low); + *value = low; return 1; + case REPOKEY_TYPE_CONSTANT: + *value = key->size; + return 1; + default: + return 0; } - return 0; } int @@ -671,6 +741,13 @@ repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep) dp = find_key_data(data, solvid, keyname, &key); if (!dp) return 0; + switch (key->type) + { + case_CHKSUM_TYPES: + break; + default: + return 0; + } *typep = key->type; return dp; } @@ -687,7 +764,7 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q) dp = find_key_data(data, solvid, keyname, &key); if (!dp) return 0; - if (key->type != REPOKEY_TYPE_IDARRAY && key->type != REPOKEY_TYPE_REL_IDARRAY) + if (key->type != REPOKEY_TYPE_IDARRAY) return 0; for (;;) { @@ -699,12 +776,30 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q) return 1; } +const void * +repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp) +{ + unsigned char *dp; + Repokey *key; + Id len; + + dp = find_key_data(data, solvid, keyname, &key); + if (!dp || key->type != REPOKEY_TYPE_BINARY) + { + *lenp = 0; + return 0; + } + dp = data_read_id(dp, &len); + *lenp = len; + return dp; +} + Id repodata_globalize_id(Repodata *data, Id id, int create) { if (!id || !data || !data->localpool) return id; - return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create); + return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create); } Id @@ -712,16 +807,111 @@ repodata_localize_id(Repodata *data, Id id, int create) { if (!id || !data || !data->localpool) return id; - return stringpool_str2id(&data->spool, id2str(data->repo->pool, id), create); + return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create); +} + +Id +repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create) +{ + if (!id || !data || !fromdata) + return id; + if (!data->localpool || !fromdata->localpool) + { + if (fromdata->localpool) + id = repodata_globalize_id(fromdata, id, create); + if (data->localpool) + id = repodata_localize_id(data, id, create); + return id; + } + /* localpool is set in both data and fromdata */ + return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create); +} + +Id +repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid) +{ + Id *ap; + if (!data->attrs) + return 0; + ap = data->attrs[solvid - data->start]; + if (!ap) + return 0; + for (; *ap; ap += 2) + { + if (data->keys[*ap].name != keyname) + continue; + if (data->keys[*ap].type == REPOKEY_TYPE_VOID) + return voidid; + if (data->keys[*ap].type == REPOKEY_TYPE_ID) + return ap[1]; + return 0; + } + return 0; +} + +const char * +repodata_lookup_dirstrarray_uninternalized(Repodata *data, Id solvid, Id keyname, Id *didp, Id *iterp) +{ + Id *ap, did; + Id iter = *iterp; + if (iter == 0) /* find key data */ + { + if (!data->attrs) + return 0; + ap = data->attrs[solvid - data->start]; + if (!ap) + return 0; + for (; *ap; ap += 2) + if (data->keys[*ap].name == keyname && data->keys[*ap].type == REPOKEY_TYPE_DIRSTRARRAY) + break; + if (!*ap) + return 0; + iter = ap[1]; + } + did = *didp; + for (ap = data->attriddata + iter; *ap; ap += 2) + { + if (did && ap[0] != did) + continue; + *didp = ap[0]; + *iterp = ap - data->attriddata + 2; + return (const char *)data->attrdata + ap[1]; + } + *iterp = 0; + return 0; } +const unsigned char * +repodata_lookup_bin_checksum_uninternalized(Repodata *data, Id solvid, Id keyname, Id *typep) +{ + Id *ap; + if (!data->attrs) + return 0; + ap = data->attrs[solvid - data->start]; + if (!ap) + return 0; + for (; *ap; ap += 2) + { + if (data->keys[*ap].name != keyname) + continue; + switch (data->keys[*ap].type) + { + case_CHKSUM_TYPES: + *typep = data->keys[*ap].type; + return (const unsigned char *)data->attrdata + ap[1]; + default: + break; + } + } + return 0; +} /************************************************************************ * data search */ -int +const char * repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags) { switch (key->type) @@ -732,7 +922,7 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f if (data && data->localpool) kv->str = stringpool_id2str(&data->spool, kv->id); else - kv->str = id2str(pool, kv->id); + kv->str = pool_id2str(pool, kv->id); if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE) { const char *s; @@ -741,26 +931,26 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f if (*s == ':' && s > kv->str) kv->str = s + 1; } - return 1; + return kv->str; case REPOKEY_TYPE_STR: - return 1; + return kv->str; case REPOKEY_TYPE_DIRSTRARRAY: if (!(flags & SEARCH_FILES)) - return 1; /* match just the basename */ + return kv->str; /* match just the basename */ + if (kv->num) + return kv->str; /* already stringified */ /* Put the full filename into kv->str. */ kv->str = repodata_dir2str(data, kv->id, kv->str); - /* And to compensate for that put the "empty" directory into - kv->id, so that later calls to repodata_dir2str on this data - come up with the same filename again. */ - kv->id = 0; - return 1; - case REPOKEY_TYPE_MD5: - case REPOKEY_TYPE_SHA1: - case REPOKEY_TYPE_SHA256: + kv->num = 1; /* mark stringification */ + return kv->str; + case_CHKSUM_TYPES: if (!(flags & SEARCH_CHECKSUMS)) return 0; /* skip em */ + if (kv->num) + return kv->str; /* already stringified */ kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str); - return 1; + kv->num = 1; /* mark stringification */ + return kv->str; default: return 0; } @@ -907,22 +1097,6 @@ repodata_setpos_kv(Repodata *data, KeyValue *kv) * data iterator functions */ -static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = { - { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, - { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE }, -}; - static inline Id * solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname) { @@ -976,31 +1150,52 @@ solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname) int datamatcher_init(Datamatcher *ma, const char *match, int flags) { + match = match ? solv_strdup(match) : 0; ma->match = match; ma->flags = flags; ma->error = 0; ma->matchdata = 0; if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX) { - ma->matchdata = sat_calloc(1, sizeof(regex_t)); + ma->matchdata = solv_calloc(1, sizeof(regex_t)); ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0)); if (ma->error) { - sat_free(ma->matchdata); + solv_free(ma->matchdata); ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR; } } + if ((flags & SEARCH_FILES) != 0 && match) + { + /* prepare basename check */ + if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND) + { + const char *p = strrchr(match, '/'); + ma->matchdata = (void *)(p ? p + 1 : match); + } + else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB) + { + const char *p; + for (p = match + strlen(match) - 1; p >= match; p--) + if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/') + break; + ma->matchdata = (void *)(p + 1); + } + } return ma->error; } void datamatcher_free(Datamatcher *ma) { + if (ma->match) + ma->match = solv_free((char *)ma->match); if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata) { regfree(ma->matchdata); - ma->matchdata = sat_free(ma->matchdata); + solv_free(ma->matchdata); } + ma->matchdata = 0; } int @@ -1011,67 +1206,67 @@ datamatcher_match(Datamatcher *ma, const char *str) { case SEARCH_SUBSTRING: if (ma->flags & SEARCH_NOCASE) - { - if (!strcasestr(str, ma->match)) - return 0; - } + return strcasestr(str, ma->match) != 0; else - { - if (!strstr(str, ma->match)) - return 0; - } - break; + return strstr(str, ma->match) != 0; case SEARCH_STRING: if (ma->flags & SEARCH_NOCASE) - { - if (strcasecmp(ma->match, str)) - return 0; - } + return !strcasecmp(ma->match, str); else - { - if (strcmp(ma->match, str)) - return 0; - } - break; + return !strcmp(ma->match, str); case SEARCH_STRINGSTART: if (ma->flags & SEARCH_NOCASE) - { - if (strncasecmp(ma->match, str, strlen(ma->match))) - return 0; - } + return !strncasecmp(ma->match, str, strlen(ma->match)); else - { - if (strncmp(ma->match, str, strlen(ma->match))) - return 0; - } - break; + return !strncmp(ma->match, str, strlen(ma->match)); case SEARCH_STRINGEND: l = strlen(str) - strlen(ma->match); if (l < 0) return 0; if (ma->flags & SEARCH_NOCASE) - { - if (strcasecmp(ma->match, str + l)) - return 0; - } + return !strcasecmp(ma->match, str + l); else - { - if (strcmp(ma->match, str + l)) - return 0; - } - break; + return !strcmp(ma->match, str + l); case SEARCH_GLOB: - if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0)) - return 0; - break; + return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0); case SEARCH_REGEX: - if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0)) + return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0); + default: + return 0; + } +} + +/* check if the matcher can match the provides basename */ + +int +datamatcher_checkbasename(Datamatcher *ma, const char *basename) +{ + int l; + const char *match = ma->matchdata; + if (!match) + return 1; + switch (ma->flags & SEARCH_STRINGMASK) + { + case SEARCH_STRING: + break; + case SEARCH_STRINGEND: + if (match != ma->match) + break; /* had slash, do exact match on basename */ + /* FALLTHROUGH */ + case SEARCH_GLOB: + /* check if the basename ends with match */ + l = strlen(basename) - strlen(match); + if (l < 0) return 0; + basename += l; break; default: - return 0; + return 1; /* maybe matches */ } - return 1; + if ((ma->flags & SEARCH_NOCASE) != 0) + return !strcasecmp(match, basename); + else + return !strcmp(match, basename); } int @@ -1110,12 +1305,12 @@ enum { di_entersub, di_leavesub, - di_nextsolvableattr, di_nextsolvablekey, - di_entersolvablekey + di_entersolvablekey, + di_nextsolvableattr }; -/* see repo.h for documentation */ +/* see dataiterator.h for documentation */ int dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags) { @@ -1146,6 +1341,16 @@ void dataiterator_init_clone(Dataiterator *di, Dataiterator *from) { *di = *from; + if (di->dupstr) + { + if (di->dupstr == di->kv.str) + di->dupstr = solv_memdup(di->dupstr, di->dupstrn); + else + { + di->dupstr = 0; + di->dupstrn = 0; + } + } memset(&di->matcher, 0, sizeof(di->matcher)); if (from->matcher.match) datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags); @@ -1181,20 +1386,20 @@ void dataiterator_set_search(Dataiterator *di, Repo *repo, Id p) { di->repo = repo; - di->repoid = -1; + di->repoid = 0; di->flags &= ~SEARCH_THISSOLVID; di->nparents = 0; di->rootlevel = 0; - di->repodataid = 0; - if (!di->pool->nrepos) + di->repodataid = 1; + if (!di->pool->urepos) { di->state = di_bye; return; } if (!repo) { - di->repoid = 0; - di->repo = di->pool->repos[0]; + di->repoid = 1; + di->repo = di->pool->repos[di->repoid]; } di->state = di_enterrepo; if (p) @@ -1230,13 +1435,15 @@ dataiterator_free(Dataiterator *di) { if (di->matcher.match) datamatcher_free(&di->matcher); + if (di->dupstr) + solv_free(di->dupstr); } -static inline unsigned char * +static unsigned char * dataiterator_find_keyname(Dataiterator *di, Id keyname) { - Id *keyp = di->keyp; - Repokey *keys = di->data->keys; + Id *keyp; + Repokey *keys = di->data->keys, *key; unsigned char *dp; for (keyp = di->keyp; *keyp; keyp++) @@ -1244,6 +1451,11 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname) break; if (!*keyp) return 0; + key = keys + *keyp; + if (key->type == REPOKEY_TYPE_DELETED) + return 0; + if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET) + return 0; /* get_data will not work, no need to forward */ dp = forward_to_key(di->data, *keyp, di->keyp, di->dp); if (!dp) return 0; @@ -1251,6 +1463,25 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname) return dp; } +static inline int +is_filelist_extension(Repodata *data) +{ + int j; + if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST)) + return 0; + for (j = 1; j < data->nkeys; j++) + if (data->keys[j].name == SOLVABLE_FILELIST) + break; + if (j == data->nkeys) + return 0; + if (data->state != REPODATA_AVAILABLE) + return 1; + for (j = 1; j < data->nkeys; j++) + if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST) + return 0; + return 1; +} + static int dataiterator_filelistcheck(Dataiterator *di) { @@ -1258,18 +1489,46 @@ dataiterator_filelistcheck(Dataiterator *di) int needcomplete = 0; Repodata *data = di->data; - if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0) + if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0) if (!di->matcher.match || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB) - || !repodata_filelistfilter_matches(di->data, di->matcher.match)) + || !repodata_filelistfilter_matches(data, di->matcher.match)) needcomplete = 1; if (data->state != REPODATA_AVAILABLE) return needcomplete ? 1 : 0; - for (j = 1; j < data->nkeys; j++) - if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST) - break; - return j == data->nkeys && !needcomplete ? 0 : 1; + if (!needcomplete) + { + /* we don't need the complete filelist, so ignore all stubs */ + if (data->repo->nrepodata == 2) + return 1; + for (j = 1; j < data->nkeys; j++) + if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST) + return 1; + return 0; + } + else + { + /* we need the complete filelist. check if we habe a filtered filelist and there's + * a extension with the complete filelist later on */ + for (j = 1; j < data->nkeys; j++) + if (data->keys[j].name == SOLVABLE_FILELIST) + break; + if (j == data->nkeys) + return 0; /* does not have filelist */ + for (j = 1; j < data->nkeys; j++) + if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST) + break; + if (j == data->nkeys) + return 1; /* this is the externsion */ + while (data - data->repo->repodata + 1 < data->repo->nrepodata) + { + data++; + if (is_filelist_extension(data)) + return 0; + } + return 1; + } } int @@ -1277,14 +1536,21 @@ dataiterator_step(Dataiterator *di) { Id schema; + if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) { + unsigned int ddpoff = di->ddp - di->vert_ddp; + di->vert_off += ddpoff; + di->vert_len -= ddpoff; + di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len); + di->vert_storestate = di->data->storestate; + if (!di->ddp) + di->state = di_nextkey; + } for (;;) { switch (di->state) { case di_enterrepo: di_enterrepo: - if (!di->repo) - goto di_bye; - if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)) + if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))) goto di_nextrepo; if (!(di->flags & SEARCH_THISSOLVID)) { @@ -1294,12 +1560,14 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_entersolvable: di_entersolvable: - if (di->repodataid >= 0) + if (di->repodataid) { - di->repodataid = 0; /* reset repodata iterator */ + di->repodataid = 1; /* reset repodata iterator */ if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames) { - di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0); + extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1]; + + di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0); di->data = 0; goto di_entersolvablekey; } @@ -1307,13 +1575,13 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_enterrepodata: di_enterrepodata: - if (di->repodataid >= 0) + if (di->repodataid) { if (di->repodataid >= di->repo->nrepodata) goto di_nextsolvable; di->data = di->repo->repodata + di->repodataid; } - if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di)) + if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di)) goto di_nextrepodata; if (!maybe_load_repodata(di->data, di->keyname)) goto di_nextrepodata; @@ -1340,7 +1608,27 @@ dataiterator_step(Dataiterator *di) case di_enterkey: di_enterkey: di->kv.entry = -1; di->key = di->data->keys + *di->keyp; - di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0); + if (!di->dp) + goto di_nextkey; + /* this is get_data() modified to store vert_ data */ + if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + Id off, len; + di->dp = data_read_id(di->dp, &off); + di->dp = data_read_id(di->dp, &len); + di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len); + di->vert_off = off; + di->vert_len = len; + di->vert_storestate = di->data->storestate; + } + else if (di->key->storage == KEY_STORAGE_INCORE) + { + di->ddp = di->dp; + if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0)) + di->dp = data_skip_key(di->data, di->dp, di->key); + } + else + di->ddp = 0; if (!di->ddp) goto di_nextkey; if (di->key->type == REPOKEY_TYPE_DELETED) @@ -1368,7 +1656,7 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_nextrepodata: di_nextrepodata: - if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata) + if (di->repodataid && ++di->repodataid < di->repo->nrepodata) goto di_enterrepodata; /* FALLTHROUGH */ @@ -1388,10 +1676,10 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_nextrepo: di_nextrepo: - if (di->repoid >= 0) + if (di->repoid > 0) { di->repoid++; - di->repodataid = 0; + di->repodataid = 1; if (di->repoid < di->pool->nrepos) { di->repo = di->pool->repos[di->repoid]; @@ -1407,7 +1695,7 @@ dataiterator_step(Dataiterator *di) case di_enterarray: di_enterarray: if (di->key->name == REPOSITORY_SOLVABLES) goto di_nextkey; - di->ddp = data_read_id(di->ddp, &di->kv.num); + di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num); di->kv.eof = 0; di->kv.entry = -1; /* FALLTHROUGH */ @@ -1468,16 +1756,6 @@ dataiterator_step(Dataiterator *di) /* special solvable attr handling follows */ - case di_nextsolvableattr: - di->kv.id = *di->idp++; - di->kv.entry++; - if (!*di->idp) - { - di->kv.eof = 1; - di->state = di_nextsolvablekey; - } - break; - case di_nextsolvablekey: di_nextsolvablekey: if (di->keyname || di->key->name == RPM_RPMDBID) goto di_enterrepodata; @@ -1486,39 +1764,55 @@ dataiterator_step(Dataiterator *di) case di_entersolvablekey: di_entersolvablekey: di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name); - if (!di->idp || !di->idp[0]) + if (!di->idp || !*di->idp) goto di_nextsolvablekey; - di->kv.id = di->idp[0]; - di->kv.num = di->idp[0]; - di->idp++; - if (!di->kv.eof && !di->idp[0]) - di->kv.eof = 1; - di->kv.entry = 0; if (di->kv.eof) - di->state = di_nextsolvablekey; - else - di->state = di_nextsolvableattr; - break; - } - - if (di->matcher.match) - { - /* simple pre-check so that we don't need to stringify */ - if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING)) { - int l = strlen(di->matcher.match) - strlen(di->kv.str); - if (l < 0 || strcmp(di->matcher.match + l, di->kv.str)) - continue; + /* not an array */ + di->kv.id = *di->idp; + di->kv.num = *di->idp; /* for rpmdbid */ + di->kv.num2 = 0; /* for rpmdbid */ + di->kv.entry = 0; + di->state = di_nextsolvablekey; + break; } - if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)) + di->kv.entry = -1; + /* FALLTHROUGH */ + + case di_nextsolvableattr: + di->state = di_nextsolvableattr; + di->kv.id = *di->idp++; + di->kv.entry++; + if (!*di->idp) + { + di->kv.eof = 1; + di->state = di_nextsolvablekey; + } + break; + + } + + if (di->matcher.match) + { + const char *str; + /* simple pre-check so that we don't need to stringify */ + if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0) + if (!datamatcher_checkbasename(&di->matcher, di->kv.str)) + continue; + if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))) { if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)) return 1; continue; } - if (!datamatcher_match(&di->matcher, di->kv.str)) + if (!datamatcher_match(&di->matcher, str)) continue; } + else + { + if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0) + repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags); + } /* found something! */ return 1; } @@ -1589,6 +1883,13 @@ dataiterator_clonepos(Dataiterator *di, Dataiterator *from) di->parents[i].kv.parent = &di->parents[i - 1].kv; di->kv.parent = &di->parents[di->nparents - 1].kv; } + di->dupstr = 0; + di->dupstrn = 0; + if (from->dupstr && from->dupstr == from->kv.str) + { + di->dupstrn = from->dupstrn; + di->dupstr = solv_memdup(from->dupstr, from->dupstrn); + } } void @@ -1681,30 +1982,31 @@ dataiterator_jump_to_solvid(Dataiterator *di, Id solvid) di->state = di_bye; return; } - di->repoid = -1; - di->data = di->repo->repodata + di->pool->pos.repodataid; - di->repodataid = -1; - di->solvid = solvid; - di->state = di_enterrepo; - di->flags |= SEARCH_THISSOLVID; - return; + di->repoid = 0; + if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) { + solvid = SOLVID_META; /* META pos hack */ + } else { + di->data = di->repo->repodata + di->pool->pos.repodataid; + di->repodataid = 0; + } } - if (solvid > 0) + else if (solvid > 0) { di->repo = di->pool->solvables[solvid].repo; - di->repoid = -1; + di->repoid = 0; } - else if (di->repoid >= 0) + if (di->repoid > 0) { - if (!di->pool->nrepos) + if (!di->pool->urepos) { di->state = di_bye; return; } - di->repo = di->pool->repos[0]; - di->repoid = 0; + di->repoid = 1; + di->repo = di->pool->repos[di->repoid]; } - di->repodataid = 0; + if (solvid != SOLVID_POS) + di->repodataid = 1; di->solvid = solvid; if (solvid) di->flags |= SEARCH_THISSOLVID; @@ -1718,8 +2020,8 @@ dataiterator_jump_to_repo(Dataiterator *di, Repo *repo) di->kv.parent = 0; di->rootlevel = 0; di->repo = repo; - di->repoid = -1; - di->repodataid = 0; + di->repoid = 0; /* 0 means stay at repo */ + di->repodataid = 1; di->solvid = 0; di->flags &= ~SEARCH_THISSOLVID; di->state = di_enterrepo; @@ -1728,11 +2030,56 @@ dataiterator_jump_to_repo(Dataiterator *di, Repo *repo) int dataiterator_match(Dataiterator *di, Datamatcher *ma) { - if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)) + const char *str; + if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))) return 0; - if (!ma) - return 1; - return datamatcher_match(ma, di->kv.str); + return ma ? datamatcher_match(ma, str) : 1; +} + +void +dataiterator_strdup(Dataiterator *di) +{ + int l = -1; + + if (!di->kv.str || di->kv.str == di->dupstr) + return; + switch (di->key->type) + { + case_CHKSUM_TYPES: + case REPOKEY_TYPE_DIRSTRARRAY: + if (di->kv.num) /* was it stringified into tmp space? */ + l = strlen(di->kv.str) + 1; + break; + default: + break; + } + if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + switch (di->key->type) + { + case REPOKEY_TYPE_STR: + case REPOKEY_TYPE_DIRSTRARRAY: + l = strlen(di->kv.str) + 1; + break; + case_CHKSUM_TYPES: + l = solv_chksum_len(di->key->type); + break; + case REPOKEY_TYPE_BINARY: + l = di->kv.num; + break; + } + } + if (l >= 0) + { + if (!di->dupstrn || di->dupstrn < l) + { + di->dupstrn = l + 16; + di->dupstr = solv_realloc(di->dupstr, di->dupstrn); + } + if (l) + memcpy(di->dupstr, di->kv.str, l); + di->kv.str = di->dupstr; + } } /************************************************************************ @@ -1751,10 +2098,10 @@ repodata_extend(Repodata *data, Id p) int new = p - data->end + 1; if (data->attrs) { - data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK); + data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK); memset(data->attrs + old, 0, new * sizeof(Id *)); } - data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK); + data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK); memset(data->incoreoffset + old, 0, new * sizeof(Id)); data->end = p + 1; } @@ -1764,11 +2111,11 @@ repodata_extend(Repodata *data, Id p) int new = data->start - p; if (data->attrs) { - data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK); + data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK); memmove(data->attrs + new, data->attrs, old * sizeof(Id *)); memset(data->attrs, 0, new * sizeof(Id *)); } - data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK); + data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK); memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id)); memset(data->incoreoffset, 0, new * sizeof(Id)); data->start = p; @@ -1788,21 +2135,21 @@ repodata_shrink(Repodata *data, int end) if (data->attrs) { for (i = 0; i < data->end - data->start; i++) - sat_free(data->attrs[i]); - data->attrs = sat_free(data->attrs); + solv_free(data->attrs[i]); + data->attrs = solv_free(data->attrs); } - data->incoreoffset = sat_free(data->incoreoffset); + data->incoreoffset = solv_free(data->incoreoffset); data->start = data->end = 0; return; } if (data->attrs) { for (i = end; i < data->end; i++) - sat_free(data->attrs[i - data->start]); - data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK); + solv_free(data->attrs[i - data->start]); + data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK); } if (data->incoreoffset) - data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK); + data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK); data->end = end; } @@ -1814,7 +2161,8 @@ repodata_extend_block(Repodata *data, Id start, Id num) return; if (!data->incoreoffset) { - data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK); + /* this also means that data->attrs is NULL */ + data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK); data->start = start; data->end = start + num; return; @@ -1830,6 +2178,7 @@ repodata_extend_block(Repodata *data, Id start, Id num) #define REPODATA_ATTRS_BLOCK 31 #define REPODATA_ATTRDATA_BLOCK 1023 #define REPODATA_ATTRIDDATA_BLOCK 63 +#define REPODATA_ATTRNUM64DATA_BLOCK 15 Id @@ -1837,10 +2186,10 @@ repodata_new_handle(Repodata *data) { if (!data->nxattrs) { - data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK); + data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK); data->nxattrs = 2; /* -1: SOLVID_META */ } - data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK); + data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK); data->xattrs[data->nxattrs] = 0; return -(data->nxattrs++); } @@ -1852,7 +2201,7 @@ repodata_get_attrp(Repodata *data, Id handle) { if (handle == SOLVID_META && !data->xattrs) { - data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK); + data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK); data->nxattrs = 2; } return data->xattrs - handle; @@ -1860,7 +2209,7 @@ repodata_get_attrp(Repodata *data, Id handle) if (handle < data->start || handle >= data->end) repodata_extend(data, handle); if (!data->attrs) - data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK); + data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK); return data->attrs + (handle - data->start); } @@ -1892,7 +2241,7 @@ repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite } i = pp - ap; } - ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK); + ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK); *app = ap; pp = ap + i; *pp++ = keyid; @@ -1922,13 +2271,19 @@ repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id) } void -repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num) +repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num) { Repokey key; key.name = keyname; key.type = REPOKEY_TYPE_NUM; key.size = 0; key.storage = KEY_STORAGE_INCORE; + if (num >= 0x80000000) + { + data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK); + data->attrnum64data[data->attrnum64datalen] = num; + num = 0x80000000 | data->attrnum64datalen++; + } repodata_set(data, solvid, &key, (Id)num); } @@ -1940,7 +2295,7 @@ repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str) if (data->localpool) id = stringpool_str2id(&data->spool, str, 1); else - id = str2id(data->repo->pool, str, 1); + id = pool_str2id(data->repo->pool, str, 1); key.name = keyname; key.type = REPOKEY_TYPE_ID; key.size = 0; @@ -1992,7 +2347,7 @@ repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str) key.type = REPOKEY_TYPE_STR; key.size = 0; key.storage = KEY_STORAGE_INCORE; - data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); + data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); memcpy(data->attrdata + data->attrdatalen, str, l); repodata_set(data, solvid, &key, data->attrdatalen); data->attrdatalen += l; @@ -2004,11 +2359,13 @@ repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len) Repokey key; unsigned char *dp; + if (len < 0) + return; key.name = keyname; key.type = REPOKEY_TYPE_BINARY; key.size = 0; key.storage = KEY_STORAGE_INCORE; - data->attrdata = sat_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK); + data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK); dp = data->attrdata + data->attrdatalen; if (len >= (1 << 14)) { @@ -2039,7 +2396,7 @@ repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrys if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen) { /* great! just append the new data */ - data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); data->attriddatalen--; /* overwrite terminating 0 */ data->lastdatalen += entrysize; return; @@ -2062,7 +2419,7 @@ repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrys key.type = keytype; key.size = 0; key.storage = KEY_STORAGE_INCORE; - data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); keyid = repodata_key2id(data, &key, 1); repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1); data->lasthandle = handle; @@ -2076,13 +2433,13 @@ repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrys if (ida + 1 == data->attriddata + data->attriddatalen) { /* this was the last entry, just append it */ - data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); data->attriddatalen--; /* overwrite terminating 0 */ } else { /* too bad. move to back. */ - data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id)); pp[1] = data->attriddatalen; data->attriddatalen += oldsize; @@ -2099,13 +2456,13 @@ repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type, Repokey key; int l; - if (!(l = sat_chksum_len(type))) + if (!(l = solv_chksum_len(type))) return; key.name = keyname; key.type = type; key.size = 0; key.storage = KEY_STORAGE_INCORE; - data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); + data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); memcpy(data->attrdata + data->attrdatalen, str, l); repodata_set(data, solvid, &key, data->attrdatalen); data->attrdatalen += l; @@ -2118,9 +2475,9 @@ repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type, unsigned char buf[64]; int l; - if (!(l = sat_chksum_len(type))) + if (!(l = solv_chksum_len(type))) return; - if (l > sizeof(buf) || sat_hex2bin(&str, buf, l) != l) + if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l) return; repodata_set_bin_checksum(data, solvid, keyname, type, buf); } @@ -2130,7 +2487,7 @@ repodata_chk2str(Repodata *data, Id type, const unsigned char *buf) { int l; - if (!(l = sat_chksum_len(type))) + if (!(l = solv_chksum_len(type))) return ""; return pool_bin2hex(data->repo->pool, buf, l); } @@ -2139,12 +2496,37 @@ repodata_chk2str(Repodata *data, Id type, const unsigned char *buf) static inline const char * evrid2vrstr(Pool *pool, Id evrid) { - const char *p, *evr = id2str(pool, evrid); + const char *p, *evr = pool_id2str(pool, evrid); if (!evr) return evr; for (p = evr; *p >= '0' && *p <= '9'; p++) ; - return p != evr && *p == ':' ? p + 1 : evr; + return p != evr && *p == ':' && p[1] ? p + 1 : evr; +} + +static inline void +repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l) +{ + Id id; + if (data->localpool) + id = stringpool_strn2id(&data->spool, str, l, 1); + else + id = pool_strn2id(data->repo->pool, str, l, 1); + repodata_set_id(data, solvid, keyname, id); +} + +static inline void +repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l) +{ + if (!str[l]) + repodata_set_str(data, solvid, keyname, str); + else + { + char *s = solv_strdup(str); + s[l] = 0; + repodata_set_str(data, solvid, keyname, s); + free(s); + } } void @@ -2180,21 +2562,14 @@ repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, c s = pool->solvables + solvid; if (dir && l) { - str = id2str(pool, s->arch); + str = pool_id2str(pool, s->arch); if (!strncmp(dir, str, l) && !str[l]) repodata_set_void(data, solvid, SOLVABLE_MEDIADIR); - else if (!dir[l]) - repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir); else - { - char *dir2 = strdup(dir); - dir2[l] = 0; - repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2); - free(dir2); - } + repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l); } fp = file; - str = id2str(pool, s->name); + str = pool_id2str(pool, s->name); l = strlen(str); if ((!l || !strncmp(fp, str, l)) && fp[l] == '-') { @@ -2204,7 +2579,7 @@ repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, c if ((!l || !strncmp(fp, str, l)) && fp[l] == '.') { fp += l + 1; - str = id2str(pool, s->arch); + str = pool_id2str(pool, s->arch); l = strlen(str); if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm")) { @@ -2216,6 +2591,128 @@ repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, c repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file); } +/* XXX: medianr is currently not stored */ +void +repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file) +{ + int l = 0; + const char *evr, *suf, *s; + + if (!dir) + { + if ((dir = strrchr(file, '/')) != 0) + { + l = dir - file; + dir = file; + file = dir + l + 1; + if (!l) + l++; + } + } + else + l = strlen(dir); + if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/')) + { + dir += 2; + l -= 2; + } + if (l == 1 && dir[0] == '.') + l = 0; + if (dir && l) + repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l); + evr = strchr(file, '-'); + if (evr) + { + for (s = evr - 1; s > file; s--) + if (*s == '-') + { + evr = s; + break; + } + } + suf = strrchr(file, '.'); + if (suf) + { + for (s = suf - 1; s > file; s--) + if (*s == '.') + { + suf = s; + break; + } + if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm")) + { + /* We accept one more item as suffix. */ + for (s = suf - 1; s > file; s--) + if (*s == '.') + { + suf = s; + break; + } + } + } + if (!evr) + suf = 0; + if (suf && evr && suf < evr) + suf = 0; + repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file)); + if (evr) + repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1)); + if (suf) + repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1); +} + +void +repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg) +{ + Pool *pool = data->repo->pool; + Solvable *s = pool->solvables + solvid; + const char *p, *sevr, *sarch, *name, *evr; + + p = strrchr(sourcepkg, '.'); + if (!p || strcmp(p, ".rpm") != 0) + { + if (*sourcepkg) + repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg); + return; + } + p--; + while (p > sourcepkg && *p != '.') + p--; + if (*p != '.' || p == sourcepkg) + return; + sarch = p-- + 1; + while (p > sourcepkg && *p != '-') + p--; + if (*p != '-' || p == sourcepkg) + return; + p--; + while (p > sourcepkg && *p != '-') + p--; + if (*p != '-' || p == sourcepkg) + return; + sevr = p + 1; + pool = s->repo->pool; + + name = pool_id2str(pool, s->name); + if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0) + repodata_set_void(data, solvid, SOLVABLE_SOURCENAME); + else + repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1)); + + evr = evrid2vrstr(pool, s->evr); + if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0) + repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR); + else + repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1)); + + if (!strcmp(sarch, "src.rpm")) + repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC); + else if (!strcmp(sarch, "nosrc.rpm")) + repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC); + else + repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1)); +} + void repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q) { @@ -2227,7 +2724,7 @@ repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q) key.size = 0; key.storage = KEY_STORAGE_INCORE; repodata_set(data, solvid, &key, data->attriddatalen); - data->attriddata = sat_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); for (i = 0; i < q->count; i++) data->attriddata[data->attriddatalen++] = q->elements[i]; data->attriddata[data->attriddatalen++] = 0; @@ -2255,7 +2752,7 @@ repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *s assert(dir); l = strlen(str) + 1; - data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); + data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK); memcpy(data->attrdata + data->attrdatalen, str, l); stroff = data->attrdatalen; data->attrdatalen += l; @@ -2288,7 +2785,7 @@ repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname, if (data->localpool) id = stringpool_str2id(&data->spool, str, 1); else - id = str2id(data->repo->pool, str, 1); + id = pool_str2id(data->repo->pool, str, 1); repodata_add_idarray(data, solvid, keyname, id); } @@ -2309,7 +2806,7 @@ repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle) } void -repodata_delete_uninternalized(Repodata *data, Id solvid, Id keyname) +repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname) { Id *pp, *ap, **app; app = repodata_get_attrp(data, solvid); @@ -2335,7 +2832,7 @@ repodata_delete_uninternalized(Repodata *data, Id solvid, Id keyname) /* XXX: does not work correctly, needs fix in iterators! */ void -repodata_delete(Repodata *data, Id solvid, Id keyname) +repodata_unset(Repodata *data, Id solvid, Id keyname) { Repokey key; key.name = keyname; @@ -2350,7 +2847,7 @@ void repodata_merge_attrs(Repodata *data, Id dest, Id src) { Id *keyp; - if (dest == src || !(keyp = data->attrs[src - data->start])) + if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start])) return; for (; *keyp; keyp += 2) repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0); @@ -2361,13 +2858,28 @@ void repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite) { Id *keyp; - if (dest == src || !(keyp = data->attrs[src - data->start])) + if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start])) return; for (; *keyp; keyp += 2) if (!keyidmap || MAPTST(keyidmap, keyp[0])) repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite); } +/* swap (uninternalized) attrs from src and dest */ +void +repodata_swap_attrs(Repodata *data, Id dest, Id src) +{ + Id *tmpattrs; + if (!data->attrs || dest == src) + return; + if (dest < data->start || dest >= data->end) + repodata_extend(data, dest); + if (src < data->start || src >= data->end) + repodata_extend(data, src); + tmpattrs = data->attrs[dest - data->start]; + data->attrs[dest - data->start] = data->attrs[src - data->start]; + data->attrs[src - data->start] = tmpattrs; +} /**********************************************************************/ @@ -2382,11 +2894,12 @@ struct extdata { }; static void -data_addid(struct extdata *xd, Id x) +data_addid(struct extdata *xd, Id sx) { + unsigned int x = (unsigned int)sx; unsigned char *dp; - xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK); + xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK); dp = xd->buf + xd->len; if (x >= (1 << 14)) @@ -2404,23 +2917,145 @@ data_addid(struct extdata *xd, Id x) } static void -data_addideof(struct extdata *xd, Id x, int eof) +data_addid64(struct extdata *xd, unsigned long long x) +{ + if (x >= 0x100000000) + { + if ((x >> 35) != 0) + { + data_addid(xd, (Id)(x >> 35)); + xd->buf[xd->len - 1] |= 128; + } + data_addid(xd, (Id)((unsigned int)x | 0x80000000)); + xd->buf[xd->len - 5] = (x >> 28) | 128; + } + else + data_addid(xd, (Id)x); +} + +static void +data_addideof(struct extdata *xd, Id sx, int eof) { - if (x >= 64) - x = (x & 63) | ((x & ~63) << 1); - data_addid(xd, (eof ? x : x | 64)); + unsigned int x = (unsigned int)sx; + unsigned char *dp; + + xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK); + dp = xd->buf + xd->len; + + if (x >= (1 << 13)) + { + if (x >= (1 << 27)) + *dp++ = (x >> 27) | 128; + if (x >= (1 << 20)) + *dp++ = (x >> 20) | 128; + *dp++ = (x >> 13) | 128; + } + if (x >= (1 << 6)) + *dp++ = (x >> 6) | 128; + *dp++ = eof ? (x & 63) : (x & 63) | 64; + xd->len = dp - xd->buf; } static void data_addblob(struct extdata *xd, unsigned char *blob, int len) { - xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK); + xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK); memcpy(xd->buf + xd->len, blob, len); xd->len += len; } /*********************************/ +/* this is to reduct memory usage when internalizing oversized repos */ +static void +compact_attrdata(Repodata *data, int entry, int nentry) +{ + int i; + unsigned int attrdatastart = data->attrdatalen; + unsigned int attriddatastart = data->attriddatalen; + if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024) + return; + for (i = entry; i < nentry; i++) + { + Id v, *attrs = data->attrs[i]; + if (!attrs) + continue; + for (; *attrs; attrs += 2) + { + switch (data->keys[*attrs].type) + { + case REPOKEY_TYPE_STR: + case REPOKEY_TYPE_BINARY: + case_CHKSUM_TYPES: + if ((unsigned int)attrs[1] < attrdatastart) + attrdatastart = attrs[1]; + break; + case REPOKEY_TYPE_DIRSTRARRAY: + for (v = attrs[1]; data->attriddata[v] ; v += 2) + if (data->attriddata[v + 1] < attrdatastart) + attrdatastart = data->attriddata[v + 1]; + /* FALLTHROUGH */ + case REPOKEY_TYPE_IDARRAY: + case REPOKEY_TYPE_DIRNUMNUMARRAY: + if ((unsigned int)attrs[1] < attriddatastart) + attriddatastart = attrs[1]; + break; + case REPOKEY_TYPE_FIXARRAY: + case REPOKEY_TYPE_FLEXARRAY: + return; + default: + break; + } + } + } +#if 0 + printf("compact_attrdata %d %d\n", entry, nentry); + printf("attrdatastart: %d\n", attrdatastart); + printf("attriddatastart: %d\n", attriddatastart); +#endif + if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024) + return; + for (i = entry; i < nentry; i++) + { + Id v, *attrs = data->attrs[i]; + if (!attrs) + continue; + for (; *attrs; attrs += 2) + { + switch (data->keys[*attrs].type) + { + case REPOKEY_TYPE_STR: + case REPOKEY_TYPE_BINARY: + case_CHKSUM_TYPES: + attrs[1] -= attrdatastart; + break; + case REPOKEY_TYPE_DIRSTRARRAY: + for (v = attrs[1]; data->attriddata[v] ; v += 2) + data->attriddata[v + 1] -= attrdatastart; + /* FALLTHROUGH */ + case REPOKEY_TYPE_IDARRAY: + case REPOKEY_TYPE_DIRNUMNUMARRAY: + attrs[1] -= attriddatastart; + break; + default: + break; + } + } + } + if (attrdatastart) + { + data->attrdatalen -= attrdatastart; + memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen); + data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK); + } + if (attriddatastart) + { + data->attriddatalen -= attriddatastart; + memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id)); + data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + } +} + /* internalalize some key into incore/vincore data */ static void @@ -2445,6 +3080,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, case REPOKEY_TYPE_VOID: case REPOKEY_TYPE_CONSTANT: case REPOKEY_TYPE_CONSTANTID: + case REPOKEY_TYPE_DELETED: break; case REPOKEY_TYPE_STR: data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1); @@ -2455,11 +3091,26 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, case REPOKEY_TYPE_SHA1: data_addblob(xd, data->attrdata + val, SIZEOF_SHA1); break; + case REPOKEY_TYPE_SHA224: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA224); + break; case REPOKEY_TYPE_SHA256: data_addblob(xd, data->attrdata + val, SIZEOF_SHA256); break; - case REPOKEY_TYPE_ID: + case REPOKEY_TYPE_SHA384: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA384); + break; + case REPOKEY_TYPE_SHA512: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA512); + break; case REPOKEY_TYPE_NUM: + if (val & 0x80000000) + { + data_addid64(xd, data->attrnum64data[val ^ 0x80000000]); + break; + } + /* FALLTHROUGH */ + case REPOKEY_TYPE_ID: case REPOKEY_TYPE_DIR: data_addid(xd, val); break; @@ -2467,7 +3118,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, { Id len; unsigned char *dp = data_read_id(data->attrdata + val, &len); - dp += len; + dp += (unsigned int)len; data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val)); } break; @@ -2496,32 +3147,34 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, schemaid = 0; for (ida = data->attriddata + val; *ida; ida++) { + Id *kp; sp = schema; - Id *kp = data->xattrs[-*ida]; + kp = data->xattrs[-*ida]; if (!kp) - continue; + continue; /* ignore empty elements */ num++; - for (;*kp; kp += 2) + for (; *kp; kp += 2) *sp++ = *kp; *sp = 0; if (!schemaid) schemaid = repodata_schema2id(data, schema, 1); else if (schemaid != repodata_schema2id(data, schema, 0)) { - pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n"); - exit(1); + pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n"); + num = 0; + break; } } + data_addid(xd, num); if (!num) break; - data_addid(xd, num); data_addid(xd, schemaid); for (ida = data->attriddata + val; *ida; ida++) { Id *kp = data->xattrs[-*ida]; if (!kp) continue; - for (;*kp; kp += 2) + for (; *kp; kp += 2) repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]); } break; @@ -2553,7 +3206,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, break; } default: - pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type); + pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type); exit(1); } if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) @@ -2565,21 +3218,66 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, } } +/* create a circular linked list of all keys that share + * the same keyname */ +static Id * +calculate_keylink(Repodata *data) +{ + int i, j; + Id *link; + Id maxkeyname = 0, *keytable = 0; + link = solv_calloc(data->nkeys, sizeof(Id)); + if (data->nkeys <= 2) + return link; + for (i = 1; i < data->nkeys; i++) + { + Id n = data->keys[i].name; + if (n >= maxkeyname) + { + keytable = solv_realloc2(keytable, n + 128, sizeof(Id)); + memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id)); + maxkeyname = n + 128; + } + j = keytable[n]; + if (j) + link[i] = link[j]; + else + j = i; + link[j] = i; + keytable[n] = i; + } + /* remove links that just point to themselfs */ + for (i = 1; i < data->nkeys; i++) + if (link[i] == i) + link[i] = 0; + solv_free(keytable); + return link; +} + void repodata_internalize(Repodata *data) { Repokey *key, solvkey; Id entry, nentry; - Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen; + Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen; + Offset *oldincoreoffs = 0; + int schemaidx; unsigned char *dp, *ndp; - int newschema, oldcount; + int neednewschema; struct extdata newincore; struct extdata newvincore; Id solvkeyid; + Id *keylink; + int haveoldkl; if (!data->attrs && !data->xattrs) return; +#if 0 + printf("repodata_internalize %d\n", data->repodataid); + printf(" attr data: %d K\n", data->attrdatalen / 1024); + printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4)); +#endif newvincore.buf = data->vincore; newvincore.len = data->vincorelen; @@ -2591,8 +3289,8 @@ repodata_internalize(Repodata *data) solvkey.storage = KEY_STORAGE_INCORE; solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0); - schema = sat_malloc2(data->nkeys, sizeof(Id)); - seen = sat_malloc2(data->nkeys, sizeof(Id)); + schema = solv_malloc2(data->nkeys, sizeof(Id)); + seen = solv_malloc2(data->nkeys, sizeof(Id)); /* Merge the data already existing (in data->schemata, ->incoredata and friends) with the new attributes in data->attrs[]. */ @@ -2601,181 +3299,246 @@ repodata_internalize(Repodata *data) data_addid(&newincore, 0); /* start data at offset 1 */ data->mainschema = 0; - data->mainschemaoffsets = sat_free(data->mainschemaoffsets); + data->mainschemaoffsets = solv_free(data->mainschemaoffsets); + keylink = calculate_keylink(data); /* join entry data */ /* we start with the meta data, entry -1 */ for (entry = -1; entry < nentry; entry++) { - memset(seen, 0, data->nkeys * sizeof(Id)); - oldschema = 0; + oldschemaid = 0; dp = data->incoredata; if (dp) { dp += entry >= 0 ? data->incoreoffset[entry] : 1; - dp = data_read_id(dp, &oldschema); + dp = data_read_id(dp, &oldschemaid); } + memset(seen, 0, data->nkeys * sizeof(Id)); #if 0 -fprintf(stderr, "oldschema %d\n", oldschema); -fprintf(stderr, "schemata %d\n", data->schemata[oldschema]); +fprintf(stderr, "oldschemaid %d\n", oldschemaid); +fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]); fprintf(stderr, "schemadata %p\n", data->schemadata); #endif - /* seen: -1: old data 0: skipped >0: id + 1 */ - newschema = 0; - oldcount = 0; + + /* seen: -1: old data, 0: skipped, >0: id + 1 */ + neednewschema = 0; sp = schema; - for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++) + haveoldkl = 0; + for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++) { if (seen[*keyp]) { - pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n"); - exit(1); + /* oops, should not happen */ + neednewschema = 1; + continue; } - seen[*keyp] = -1; + seen[*keyp] = -1; /* use old marker */ *sp++ = *keyp; - oldcount++; + if (keylink[*keyp]) + haveoldkl = 1; /* potential keylink conflict */ } - if (entry >= 0) - keyp = data->attrs ? data->attrs[entry] : 0; - else + + /* strip solvables key */ + if (entry < 0 && solvkeyid && seen[solvkeyid]) { - /* strip solvables key */ *sp = 0; for (sp = keyp = schema; *sp; sp++) if (*sp != solvkeyid) *keyp++ = *sp; - else - oldcount--; sp = keyp; seen[solvkeyid] = 0; - keyp = data->xattrs ? data->xattrs[1] : 0; + neednewschema = 1; } + + /* add new entries */ + if (entry >= 0) + keyp = data->attrs ? data->attrs[entry] : 0; + else + keyp = data->xattrs ? data->xattrs[1] : 0; if (keyp) for (; *keyp; keyp += 2) { if (!seen[*keyp]) { - newschema = 1; + neednewschema = 1; *sp++ = *keyp; + if (haveoldkl && keylink[*keyp]) /* this should be pretty rare */ + { + Id kl; + for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl]) + if (seen[kl] == -1) + { + /* replacing old key kl, remove from schema and seen */ + Id *osp; + for (osp = schema; osp < sp; osp++) + if (*osp == kl) + { + memmove(osp, osp + 1, (sp - osp) * sizeof(Id)); + sp--; + seen[kl] = 0; + break; + } + } + } } seen[*keyp] = keyp[1] + 1; } + + /* add solvables key if needed */ if (entry < 0 && data->end != data->start) { - *sp++ = solvkeyid; - newschema = 1; + *sp++ = solvkeyid; /* always last in schema */ + neednewschema = 1; } + + /* commit schema */ *sp = 0; - if (newschema) + if (neednewschema) /* Ideally we'd like to sort the new schema here, to ensure - schema equality independend of the ordering. We can't do that - yet. For once see below (old ids need to come before new ids). - An additional difficulty is that we also need to move - the values with the keys. */ + schema equality independend of the ordering. */ schemaid = repodata_schema2id(data, schema, 1); else - schemaid = oldschema; + schemaid = oldschemaid; + + if (entry < 0) + { + data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id)); + data->mainschema = schemaid; + } + + /* find offsets in old incore data */ + if (oldschemaid) + { + Id *lastneeded = 0; + for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++) + if (seen[*sp] == -1) + lastneeded = sp + 1; + if (lastneeded) + { + if (!oldincoreoffs) + oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset)); + for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++) + { + /* Skip the data associated with this old key. */ + key = data->keys + *sp; + ndp = dp; + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + ndp = data_skip(ndp, REPOKEY_TYPE_ID); + ndp = data_skip(ndp, REPOKEY_TYPE_ID); + } + else if (key->storage == KEY_STORAGE_INCORE) + ndp = data_skip_key(data, ndp, key); + oldincoreoffs[*sp * 2] = dp - data->incoredata; + oldincoreoffs[*sp * 2 + 1] = ndp - dp; + dp = ndp; + } + } + } + /* just copy over the complete old entry (including the schemaid) if there was no new data */ + if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp) + { + ndp = data->incoredata + data->incoreoffset[entry]; + data->incoreoffset[entry] = newincore.len; + data_addblob(&newincore, ndp, dp - ndp); + goto entrydone; + } /* Now create data blob. We walk through the (possibly new) schema and either copy over old data, or insert the new. */ - /* XXX Here we rely on the fact that the (new) schema has the form - o1 o2 o3 o4 ... | n1 n2 n3 ... - (oX being the old keyids (possibly overwritten), and nX being - the new keyids). This rules out sorting the keyids in order - to ensure a small schema count. */ if (entry >= 0) data->incoreoffset[entry] = newincore.len; data_addid(&newincore, schemaid); - if (entry == -1) - { - data->mainschema = schemaid; - data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id)); - } - keypstart = data->schemadata + data->schemata[schemaid]; - for (keyp = keypstart; *keyp; keyp++) + + /* we don't use a pointer to the schemadata here as repodata_serialize_key + * may call repodata_schema2id() which might realloc our schemadata */ + for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++) { - if (entry == -1) - data->mainschemaoffsets[keyp - keypstart] = newincore.len; - if (*keyp == solvkeyid) + if (entry < 0) { - /* add flexarray entry count */ - data_addid(&newincore, data->end - data->start); - break; - } - key = data->keys + *keyp; -#if 0 - fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type)); -#endif - ndp = dp; - if (oldcount) - { - /* Skip the data associated with this old key. */ - if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len; + if (keyid == solvkeyid) { - ndp = data_skip(dp, REPOKEY_TYPE_ID); - ndp = data_skip(ndp, REPOKEY_TYPE_ID); + /* add flexarray entry count */ + data_addid(&newincore, data->end - data->start); + break; /* always the last entry */ } - else if (key->storage == KEY_STORAGE_INCORE) - ndp = data_skip_key(data, dp, key); - oldcount--; } - if (seen[*keyp] == -1) + if (seen[keyid] == -1) { - /* If this key was an old one _and_ was not overwritten with - a different value copy over the old value (we skipped it - above). */ - if (dp != ndp) - data_addblob(&newincore, dp, ndp - dp); - seen[*keyp] = 0; + if (oldincoreoffs[keyid * 2 + 1]) + data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]); } - else if (seen[*keyp]) + else if (seen[keyid]) + repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1); + } + +entrydone: + /* free memory */ + if (entry >= 0 && data->attrs) + { + if (data->attrs[entry]) + data->attrs[entry] = solv_free(data->attrs[entry]); + if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry) { - /* Otherwise we have a new value. Parse it into the internal - form. */ - repodata_serialize_key(data, &newincore, &newvincore, - schema, key, seen[*keyp] - 1); + compact_attrdata(data, entry + 1, nentry); /* try to free some memory */ +#if 0 + printf(" attr data: %d K\n", data->attrdatalen / 1024); + printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4)); + printf(" incore data: %d K\n", newincore.len / 1024); + printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024); + /* malloc_stats(); */ +#endif } - dp = ndp; } - if (entry >= 0 && data->attrs && data->attrs[entry]) - data->attrs[entry] = sat_free(data->attrs[entry]); } /* free all xattrs */ for (entry = 0; entry < data->nxattrs; entry++) if (data->xattrs[entry]) - sat_free(data->xattrs[entry]); - data->xattrs = sat_free(data->xattrs); + solv_free(data->xattrs[entry]); + data->xattrs = solv_free(data->xattrs); data->nxattrs = 0; data->lasthandle = 0; data->lastkey = 0; data->lastdatalen = 0; - sat_free(schema); - sat_free(seen); + solv_free(schema); + solv_free(seen); + solv_free(keylink); + solv_free(oldincoreoffs); repodata_free_schemahash(data); - sat_free(data->incoredata); + solv_free(data->incoredata); data->incoredata = newincore.buf; data->incoredatalen = newincore.len; data->incoredatafree = 0; - sat_free(data->vincore); + solv_free(data->vincore); data->vincore = newvincore.buf; data->vincorelen = newvincore.len; - data->attrs = sat_free(data->attrs); - data->attrdata = sat_free(data->attrdata); - data->attriddata = sat_free(data->attriddata); + data->attrs = solv_free(data->attrs); + data->attrdata = solv_free(data->attrdata); + data->attriddata = solv_free(data->attriddata); + data->attrnum64data = solv_free(data->attrnum64data); data->attrdatalen = 0; data->attriddatalen = 0; + data->attrnum64datalen = 0; +#if 0 + printf("repodata_internalize %d done\n", data->repodataid); + printf(" incore data: %d K\n", data->incoredatalen / 1024); +#endif } void repodata_disable_paging(Repodata *data) { if (maybe_load_repodata(data, 0)) - repopagestore_disable_paging(&data->store); + { + repopagestore_disable_paging(&data->store); + data->storestate++; + } } static void @@ -2783,7 +3546,9 @@ repodata_load_stub(Repodata *data) { Repo *repo = data->repo; Pool *pool = repo->pool; - int r; + int r, i; + struct _Pool_tmpspace oldtmpspace; + Datapos oldpos; if (!pool->loadcallback) { @@ -2791,11 +3556,54 @@ repodata_load_stub(Repodata *data) return; } data->state = REPODATA_LOADING; + + /* save tmp space and pos */ + oldtmpspace = pool->tmpspace; + memset(&pool->tmpspace, 0, sizeof(pool->tmpspace)); + oldpos = pool->pos; + r = pool->loadcallback(pool, data, pool->loadcallbackdata); + + /* restore tmp space and pos */ + for (i = 0; i < POOL_TMPSPACEBUF; i++) + solv_free(pool->tmpspace.buf[i]); + pool->tmpspace = oldtmpspace; + if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid) + memset(&oldpos, 0, sizeof(oldpos)); + pool->pos = oldpos; + data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR; } -void +static inline void +repodata_add_stubkey(Repodata *data, Id keyname, Id keytype) +{ + Repokey xkey; + + xkey.name = keyname; + xkey.type = keytype; + xkey.storage = KEY_STORAGE_INCORE; + xkey.size = 0; + repodata_key2id(data, &xkey, 1); +} + +static Repodata * +repodata_add_stub(Repodata **datap) +{ + Repodata *data = *datap; + Repo *repo = data->repo; + Id repodataid = data - repo->repodata; + Repodata *sdata = repo_add_repodata(repo, 0); + data = repo->repodata + repodataid; + if (data->end > data->start) + repodata_extend_block(sdata, data->start, data->end - data->start); + sdata->state = REPODATA_STUB; + sdata->loadcallback = repodata_load_stub; + *datap = data; + return sdata; +} + +Repodata * repodata_create_stubs(Repodata *data) { Repo *repo = data->repo; @@ -2805,38 +3613,26 @@ repodata_create_stubs(Repodata *data) Dataiterator di; Id xkeyname = 0; int i, cnt = 0; - int repodataid; - int datastart, dataend; - repodataid = data - repo->repodata; - datastart = data->start; - dataend = data->end; dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0); while (dataiterator_step(&di)) - { - if (di.data - repo->repodata != repodataid) - continue; + if (di.data == data) cnt++; - } dataiterator_free(&di); if (!cnt) - return; - stubdataids = sat_calloc(cnt, sizeof(*stubdataids)); + return data; + stubdataids = solv_calloc(cnt, sizeof(*stubdataids)); for (i = 0; i < cnt; i++) { - sdata = repo_add_repodata(repo, 0); - if (dataend > datastart) - repodata_extend_block(sdata, datastart, dataend - datastart); + sdata = repodata_add_stub(&data); stubdataids[i] = sdata - repo->repodata; - sdata->state = REPODATA_STUB; - sdata->loadcallback = repodata_load_stub; } i = 0; dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0); sdata = 0; while (dataiterator_step(&di)) { - if (di.data - repo->repodata != repodataid) + if (di.data != data) continue; if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents) { @@ -2860,32 +3656,27 @@ repodata_create_stubs(Repodata *data) repodata_set_void(sdata, SOLVID_META, di.key->name); break; case REPOKEY_TYPE_NUM: - repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num); + repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv)); break; - case REPOKEY_TYPE_MD5: - case REPOKEY_TYPE_SHA1: - case REPOKEY_TYPE_SHA256: + case_CHKSUM_TYPES: repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str); break; case REPOKEY_TYPE_IDARRAY: repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id); if (di.key->name == REPOSITORY_KEYS) { - Repokey xkey; - if (!xkeyname) { if (!di.kv.eof) xkeyname = di.kv.id; - continue; } - xkey.name = xkeyname; - xkey.type = di.kv.id; - xkey.storage = KEY_STORAGE_INCORE; - xkey.size = 0; - repodata_key2id(sdata, &xkey, 1); - xkeyname = 0; + else + { + repodata_add_stubkey(sdata, xkeyname, di.kv.id); + xkeyname = 0; + } } + break; default: break; } @@ -2893,9 +3684,13 @@ repodata_create_stubs(Repodata *data) dataiterator_free(&di); for (i = 0; i < cnt; i++) repodata_internalize(repo->repodata + stubdataids[i]); - sat_free(stubdataids); + solv_free(stubdataids); + return data; +} + +unsigned int +repodata_memused(Repodata *data) +{ + return data->incoredatalen + data->vincorelen; } -/* -vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: -*/