From 5fa45fa3e131be408a4b6e133cbe40bc83e0c613 Mon Sep 17 00:00:00 2001 From: Tomas Mlcoch Date: Wed, 5 Sep 2012 13:48:57 +0200 Subject: [PATCH] Fix bug in optimalization for memory saving. --- src/createrepo_c.c | 18 ++++++++++-------- src/load_metadata.c | 29 +++++++++++++++++++++++++---- src/load_metadata.h | 14 ++++++-------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/createrepo_c.c b/src/createrepo_c.c index 1478d76..a796e53 100644 --- a/src/createrepo_c.c +++ b/src/createrepo_c.c @@ -403,8 +403,8 @@ main(int argc, char **argv) // Recursive walk - GHashTable *pkglist_ht = g_hash_table_new(g_str_hash, g_str_equal); - /* ^^^ Hashtable with basenames of files which will be processed */ + GSList *current_pkglist = NULL; + /* ^^^ List with basenames of files which will be processed */ int package_count = 0; if (!(cmd_options->include_pkgs)) { @@ -477,7 +477,7 @@ main(int argc, char **argv) task->path = g_strdup(dirname); if (output_pkg_list) fprintf(output_pkg_list, "%s\n", repo_relative_path); - g_hash_table_insert(pkglist_ht, task->filename, NULL); + current_pkglist = g_slist_prepend(current_pkglist, task->filename); // TODO: One common path for all tasks with the same path? g_thread_pool_push(pool, task, NULL); package_count++; @@ -527,7 +527,7 @@ main(int argc, char **argv) task->path = strndup(relative_path, x); // packages/i386/ if (output_pkg_list) fprintf(output_pkg_list, "%s\n", relative_path); - g_hash_table_insert(pkglist_ht, task->filename, NULL); + current_pkglist = g_slist_prepend(current_pkglist, task->filename); g_thread_pool_push(pool, task, NULL); package_count++; } @@ -558,7 +558,9 @@ main(int argc, char **argv) else old_metadata_location = cr_get_metadata_location(in_dir, 1); - ret = cr_load_xml_metadata(old_metadata, old_metadata_location, pkglist_ht); + ret = cr_load_xml_metadata(old_metadata, + old_metadata_location, + current_pkglist); if (ret == CR_LOAD_METADATA_OK) g_debug("Old metadata from: %s - loaded", out_dir); else @@ -571,7 +573,7 @@ main(int argc, char **argv) g_message("Loading metadata from: %s", path); ret = cr_locate_and_load_xml_metadata(old_metadata, path, - pkglist_ht); + current_pkglist); if (ret == CR_LOAD_METADATA_OK) g_debug("Metadata from md-path %s - loaded", path); else @@ -582,8 +584,8 @@ main(int argc, char **argv) g_hash_table_size(old_metadata->ht)); } - g_hash_table_destroy(pkglist_ht); - pkglist_ht = NULL; + g_slist_free(current_pkglist); + current_pkglist = NULL; // Copy groupfile diff --git a/src/load_metadata.c b/src/load_metadata.c index c236475..6827c6a 100644 --- a/src/load_metadata.c +++ b/src/load_metadata.c @@ -570,6 +570,9 @@ pri_end_handler(void *data, const char *el) basename, NULL, NULL); + if (store) + // remove pkg from allowed packages + g_hash_table_remove(ppd->pkglist, basename); } if (ppd->chunk) @@ -1014,12 +1017,13 @@ oth_end_handler(void *data, const char *el) int load_xml_files(GHashTable *hashtable, const char *primary_xml_path, const char *filelists_xml_path, const char *other_xml_path, - GStringChunk *chunk, GHashTable *pkglist) + GStringChunk *chunk, GSList *pkglist) { cr_CompressionType compression_type; CR_FILE *pri_xml_cwfile, *fil_xml_cwfile, *oth_xml_cwfile; XML_Parser pri_p, fil_p, oth_p; struct ParserData parser_data; + GHashTable *pkglist_ht = NULL; // Detect compression type @@ -1049,6 +1053,22 @@ load_xml_files(GHashTable *hashtable, const char *primary_xml_path, } + // Create hashtable from pkglist + // This hashtable is used for checking if the metadata of the package + // should be included. + // Purpose is to save memory - We load only metadata about + // packages which we probably will use + // This hashtable is modified "on the fly" - If we found and load + // a metadata about the package, we remove its record from the hashtable. + // So if we met the metadata for this package again we will ignore it. + + if (pkglist) { + GSList *elem; + pkglist_ht = g_hash_table_new(g_str_hash, g_str_equal); + for (elem = pkglist; elem; elem = g_slist_next(elem)) + g_hash_table_insert(pkglist_ht, elem->data, NULL); + } + // Prepare parsers parser_data.current_string = g_string_sized_new(1024); @@ -1058,7 +1078,7 @@ load_xml_files(GHashTable *hashtable, const char *primary_xml_path, parser_data.last_elem = NONE_ELEM; parser_data.error = FALSE; parser_data.chunk = chunk; - parser_data.pkglist = pkglist; + parser_data.pkglist = pkglist_ht; pri_p = XML_ParserCreate(NULL); XML_SetUserData(pri_p, (void *) &parser_data); @@ -1184,6 +1204,7 @@ cleanup: // Cleanup + if (pkglist_ht) g_hash_table_destroy(pkglist_ht); XML_ParserFree(pri_p); XML_ParserFree(fil_p); XML_ParserFree(oth_p); @@ -1206,7 +1227,7 @@ cleanup: int cr_load_xml_metadata(cr_Metadata md, struct cr_MetadataLocation *ml, - GHashTable *pkglist) + GSList *pkglist) { if (!md || !ml) return CR_LOAD_METADATA_ERR; @@ -1286,7 +1307,7 @@ cr_load_xml_metadata(cr_Metadata md, int cr_locate_and_load_xml_metadata(cr_Metadata md, const char *repopath, - GHashTable *pkglist) + GSList *pkglist) { if (!md || !repopath) return CR_LOAD_METADATA_ERR; diff --git a/src/load_metadata.h b/src/load_metadata.h index e3ac809..7ee566c 100644 --- a/src/load_metadata.h +++ b/src/load_metadata.h @@ -83,27 +83,25 @@ void cr_destroy_metadata(cr_Metadata md); * Load metadata from the specified location. * @param md metadata object * @param ml metadata location - * @param pkglist load only packages which base filename is as a key - * in this hashtable (value is ignored). If param is - * NULL all packages are loaded. + * @param pkglist load only packages which base filename is in this + * list. If param is NULL all packages are loaded. * @return return code CR_LOAD_METADATA_OK or CR_LOAD_METADATA_ERR */ int cr_load_xml_metadata(cr_Metadata md, struct cr_MetadataLocation *ml, - GHashTable *pkglist); + GSList *pkglist); /** \ingroup load_metadata * Locate and load metadata from the specified path. * @param md metadata object * @param repopath path to repo (to directory with repodata/ subdir) - * @param pkglist load only packages which base filename is as a key - * in this hashtable (value is ignored). If param is - * NULL all packages are loaded. + * @param pkglist load only packages which base filename is in this + * list. If param is NULL all packages are loaded. * @return return code CR_LOAD_METADATA_OK or CR_LOAD_METADATA_ERR */ int cr_locate_and_load_xml_metadata(cr_Metadata md, const char *repopath, - GHashTable *pkglist); + GSList *pkglist); #ifdef __cplusplus } -- 2.7.4