From 6ac8dc123cd55e692134723d9a2e364d42926b4f Mon Sep 17 00:00:00 2001 From: Tomas Mlcoch Date: Wed, 25 Jun 2014 16:09:11 +0200 Subject: [PATCH] Implements support for --cachedir (interfaces of parsepkg and parsehdr module were slightly modified) --- src/dumper_thread.c | 79 +++++++++++++++++++++++++++++++++++++++++++++--- src/parsehdr.c | 52 ++++++++++++++++++------------- src/parsehdr.h | 10 ++++++ src/parsepkg.c | 7 +++-- src/parsepkg.h | 5 +++ src/python/parsepkg-py.c | 4 ++- 6 files changed, 128 insertions(+), 29 deletions(-) diff --git a/src/dumper_thread.c b/src/dumper_thread.c index d131cf2..de098e5 100644 --- a/src/dumper_thread.c +++ b/src/dumper_thread.c @@ -26,14 +26,15 @@ #include #include #include +#include "checksum.h" #include "dumper_thread.h" #include "error.h" #include "misc.h" #include "parsepkg.h" #include "xml_dump.h" -#define MAX_TASK_BUFFER_LEN 20 - +#define MAX_TASK_BUFFER_LEN 20 +#define CACHEDCHKSUM_BUFFER_LEN 2048 struct BufferedTask { long id; // ID of the task @@ -143,17 +144,79 @@ write_pkg(long id, static char * get_checksum(const char *filename, cr_ChecksumType type, + cr_Package *pkg, const char *cachedir, GError **err) { GError *tmp_err = NULL; char *checksum = NULL; + char *cachefn = NULL; + + if (cachedir) { + // Prepare cache fn + char *key, *cachefilename; + cr_ChecksumCtx *ctx = cr_checksum_new(type, err); + if (!ctx) return NULL; + + if (pkg->siggpg) + cr_checksum_update(ctx, pkg->siggpg->data, pkg->siggpg->size, NULL); + if (pkg->sigpgp) + cr_checksum_update(ctx, pkg->sigpgp->data, pkg->sigpgp->size, NULL); + if (pkg->hdrid) + cr_checksum_update(ctx, pkg->hdrid, strlen(pkg->hdrid), NULL); + + key = cr_checksum_final(ctx, err); + if (!key) return NULL; + + cachefn = g_strdup_printf("%s%s-%s-%"G_GINT64_FORMAT"-%"G_GINT64_FORMAT, + cachedir, + cr_get_filename(pkg->location_href), + key, pkg->size_installed, pkg->time_file); + free(key); + + // Try to load checksum + FILE *f = fopen(cachefn, "r"); + if (f) { + char buf[CACHEDCHKSUM_BUFFER_LEN]; + size_t readed = fread(buf, 1, CACHEDCHKSUM_BUFFER_LEN, f); + if (!ferror(f) && readed > 0) { + checksum = g_strndup(buf, readed); + } + fclose(f); + } + + if (checksum) { + g_debug("Cached checksum used: %s: \"%s\"", cachefn, checksum); + goto exit; + } + } + // Calculate checksum checksum = cr_checksum_file(filename, type, &tmp_err); if (!checksum) { g_propagate_prefixed_error(err, tmp_err, "Error while checksum calculation: "); + goto exit; } + + // Cache the checksum value + if (cachefn && !g_file_test(cachefn, G_FILE_TEST_EXISTS)) { + gchar *template = g_strconcat(cachefn, "-XXXXXX", NULL); + gint fd = g_mkstemp(template); + if (fd < 0) { + g_free(template); + goto exit; + } + write(fd, checksum, strlen(checksum)); + close(fd); + if (g_rename(template, cachefn) == -1) + g_remove(template); + g_free(template); + } + +exit: + g_free(cachefn); + return checksum; } @@ -165,6 +228,7 @@ load_rpm(const char *filename, const char *location_base, int changelog_limit, struct stat *stat_buf, + cr_HeaderReadingFlags hdrrflags, GError **err) { cr_Package *pkg = NULL; @@ -174,7 +238,7 @@ load_rpm(const char *filename, assert(!err || *err == NULL); // Get a package object - pkg = cr_package_from_rpm_base(filename, changelog_limit, err); + pkg = cr_package_from_rpm_base(filename, changelog_limit, hdrrflags, err); if (!pkg) goto errexit; @@ -203,7 +267,7 @@ load_rpm(const char *filename, } // Compute checksum - char *checksum = get_checksum(filename, checksum_type, + char *checksum = get_checksum(filename, checksum_type, pkg, checksum_cachedir, &tmp_err); if (!checksum) goto errexit; @@ -239,6 +303,7 @@ cr_dumper_thread(gpointer data, gpointer user_data) cr_Package *pkg = NULL; // Package from file struct stat stat_buf; // Struct with info from stat() on file struct cr_XmlStruct res; // Structure for generated XML + cr_HeaderReadingFlags hdrrflags = CR_HDRR_NONE; struct UserData *udata = (struct UserData *) user_data; struct PoolTask *task = (struct PoolTask *) data; @@ -248,6 +313,10 @@ cr_dumper_thread(gpointer data, gpointer user_data) const char *location_href = task->full_path + udata->repodir_name_len; const char *location_base = udata->location_base; + // If --cachedir is used, load signatures and hdrid from packages too + if (udata->checksum_cachedir) + hdrrflags = CR_HDRR_LOADHDRID | CR_HDRR_LOADSIGNATURES; + // Get stat info about file if (udata->old_metadata && !(udata->skip_stat)) { if (stat(task->full_path, &stat_buf) == -1) { @@ -294,7 +363,7 @@ cr_dumper_thread(gpointer data, gpointer user_data) pkg = load_rpm(task->full_path, udata->checksum_type, udata->checksum_cachedir, location_href, udata->location_base, udata->changelog_limit, - NULL, &tmp_err); + NULL, hdrrflags, &tmp_err); assert(pkg || tmp_err); if (!pkg) { diff --git a/src/parsehdr.c b/src/parsehdr.c index 7a81829..928d8fd 100644 --- a/src/parsehdr.c +++ b/src/parsehdr.c @@ -138,7 +138,10 @@ cr_compare_dependency(const char *dep1, const char *dep2) cr_Package * -cr_package_from_header(Header hdr, int changelog_limit, GError **err) +cr_package_from_header(Header hdr, + int changelog_limit, + cr_HeaderReadingFlags hdrrflags, + GError **err) { cr_Package *pkg; @@ -555,30 +558,37 @@ cr_package_from_header(Header hdr, int changelog_limit, GError **err) // Keys and hdrid (data used for caching when the --cachedir is specified) // - pkg->hdrid = cr_safe_string_chunk_insert(pkg->chunk, - headerGetString(hdr, RPMTAG_HDRID)); + if (hdrrflags & CR_HDRR_LOADHDRID) + pkg->hdrid = cr_safe_string_chunk_insert(pkg->chunk, + headerGetString(hdr, RPMTAG_HDRID)); - rpmtd gpgtd = rpmtdNew(); - rpmtd pgptd = rpmtdNew(); + if (hdrrflags & CR_HDRR_LOADSIGNATURES) { + rpmtd gpgtd = rpmtdNew(); + rpmtd pgptd = rpmtdNew(); - if (headerGet(hdr, RPMTAG_SIGGPG, gpgtd, flags)) { - pkg->siggpg = cr_binary_data_new(); - pkg->siggpg->size = gpgtd->count; - pkg->siggpg->data = g_string_chunk_insert_len(pkg->chunk, - gpgtd->data, - gpgtd->count); - } + if (headerGet(hdr, RPMTAG_SIGGPG, gpgtd, hdrrflags) + && gpgtd->count > 0) + { + pkg->siggpg = cr_binary_data_new(); + pkg->siggpg->size = gpgtd->count; + pkg->siggpg->data = g_string_chunk_insert_len(pkg->chunk, + gpgtd->data, + gpgtd->count); + } - if (headerGet(hdr, RPMTAG_SIGPGP, pgptd, flags)) { - pkg->sigpgp = cr_binary_data_new(); - pkg->sigpgp->size = pgptd->count; - pkg->sigpgp->data = g_string_chunk_insert_len(pkg->chunk, - pgptd->data, - pgptd->count); - } + if (headerGet(hdr, RPMTAG_SIGPGP, pgptd, hdrrflags) + && pgptd->count > 0) + { + pkg->sigpgp = cr_binary_data_new(); + pkg->sigpgp->size = pgptd->count; + pkg->sigpgp->data = g_string_chunk_insert_len(pkg->chunk, + pgptd->data, + pgptd->count); + } - rpmtdFree(gpgtd); - rpmtdFree(pgptd); + rpmtdFree(gpgtd); + rpmtdFree(pgptd); + } return pkg; } diff --git a/src/parsehdr.h b/src/parsehdr.h index 568a3fb..0fb1bd5 100644 --- a/src/parsehdr.h +++ b/src/parsehdr.h @@ -34,15 +34,25 @@ extern "C" { * @{ */ +/** Flags + */ +typedef enum { + CR_HDRR_NONE = (1 << 0), + CR_HDRR_LOADHDRID = (1 << 1), /*!< Load hdrid */ + CR_HDRR_LOADSIGNATURES = (1 << 2), /*!< Load siggpg and siggpg */ +} cr_HeaderReadingFlags; + /** Read data from header and return filled cr_Package structure. * All const char * params could be NULL. * @param hdr Header * @param changelog_limit number of changelog entries + * @param flags Flags for header reading * @param err GError ** * @return Newly allocated cr_Package or NULL on error */ cr_Package *cr_package_from_header(Header hdr, int changelog_limit, + cr_HeaderReadingFlags flags, GError **err); /** @} */ diff --git a/src/parsepkg.c b/src/parsepkg.c index 05d8fa1..38f8c09 100644 --- a/src/parsepkg.c +++ b/src/parsepkg.c @@ -123,6 +123,7 @@ read_header(const char *filename, Header *hdr, GError **err) cr_Package * cr_package_from_rpm_base(const char *filename, int changelog_limit, + cr_HeaderReadingFlags flags, GError **err) { Header hdr; @@ -134,7 +135,7 @@ cr_package_from_rpm_base(const char *filename, if (!read_header(filename, &hdr, err)) return NULL; - pkg = cr_package_from_header(hdr, changelog_limit, err); + pkg = cr_package_from_header(hdr, changelog_limit, flags, err); headerFree(hdr); return pkg; } @@ -146,6 +147,7 @@ cr_package_from_rpm(const char *filename, const char *location_base, int changelog_limit, struct stat *stat_buf, + cr_HeaderReadingFlags flags, GError **err) { cr_Package *pkg = NULL; @@ -155,7 +157,7 @@ cr_package_from_rpm(const char *filename, assert(!err || *err == NULL); // Get a package object - pkg = cr_package_from_rpm_base(filename, changelog_limit, err); + pkg = cr_package_from_rpm_base(filename, changelog_limit, flags, err); if (!pkg) goto errexit; @@ -239,6 +241,7 @@ cr_xml_from_rpm(const char *filename, location_base, changelog_limit, stat_buf, + CR_HDRR_NONE, err); if (!pkg) return result; diff --git a/src/parsepkg.h b/src/parsepkg.h index 2e8f82e..6213698 100644 --- a/src/parsepkg.h +++ b/src/parsepkg.h @@ -26,6 +26,7 @@ extern "C" { #include #include "checksum.h" +#include "parsehdr.h" #include "package.h" /** \defgroup parsepkg Package parser API. @@ -49,12 +50,14 @@ void cr_package_parser_cleanup(); * are not filled. * @param filename filename * @param changelog_limit number of changelogs that will be loaded + * @param flags Flags for header reading * @param err GError ** * @return cr_Package or NULL on error */ cr_Package * cr_package_from_rpm_base(const char *filename, int changelog_limit, + cr_HeaderReadingFlags flags, GError **err); /** Generate a package object from a package file. @@ -65,6 +68,7 @@ cr_package_from_rpm_base(const char *filename, * @param changelog_limit number of changelog entries * @param stat_buf struct stat of the filename * (optional - could be NULL) + * @param flags Flags for header reading * @param err GError ** * @return cr_Package or NULL on error */ @@ -74,6 +78,7 @@ cr_Package *cr_package_from_rpm(const char *filename, const char *location_base, int changelog_limit, struct stat *stat_buf, + cr_HeaderReadingFlags flags, GError **err); /** Generate XML for the specified package. diff --git a/src/python/parsepkg-py.c b/src/python/parsepkg-py.c index bd6cc46..936d2fb 100644 --- a/src/python/parsepkg-py.c +++ b/src/python/parsepkg-py.c @@ -38,6 +38,7 @@ py_package_from_rpm(PyObject *self, PyObject *args) int checksum_type, changelog_limit; char *filename, *location_href, *location_base; GError *tmp_err = NULL; + cr_HeaderReadingFlags flags = CR_HDRR_NONE; // TODO - support for flags if (!PyArg_ParseTuple(args, "sizzi:py_package_from_rpm", &filename, @@ -49,7 +50,8 @@ py_package_from_rpm(PyObject *self, PyObject *args) } pkg = cr_package_from_rpm(filename, checksum_type, location_href, - location_base, changelog_limit, NULL, &tmp_err); + location_base, changelog_limit, NULL, + flags, &tmp_err); if (tmp_err) { nice_exception(&tmp_err, "Cannot load %s: ", filename); return NULL; -- 2.7.4