From 087f3bf267e6ba92c01879c434299a667538f87c Mon Sep 17 00:00:00 2001 From: Tomas Mlcoch Date: Thu, 31 Oct 2013 11:12:12 +0100 Subject: [PATCH] Support for other then primary, filelists and other metadata --- deltarepo/deltarepo/applicator.py | 31 ++++++++++++++++-- deltarepo/deltarepo/delta_plugins.py | 61 ++++++++++++++++++++++++++++++++++++ deltarepo/deltarepo/generator.py | 35 +++++++++++++++++++-- 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/deltarepo/deltarepo/applicator.py b/deltarepo/deltarepo/applicator.py index a5bfa6e..2b0fdda 100644 --- a/deltarepo/deltarepo/applicator.py +++ b/deltarepo/deltarepo/applicator.py @@ -12,7 +12,7 @@ import os import shutil import createrepo_c as cr from deltarepo.common import LoggingInterface, Metadata, RemovedXml -from deltarepo.delta_plugins import PLUGINS +from deltarepo.delta_plugins import PLUGINS, GENERAL_PLUGIN from deltarepo.errors import DeltaRepoError, DeltaRepoPluginError __all__ = ['DeltaRepoApplicator'] @@ -243,8 +243,35 @@ class DeltaRepoApplicator(LoggingInterface): used_plugins.add(plugin) plugin_used = True - # TODO: # Process rest of the metadata files + metadata_objects = {} + for rectype, rec in self.delta_records.items(): + if rectype in ("primary_db", "filelists_db", "other_db", "removed"): + # Skip databases and removed + continue + + if rectype not in processed_metadata: + metadata_object = self._new_metadata(rectype) + if metadata_object is not None: + self._debug("To be processed by general delta plugin: %s" \ + % rectype) + metadata_objects[rectype] = metadata_object + else: + self._debug("Not processed: %s - SKIP", rectype) + + if metadata_objects: + # Use the plugin + self._debug("Plugin {0}: Active".format(GENERAL_PLUGIN.NAME)) + plugin_instance = GENERAL_PLUGIN() + plugin_instance.apply(metadata_objects, self.bundle) + + for md in metadata_objects.values(): + self._debug("Plugin {0}: Processed \"{1}\" delta record "\ + "which produced:".format( + GENERAL_PLUGIN.NAME, md.metadata_type)) + for repomd_record in md.generated_repomd_records: + self._debug(" - {0}".format(repomd_record.type)) + self.new_repomd.set_record(repomd_record) self._debug("Used plugins: {0}".format([p.NAME for p in used_plugins])) diff --git a/deltarepo/deltarepo/delta_plugins.py b/deltarepo/deltarepo/delta_plugins.py index c675a7b..8a1c670 100644 --- a/deltarepo/deltarepo/delta_plugins.py +++ b/deltarepo/deltarepo/delta_plugins.py @@ -1,10 +1,12 @@ import os import os.path +import shutil import hashlib import createrepo_c as cr from deltarepo.errors import DeltaRepoError, DeltaRepoPluginError PLUGINS = [] +GENERAL_PLUGIN = None class DeltaRepoPlugin(object): @@ -68,6 +70,65 @@ class DeltaRepoPlugin(object): def gen(self, metadata, bundle): raise NotImplementedError("Not implemented") + +class GeneralDeltaRepoPlugin(DeltaRepoPlugin): + + NAME = "GeneralDeltaPlugin" + VERSION = 1 + METADATA = [] + APPLY_REQUIRED_BUNDLE_KEYS = ["removed_obj", + "unique_md_filenames"] + APPLY_BUNDLE_CONTRIBUTION = [] + GEN_REQUIRED_BUNDLE_KEYS = ["removed_obj", + "unique_md_filenames"] + GEN_BUNDLE_CONTRIBUTION = [] + + def _path(self, path, record): + """Return path to the repodata file.""" + return os.path.join(path, record.location_href) + + def apply(self, metadata, bundle): + + # Get info from bundle + removed_obj = bundle["removed_obj"] + unique_md_filenames = bundle["unique_md_filenames"] + + # + for md in metadata.values(): + md.new_fn = os.path.join(md.out_dir, os.path.basename(md.delta_fn)) + shutil.copy2(md.delta_fn, md.new_fn) + + # Prepare repomd record of xml file + rec = cr.RepomdRecord(md.metadata_type, md.new_fn) + rec.fill(md.checksum_type) + if unique_md_filenames: + rec.rename_file() + + md.generated_repomd_records.append(rec) + + def gen(self, metadata, bundle): + + ## TODO: Compress uncompressed data + + # Get info from bundle + removed_obj = bundle["removed_obj"] + unique_md_filenames = bundle["unique_md_filenames"] + + for md in metadata.values(): + md.delta_fn = os.path.join(md.out_dir, os.path.basename(md.new_fn)) + shutil.copy2(md.new_fn, md.delta_fn) + + # Prepare repomd record of xml file + rec = cr.RepomdRecord(md.metadata_type, md.delta_fn) + rec.fill(md.checksum_type) + if unique_md_filenames: + rec.rename_file() + + md.generated_repomd_records.append(rec) + +GENERAL_PLUGIN = GeneralDeltaRepoPlugin + + class MainDeltaRepoPlugin(DeltaRepoPlugin): NAME = "MainDeltaPlugin" diff --git a/deltarepo/deltarepo/generator.py b/deltarepo/deltarepo/generator.py index d69e69d..68ec580 100644 --- a/deltarepo/deltarepo/generator.py +++ b/deltarepo/deltarepo/generator.py @@ -12,7 +12,7 @@ import os import shutil import createrepo_c as cr from deltarepo.common import LoggingInterface, Metadata, RemovedXml -from deltarepo.delta_plugins import PLUGINS +from deltarepo.delta_plugins import PLUGINS, GENERAL_PLUGIN from deltarepo.errors import DeltaRepoError, DeltaRepoPluginError __all__ = ['DeltaRepoGenerator'] @@ -146,6 +146,10 @@ class DeltaRepoGenerator(LoggingInterface): return metadata + def _run_plugin(plugin, metadata_objects): + # TODO XXX + pass + def gen(self): # Prepare output path @@ -228,8 +232,35 @@ class DeltaRepoGenerator(LoggingInterface): used_plugins.add(plugin) plugin_used = True - # TODO: # Process rest of the metadata files + metadata_objects = {} + for rectype, rec in self.new_records.items(): + if rectype in ("primary_db", "filelists_db", "other_db"): + # Skip databases + continue + + if rectype not in processed_metadata: + metadata_object = self._new_metadata(rectype) + if metadata_object is not None: + self._debug("To be processed by general delta plugin: %s" \ + % rectype) + metadata_objects[rectype] = metadata_object + else: + self._debug("Not processed: %s - SKIP", rectype) + + if metadata_objects: + # Use the plugin + self._debug("Plugin {0}: Active".format(GENERAL_PLUGIN.NAME)) + plugin_instance = GENERAL_PLUGIN() + plugin_instance.gen(metadata_objects, self.bundle) + + for md in metadata_objects.values(): + self._debug("Plugin {0}: Processed \"{1}\" delta record "\ + "which produced:".format( + GENERAL_PLUGIN.NAME, md.metadata_type)) + for repomd_record in md.generated_repomd_records: + self._debug(" - {0}".format(repomd_record.type)) + self.delta_repomd.set_record(repomd_record) # Write out removed.xml self._debug("Writing removed.xml ...") -- 2.7.4