From dd42e4f1689c628222fdf4f4ba83229e103e8207 Mon Sep 17 00:00:00 2001 From: "biao716.wang" Date: Tue, 16 Aug 2022 16:30:49 +0900 Subject: [PATCH] Support f2fs option and check if kernel support f2s Change-Id: I4aa333d961296073661e27c24593f651f7d372bc Signed-off-by: biao716.wang --- mic/3rdparty/pykickstart/base.py | 2 +- mic/3rdparty/pykickstart/commands/snapshot.py | 6 +- mic/3rdparty/pykickstart/options.py | 2 +- mic/3rdparty/pykickstart/sections.py | 2 +- mic/archive.py | 4 +- mic/cmd_chroot.py | 2 +- mic/cmd_create.py | 2 +- mic/conf.py | 2 +- mic/helpformat.py | 2 +- mic/imager/loop.py | 6 ++ mic/kickstart/custom_commands/partition.py | 5 ++ mic/msger.py | 2 +- mic/plugin.py | 2 +- mic/utils/fs_related.py | 124 ++++++++++++++++++++++++++ mic/utils/misc.py | 6 ++ plugins/backend/yumpkgmgr.py | 4 +- plugins/backend/zypppkgmgr.py | 4 +- plugins/imager/loop_plugin.py | 3 + plugins/imager/raw_plugin.py | 4 +- tests/test_baseimager.py | 2 +- 20 files changed, 164 insertions(+), 22 deletions(-) diff --git a/mic/3rdparty/pykickstart/base.py b/mic/3rdparty/pykickstart/base.py index b1b7c19..31f7301 100644 --- a/mic/3rdparty/pykickstart/base.py +++ b/mic/3rdparty/pykickstart/base.py @@ -178,7 +178,7 @@ class KickstartCommand(KickstartObject): of objects (like partitions, network devices, etc.) and need to populate a Data object. """ - for (key, val) in vars(namespace).items(): + for (key, val) in list(vars(namespace).items()): if val is not None: setattr(obj, key, val) diff --git a/mic/3rdparty/pykickstart/commands/snapshot.py b/mic/3rdparty/pykickstart/commands/snapshot.py index 22d687c..c002a58 100644 --- a/mic/3rdparty/pykickstart/commands/snapshot.py +++ b/mic/3rdparty/pykickstart/commands/snapshot.py @@ -100,7 +100,7 @@ class F26_Snapshot(KickstartCommand): help=""" Name of the newly created snapshot.""") # Show all possible options in meta message - meta_msg = "<%s>" % ("|".join(self.whenMap.keys())) + meta_msg = "<%s>" % ("|".join(list(self.whenMap.keys()))) op.add_argument("--when", metavar=meta_msg, type=self._when_cb, version=F26, required=True, help=""" You can specify two possible values: ``pre-install`` and ``post-install``. @@ -132,9 +132,9 @@ class F26_Snapshot(KickstartCommand): raise KickstartParseError(msg, lineno=self.lineno) # Check if value in a '--when' param is valid - if snap_data.when != "" and snap_data.when not in self.whenMap.values(): + if snap_data.when != "" and snap_data.when not in list(self.whenMap.values()): msg = (_("Snapshot when param must have one of these values %s!") % - self.whenMap.keys()) + list(self.whenMap.keys())) raise KickstartParseError(msg, lineno=self.lineno) return snap_data diff --git a/mic/3rdparty/pykickstart/options.py b/mic/3rdparty/pykickstart/options.py index c3981e5..f3bcbef 100644 --- a/mic/3rdparty/pykickstart/options.py +++ b/mic/3rdparty/pykickstart/options.py @@ -55,7 +55,7 @@ from pykickstart.version import versionToLongString from pykickstart.i18n import _ def commaSplit(value): - return list(filter(None, [v.strip() for v in value.split(',')])) + return list([_f for _f in [v.strip() for v in value.split(',')] if _f]) def ksboolean(value): try: diff --git a/mic/3rdparty/pykickstart/sections.py b/mic/3rdparty/pykickstart/sections.py index 2aafd14..e793dcb 100644 --- a/mic/3rdparty/pykickstart/sections.py +++ b/mic/3rdparty/pykickstart/sections.py @@ -822,7 +822,7 @@ class PackageSection(Section): for arg in args: for option, new_option in \ - {"--instLangs": "--inst-langs", "--excludeWeakdeps": "--exclude-weakdeps"}.items(): + list({"--instLangs": "--inst-langs", "--excludeWeakdeps": "--exclude-weakdeps"}.items()): if option in arg: warnings.warn(_("The %(option)s option on line %(lineno)s will be deprecated in " "future releases. Please modify your kickstart file to replace " diff --git a/mic/archive.py b/mic/archive.py index 40c64b0..0d9a3d0 100644 --- a/mic/archive.py +++ b/mic/archive.py @@ -222,7 +222,7 @@ def decompress(file_path, decompress_format=None): raise OSError ("can't decompress a file not existed: '%s'" % file_path) (file_name, file_ext) = os.path.splitext(file_path) - for key, suffixes in _COMPRESS_SUFFIXES.items(): + for key, suffixes in list(_COMPRESS_SUFFIXES.items()): if file_ext in suffixes: file_ext = key break @@ -402,7 +402,7 @@ def make_archive(archive_name, target_name): if not os.path.exists(target_name): raise OSError ("archive object does not exist: '%s'" % target_name) - for aformat, suffixes in _ARCHIVE_SUFFIXES.items(): + for aformat, suffixes in list(_ARCHIVE_SUFFIXES.items()): if list(filter(archive_name.endswith, suffixes)): archive_format = aformat break diff --git a/mic/cmd_chroot.py b/mic/cmd_chroot.py index fd42d78..ecfc66d 100755 --- a/mic/cmd_chroot.py +++ b/mic/cmd_chroot.py @@ -53,7 +53,7 @@ def main(parser, args, argv): configmgr.chroot['saveto'] = args.saveto imagetype = misc.get_image_type(targetimage) - if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"): + if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg", "f2fsimg"): imagetype = "loop" chrootclass = None diff --git a/mic/cmd_create.py b/mic/cmd_create.py index 75270ec..ce18ed5 100755 --- a/mic/cmd_create.py +++ b/mic/cmd_create.py @@ -85,7 +85,7 @@ def main(parser, args, argv): #check the imager type createrClass = None - for subcmd, klass in pluginmgr.get_plugins('imager').items(): + for subcmd, klass in list(pluginmgr.get_plugins('imager').items()): if subcmd == args.subcommand and hasattr(klass, 'do_create'): createrClass = klass diff --git a/mic/conf.py b/mic/conf.py index c864e8f..d68d086 100755 --- a/mic/conf.py +++ b/mic/conf.py @@ -120,7 +120,7 @@ class ConfigMgr(object): self.__siteconf = None # initialize the values with defaults - for sec, vals in self.DEFAULTS.items(): + for sec, vals in list(self.DEFAULTS.items()): setattr(self, sec, vals) def __set_siteconf(self, siteconf): diff --git a/mic/helpformat.py b/mic/helpformat.py index 799af46..f215cf1 100755 --- a/mic/helpformat.py +++ b/mic/helpformat.py @@ -35,7 +35,7 @@ class MICHelpFormatter(RawDescriptionHelpFormatter): """Collect aliases.""" if action.choices: - for item, parser in action.choices.items(): + for item, parser in list(action.choices.items()): self._aliases[str(item)] = parser.get_default('alias') return super(MICHelpFormatter, self).add_argument(action) diff --git a/mic/imager/loop.py b/mic/imager/loop.py index 8056ffb..27ac06a 100755 --- a/mic/imager/loop.py +++ b/mic/imager/loop.py @@ -167,6 +167,7 @@ class LoopImageCreator(BaseImageCreator): 'fsopts': part.fsopts or None, 'aft_fstype': aft_fstype or None, 'extopts': part.extopts or None, + 'f2fsopts': part.f2fsopts or None, 'vdfsopts': part.vdfsopts or None, 'squashfsopts': part.squashfsopts or None, 'cpioopts': part.cpioopts or None, @@ -354,6 +355,7 @@ class LoopImageCreator(BaseImageCreator): "name": imgname, "size": self.__image_size or 4096, "fstype": self.__fstype or "ext3", + "f2fsopts": None, "extopts": None, "loop": None, "uuid": None, @@ -376,6 +378,8 @@ class LoopImageCreator(BaseImageCreator): MyDiskMount = fs.BtrfsDiskMount elif fstype in ("vfat", "msdos"): MyDiskMount = fs.VfatDiskMount + elif fstype == "f2fs": + MyDiskMount = fs.F2fsDiskMount else: raise MountError('Cannot support fstype: %s' % fstype) @@ -391,6 +395,8 @@ class LoopImageCreator(BaseImageCreator): if fstype in ("ext2", "ext3", "ext4"): loop['loop'].extopts = loop['extopts'] + elif fstype == "f2fs": + loop['loop'].f2fsopts = loop['f2fsopts'] try: msger.verbose('Mounting image "%s" on "%s"' % (imgname, mp)) diff --git a/mic/kickstart/custom_commands/partition.py b/mic/kickstart/custom_commands/partition.py index f8c20ea..cf941d3 100755 --- a/mic/kickstart/custom_commands/partition.py +++ b/mic/kickstart/custom_commands/partition.py @@ -26,6 +26,7 @@ class Mic_PartData(FC4_PartData): self.deleteRemovedAttrs() self.align = kwargs.get("align", None) self.extopts = kwargs.get("extopts", None) + self.f2fsopts = kwargs.get("f2fsopts", None) self.part_type = kwargs.get("part_type", None) self.uuid = kwargs.get("uuid", None) self.exclude_image = kwargs.get("exclude_from_image", False) @@ -42,6 +43,8 @@ class Mic_PartData(FC4_PartData): retval += " --align" if self.extopts: retval += " --extoptions=%s" % self.extopts + if self.f2fsopts: + retval += " --f2fsoptions=%s" % self.f2fsopts if self.part_type: retval += " --part-type=%s" % self.part_type if self.uuid: @@ -72,6 +75,8 @@ class Mic_Partition(FC4_Partition): default=None, version=FC4, help='align') op.add_argument("--extoptions", type=str, action="store", dest="extopts", default=None, version=FC4, help='extoptions') + op.add_argument("--f2fsoptions", type=str, action="store", dest="f2fsopts", + default=None, version=FC4, help='f2fsoptions') op.add_argument("--part-type", type=str, action="store", dest="part_type", default=None, version=FC4, help='part-type') op.add_argument("--uuid", dest="uuid", action="store", type=str, version=FC4, help='uuid') diff --git a/mic/msger.py b/mic/msger.py index a260499..99e4c94 100644 --- a/mic/msger.py +++ b/mic/msger.py @@ -382,7 +382,7 @@ def pause(msg=None): """ Pause for any key """ if msg is None: msg = "press ANY KEY to continue ..." - input(msg) + eval(input(msg)) def set_logfile(logfile, mode='w'): """ Set logfile path to the MIC logger """ diff --git a/mic/plugin.py b/mic/plugin.py index e8ad9e8..3cd9bc8 100644 --- a/mic/plugin.py +++ b/mic/plugin.py @@ -62,7 +62,7 @@ class PluginMgr(object): # the value True/False means "loaded" def _load_all(self): - for (pdir, loaded) in self.plugin_dirs.items(): + for (pdir, loaded) in list(self.plugin_dirs.items()): if loaded: continue sys.path.insert(0, pdir) diff --git a/mic/utils/fs_related.py b/mic/utils/fs_related.py index 35d49d2..f3d16e4 100755 --- a/mic/utils/fs_related.py +++ b/mic/utils/fs_related.py @@ -446,6 +446,130 @@ class DiskMount(Mount): self.mounted = True +class F2fsDiskMount(DiskMount): + """A DiskMount object that is able to format/resize f2fs filesystems.""" + def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid=None): + self.__check_f2fs() + DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) + self.blocksize = blocksize + self.fslabel = fslabel.replace("/", "") + self.uuid = fsuuid or None + self.skipformat = skipformat + self.fsopts = fsopts + self.__f2fsopts = None + self.blkidcmd = find_binary_path("blkid") + self.dumpe2fs = find_binary_path("dump." + self.fstype) + self.fsckcmd = find_binary_path("fsck." + self.fstype) + self.resizecmd = find_binary_path("resize." + self.fstype) + + def __get_f2fsopts(self): + return self.__f2f2opts + + def __set_f2fsopts(self, val): + if val is None: + self.__f2fsopts = None + else: + self.__f2fsopts = val + f2fsopts = property(__get_f2fsopts, __set_f2fsopts) + + def __check_f2fs(self): + found = False + """ Need to load f2fs module to mount it """ + load_module("f2fs") + for line in open("/proc/filesystems"): + if line.find("f2fs") > -1: + found = True + break + if not found: + raise MountError("Your system can't mount f2fs filesystem, please make sure your kernel has f2fs support and the module f2fs.ko has been loaded.") + + def __parse_field(self, output, field): + for line in output.split(" "): + if line.startswith(field + "="): + return line[len(field) + 1:].strip().replace("\"", "") + + raise KeyError("Failed to find field '%s' in output" % field) + + def __format_filesystem(self): + if self.skipformat: + msger.debug("Skip filesystem format.") + return + + msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) + + cmdlist = [self.mkfscmd, "-l", self.fslabel] + if self.__f2fsopts: + cmdlist.extend(self._f2fsopts.split()) + cmdlist.extend([self.disk.device]) + + rc, errout = runner.runtool(cmdlist, catch=2) + if rc != 0: + raise MountError("Error creating %s filesystem on disk %s:\n%s" % + (self.fstype, self.disk.device, errout)) + + self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID") + + def __resize_filesystem(self, size = None): + msger.info("Resizing filesystem ...") + current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] + + if size is None: + size = self.disk.size + + if size == current_size: + return + + if size > current_size: + self.disk.expand(size=size) + + self.__fsck() + + return size + + def __create(self): + resize = False + if not self.disk.fixed() and self.disk.exists(): + resize = True + + self.disk.create() + + if resize: + self.__resize_filesystem() + else: + self.__format_filesystem() + + def mount(self, options = None, init_expand = False): + self.__create() + if init_expand: + expand_size = int(self.disk.size * 1.5) + msger.info("Initial partition size expanded : %ld -> %ld" % (self.disk.size, expand_size)) + self.__resize_filesystem(expand_size) + self.disk.reread_size() + DiskMount.mount(self, options) + + def __fsck(self): + msger.info("Checking filesystem %s" % self.disk.lofile) + runner.quiet([self.fsckcmd, self.disk.lofile]) + + def __get_size_from_filesystem(self): + return self.disk.size + + def __resize_to_minimal(self): + msger.info("Resizing filesystem to minimal ...") + self.__fsck() + + return self.__get_size_from_filesystem() + + def resparse(self, size = None): + self.cleanup() + if size == 0: + minsize = 0 + else: + minsize = self.__resize_to_minimal() + self.disk.truncate(minsize) + self.__resize_filesystem(size) + return minsize + class ExtDiskMount(DiskMount): """A DiskMount object that is able to format/resize ext[23] filesystems.""" def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid=None): diff --git a/mic/utils/misc.py b/mic/utils/misc.py index 36294a3..0ac617b 100755 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -371,6 +371,12 @@ def get_image_type(path): if file_header[0:len(vdi_flag)] == vdi_flag: return maptab["vdi"] + #Checking f2fs fs type. + blkidcmd = find_binary_path("blkid") + out = runner.outs([blkidcmd, '-o', 'value', '-s', 'TYPE', path]) + if out == "f2fs": + return "f2fsimg" + output = runner.outs(['file', path]) isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*") usbimgptn = re.compile(r".*x86 boot sector.*active.*") diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index 8a3b228..850dc38 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -233,7 +233,7 @@ class Yum(BackendPlugin, yum.YumBase): # we also need to remove from the conditionals # dict so that things don't get pulled back in as a result # of them. yes, this is ugly. conditionals should die. - for req, pkgs in self.tsInfo.conditionals.items(): + for req, pkgs in list(self.tsInfo.conditionals.items()): if x in pkgs: pkgs.remove(x) self.tsInfo.conditionals[req] = pkgs @@ -283,7 +283,7 @@ class Yum(BackendPlugin, yum.YumBase): repo.mirrorlist = _varSubstitute(mirrorlist) conf = yum.config.RepoConf() - for k, v in conf.items(): + for k, v in list(conf.items()): if v or not hasattr(repo, k): repo.setAttribute(k, v) diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index 60a4a7d..75324ea 100644 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -321,9 +321,7 @@ class Zypp(BackendPlugin): if found: if include == ksparser.GROUP_REQUIRED: - list(map( - lambda p: self.deselectPackage(p), - list(grp.default_packages.keys()))) + list([self.deselectPackage(p) for p in list(grp.default_packages.keys())]) return None else: diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index beba62c..dc0bfd9 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -166,6 +166,9 @@ class LoopPlugin(ImagerPlugin): elif imgtype in ("ext3fsimg", "ext4fsimg"): fstype = imgtype[:4] myDiskMount = fs_related.ExtDiskMount + elif imgtype == "f2fsimg": + fstype = "f2fs" + myDiskMount = fs_related.F2fsDiskMount else: raise errors.CreatorError("Unsupported filesystem type: %s" \ % imgtype) diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index bb6661c..e5b35dc 100644 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -143,7 +143,7 @@ class RawPlugin(ImagerPlugin): # not recognize properly. # TODO: Can we make better assumption? fstype = "btrfs" - elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs" ]: + elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs", "f2fs" ]: fstype = partition_info[5] elif partition_info[5] in [ "fat16", "fat32" ]: fstype = "vfat" @@ -155,7 +155,7 @@ class RawPlugin(ImagerPlugin): if rootpart and rootpart == line[0]: mountpoint = '/' - elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs" ]: + elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs", "f2fs" ]: # TODO: Check that this is actually the valid root partition from /etc/fstab mountpoint = "/" root_mounted = True diff --git a/tests/test_baseimager.py b/tests/test_baseimager.py index fe39e7d..a4f66e8 100644 --- a/tests/test_baseimager.py +++ b/tests/test_baseimager.py @@ -85,7 +85,7 @@ class BaseImgrTest(unittest.TestCase): creatoropts['pkgmgr'] = backend cfgmgr._ksconf = KSCONF pkgmgr = None - for (key, pcls) in pluginmgr.PluginMgr().get_plugins('backend').items(): + for (key, pcls) in list(pluginmgr.PluginMgr().get_plugins('backend').items()): if key == creatoropts['pkgmgr']: pkgmgr = pcls break -- 2.7.4