Support f2fs option and check if kernel support f2s 89/279689/2
authorbiao716.wang <biao716.wang@samsung.com>
Tue, 16 Aug 2022 07:30:49 +0000 (16:30 +0900)
committerbiao716.wang <biao716.wang@samsung.com>
Tue, 16 Aug 2022 07:33:31 +0000 (16:33 +0900)
Change-Id: I4aa333d961296073661e27c24593f651f7d372bc
Signed-off-by: biao716.wang <biao716.wang@samsung.com>
20 files changed:
mic/3rdparty/pykickstart/base.py
mic/3rdparty/pykickstart/commands/snapshot.py
mic/3rdparty/pykickstart/options.py
mic/3rdparty/pykickstart/sections.py
mic/archive.py
mic/cmd_chroot.py
mic/cmd_create.py
mic/conf.py
mic/helpformat.py
mic/imager/loop.py
mic/kickstart/custom_commands/partition.py
mic/msger.py
mic/plugin.py
mic/utils/fs_related.py
mic/utils/misc.py
plugins/backend/yumpkgmgr.py
plugins/backend/zypppkgmgr.py
plugins/imager/loop_plugin.py
plugins/imager/raw_plugin.py
tests/test_baseimager.py

index b1b7c19..31f7301 100644 (file)
@@ -178,7 +178,7 @@ class KickstartCommand(KickstartObject):
            of objects (like partitions, network devices, etc.) and need to populate
            a Data object.
         """
-        for (key, val) in vars(namespace).items():
+        for (key, val) in list(vars(namespace).items()):
             if val is not None:
                 setattr(obj, key, val)
 
index 22d687c..c002a58 100644 (file)
@@ -100,7 +100,7 @@ class F26_Snapshot(KickstartCommand):
                         help="""
                         Name of the newly created snapshot.""")
         # Show all possible options in meta message
-        meta_msg = "<%s>" % ("|".join(self.whenMap.keys()))
+        meta_msg = "<%s>" % ("|".join(list(self.whenMap.keys())))
         op.add_argument("--when", metavar=meta_msg, type=self._when_cb, version=F26,
                         required=True, help="""
                         You can specify two possible values: ``pre-install`` and ``post-install``.
@@ -132,9 +132,9 @@ class F26_Snapshot(KickstartCommand):
             raise KickstartParseError(msg, lineno=self.lineno)
 
         # Check if value in a '--when' param is valid
-        if snap_data.when != "" and snap_data.when not in self.whenMap.values():
+        if snap_data.when != "" and snap_data.when not in list(self.whenMap.values()):
             msg = (_("Snapshot when param must have one of these values %s!") %
-                   self.whenMap.keys())
+                   list(self.whenMap.keys()))
             raise KickstartParseError(msg, lineno=self.lineno)
 
         return snap_data
index c3981e5..f3bcbef 100644 (file)
@@ -55,7 +55,7 @@ from pykickstart.version import versionToLongString
 from pykickstart.i18n import _
 
 def commaSplit(value):
-    return list(filter(None, [v.strip() for v in value.split(',')]))
+    return list([_f for _f in [v.strip() for v in value.split(',')] if _f])
 
 def ksboolean(value):
     try:
index 2aafd14..e793dcb 100644 (file)
@@ -822,7 +822,7 @@ class PackageSection(Section):
 
         for arg in args:
             for option, new_option in \
-                {"--instLangs": "--inst-langs", "--excludeWeakdeps": "--exclude-weakdeps"}.items():
+                list({"--instLangs": "--inst-langs", "--excludeWeakdeps": "--exclude-weakdeps"}.items()):
                 if option in arg:
                     warnings.warn(_("The %(option)s option on line %(lineno)s will be deprecated in "
                                     "future releases. Please modify your kickstart file to replace "
index 40c64b0..0d9a3d0 100644 (file)
@@ -222,7 +222,7 @@ def decompress(file_path, decompress_format=None):
         raise OSError ("can't decompress a file not existed: '%s'" % file_path)
 
     (file_name, file_ext) = os.path.splitext(file_path)
-    for key, suffixes in _COMPRESS_SUFFIXES.items():
+    for key, suffixes in list(_COMPRESS_SUFFIXES.items()):
         if file_ext in suffixes:
             file_ext = key
             break
@@ -402,7 +402,7 @@ def make_archive(archive_name, target_name):
     if not os.path.exists(target_name):
         raise OSError ("archive object does not exist: '%s'" % target_name)
 
-    for aformat, suffixes in _ARCHIVE_SUFFIXES.items():
+    for aformat, suffixes in list(_ARCHIVE_SUFFIXES.items()):
         if list(filter(archive_name.endswith, suffixes)):
             archive_format = aformat
             break
index fd42d78..ecfc66d 100755 (executable)
@@ -53,7 +53,7 @@ def main(parser, args, argv):
     configmgr.chroot['saveto'] = args.saveto
 
     imagetype = misc.get_image_type(targetimage)
-    if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"):
+    if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg", "f2fsimg"):
         imagetype = "loop"
 
     chrootclass = None
index 75270ec..ce18ed5 100755 (executable)
@@ -85,7 +85,7 @@ def main(parser, args, argv):
 
     #check the imager type
     createrClass = None
-    for subcmd, klass in pluginmgr.get_plugins('imager').items():
+    for subcmd, klass in list(pluginmgr.get_plugins('imager').items()):
         if subcmd == args.subcommand and hasattr(klass, 'do_create'):
             createrClass = klass
 
index c864e8f..d68d086 100755 (executable)
@@ -120,7 +120,7 @@ class ConfigMgr(object):
         self.__siteconf = None
 
         # initialize the values with defaults
-        for sec, vals in self.DEFAULTS.items():
+        for sec, vals in list(self.DEFAULTS.items()):
             setattr(self, sec, vals)
 
     def __set_siteconf(self, siteconf):
index 799af46..f215cf1 100755 (executable)
@@ -35,7 +35,7 @@ class MICHelpFormatter(RawDescriptionHelpFormatter):
         """Collect aliases."""
 
         if action.choices:
-            for item, parser in action.choices.items():
+            for item, parser in list(action.choices.items()):
                 self._aliases[str(item)] = parser.get_default('alias')
 
         return super(MICHelpFormatter, self).add_argument(action)
index 8056ffb..27ac06a 100755 (executable)
@@ -167,6 +167,7 @@ class LoopImageCreator(BaseImageCreator):
                     'fsopts': part.fsopts or None,
                     'aft_fstype': aft_fstype or None,
                     'extopts': part.extopts or None,
+                    'f2fsopts': part.f2fsopts or None,
                     'vdfsopts': part.vdfsopts or None,
                     'squashfsopts': part.squashfsopts or None,
                     'cpioopts': part.cpioopts or None,
@@ -354,6 +355,7 @@ class LoopImageCreator(BaseImageCreator):
                  "name": imgname,
                  "size": self.__image_size or 4096,
                  "fstype": self.__fstype or "ext3",
+                 "f2fsopts": None,
                  "extopts": None,
                  "loop": None,
                  "uuid": None,
@@ -376,6 +378,8 @@ class LoopImageCreator(BaseImageCreator):
                 MyDiskMount = fs.BtrfsDiskMount
             elif fstype in ("vfat", "msdos"):
                 MyDiskMount = fs.VfatDiskMount
+            elif fstype == "f2fs":
+                MyDiskMount = fs.F2fsDiskMount
             else:
                 raise MountError('Cannot support fstype: %s' % fstype)
 
@@ -391,6 +395,8 @@ class LoopImageCreator(BaseImageCreator):
 
             if fstype in ("ext2", "ext3", "ext4"):
                 loop['loop'].extopts = loop['extopts']
+            elif fstype == "f2fs":
+                loop['loop'].f2fsopts = loop['f2fsopts']
 
             try:
                 msger.verbose('Mounting image "%s" on "%s"' % (imgname, mp))
index f8c20ea..cf941d3 100755 (executable)
@@ -26,6 +26,7 @@ class Mic_PartData(FC4_PartData):
         self.deleteRemovedAttrs()
         self.align = kwargs.get("align", None)
         self.extopts = kwargs.get("extopts", None)
+        self.f2fsopts = kwargs.get("f2fsopts", None)
         self.part_type = kwargs.get("part_type", None)
         self.uuid = kwargs.get("uuid", None)
         self.exclude_image = kwargs.get("exclude_from_image", False)
@@ -42,6 +43,8 @@ class Mic_PartData(FC4_PartData):
             retval += " --align"
         if self.extopts:
             retval += " --extoptions=%s" % self.extopts
+        if self.f2fsopts:
+            retval += " --f2fsoptions=%s" % self.f2fsopts
         if self.part_type:
             retval += " --part-type=%s" % self.part_type
         if self.uuid:
@@ -72,6 +75,8 @@ class Mic_Partition(FC4_Partition):
                       default=None, version=FC4, help='align')
         op.add_argument("--extoptions", type=str, action="store", dest="extopts",
                       default=None, version=FC4, help='extoptions')
+        op.add_argument("--f2fsoptions", type=str, action="store", dest="f2fsopts",
+                      default=None, version=FC4, help='f2fsoptions')
         op.add_argument("--part-type", type=str, action="store", dest="part_type",
                       default=None, version=FC4, help='part-type')
         op.add_argument("--uuid", dest="uuid", action="store", type=str, version=FC4, help='uuid')
index a260499..99e4c94 100644 (file)
@@ -382,7 +382,7 @@ def pause(msg=None):
     """ Pause for any key """
     if msg is None:
         msg = "press ANY KEY to continue ..."
-    input(msg)
+    eval(input(msg))
 
 def set_logfile(logfile, mode='w'):
     """ Set logfile path to the MIC logger """
index e8ad9e8..3cd9bc8 100644 (file)
@@ -62,7 +62,7 @@ class PluginMgr(object):
             # the value True/False means "loaded"
 
     def _load_all(self):
-        for (pdir, loaded) in self.plugin_dirs.items():
+        for (pdir, loaded) in list(self.plugin_dirs.items()):
             if loaded: continue
 
             sys.path.insert(0, pdir)
index 35d49d2..f3d16e4 100755 (executable)
@@ -446,6 +446,130 @@ class DiskMount(Mount):
 
         self.mounted = True
 
+class F2fsDiskMount(DiskMount):
+    """A DiskMount object that is able to format/resize f2fs filesystems."""
+    def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid=None):
+        self.__check_f2fs()
+        DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
+        self.blocksize = blocksize
+        self.fslabel = fslabel.replace("/", "")
+        self.uuid = fsuuid or None
+        self.skipformat = skipformat
+        self.fsopts = fsopts
+        self.__f2fsopts = None
+        self.blkidcmd = find_binary_path("blkid")
+        self.dumpe2fs = find_binary_path("dump." + self.fstype)
+        self.fsckcmd = find_binary_path("fsck." + self.fstype)
+        self.resizecmd = find_binary_path("resize." + self.fstype)
+
+    def __get_f2fsopts(self):
+        return self.__f2f2opts
+
+    def __set_f2fsopts(self, val):
+        if val is None:
+            self.__f2fsopts = None
+        else:
+            self.__f2fsopts = val
+    f2fsopts = property(__get_f2fsopts, __set_f2fsopts)
+
+    def __check_f2fs(self):
+        found = False
+        """ Need to load f2fs module to mount it """
+        load_module("f2fs")
+        for line in open("/proc/filesystems"):
+            if line.find("f2fs") > -1:
+                found = True
+                break
+        if not found:
+            raise MountError("Your system can't mount f2fs filesystem, please make sure your kernel has f2fs support and the module f2fs.ko has been loaded.")
+
+    def __parse_field(self, output, field):
+        for line in output.split(" "):
+            if line.startswith(field + "="):
+                return line[len(field) + 1:].strip().replace("\"", "")
+
+        raise KeyError("Failed to find field '%s' in output" % field)
+
+    def __format_filesystem(self):
+        if self.skipformat:
+            msger.debug("Skip filesystem format.")
+            return
+
+        msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
+
+        cmdlist = [self.mkfscmd, "-l", self.fslabel]
+        if self.__f2fsopts:
+            cmdlist.extend(self._f2fsopts.split())
+        cmdlist.extend([self.disk.device])
+
+        rc, errout = runner.runtool(cmdlist, catch=2)
+        if rc != 0:
+            raise MountError("Error creating %s filesystem on disk %s:\n%s" %
+                             (self.fstype, self.disk.device, errout))
+
+        self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID")
+
+    def __resize_filesystem(self, size = None):
+        msger.info("Resizing filesystem ...")
+        current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
+
+        if size is None:
+            size = self.disk.size
+
+        if size == current_size:
+            return
+
+        if size > current_size:
+            self.disk.expand(size=size)
+
+        self.__fsck()
+
+        return size
+
+    def __create(self):
+        resize = False
+        if not self.disk.fixed() and self.disk.exists():
+            resize = True
+
+        self.disk.create()
+
+        if resize:
+            self.__resize_filesystem()
+        else:
+            self.__format_filesystem()
+
+    def mount(self, options = None, init_expand = False):
+        self.__create()
+        if init_expand:
+            expand_size = int(self.disk.size * 1.5)
+            msger.info("Initial partition size expanded : %ld -> %ld" % (self.disk.size, expand_size))
+            self.__resize_filesystem(expand_size)
+            self.disk.reread_size()
+        DiskMount.mount(self, options)
+
+    def __fsck(self):
+        msger.info("Checking filesystem %s" % self.disk.lofile)
+        runner.quiet([self.fsckcmd, self.disk.lofile])
+
+    def __get_size_from_filesystem(self):
+        return self.disk.size
+
+    def __resize_to_minimal(self):
+        msger.info("Resizing filesystem to minimal ...")
+        self.__fsck()
+
+        return self.__get_size_from_filesystem()
+
+    def resparse(self, size = None):
+        self.cleanup()
+        if size == 0:
+            minsize = 0
+        else:
+            minsize = self.__resize_to_minimal()
+            self.disk.truncate(minsize)
+            self.__resize_filesystem(size)
+        return minsize
+
 class ExtDiskMount(DiskMount):
     """A DiskMount object that is able to format/resize ext[23] filesystems."""
     def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid=None):
index 36294a3..0ac617b 100755 (executable)
@@ -371,6 +371,12 @@ def get_image_type(path):
     if file_header[0:len(vdi_flag)] == vdi_flag:
         return maptab["vdi"]
 
+    #Checking f2fs fs type.
+    blkidcmd = find_binary_path("blkid")
+    out = runner.outs([blkidcmd, '-o', 'value', '-s', 'TYPE', path])
+    if out == "f2fs":
+        return "f2fsimg"
+
     output = runner.outs(['file', path])
     isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
     usbimgptn = re.compile(r".*x86 boot sector.*active.*")
index 8a3b228..850dc38 100644 (file)
@@ -233,7 +233,7 @@ class Yum(BackendPlugin, yum.YumBase):
                 # we also need to remove from the conditionals
                 # dict so that things don't get pulled back in as a result
                 # of them.  yes, this is ugly.  conditionals should die.
-                for req, pkgs in self.tsInfo.conditionals.items():
+                for req, pkgs in list(self.tsInfo.conditionals.items()):
                     if x in pkgs:
                         pkgs.remove(x)
                         self.tsInfo.conditionals[req] = pkgs
@@ -283,7 +283,7 @@ class Yum(BackendPlugin, yum.YumBase):
             repo.mirrorlist = _varSubstitute(mirrorlist)
 
         conf = yum.config.RepoConf()
-        for k, v in conf.items():
+        for k, v in list(conf.items()):
             if v or not hasattr(repo, k):
                 repo.setAttribute(k, v)
 
index 60a4a7d..75324ea 100644 (file)
@@ -321,9 +321,7 @@ class Zypp(BackendPlugin):
 
         if found:
             if include == ksparser.GROUP_REQUIRED:
-                list(map(
-                    lambda p: self.deselectPackage(p),
-                    list(grp.default_packages.keys())))
+                list([self.deselectPackage(p) for p in list(grp.default_packages.keys())])
 
             return None
         else:
index beba62c..dc0bfd9 100644 (file)
@@ -166,6 +166,9 @@ class LoopPlugin(ImagerPlugin):
         elif imgtype in ("ext3fsimg", "ext4fsimg"):
             fstype = imgtype[:4]
             myDiskMount = fs_related.ExtDiskMount
+        elif imgtype == "f2fsimg":
+            fstype = "f2fs"
+            myDiskMount = fs_related.F2fsDiskMount
         else:
             raise errors.CreatorError("Unsupported filesystem type: %s" \
                                       % imgtype)
index bb6661c..e5b35dc 100644 (file)
@@ -143,7 +143,7 @@ class RawPlugin(ImagerPlugin):
                 # not recognize properly.
                 # TODO: Can we make better assumption?
                 fstype = "btrfs"
-            elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs" ]:
+            elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs", "f2fs" ]:
                 fstype = partition_info[5]
             elif partition_info[5] in [ "fat16", "fat32" ]:
                 fstype = "vfat"
@@ -155,7 +155,7 @@ class RawPlugin(ImagerPlugin):
 
             if rootpart and rootpart == line[0]:
                 mountpoint = '/'
-            elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs" ]:
+            elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs", "f2fs" ]:
                 # TODO: Check that this is actually the valid root partition from /etc/fstab
                 mountpoint = "/"
                 root_mounted = True
index fe39e7d..a4f66e8 100644 (file)
@@ -85,7 +85,7 @@ class BaseImgrTest(unittest.TestCase):
         creatoropts['pkgmgr'] = backend
         cfgmgr._ksconf =  KSCONF
         pkgmgr = None
-        for (key, pcls) in pluginmgr.PluginMgr().get_plugins('backend').items():
+        for (key, pcls) in list(pluginmgr.PluginMgr().get_plugins('backend').items()):
             if key == creatoropts['pkgmgr']:
                 pkgmgr = pcls
                 break