From cdfc17622a94943d667d3f608169ae690078387a Mon Sep 17 00:00:00 2001 From: "yuhuan.yang" Date: Tue, 17 Oct 2017 00:42:28 +0800 Subject: [PATCH] Another method of install tpk. Change-Id: Ibe2f1cfaa7625ab1ea938feda5df6406beb267e6 --- mic/3rdparty/pykickstart/base.py | 3 +- mic/3rdparty/pykickstart/handlers/control.py | 3 ++ mic/3rdparty/pykickstart/parser.py | 24 +++++++++++++-- mic/3rdparty/pykickstart/sections.py | 19 +++++++++++- mic/conf.py | 9 ++++++ mic/imager/baseimager.py | 45 ++++++++++++++++++++++++++-- mic/kickstart/__init__.py | 30 ++++++++++++++++++- mic/kickstart/custom_commands/__init__.py | 4 ++- mic/kickstart/custom_commands/micrepo.py | 24 +++++++++++++++ mic/rt_util.py | 2 ++ plugins/imager/fs_plugin.py | 1 + plugins/imager/loop_plugin.py | 1 + plugins/imager/qcow_plugin.py | 1 + plugins/imager/raw_plugin.py | 1 + 14 files changed, 159 insertions(+), 8 deletions(-) diff --git a/mic/3rdparty/pykickstart/base.py b/mic/3rdparty/pykickstart/base.py index e6c8f56..e3597df 100644 --- a/mic/3rdparty/pykickstart/base.py +++ b/mic/3rdparty/pykickstart/base.py @@ -46,7 +46,7 @@ import types import warnings from pykickstart.errors import * from pykickstart.ko import * -from pykickstart.parser import Packages +from pykickstart.parser import Packages,TpkPackages from pykickstart.version import versionToString ### @@ -253,6 +253,7 @@ class BaseHandler(KickstartObject): # everything else I can think of. self.scripts = [] self.packages = Packages() + self.tpk_packages = TpkPackages() self.platform = "" # These will be set by the dispatcher. diff --git a/mic/3rdparty/pykickstart/handlers/control.py b/mic/3rdparty/pykickstart/handlers/control.py index d8c8f2b..119e32f 100644 --- a/mic/3rdparty/pykickstart/handlers/control.py +++ b/mic/3rdparty/pykickstart/handlers/control.py @@ -19,6 +19,7 @@ # from pykickstart.version import * from pykickstart.commands import * +from mic.kickstart.custom_commands.micrepo import * # This map is keyed on kickstart syntax version as provided by # pykickstart.version. Within each sub-dict is a mapping from command name @@ -217,6 +218,7 @@ commandMap = { "raid": raid.FC5_Raid, "reboot": reboot.FC6_Reboot, "repo": repo.FC6_Repo, + "tpk_repo": Mic_Tpk_Repo, "rootpw": rootpw.FC3_RootPw, "selinux": selinux.FC3_SELinux, "services": services.FC6_Services, @@ -1087,6 +1089,7 @@ dataMap = { "PartData": partition.FC4_PartData, "RaidData": raid.FC5_RaidData, "RepoData": repo.FC6_RepoData, + "TpkRepoData": Mic_Tpk_RepoData, "UserData": user.FC6_UserData, "VolGroupData": volgroup.FC3_VolGroupData, "ZFCPData": zfcp.FC3_ZFCPData, diff --git a/mic/3rdparty/pykickstart/parser.py b/mic/3rdparty/pykickstart/parser.py index cf4db4c..f09f925 100644 --- a/mic/3rdparty/pykickstart/parser.py +++ b/mic/3rdparty/pykickstart/parser.py @@ -30,7 +30,6 @@ This module exports several important classes: KickstartParser - The kickstart file parser state machine. """ - from collections import Iterator import os import shlex @@ -293,6 +292,7 @@ class Packages(KickstartObject): self.groupList = [] self.handleMissing = constants.KS_MISSING_PROMPT self.packageList = [] + self.tpk_packageList = [] self.instLangs = None def __str__(self): @@ -407,7 +407,25 @@ class Packages(KickstartObject): self.packageList = list(existingPackageSet) self.excludedList = list(existingExcludedSet) - +class TpkPackages(KickstartObject): + """A class representing the %tpk_packages section of the kickstart file.""" + def __init__(self, *args, **kwargs): + KickstartObject.__init__(self, *args, **kwargs) + self.tpk_packageList = [] + def __str__(self): + tpk_pkgs = "" + retval = "\n%tpk_packages" + p = self.packageList + p.sort() + for pkg in p: + tpk_pkgs += "%s\n" % pkg + return retval + "\n" +tpk_pkgs + def add(self, tpkPackageList): + tpk_PackageSet = set(self.tpk_packageList) + for tpk_pkg in tpkPackageList: + stripped = tpk_pkg.strip() + tpk_PackageSet.add(stripped) + self.tpk_packageList = list(tpk_PackageSet) ### ### PARSER ### @@ -703,3 +721,5 @@ class KickstartParser: self.registerSection(TracebackScriptSection(self.handler, dataObj=Script)) self.registerSection(RunScriptSection(self.handler, dataObj=Script)) self.registerSection(PackageSection(self.handler)) + self.registerSection(TpkPackageSection(self.handler)) + diff --git a/mic/3rdparty/pykickstart/sections.py b/mic/3rdparty/pykickstart/sections.py index 51cba3b..223c440 100644 --- a/mic/3rdparty/pykickstart/sections.py +++ b/mic/3rdparty/pykickstart/sections.py @@ -208,7 +208,6 @@ class PackageSection(Section): (h, s, t) = line.partition('#') line = h.rstrip() - self.handler.packages.add([line]) def handleHeader(self, lineno, args): @@ -247,3 +246,21 @@ class PackageSection(Section): if opts.instLangs: self.handler.packages.instLangs = opts.instLangs + +class TpkPackageSection(Section): + sectionOpen = "%tpk_packages" + + def handleLine(self, line): + if not self.handler: + return + + (h, s, t) = line.partition('#') + line = h.rstrip() + self.handler.tpk_packages.add([line]) + + def handleHeader(self, lineno, args): + """Process the arguments to the %tpk_packages header and set attributes + on the Version's TpkPackages instance appropriate. This method may be + overridden in a subclass if necessary. + """ + Section.handleHeader(self, lineno, args) diff --git a/mic/conf.py b/mic/conf.py index d56d3cb..40d4a64 100755 --- a/mic/conf.py +++ b/mic/conf.py @@ -71,6 +71,7 @@ class ConfigMgr(object): "check_pkgs": [], "repourl": {}, "localrepos": [], # save localrepos + "localtpkrepos":[], "runtime": "bootstrap", "extrarepos": {}, "ignore_ksrepo": False, @@ -241,6 +242,14 @@ class ConfigMgr(object): self.create['repomd'] = misc.get_metadata_from_repos( ksrepos, self.create['cachedir']) + kstpkrepos = kickstart.get_tpkrepos(ks) + if kstpkrepos: + for tpk_repo in kstpkrepos: + if hasattr(tpk_repo,'baseurl') and tpk_repo.baseurl.startswith("file:"): + tpk_repourl = tpk_repo.baseurl.replace('file:','') + tpk_repourl = "/%s" % tpk_repourl.lstrip('/') + self.create['localtpkrepos'].append(tpk_repourl) + msger.raw(" DONE") target_archlist, archlist = misc.get_arch(self.create['repomd']) diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index ec2edb5..b10f2ed 100755 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -1,4 +1,3 @@ - #!/usr/bin/python -tt # # Copyright (c) 2007 Red Hat Inc. @@ -28,7 +27,6 @@ import tarfile import glob import json from datetime import datetime - import rpm import time from mic import kickstart @@ -38,6 +36,7 @@ from mic.utils import misc, grabber, runner, fs_related as fs from mic.chroot import kill_proc_inchroot from mic.archive import get_archive_suffixes from mic.conf import configmgr +from mic.utils.grabber import myurlgrab #post script max run time MAX_RUN_TIME = 120 @@ -1088,6 +1087,7 @@ class BaseImageCreator(object): if 'debuginfo' in self.install_pkgs: pkg_manager.install_debuginfo = True + for repo in kickstart.get_repos(self.ks, repo_urls, self.ignore_ksrepo): (name, baseurl, mirrorlist, inc, exc, proxy, proxy_username, proxy_password, debuginfo, @@ -1154,6 +1154,47 @@ class BaseImageCreator(object): except: pass + def tpkinstall(self): + if self.ks: + tpk_pkgs = kickstart.get_tpkpackages(self.ks) + tpk_repoList = kickstart.get_tpkrepos(self.ks) + if tpk_repoList and tpk_pkgs: + tpk_dir = "/usr/apps/.preload-tpk" + fs.makedirs(self._instroot + "/usr/apps") + fs.makedirs(self._instroot + tpk_dir) + for pkg in tpk_pkgs: + flag = 0 + for tpk_repo in tpk_repoList: + if hasattr(tpk_repo,'baseurl') and tpk_repo.baseurl.startswith("file:"): + tpk_repourl = tpk_repo.baseurl.replace('file:','') + tpk_repourl = "/%s" % tpk_repourl.lstrip('/') + tpk_pkgpath = tpk_repourl + "/"+ pkg + if os.path.isfile(tpk_pkgpath): + shutil.copy(tpk_pkgpath,self._instroot + tpk_dir) + flag = 1 + break + elif hasattr(tpk_repo,'baseurl'): + url = tpk_repo.baseurl.join(pkg) + filename = self._instroot+tpk_dir+"/"+pkg + if tpk_repo.baseurl.startswith("http:"): + import urllib + status = urllib.urlopen(url).code + if status == 200: + filename = myurlgrab(url.full, filename, None) + flag = 1 + break + elif status == 404 or status == None: + continue + #url is ok, then download, url wrong, check other url. + elif tpk_repo.baseurl.startswith("https:") : + try: + flag = 1 + filename = myurlgrab(url.full, filename, None) + except CreatorError: + continue + if flag == 0: + raise CreatorError("Tpk package missing.") + def postinstall(self): self.copy_attachment() diff --git a/mic/kickstart/__init__.py b/mic/kickstart/__init__.py index 76cc525..fd2806d 100755 --- a/mic/kickstart/__init__.py +++ b/mic/kickstart/__init__.py @@ -100,11 +100,13 @@ def read_kickstart(path): using_version = ksversion.DEVEL commandMap[using_version]["desktop"] = desktop.Mic_Desktop commandMap[using_version]["repo"] = micrepo.Mic_Repo + commandMap[using_version]["tpk_repo"] = micrepo.Mic_Tpk_Repo commandMap[using_version]["bootloader"] = micboot.Mic_Bootloader commandMap[using_version]["part"] = partition.Mic_Partition commandMap[using_version]["partition"] = partition.Mic_Partition commandMap[using_version]["installerfw_plugins"] = installerfw.Mic_installerfw dataMap[using_version]["RepoData"] = micrepo.Mic_RepoData + dataMap[using_version]["Tpk_RepoData"] = micrepo.Mic_Tpk_RepoData dataMap[using_version]["PartData"] = partition.Mic_PartData superclass = ksversion.returnClassForVersion(version=using_version) @@ -772,9 +774,32 @@ def get_repos(ks, repo_urls=None, ignore_ksrepo=False): repo.get('user', None), repo.get('passwd', None)) repos[name] = Repo(**repo) - return repos.values() +TpkRepoType = collections.namedtuple("TpkRepo", + "name, baseurl,proxy,proxy_username,proxy_password,ssl_verify") + +def TpkRepo(name, baseurl, proxy=None,proxy_username=None, proxy_password=None,ssl_verify=None): + return TpkRepoType(name, baseurl,proxy,proxy_username,proxy_password,ssl_verify) + + +def get_tpkrepos(ks): + tpkrepos = {} + for tpkrepodata in ks.handler.tpk_repo.tpkRepoList: + tpkrepo = {} + for field in TpkRepoType._fields: + if hasattr(tpkrepodata, field) and getattr(tpkrepodata, field): + tpkrepo[field] = getattr(tpkrepodata, field) + + if hasattr(tpkrepodata, 'baseurl') and getattr(tpkrepodata, 'baseurl'): + tpkrepo['baseurl'] = SafeURL(getattr(tpkrepodata, 'baseurl'),getattr(tpkrepodata, 'user',None),getattr(tpkrepodata, 'passwd',None)) + + if 'name' in tpkrepo: + tpkrepos[tpkrepo['name']] = TpkRepo(**tpkrepo) + + return tpkrepos.values() + + def convert_method_to_repo(ks): try: ks.handler.repo.methodToRepo() @@ -790,6 +815,9 @@ def get_pre_packages(ks, required=()): def get_packages(ks, required=()): return ks.handler.packages.packageList + list(required) +def get_tpkpackages(ks): + return ks.handler.tpk_packages.tpk_packageList + def get_groups(ks, required=()): return ks.handler.packages.groupList + list(required) diff --git a/mic/kickstart/custom_commands/__init__.py b/mic/kickstart/custom_commands/__init__.py index 5f4c440..381c917 100644 --- a/mic/kickstart/custom_commands/__init__.py +++ b/mic/kickstart/custom_commands/__init__.py @@ -1,5 +1,5 @@ from desktop import Mic_Desktop -from micrepo import Mic_Repo, Mic_RepoData +from micrepo import Mic_Repo, Mic_RepoData,Mic_Tpk_Repo, Mic_Tpk_RepoData from partition import Mic_Partition from installerfw import Mic_installerfw @@ -7,6 +7,8 @@ __all__ = ( "Mic_Desktop", "Mic_Repo", "Mic_RepoData", + "Mic_Tpk_Repo", + "Mic_Tpk_RepoData", "Mic_Partition", "Mic_installerfw", ) diff --git a/mic/kickstart/custom_commands/micrepo.py b/mic/kickstart/custom_commands/micrepo.py index b38ae77..25cdae9 100644 --- a/mic/kickstart/custom_commands/micrepo.py +++ b/mic/kickstart/custom_commands/micrepo.py @@ -50,6 +50,14 @@ class Mic_RepoData(F14_RepoData): return retval +class Mic_Tpk_RepoData(Mic_RepoData): + "Mic customized tpk repo data" + def __init__(self, *args, **kw): + Mic_RepoData.__init__(self, *args, **kw) + def __str__(self): + retval = Mic_RepoData._getArgsAsStr(self) + retval = "tpk_repo " + retval + return retval class Mic_Repo(F14_Repo): "Mic customized repo command" @@ -71,3 +79,19 @@ class Mic_Repo(F14_Repo): op.add_option("--priority", type="int") op.add_option("--ssl_verify", default=None) return op + + +class Mic_Tpk_Repo(Mic_Repo): + def __init__(self, writePriority=0, *args, **kwargs): + Mic_Repo.__init__(self, writePriority, *args, **kwargs) + self.op = self._getParser() + + self.tpkRepoList = kwargs.get("tpkRepoList", []) + + def _getParser(self): + op = Mic_Repo._getParser(self) + return op + + def dataList(self): + return self.tpkRepoList + diff --git a/mic/rt_util.py b/mic/rt_util.py index 3897efc..316375d 100644 --- a/mic/rt_util.py +++ b/mic/rt_util.py @@ -144,6 +144,8 @@ def get_bindmounts(cropts): for lrepo in cropts['localrepos']: binddirs.append(lrepo) + for ltpkrepo in cropts['localtpkrepos']: + binddirs.append(ltpkrepo) bindlist = map(expath, filter(None, binddirs)) bindlist += map(os.path.dirname, map(expath, filter(None, bindfiles))) diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index b2701a0..c639211 100755 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -93,6 +93,7 @@ class FsPlugin(ImagerPlugin): creator.check_depend_tools() creator.mount(None, creatoropts["cachedir"]) creator.install() + creator.tpkinstall() #Download the source packages ###private options if args.include_src: installed_pkgs = creator.get_installed_packages() diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index 6034a29..0b94f0e 100755 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -101,6 +101,7 @@ class LoopPlugin(ImagerPlugin): creator.check_depend_tools() creator.mount(None, creatoropts["cachedir"]) creator.install() + creator.tpkinstall() creator.configure(creatoropts["repomd"]) creator.copy_kernel() creator.create_cpio_image() diff --git a/plugins/imager/qcow_plugin.py b/plugins/imager/qcow_plugin.py index a81e291..d6758c5 100755 --- a/plugins/imager/qcow_plugin.py +++ b/plugins/imager/qcow_plugin.py @@ -132,6 +132,7 @@ class QcowPlugin(ImagerPlugin): creator.check_depend_tools() creator.mount(None, creatoropts["cachedir"]) creator.install() + creator.tpkinstall() creator.configure(creatoropts["repomd"]) creator.copy_kernel() creator.create_cpio_image() diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index 1524e74..e954b7b 100755 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -99,6 +99,7 @@ class RawPlugin(ImagerPlugin): creator.check_depend_tools() creator.mount(None, creatoropts["cachedir"]) creator.install() + creator.tpkinstall() creator.configure(creatoropts["repomd"]) creator.copy_kernel() creator.unmount() -- 2.7.4