From: Huang Hao Date: Thu, 5 Dec 2013 05:25:51 +0000 (+0800) Subject: Support encode form of baseurl for repo directive. X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a205b424c907fb2f3474da8d0095c69e92301e3c;p=tools%2Fmic.git Support encode form of baseurl for repo directive. If user has special characters like '@' in his/her password, it can be write as encode form embeded into url. By using zypp.Url class which is already support encoded password, it by default hide password part, so we also gain the feature that password won't be printed to console. Fix: 1499 Change-Id: I4243f36df491cb6eb80f04334962a2ed6030acbe --- diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index f04785a..d7c4d27 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -965,7 +965,7 @@ class BaseImageCreator(object): if not os.path.exists(fpath): # download pkgs try: - fpath = grabber.myurlgrab(url, fpath, proxies, None) + fpath = grabber.myurlgrab(url.full, fpath, proxies, None) except CreatorError: raise diff --git a/mic/kickstart/__init__.py b/mic/kickstart/__init__.py index 392a05e..c639e7a 100644 --- a/mic/kickstart/__init__.py +++ b/mic/kickstart/__init__.py @@ -33,6 +33,7 @@ from pykickstart.handlers.control import dataMap from mic import msger from mic.utils import errors, misc, runner, fs_related as fs from custom_commands import desktop, micrepo, micboot, partition, installerfw +from mic.utils.safeurl import SafeURL AUTH_URL_PTN = r"(?P.*)://(?P.*)(:?P.*)?@(?P.*)" @@ -731,6 +732,8 @@ def get_repos(ks, repo_urls=None): baseurl = repo_urls[repo.name] mirrorlist = None + baseurl = SafeURL(baseurl) + if repos.has_key(repo.name): msger.warning("Overriding already specified repo %s" %(repo.name,)) diff --git a/mic/utils/grabber.py b/mic/utils/grabber.py index 8c7d9d2..a0a0ae1 100644 --- a/mic/utils/grabber.py +++ b/mic/utils/grabber.py @@ -9,6 +9,7 @@ import termios from mic import msger from mic.utils import runner from mic.utils.errors import CreatorError +from mic.utils.safeurl import SafeURL from urlgrabber import grabber from urlgrabber import __version__ as grabber_version @@ -30,6 +31,8 @@ def myurlgrab(url, filename, proxies, progress_obj = None): else: try: + # cast url to str here, sometimes it can be unicode, + # but pycurl only accept str filename = g.urlgrab(url=str(url), filename=filename, ssl_verify_host=False, @@ -39,9 +42,14 @@ def myurlgrab(url, filename, proxies, progress_obj = None): quote=0, progress_obj=progress_obj) except grabber.URLGrabError, err: + tmp = SafeURL(url) msg = str(err) + if msg.find(url) < 0: - msg += ' on %s' % url + msg += ' on %s' % tmp + else: + msg = msg.replace(url, tmp) + raise CreatorError(msg) return filename diff --git a/mic/utils/misc.py b/mic/utils/misc.py index b3302b0..48562e9 100644 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -47,6 +47,7 @@ from mic.utils.grabber import myurlgrab from mic.utils.proxy import get_proxy_for from mic.utils import runner from mic.utils import rpmmisc +from mic.utils.safeurl import SafeURL RPM_RE = re.compile("(.*)\.(.*) (.*)-(.*)") @@ -559,13 +560,15 @@ def get_repostrs_from_ks(ks): if 'name' not in repo: repo['name'] = _get_temp_reponame(repodata.baseurl) + if hasattr(repodata, 'baseurl') and getattr(repodata, 'baseurl'): + repo['baseurl'] = SafeURL(getattr(repodata, 'baseurl')) kickstart_repos.append(repo) return kickstart_repos def _get_uncompressed_data_from_url(url, filename, proxies): - filename = myurlgrab(url, filename, proxies) + filename = myurlgrab(url.full, filename, proxies) suffix = None if filename.endswith(".gz"): suffix = ".gz" @@ -579,7 +582,7 @@ def _get_uncompressed_data_from_url(url, filename, proxies): def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, sumtype=None, checksum=None): - url = os.path.join(baseurl, filename) + url = baseurl.join(filename) filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename))) if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"): filename = os.path.splitext(filename_tmp)[0] @@ -604,7 +607,6 @@ def get_metadata_from_repos(repos, cachedir): reponame = repo['name'] baseurl = repo['baseurl'] - if 'proxy' in repo: proxy = repo['proxy'] else: @@ -612,12 +614,12 @@ def get_metadata_from_repos(repos, cachedir): proxies = None if proxy: - proxies = {str(baseurl.split(":")[0]):str(proxy)} + proxies = {str(baseurl.split(":")[0]): str(proxy)} makedirs(os.path.join(cachedir, reponame)) - url = os.path.join(baseurl, "repodata/repomd.xml") + url = baseurl.join("repodata/repomd.xml") filename = os.path.join(cachedir, reponame, 'repomd.xml') - repomd = myurlgrab(url, filename, proxies) + repomd = myurlgrab(url.full, filename, proxies) try: root = xmlparse(repomd) except SyntaxError: @@ -818,7 +820,7 @@ def get_package(pkg, repometadata, arch = None): con.close() if target_repo: makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"])) - url = os.path.join(target_repo["baseurl"], pkgpath) + url = target_repo["baseurl"].join(pkgpath) filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath))) if os.path.exists(filename): ret = rpmmisc.checkRpmIntegrity('rpm', filename) @@ -829,7 +831,7 @@ def get_package(pkg, repometadata, arch = None): (os.path.basename(filename), filename)) os.unlink(filename) - pkg = myurlgrab(str(url), filename, target_repo["proxies"]) + pkg = myurlgrab(url.full, filename, target_repo["proxies"]) return pkg else: return None diff --git a/mic/utils/safeurl.py b/mic/utils/safeurl.py new file mode 100644 index 0000000..dc0868d --- /dev/null +++ b/mic/utils/safeurl.py @@ -0,0 +1,61 @@ +"safe url" +import os + +from zypp import Url + + +class SafeURL(str): + "URL wrapper which won't show password out" + def __new__(cls, urlstring): + # sometimes we get unicode here, but zypp don't accept unicode string + urlstring = str(urlstring) + safe = Url(urlstring) + safe.setUsername('') + safe.setPassword('') + + safeurlstring = str(safe) + if safeurlstring.startswith("file:/"): + # zypp.Url converts file:///path/to/file to file:/path/to/file + safeurlstring = "file://" + safeurlstring[len("file:"):] + + instance = super(SafeURL, cls).__new__(cls, safeurlstring) + instance.url = Url(urlstring) + return instance + + def join(self, *path): + """ + Returns a new SafeURL with new path. Search part is removed since + after join path is changed, keep the same search part is useless. + """ + urlstring = self.without_search().full.rstrip('/') + return SafeURL(os.path.join(urlstring, *path)) + + @property + def full(self): + "Returns full url string with auth info" + fullstring = self.url.asCompleteString() + if fullstring.startswith("file:/"): + fullstring = "file://" + fullstring[len("file:/"):] + return fullstring + + def without_search(self): + "Returns a SafeURL without search part" + urlstring = self.full + idx = urlstring.find('?') + return self if idx == -1 else SafeURL(urlstring[:idx]) + + @property + def scheme(self): + return self.url.getScheme() + + @property + def user(self): + return self.url.getUsername() + + @property + def password(self): + return self.url.getPassword() + + @property + def host(self): + return self.url.getHost() diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index ad04b77..d4bf31f 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -273,7 +273,7 @@ class Yum(BackendPlugin, yum.YumBase): repo.proxy_password = proxy_password if url: - repo.baseurl.append(_varSubstitute(url)) + repo.baseurl.append(_varSubstitute(url.full)) if mirrorlist: repo.mirrorlist = _varSubstitute(mirrorlist) diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index 0ec2d45..a83fa92 100644 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -370,7 +370,7 @@ class Zypp(BackendPlugin): repo_info.setEnabled(repo.enabled) repo_info.setAutorefresh(repo.autorefresh) repo_info.setKeepPackages(repo.keeppackages) - baseurl = zypp.Url(repo.baseurl[0]) + baseurl = repo.baseurl[0].url if not ssl_verify: baseurl.setQueryParam("ssl_verify", "no") if proxy: @@ -409,7 +409,6 @@ class Zypp(BackendPlugin): else: baseurl.setQueryParam ("proxy", "_none_") - repo.baseurl[0] = baseurl.asCompleteString() self.repos.append(repo) repo_info.addBaseUrl(baseurl) @@ -770,7 +769,7 @@ class Zypp(BackendPlugin): proxies = self.get_proxies(po) try: - filename = myurlgrab(url, filename, proxies, progress_obj) + filename = myurlgrab(url.full, filename, proxies, progress_obj) except CreatorError: self.close() raise @@ -952,18 +951,12 @@ class Zypp(BackendPlugin): except IndexError: return None - baseurl = repo.baseurl[0] - - index = baseurl.find("?") - if index > -1: - baseurl = baseurl[:index] - location = pobj.location() location = str(location.filename()) if location.startswith("./"): location = location[2:] - return os.path.join(baseurl, location) + return repo.baseurl[0].join(location) def package_url(self, pkgname):