Support encode form of baseurl for repo directive.

If user has special characters like '@' in his/her password, it
can be write as encode form embeded into url. By using zypp.Url
class which is already support encoded password, it by default
hide password part, so we also gain the feature that password
won't be printed to console.

Fix: 1499

Change-Id: I4243f36df491cb6eb80f04334962a2ed6030acbe

diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py
index f04785a..d7c4d27 100644 (file)
--- a/mic/imager/baseimager.py
+++ b/mic/imager/baseimager.py
@@ -965,7 +965,7 @@ class BaseImageCreator(object):
             if not os.path.exists(fpath):
                 # download pkgs
                 try:
-                    fpath = grabber.myurlgrab(url, fpath, proxies, None)
+                    fpath = grabber.myurlgrab(url.full, fpath, proxies, None)
                 except CreatorError:
                     raise
 

diff --git a/mic/kickstart/__init__.py b/mic/kickstart/__init__.py
index 392a05e..c639e7a 100644 (file)
--- a/mic/kickstart/__init__.py
+++ b/mic/kickstart/__init__.py
@@ -33,6 +33,7 @@ from pykickstart.handlers.control import dataMap
 from mic import msger
 from mic.utils import errors, misc, runner, fs_related as fs
 from custom_commands import desktop, micrepo, micboot, partition, installerfw
+from mic.utils.safeurl import SafeURL
 
 
 AUTH_URL_PTN = r"(?P<scheme>.*)://(?P<username>.*)(:?P<password>.*)?@(?P<url>.*)"
@@ -731,6 +732,8 @@ def get_repos(ks, repo_urls=None):
             baseurl = repo_urls[repo.name]
             mirrorlist = None
 
+        baseurl = SafeURL(baseurl)
+
         if repos.has_key(repo.name):
             msger.warning("Overriding already specified repo %s" %(repo.name,))
 

diff --git a/mic/utils/grabber.py b/mic/utils/grabber.py
index 8c7d9d2..a0a0ae1 100644 (file)
--- a/mic/utils/grabber.py
+++ b/mic/utils/grabber.py
@@ -9,6 +9,7 @@ import termios
 from mic import msger
 from mic.utils import runner
 from mic.utils.errors import CreatorError
+from mic.utils.safeurl import SafeURL
 
 from urlgrabber import grabber
 from urlgrabber import __version__ as grabber_version
@@ -30,6 +31,8 @@ def myurlgrab(url, filename, proxies, progress_obj = None):
 
     else:
         try:
+            # cast url to str here, sometimes it can be unicode,
+            # but pycurl only accept str
             filename = g.urlgrab(url=str(url),
                                  filename=filename,
                                  ssl_verify_host=False,
@@ -39,9 +42,14 @@ def myurlgrab(url, filename, proxies, progress_obj = None):
                                  quote=0,
                                  progress_obj=progress_obj)
         except grabber.URLGrabError, err:
+            tmp = SafeURL(url)
             msg = str(err)
+
             if msg.find(url) < 0:
-                msg += ' on %s' % url
+                msg += ' on %s' % tmp
+            else:
+                msg = msg.replace(url, tmp)
+
             raise CreatorError(msg)
 
     return filename

diff --git a/mic/utils/misc.py b/mic/utils/misc.py
index b3302b0..48562e9 100644 (file)
--- a/mic/utils/misc.py
+++ b/mic/utils/misc.py
@@ -47,6 +47,7 @@ from mic.utils.grabber import myurlgrab
 from mic.utils.proxy import get_proxy_for
 from mic.utils import runner
 from mic.utils import rpmmisc
+from mic.utils.safeurl import SafeURL
 
 
 RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
@@ -559,13 +560,15 @@ def get_repostrs_from_ks(ks):
 
         if 'name' not in repo:
             repo['name'] = _get_temp_reponame(repodata.baseurl)
+        if hasattr(repodata, 'baseurl') and getattr(repodata, 'baseurl'):
+            repo['baseurl'] = SafeURL(getattr(repodata, 'baseurl'))
 
         kickstart_repos.append(repo)
 
     return kickstart_repos
 
 def _get_uncompressed_data_from_url(url, filename, proxies):
-    filename = myurlgrab(url, filename, proxies)
+    filename = myurlgrab(url.full, filename, proxies)
     suffix = None
     if filename.endswith(".gz"):
         suffix = ".gz"
@@ -579,7 +582,7 @@ def _get_uncompressed_data_from_url(url, filename, proxies):
 
 def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
                             sumtype=None, checksum=None):
-    url = os.path.join(baseurl, filename)
+    url = baseurl.join(filename)
     filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
     if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
         filename = os.path.splitext(filename_tmp)[0]
@@ -604,7 +607,6 @@ def get_metadata_from_repos(repos, cachedir):
         reponame = repo['name']
         baseurl  = repo['baseurl']
 
-
         if 'proxy' in repo:
             proxy = repo['proxy']
         else:
@@ -612,12 +614,12 @@ def get_metadata_from_repos(repos, cachedir):
 
         proxies = None
         if proxy:
-            proxies = {str(baseurl.split(":")[0]):str(proxy)}
+            proxies = {str(baseurl.split(":")[0]): str(proxy)}
 
         makedirs(os.path.join(cachedir, reponame))
-        url = os.path.join(baseurl, "repodata/repomd.xml")
+        url = baseurl.join("repodata/repomd.xml")
         filename = os.path.join(cachedir, reponame, 'repomd.xml')
-        repomd = myurlgrab(url, filename, proxies)
+        repomd = myurlgrab(url.full, filename, proxies)
         try:
             root = xmlparse(repomd)
         except SyntaxError:
@@ -818,7 +820,7 @@ def get_package(pkg, repometadata, arch = None):
             con.close()
     if target_repo:
         makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
-        url = os.path.join(target_repo["baseurl"], pkgpath)
+        url = target_repo["baseurl"].join(pkgpath)
         filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
         if os.path.exists(filename):
             ret = rpmmisc.checkRpmIntegrity('rpm', filename)
@@ -829,7 +831,7 @@ def get_package(pkg, repometadata, arch = None):
                           (os.path.basename(filename), filename))
             os.unlink(filename)
 
-        pkg = myurlgrab(str(url), filename, target_repo["proxies"])
+        pkg = myurlgrab(url.full, filename, target_repo["proxies"])
         return pkg
     else:
         return None

diff --git a/mic/utils/safeurl.py b/mic/utils/safeurl.py
new file mode 100644 (file)
index 0000000..dc0868d
--- /dev/null
+++ b/mic/utils/safeurl.py
@@ -0,0 +1,61 @@
+"safe url"
+import os
+
+from zypp import Url
+
+
+class SafeURL(str):
+    "URL wrapper which won't show password out"
+    def __new__(cls, urlstring):
+        # sometimes we get unicode here, but zypp don't accept unicode string
+        urlstring = str(urlstring)
+        safe = Url(urlstring)
+        safe.setUsername('')
+        safe.setPassword('')
+
+        safeurlstring = str(safe)
+        if safeurlstring.startswith("file:/"):
+            # zypp.Url converts file:///path/to/file to file:/path/to/file
+            safeurlstring = "file://" + safeurlstring[len("file:"):]
+
+        instance = super(SafeURL, cls).__new__(cls, safeurlstring)
+        instance.url = Url(urlstring)
+        return instance
+
+    def join(self, *path):
+        """
+        Returns a new SafeURL with new path. Search part is removed since
+        after join path is changed, keep the same search part is useless.
+        """
+        urlstring = self.without_search().full.rstrip('/')
+        return SafeURL(os.path.join(urlstring, *path))
+
+    @property
+    def full(self):
+        "Returns full url string with auth info"
+        fullstring = self.url.asCompleteString()
+        if fullstring.startswith("file:/"):
+            fullstring = "file://" + fullstring[len("file:/"):]
+        return fullstring
+
+    def without_search(self):
+        "Returns a SafeURL without search part"
+        urlstring = self.full
+        idx = urlstring.find('?')
+        return self if idx == -1 else SafeURL(urlstring[:idx])
+
+    @property
+    def scheme(self):
+        return self.url.getScheme()
+
+    @property
+    def user(self):
+        return self.url.getUsername()
+
+    @property
+    def password(self):
+        return self.url.getPassword()
+
+    @property
+    def host(self):
+        return self.url.getHost()

diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py
index ad04b77..d4bf31f 100644 (file)
--- a/plugins/backend/yumpkgmgr.py
+++ b/plugins/backend/yumpkgmgr.py
@@ -273,7 +273,7 @@ class Yum(BackendPlugin, yum.YumBase):
         repo.proxy_password = proxy_password
 
         if url:
-            repo.baseurl.append(_varSubstitute(url))
+            repo.baseurl.append(_varSubstitute(url.full))
 
         if mirrorlist:
             repo.mirrorlist = _varSubstitute(mirrorlist)

diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py
index 0ec2d45..a83fa92 100644 (file)
--- a/plugins/backend/zypppkgmgr.py
+++ b/plugins/backend/zypppkgmgr.py
@@ -370,7 +370,7 @@ class Zypp(BackendPlugin):
             repo_info.setEnabled(repo.enabled)
             repo_info.setAutorefresh(repo.autorefresh)
             repo_info.setKeepPackages(repo.keeppackages)
-            baseurl = zypp.Url(repo.baseurl[0])
+            baseurl = repo.baseurl[0].url
             if not ssl_verify:
                 baseurl.setQueryParam("ssl_verify", "no")
             if proxy:
@@ -409,7 +409,6 @@ class Zypp(BackendPlugin):
             else:
                 baseurl.setQueryParam ("proxy", "_none_")
 
-            repo.baseurl[0] = baseurl.asCompleteString()
             self.repos.append(repo)
 
             repo_info.addBaseUrl(baseurl)
@@ -770,7 +769,7 @@ class Zypp(BackendPlugin):
             proxies = self.get_proxies(po)
 
             try:
-                filename = myurlgrab(url, filename, proxies, progress_obj)
+                filename = myurlgrab(url.full, filename, proxies, progress_obj)
             except CreatorError:
                 self.close()
                 raise
@@ -952,18 +951,12 @@ class Zypp(BackendPlugin):
         except IndexError:
             return None
 
-        baseurl = repo.baseurl[0]
-
-        index = baseurl.find("?")
-        if index > -1:
-            baseurl = baseurl[:index]
-
         location = pobj.location()
         location = str(location.filename())
         if location.startswith("./"):
             location = location[2:]
 
-        return os.path.join(baseurl, location)
+        return repo.baseurl[0].join(location)
 
     def package_url(self, pkgname):