Attempt to bypass HTTP caches when fetching repomd.xml

If a repository and its repomd.xml changes frequently it may look
inconsistent to clients. Unlike other files in a repository repomd.xml
does not change its name along with contents like RPM packages and
metadata files with hashes in filenames. To get the most up-to-date
version additional HTTP headers (Pragma, Cache-Control) need to be passed.

Fixes: DEVT-110

Change-Id: I6a20f829d3a5331f9e99820fa46cdb497a5348ee
Signed-off-by: Łukasz Stelmach <l.stelmach@samsung.com>
Cc: Zhang, Qiang Z <qiang.z.zhang@intel.com>

diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py
index e85af25..c80a209 100644 (file)
--- a/gitbuildsys/utils.py
+++ b/gitbuildsys/utils.py
@@ -184,7 +184,7 @@ class URLGrabber(object):
         #curl.setopt(pycurl.VERBOSE, 1)
         self.curl = curl
 
-    def change_url(self, url, outfile, user, passwd):
+    def change_url(self, url, outfile, user, passwd, no_cache=False):
         '''change options for individual url'''
 
         curl = self.curl
@@ -196,6 +196,12 @@ class URLGrabber(object):
             if passwd:
                 userpwd = '%s:%s' % (user, passwd)
             curl.setopt(pycurl.USERPWD, userpwd)
+        httpheader = []
+        if no_cache:
+            httpheader.append('Pragma: no-cache')
+            httpheader.append('Cache-Control: no-cache')
+            log.debug("disable HTTP caching")
+        curl.setopt(pycurl.HTTPHEADER, httpheader)
 
     def perform(self):
         '''do the real Curl perform work'''
@@ -245,13 +251,13 @@ class URLGrabber(object):
         self.curl.close()
         self.curl = None
 
-    def grab(self, url, filename, user=None, passwd=None):
+    def grab(self, url, filename, user=None, passwd=None, no_cache=False):
         """Grab url to file."""
 
         log.debug("fetching %s => %s" % (url, filename))
 
         with open(filename, 'w') as outfile:
-            self.change_url(url, outfile, user, passwd)
+            self.change_url(url, outfile, user, passwd, no_cache)
             self.perform()
 
 
@@ -328,7 +334,7 @@ class RepoParser(object):
                 if self.is_standard_repo(repourl):
                     self.repourls[arch].append(repourl)
 
-    def fetch(self, url):
+    def fetch(self, url, no_cache=False):
         """
         Fetch url.
         Returns: file name if fetch succeds, else None.
@@ -336,7 +342,7 @@ class RepoParser(object):
         fname = os.path.join(self.cachedir, os.path.basename(url))
 
         try:
-            self.urlgrabber.grab(url, fname, url.user, url.passwd)
+            self.urlgrabber.grab(url, fname, url.user, url.passwd, no_cache)
         except PageNotFound:
             return
 
@@ -346,7 +352,7 @@ class RepoParser(object):
         """Check if repo is standard repo with repodata/repomd.xml exist."""
 
         repomd_url = repo.pathjoin('repodata/repomd.xml')
-        return not not self.fetch(repomd_url)
+        return not not self.fetch(repomd_url, no_cache=True)
 
     def _fetch_build_meta(self, latest_repo_url):
         """Fetch and parse build.xml."""