gbp/scripts/common/pq.py

   1 # vim: set fileencoding=utf-8 :
   2 #
   3 # (C) 2011 Guido Günther <agx@sigxcpu.org>
   4 # (C) 2012 Intel Corporation <markus.lehtonen@linux.intel.com>
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18 #
  19 """Common functionality for Debian and RPM patchqueue management"""
  20
  21 import re
  22 import os
  23 import subprocess
  24 import datetime
  25 import pwd
  26 import socket
  27 import time
  28 from email.message import Message
  29 from email.header import Header
  30 from email.charset import Charset, QP
  31
  32 from gbp.git import GitRepositoryError
  33 from gbp.git.modifier import GitModifier, GitTz
  34 from gbp.errors import GbpError
  35 import gbp.log
  36
  37 DEFAULT_PQ_BRANCH_NAME = "patch-queue/%(branch)s"
  38
  39
  40 def pq_branch_match(branch, pq_fmt_str):
  41     """
  42     Match branch name with pq branch name pattern
  43
  44     >>> pq_branch_match('patch-queue/foo', 'patch-queue/%(br)s').groupdict()
  45     {'br': 'foo'}
  46     >>> pq_branch_match('pq/foo/bar', 'pq/%(br)s/baz')
  47     >>> pq_branch_match('pq/foo/bar', 'pq/%(br)s/bar').groupdict()
  48     {'br': 'foo'}
  49     >>> pq_branch_match('foo/bar/1.0/pq', 'foo/%(br)s/%(ver)s/pq').groupdict()
  50     {'ver': '1.0', 'br': 'bar'}
  51     """
  52     pq_re = '^%s$' % re.sub('%\(([a-z_\-]+)\)s', r'(?P<\1>\S+)', pq_fmt_str)
  53     return  re.match(pq_re, branch)
  54
  55
  56 def is_pq_branch(branch, options):
  57     """
  58     is branch a patch-queue branch?
  59
  60     >>> from optparse import OptionParser
  61     >>> (opts, args) = OptionParser().parse_args([])
  62     >>> is_pq_branch("foo", opts)
  63     False
  64     >>> is_pq_branch("patch-queue/foo", opts)
  65     True
  66     >>> opts.pq_branch = "%(branch)s/development"
  67     >>> is_pq_branch("foo/development/bar", opts)
  68     False
  69     >>> is_pq_branch("bar/foo/development", opts)
  70     True
  71     >>> opts.pq_branch = "development"
  72     >>> is_pq_branch("development", opts)
  73     True
  74     >>> opts.pq_branch = "my/%(branch)s/pq"
  75     >>> is_pq_branch("my/foo/pqb", opts)
  76     False
  77     >>> is_pq_branch("my/foo/pq", opts)
  78     True
  79     >>> opts.pq_branch = "my/%(branch)s/%(version)s"
  80     >>> is_pq_branch("my/foo", opts)
  81     False
  82     >>> is_pq_branch("my/foo/1.0", opts)
  83     True
  84     """
  85     pq_format_str = (options.pq_branch if hasattr(options, 'pq_branch')
  86                                        else DEFAULT_PQ_BRANCH_NAME)
  87     if pq_branch_match(branch, pq_format_str):
  88         return True
  89     return False
  90
  91
  92 def pq_branch_name(branch, options, extra_keys=None):
  93     """
  94     get the patch queue branch corresponding to branch
  95
  96     >>> from optparse import OptionParser
  97     >>> (opts, args) = OptionParser().parse_args([])
  98     >>> pq_branch_name("patch-queue/master", opts)
  99     >>> pq_branch_name("foo", opts)
 100     'patch-queue/foo'
 101     >>> opts.pq_branch = "%(branch)s/development"
 102     >>> pq_branch_name("foo", opts)
 103     'foo/development'
 104     >>> opts.pq_branch = "development"
 105     >>> pq_branch_name("foo", opts)
 106     'development'
 107     >>> opts.pq_branch = "pq/%(branch)s/%(ver)s"
 108     >>> pq_branch_name("foo", opts, {'ver': '1.0'})
 109     'pq/foo/1.0'
 110     """
 111     pq_format_str = (options.pq_branch if hasattr(options, 'pq_branch')
 112                                        else DEFAULT_PQ_BRANCH_NAME)
 113     format_fields = {'branch': branch}
 114     if extra_keys:
 115         format_fields.update(extra_keys)
 116     if not is_pq_branch(branch, options):
 117         return pq_format_str % format_fields
 118
 119
 120 def pq_branch_base(pq_branch, options):
 121     """
 122     Get the branch corresponding to the given patch queue branch.
 123     Returns the packaging/debian branch if pq format string doesn't contain
 124     '%(branch)s' key.
 125
 126     >>> from optparse import OptionParser
 127     >>> (opts, args) = OptionParser().parse_args([])
 128     >>> opts.packaging_branch = "packaging"
 129     >>> pq_branch_base("patch-queue/master", opts)
 130     'master'
 131     >>> pq_branch_base("foo", opts)
 132     >>> opts.pq_branch = "my/%(branch)s/development"
 133     >>> pq_branch_base("foo/development", opts)
 134     >>> pq_branch_base("my/foo/development/bar", opts)
 135     >>> pq_branch_base("my/foo/development", opts)
 136     'foo'
 137     >>> opts.pq_branch = "development"
 138     >>> pq_branch_base("foo/development", opts)
 139     >>> pq_branch_base("development", opts)
 140     'packaging'
 141     """
 142     pq_format_str = (options.pq_branch if hasattr(options, 'pq_branch')
 143                                        else DEFAULT_PQ_BRANCH_NAME)
 144     m = pq_branch_match(pq_branch, pq_format_str)
 145     if m:
 146         if 'branch' in m.groupdict():
 147             return m.group('branch')
 148         return options.packaging_branch
 149
 150
 151 def parse_gbp_commands(info, cmd_tag, noarg_cmds, arg_cmds):
 152     """Parse gbp commands from commit message"""
 153     cmd_re = re.compile(r'^%s:\s*(?P<cmd>[a-z-]+)(\s+(?P<args>\S.*))?' %
 154                             cmd_tag, flags=re.I)
 155     commands = {}
 156     other_lines = []
 157     for line in info['body'].splitlines():
 158         match = re.match(cmd_re, line)
 159         if match:
 160             cmd = match.group('cmd').lower()
 161             if arg_cmds and cmd in arg_cmds:
 162                 if match.group('args'):
 163                     commands[cmd] = match.group('args')
 164                 else:
 165                     gbp.log.warn("Ignoring gbp-command '%s' in commit %s: "
 166                                  "missing cmd arguments" % (line, info['id']))
 167             elif noarg_cmds and cmd in noarg_cmds:
 168                 commands[cmd] = match.group('args')
 169             else:
 170                 gbp.log.warn("Ignoring unknown gbp-command '%s' in commit %s"
 171                                 % (line, info['id']))
 172         else:
 173             other_lines.append(line)
 174     return commands, other_lines
 175
 176
 177 def patch_path_filter(file_status, exclude_regex=None):
 178     """
 179     Create patch include paths, i.e. a "negation" of the exclude paths.
 180     """
 181     if exclude_regex:
 182         include_paths = []
 183         for file_list in list(file_status.values()):
 184             for fname in file_list:
 185                 if not re.match(exclude_regex, fname):
 186                     include_paths.append(fname)
 187     else:
 188         include_paths = ['.']
 189
 190     return include_paths
 191
 192
 193 def write_patch_file(filename, commit_info, diff):
 194     """Write patch file"""
 195     if not diff:
 196         gbp.log.debug("I won't generate empty diff %s" % filename)
 197         return None
 198     try:
 199         with open(filename, 'w') as patch:
 200             msg = Message()
 201             charset = Charset('utf-8')
 202             charset.body_encoding = None
 203             charset.header_encoding = QP
 204
 205             # Write headers
 206             name = commit_info['author']['name']
 207             email = commit_info['author']['email']
 208             # Git compat: put name in quotes if special characters found
 209             if re.search("[,.@()\[\]\\\:;]", name):
 210                 name = '"%s"' % name
 211             from_header = Header(str(name, 'utf-8'), charset, 77, 'from')
 212             from_header.append(str('<%s>' % email))
 213             msg['From'] = from_header
 214             date = commit_info['author'].datetime
 215             datestr = date.strftime('%a, %-d %b %Y %H:%M:%S %z')
 216             msg['Date'] = Header(str(datestr, 'utf-8'), charset, 77, 'date')
 217             msg['Subject'] = Header(str(commit_info['subject'], 'utf-8'),
 218                                     charset, 77, 'subject')
 219             # Write message body
 220             if commit_info['body']:
 221                 # Strip extra linefeeds
 222                 body = commit_info['body'].rstrip() + '\n'
 223                 try:
 224                     msg.set_payload(body.encode('ascii'))
 225                 except UnicodeDecodeError:
 226                     msg.set_payload(body, charset)
 227             patch.write(msg.as_string(unixfrom=False))
 228
 229             # Write diff
 230             patch.write('---\n')
 231             patch.write(diff)
 232     except IOError as err:
 233         raise GbpError('Unable to create patch file: %s' % err)
 234     return filename
 235
 236
 237 def format_patch(outdir, repo, commit_info, series, numbered=True,
 238                  path_exclude_regex=None, topic=''):
 239     """Create patch of a single commit"""
 240
 241     # Determine filename and path
 242     outdir = os.path.join(outdir, topic)
 243     if not os.path.exists(outdir):
 244         os.makedirs(outdir)
 245     num_prefix = '%04d-' % (len(series) + 1)
 246     suffix = '.patch'
 247     base_maxlen = 63 - len(num_prefix) - len(suffix)
 248     base = commit_info['patchname'][:base_maxlen]
 249     filename = (num_prefix if numbered else '') + base + suffix
 250     filepath = os.path.join(outdir, filename)
 251     # Make sure that we don't overwrite existing patches in the series
 252     if filepath in series:
 253         presuffix = '-%d' % len(series)
 254         base = base[:base_maxlen-len(presuffix)] + presuffix
 255         filename = (num_prefix if numbered else '') + base + suffix
 256         filepath = os.path.join(outdir, filename)
 257
 258     # Determine files to include
 259     paths = patch_path_filter(commit_info['files'], path_exclude_regex)
 260
 261     # Finally, create the patch
 262     patch = None
 263     if paths:
 264         diff = repo.diff('%s^!' % commit_info['id'], paths=paths, stat=80,
 265                          summary=True, text=True)
 266         patch = write_patch_file(filepath, commit_info, diff)
 267         if patch:
 268             series.append(patch)
 269     return patch
 270
 271
 272 def format_diff(outdir, filename, repo, start, end, path_exclude_regex=None):
 273     """Create a patch of diff between two repository objects"""
 274
 275     info = {'author': get_author(repo)}
 276     now = datetime.datetime.now().replace(tzinfo=GitTz(-time.timezone))
 277     info['author'].set_date(now)
 278     info['subject'] = "Raw diff %s..%s" % (start, end)
 279     info['body'] = ("Raw diff between %s '%s' and\n%s '%s'\n" %
 280                     (repo.get_obj_type(start), start,
 281                     repo.get_obj_type(end), end))
 282     if not filename:
 283         filename = '%s-to-%s.diff' % (start, end)
 284     filename = os.path.join(outdir, filename)
 285
 286     file_status = repo.diff_status(start, end)
 287     paths = patch_path_filter(file_status, path_exclude_regex)
 288     if paths:
 289         diff = repo.diff(start, end, paths=paths, stat=80, summary=True,
 290                          text=True)
 291         return write_patch_file(filename, info, diff)
 292     return None
 293
 294
 295 def get_author(repo):
 296     """Determine author name and email"""
 297     author = GitModifier()
 298     if repo:
 299         author = repo.get_author_info()
 300
 301     passwd_data = pwd.getpwuid(os.getuid())
 302     if not author.name:
 303         # On some distros (Ubuntu, at least) the gecos field has it's own
 304         # internal structure of comma-separated fields
 305         author.name = passwd_data.pw_gecos.split(',')[0].strip()
 306         if not author.name:
 307             author.name = passwd_data.pw_name
 308     if not author.email:
 309         if 'EMAIL' in os.environ:
 310             author.email = os.environ['EMAIL']
 311         else:
 312             author.email = "%s@%s" % (passwd_data.pw_name, socket.getfqdn())
 313
 314     return author
 315
 316
 317 def get_maintainer_from_control(repo):
 318     """Get the maintainer from the control file"""
 319     control = os.path.join(repo.path, 'debian', 'control')
 320
 321     cmd = 'sed -n -e \"s/Maintainer: \\+\\(.*\\)/\\1/p\" %s' % control
 322     cmdout = subprocess.Popen(cmd, shell=True,
 323                               stdout=subprocess.PIPE).stdout.readlines()
 324
 325     if len(cmdout) > 0:
 326         maintainer = cmdout[0].strip()
 327         m = re.match('(?P<name>.*[^ ]) *<(?P<email>.*)>', maintainer)
 328         if m:
 329             return GitModifier(m.group('name'), m.group('email'))
 330
 331     return GitModifier()
 332
 333
 334 def switch_to_pq_branch(repo, branch, options, name_keys=None):
 335     """
 336     Switch to patch-queue branch if not already there, create it if it
 337     doesn't exist yet
 338     """
 339     if is_pq_branch(branch, options):
 340         return
 341
 342     pq_branch = pq_branch_name(branch, options, name_keys)
 343     if not repo.has_branch(pq_branch):
 344         try:
 345             repo.create_branch(pq_branch)
 346         except GitRepositoryError:
 347             raise GbpError("Cannot create patch-queue branch '%s'. "
 348                            "Try 'rebase' instead." % pq_branch)
 349
 350     gbp.log.info("Switching to '%s'" % pq_branch)
 351     repo.set_branch(pq_branch)
 352
 353
 354 def apply_single_patch(repo, branch, patch, fallback_author, options):
 355     switch_to_pq_branch(repo, branch, options)
 356     topic = None if not hasattr(options, 'topic') else options.topic
 357     apply_and_commit_patch(repo, patch, fallback_author, topic)
 358
 359
 360 def apply_and_commit_patch(repo, patch, fallback_author, topic=None):
 361     """apply a single patch 'patch', add topic 'topic' and commit it"""
 362     author = {'name': patch.author,
 363               'email': patch.email,
 364               'date': patch.date }
 365
 366     patch_fn = os.path.basename(patch.path)
 367     if not (author['name'] and author['email']):
 368         if fallback_author and fallback_author['name']:
 369             author = fallback_author
 370             gbp.log.warn("Patch '%s' has no authorship information, using "
 371                          "'%s <%s>'" % (patch_fn, author['name'],
 372                                         author['email']))
 373         else:
 374             gbp.log.warn("Patch '%s' has no authorship information" % patch_fn)
 375
 376     repo.apply_patch(patch.path, strip=patch.strip)
 377     tree = repo.write_tree()
 378     msg = "%s\n\n%s" % (patch.subject, patch.long_desc)
 379     if topic:
 380         msg += "\nGbp-Pq-Topic: %s" % topic
 381     commit = repo.commit_tree(tree, msg, [repo.head], author=author)
 382     repo.update_ref('HEAD', commit, msg="gbp-pq import %s" % patch.path)
 383
 384
 385 def drop_pq(repo, branch, options, name_keys=None):
 386     if is_pq_branch(branch, options):
 387         gbp.log.err("On a patch-queue branch, can't drop it.")
 388         raise GbpError
 389     else:
 390         pq_branch = pq_branch_name(branch, options, name_keys)
 391
 392     if repo.has_branch(pq_branch):
 393         repo.delete_branch(pq_branch)
 394         gbp.log.info("Dropped branch '%s'." % pq_branch)
 395     else:
 396         gbp.log.info("No patch queue branch found - doing nothing.")