1 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright 2020 Google LLC
5 """Talks to the patchwork service to figure out what patches have been reviewed
6 and commented on. Provides a way to display review tags and comments.
7 Allows creation of a new branch based on the old but with the review tags
8 collected from patchwork.
12 import concurrent.futures
13 from itertools import repeat
19 from patman import patchstream
20 from patman.patchstream import PatchStream
21 from patman import terminal
22 from patman import tout
24 # Patches which are part of a multi-patch series are shown with a prefix like
25 # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
26 # part is optional. This decodes the string into groups. For single patches
27 # the [] part is not present:
28 # Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
29 RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
31 # This decodes the sequence string into a patch number and patch count
32 RE_SEQ = re.compile(r'(\d+)/(\d+)')
35 """Convert a list of strings into integers, using 0 if not an integer
38 vals (list): List of strings
41 list: List of integers, one for each input string
43 out = [int(val) if val.isdigit() else 0 for val in vals]
48 """Models a patch in patchwork
50 This class records information obtained from patchwork
52 Some of this information comes from the 'Patch' column:
54 [RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
56 This shows the prefix, version, seq, count and subject.
58 The other properties come from other columns in the display.
61 pid (str): ID of the patch (typically an integer)
62 seq (int): Sequence number within series (1=first) parsed from sequence
64 count (int): Number of patches in series, parsed from sequence string
65 raw_subject (str): Entire subject line, e.g.
66 "[1/2,v2] efi_loader: Sort header file ordering"
67 prefix (str): Prefix string or None (e.g. 'RFC')
68 version (str): Version string or None (e.g. 'v2')
69 raw_subject (str): Raw patch subject
70 subject (str): Patch subject with [..] part removed (same as commit
73 def __init__(self, pid):
75 self.id = pid # Use 'id' to match what the Rest API provides
80 self.raw_subject = None
83 # These make us more like a dictionary
84 def __setattr__(self, name, value):
87 def __getattr__(self, name):
91 return hash(frozenset(self.items()))
94 return self.raw_subject
96 def parse_subject(self, raw_subject):
97 """Parse the subject of a patch into its component parts
99 See RE_PATCH for details. The parsed info is placed into seq, count,
100 prefix, version, subject
103 raw_subject (str): Subject string to parse
106 ValueError: the subject cannot be parsed
108 self.raw_subject = raw_subject.strip()
109 mat = RE_PATCH.search(raw_subject.strip())
111 raise ValueError("Cannot parse subject '%s'" % raw_subject)
112 self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
113 mat_seq = RE_SEQ.match(seq_info) if seq_info else False
115 self.version = seq_info
117 if self.version and not self.version.startswith('v'):
118 self.prefix = self.version
122 self.seq = int(mat_seq.group(1))
123 self.count = int(mat_seq.group(2))
130 """Represents a single review email collected in Patchwork
132 Patches can attract multiple reviews. Each consists of an author/date and
133 a variable number of 'snippets', which are groups of quoted and unquoted
136 def __init__(self, meta, snippets):
137 """Create new Review object
140 meta (str): Text containing review author and date
141 snippets (list): List of snippets in th review, each a list of text
144 self.meta = ' : '.join([line for line in meta.splitlines() if line])
145 self.snippets = snippets
147 def compare_with_series(series, patches):
148 """Compare a list of patches with a series it came from
150 This prints any problems as warnings
153 series (Series): Series to compare against
154 patches (:type: list of Patch): list of Patch objects to compare with
159 key: Commit number (0...n-1)
160 value: Patch object for that commit
162 key: Patch number (0...n-1)
163 value: Commit object for that patch
165 # Check the names match
167 patch_for_commit = {}
168 all_patches = set(patches)
169 for seq, cmt in enumerate(series.commits):
170 pmatch = [p for p in all_patches if p.subject == cmt.subject]
172 patch_for_commit[seq] = pmatch[0]
173 all_patches.remove(pmatch[0])
174 elif len(pmatch) > 1:
175 warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
176 (seq + 1, cmt.subject,
177 '\n '.join([p.subject for p in pmatch])))
179 warnings.append("Cannot find patch for commit %d ('%s')" %
180 (seq + 1, cmt.subject))
183 # Check the names match
184 commit_for_patch = {}
185 all_commits = set(series.commits)
186 for seq, patch in enumerate(patches):
187 cmatch = [c for c in all_commits if c.subject == patch.subject]
189 commit_for_patch[seq] = cmatch[0]
190 all_commits.remove(cmatch[0])
191 elif len(cmatch) > 1:
192 warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
193 (seq + 1, patch.subject,
194 '\n '.join([c.subject for c in cmatch])))
196 warnings.append("Cannot find commit for patch %d ('%s')" %
197 (seq + 1, patch.subject))
199 return patch_for_commit, commit_for_patch, warnings
201 def call_rest_api(url, subpath):
202 """Call the patchwork API and return the result as JSON
205 url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
206 subpath (str): URL subpath to use
212 ValueError: the URL could not be read
214 full_url = '%s/api/1.2/%s' % (url, subpath)
215 response = requests.get(full_url)
216 if response.status_code != 200:
217 raise ValueError("Could not read URL '%s'" % full_url)
218 return response.json()
220 def collect_patches(series, series_id, url, rest_api=call_rest_api):
221 """Collect patch information about a series from patchwork
223 Uses the Patchwork REST API to collect information provided by patchwork
224 about the status of each patch.
227 series (Series): Series object corresponding to the local branch
228 containing the series
229 series_id (str): Patch series ID number
230 url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
231 rest_api (function): API function to call to access Patchwork, for
235 list: List of patches sorted by sequence number, each a Patch object
238 ValueError: if the URL could not be read or the web page does not follow
239 the expected structure
241 data = rest_api(url, 'series/%s/' % series_id)
243 # Get all the rows, which are patches
244 patch_dict = data['patches']
245 count = len(patch_dict)
246 num_commits = len(series.commits)
247 if count != num_commits:
248 tout.warning('Warning: Patchwork reports %d patches, series has %d' %
249 (count, num_commits))
253 # Work through each row (patch) one at a time, collecting the information
255 for pw_patch in patch_dict:
256 patch = Patch(pw_patch['id'])
257 patch.parse_subject(pw_patch['name'])
258 patches.append(patch)
260 tout.warning(' (total of %d warnings)' % warn_count)
262 # Sort patches by patch number
263 patches = sorted(patches, key=lambda x: x.seq)
266 def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, url,
267 rest_api=call_rest_api):
268 """Find new rtags collected by patchwork that we don't know about
270 This is designed to be run in parallel, once for each commit/patch
273 new_rtag_list (list): New rtags are written to new_rtag_list[seq]
275 key: Response tag (e.g. 'Reviewed-by')
276 value: Set of people who gave that response, each a name/email
278 review_list (list): New reviews are written to review_list[seq]
280 List of reviews for the patch, each a Review
281 seq (int): Position in new_rtag_list to update
282 cmt (Commit): Commit object for this commit
283 patch (Patch): Corresponding Patch object for this patch
284 url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
285 rest_api (function): API function to call to access Patchwork, for
291 # Get the content for the patch email itself as well as all comments
292 data = rest_api(url, 'patches/%s/' % patch.id)
293 pstrm = PatchStream.process_text(data['content'], True)
295 rtags = collections.defaultdict(set)
296 for response, people in pstrm.commit.rtags.items():
297 rtags[response].update(people)
299 data = rest_api(url, 'patches/%s/comments/' % patch.id)
303 pstrm = PatchStream.process_text(comment['content'], True)
305 submitter = comment['submitter']
306 person = '%s <%s>' % (submitter['name'], submitter['email'])
307 reviews.append(Review(person, pstrm.snippets))
308 for response, people in pstrm.commit.rtags.items():
309 rtags[response].update(people)
311 # Find the tags that are not in the commit
312 new_rtags = collections.defaultdict(set)
313 base_rtags = cmt.rtags
314 for tag, people in rtags.items():
316 is_new = (tag not in base_rtags or
317 who not in base_rtags[tag])
319 new_rtags[tag].add(who)
320 new_rtag_list[seq] = new_rtags
321 review_list[seq] = reviews
323 def show_responses(rtags, indent, is_new):
324 """Show rtags collected
327 rtags (dict): review tags to show
328 key: Response tag (e.g. 'Reviewed-by')
329 value: Set of people who gave that response, each a name/email string
330 indent (str): Indentation string to write before each line
331 is_new (bool): True if this output should be highlighted
334 int: Number of review tags displayed
336 col = terminal.Color()
338 for tag in sorted(rtags.keys()):
340 for who in sorted(people):
341 terminal.Print(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
342 newline=False, colour=col.GREEN, bright=is_new)
343 terminal.Print(who, colour=col.WHITE, bright=is_new)
347 def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
349 """Create a new branch with review tags added
352 series (Series): Series object for the existing branch
353 new_rtag_list (list): List of review tags to add, one for each commit,
355 key: Response tag (e.g. 'Reviewed-by')
356 value: Set of people who gave that response, each a name/email
358 branch (str): Existing branch to update
359 dest_branch (str): Name of new branch to create
360 overwrite (bool): True to force overwriting dest_branch if it exists
361 repo (pygit2.Repository): Repo to use (use None unless testing)
364 int: Total number of review tags added across all commits
367 ValueError: if the destination branch name is the same as the original
368 branch, or it already exists and @overwrite is False
370 if branch == dest_branch:
372 'Destination branch must not be the same as the original branch')
374 repo = pygit2.Repository('.')
375 count = len(series.commits)
376 new_br = repo.branches.get(dest_branch)
379 raise ValueError("Branch '%s' already exists (-f to overwrite)" %
384 target = repo.revparse_single('%s~%d' % (branch, count))
385 repo.branches.local.create(dest_branch, target)
388 for seq in range(count):
389 parent = repo.branches.get(dest_branch)
390 cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
392 repo.merge_base(cherry.oid, parent.target)
393 base_tree = cherry.parents[0].tree
395 index = repo.merge_trees(base_tree, parent, cherry)
396 tree_id = index.write_tree(repo)
399 if new_rtag_list[seq]:
400 for tag, people in new_rtag_list[seq].items():
402 lines.append('%s: %s' % (tag, who))
404 message = patchstream.insert_tags(cherry.message.rstrip(),
408 parent.name, cherry.author, cherry.committer, message, tree_id,
412 def check_patchwork_status(series, series_id, branch, dest_branch, force,
413 show_comments, url, rest_api=call_rest_api,
415 """Check the status of a series on Patchwork
417 This finds review tags and comments for a series in Patchwork, displaying
418 them to show what is new compared to the local series.
421 series (Series): Series object for the existing branch
422 series_id (str): Patch series ID number
423 branch (str): Existing branch to update, or None
424 dest_branch (str): Name of new branch to create, or None
425 force (bool): True to force overwriting dest_branch if it exists
426 show_comments (bool): True to show the comments on each patch
427 url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
428 rest_api (function): API function to call to access Patchwork, for
430 test_repo (pygit2.Repository): Repo to use (use None unless testing)
432 patches = collect_patches(series, series_id, url, rest_api)
433 col = terminal.Color()
434 count = len(series.commits)
435 new_rtag_list = [None] * count
436 review_list = [None] * count
438 patch_for_commit, _, warnings = compare_with_series(series, patches)
439 for warn in warnings:
442 patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
444 with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
445 futures = executor.map(
446 find_new_responses, repeat(new_rtag_list), repeat(review_list),
447 range(count), series.commits, patch_list, repeat(url),
449 for fresponse in futures:
451 raise fresponse.exception()
454 for seq, cmt in enumerate(series.commits):
455 patch = patch_for_commit.get(seq)
458 terminal.Print('%3d %s' % (patch.seq, patch.subject[:50]),
460 cmt = series.commits[seq]
461 base_rtags = cmt.rtags
462 new_rtags = new_rtag_list[seq]
465 show_responses(base_rtags, indent, False)
466 num_to_add += show_responses(new_rtags, indent, True)
468 for review in review_list[seq]:
469 terminal.Print('Review: %s' % review.meta, colour=col.RED)
470 for snippet in review.snippets:
472 quoted = line.startswith('>')
473 terminal.Print(' %s' % line,
474 colour=col.MAGENTA if quoted else None)
477 terminal.Print("%d new response%s available in patchwork%s" %
478 (num_to_add, 's' if num_to_add != 1 else '',
480 else ' (use -d to write them to a new branch)'))
483 num_added = create_branch(series, new_rtag_list, branch,
484 dest_branch, force, test_repo)
486 "%d response%s added from patchwork into new branch '%s'" %
487 (num_added, 's' if num_added != 1 else '', dest_branch))