1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2011 The Chromium OS Authors.
12 from patman import command
13 from patman import commit
14 from patman import gitutil
15 from patman.series import Series
17 # Tags that we detect and remove
18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
19 '|Reviewed-on:|Commit-\w*:')
21 # Lines which are allowed after a TEST= line
22 re_allowed_after_test = re.compile('^Signed-off-by:')
25 re_signoff = re.compile('^Signed-off-by: *(.*)')
28 re_cover = re.compile('^Cover-([a-z-]*): *(.*)')
31 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
33 # Change-Id will be used to generate the Message-Id and then be stripped
34 re_change_id = re.compile('^Change-Id: *(.*)')
37 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
39 # Commit tags that we want to collect and keep
40 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
42 # The start of a new commit in the git log
43 re_commit = re.compile('^commit ([0-9a-f]*)$')
45 # We detect these since checkpatch doesn't always do it
46 re_space_before_tab = re.compile('^[+].* \t')
48 # Match indented lines for changes
49 re_leading_whitespace = re.compile('^\s')
51 # States we can be in - can we use range() and still have comments?
52 STATE_MSG_HEADER = 0 # Still in the message header
53 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
54 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
55 STATE_DIFFS = 3 # In the diff part (past --- line)
58 """Class for detecting/injecting tags in a patch or series of patches
60 We support processing the output of 'git log' to read out the tags we
61 are interested in. We can also process a patch file in order to remove
62 unwanted tags or inject additional ones. These correspond to the two
65 def __init__(self, series, name=None, is_log=False):
66 self.skip_blank = False # True to skip a single blank line
67 self.found_test = False # Found a TEST= line
68 self.lines_after_test = 0 # Number of lines found after TEST=
69 self.warn = [] # List of warnings we have collected
70 self.linenum = 1 # Output line number we are up to
71 self.in_section = None # Name of start...END section we are in
72 self.notes = [] # Series notes
73 self.section = [] # The current section...END section
74 self.series = series # Info about the patch series
75 self.is_log = is_log # True if indent like git log
76 self.in_change = None # Name of the change list we are in
77 self.change_version = 0 # Non-zero if we are in a change list
78 self.change_lines = [] # Lines of the current change
79 self.blank_count = 0 # Number of blank lines stored up
80 self.state = STATE_MSG_HEADER # What state are we in?
81 self.signoff = [] # Contents of signoff line
82 self.commit = None # Current commit
84 def AddToSeries(self, line, name, value):
85 """Add a new Series-xxx tag.
87 When a Series-xxx tag is detected, we come here to record it, if we
88 are scanning a 'git log'.
91 line: Source line containing tag (useful for debug/error messages)
92 name: Tag name (part after 'Series-')
93 value: Tag value (part after 'Series-xxx: ')
96 self.in_section = name
97 self.skip_blank = False
99 self.series.AddTag(self.commit, line, name, value)
101 def AddToCommit(self, line, name, value):
102 """Add a new Commit-xxx tag.
104 When a Commit-xxx tag is detected, we come here to record it.
107 line: Source line containing tag (useful for debug/error messages)
108 name: Tag name (part after 'Commit-')
109 value: Tag value (part after 'Commit-xxx: ')
112 self.in_section = 'commit-' + name
113 self.skip_blank = False
115 def AddCommitRtag(self, rtag_type, who):
116 """Add a response tag to the current commit
119 key: rtag type (e.g. 'Reviewed-by')
120 who: Person who gave that rtag, e.g. 'Fred Bloggs <fred@bloggs.org>'
122 self.commit.AddRtag(rtag_type, who)
124 def CloseCommit(self):
125 """Save the current commit into our commit list, and reset our state"""
126 if self.commit and self.is_log:
127 self.series.AddCommit(self.commit)
129 # If 'END' is missing in a 'Cover-letter' section, and that section
130 # happens to show up at the very end of the commit message, this is
131 # the chance for us to fix it up.
132 if self.in_section == 'cover' and self.is_log:
133 self.series.cover = self.section
134 self.in_section = None
135 self.skip_blank = True
138 def ParseVersion(self, value, line):
139 """Parse a version from a *-changes tag
142 value: Tag value (part after 'xxx-changes: '
143 line: Source line containing tag
146 The version as an integer
150 except ValueError as str:
151 raise ValueError("%s: Cannot decode version info '%s'" %
152 (self.commit.hash, line))
154 def FinalizeChange(self):
155 """Finalize a (multi-line) change and add it to the series or commit"""
156 if not self.change_lines:
158 change = '\n'.join(self.change_lines)
160 if self.in_change == 'Series':
161 self.series.AddChange(self.change_version, self.commit, change)
162 elif self.in_change == 'Cover':
163 self.series.AddChange(self.change_version, None, change)
164 elif self.in_change == 'Commit':
165 self.commit.AddChange(self.change_version, change)
166 self.change_lines = []
168 def ProcessLine(self, line):
169 """Process a single line of a patch file or commit log
171 This process a line and returns a list of lines to output. The list
172 may be empty or may contain multiple output lines.
174 This is where all the complicated logic is located. The class's
175 state is used to move between different states and detect things
178 We can be in one of two modes:
179 self.is_log == True: This is 'git log' mode, where most output is
180 indented by 4 characters and we are scanning for tags
182 self.is_log == False: This is 'patch' mode, where we already have
183 all the tags, and are processing patches to remove junk we
184 don't want, and add things we think are required.
187 line: text line to process
190 list of output lines, or [] if nothing should be output
192 # Initially we have no output. Prepare the input line string
194 line = line.rstrip('\n')
196 commit_match = re_commit.match(line) if self.is_log else None
202 # Handle state transition and skipping blank lines
203 series_tag_match = re_series_tag.match(line)
204 change_id_match = re_change_id.match(line)
205 commit_tag_match = re_commit_tag.match(line)
206 cover_match = re_cover.match(line)
207 signoff_match = re_signoff.match(line)
208 leading_whitespace_match = re_leading_whitespace.match(line)
210 if self.state == STATE_PATCH_HEADER:
211 tag_match = re_tag.match(line)
212 is_blank = not line.strip()
214 if (self.state == STATE_MSG_HEADER
215 or self.state == STATE_PATCH_SUBJECT):
218 # We don't have a subject in the text stream of patch files
219 # It has its own line with a Subject: tag
220 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
223 self.state = STATE_MSG_HEADER
225 # If a tag is detected, or a new commit starts
226 if series_tag_match or commit_tag_match or change_id_match or \
227 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
228 # but we are already in a section, this means 'END' is missing
229 # for that section, fix it up.
231 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
232 if self.in_section == 'cover':
233 self.series.cover = self.section
234 elif self.in_section == 'notes':
236 self.series.notes += self.section
237 elif self.in_section == 'commit-notes':
239 self.commit.notes += self.section
241 self.warn.append("Unknown section '%s'" % self.in_section)
242 self.in_section = None
243 self.skip_blank = True
245 # but we are already in a change list, that means a blank line
246 # is missing, fix it up.
248 self.warn.append("Missing 'blank line' in section '%s-changes'" % self.in_change)
249 self.FinalizeChange()
250 self.in_change = None
251 self.change_version = 0
253 # If we are in a section, keep collecting lines until we see END
256 if self.in_section == 'cover':
257 self.series.cover = self.section
258 elif self.in_section == 'notes':
260 self.series.notes += self.section
261 elif self.in_section == 'commit-notes':
263 self.commit.notes += self.section
265 self.warn.append("Unknown section '%s'" % self.in_section)
266 self.in_section = None
267 self.skip_blank = True
270 self.section.append(line)
272 # If we are not in a section, it is an unexpected END
274 raise ValueError("'END' wihout section")
276 # Detect the commit subject
277 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
278 self.commit.subject = line
280 # Detect the tags we want to remove, and skip blank lines
281 elif re_remove.match(line) and not commit_tag_match:
282 self.skip_blank = True
284 # TEST= should be the last thing in the commit, so remove
285 # everything after it
286 if line.startswith('TEST='):
287 self.found_test = True
288 elif self.skip_blank and is_blank:
289 self.skip_blank = False
291 # Detect Cover-xxx tags
293 name = cover_match.group(1)
294 value = cover_match.group(2)
296 self.in_section = 'cover'
297 self.skip_blank = False
298 elif name == 'letter-cc':
299 self.AddToSeries(line, 'cover-cc', value)
300 elif name == 'changes':
301 self.in_change = 'Cover'
302 self.change_version = self.ParseVersion(value, line)
304 # If we are in a change list, key collected lines until a blank one
307 # Blank line ends this change list
308 self.FinalizeChange()
309 self.in_change = None
310 self.change_version = 0
312 self.FinalizeChange()
313 self.in_change = None
314 self.change_version = 0
315 out = self.ProcessLine(line)
317 if not leading_whitespace_match:
318 self.FinalizeChange()
319 self.change_lines.append(line)
320 self.skip_blank = False
322 # Detect Series-xxx tags
323 elif series_tag_match:
324 name = series_tag_match.group(1)
325 value = series_tag_match.group(2)
326 if name == 'changes':
327 # value is the version number: e.g. 1, or 2
328 self.in_change = 'Series'
329 self.change_version = self.ParseVersion(value, line)
331 self.AddToSeries(line, name, value)
332 self.skip_blank = True
334 # Detect Change-Id tags
335 elif change_id_match:
336 value = change_id_match.group(1)
338 if self.commit.change_id:
339 raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
340 (self.commit.hash, self.commit.change_id, value))
341 self.commit.change_id = value
342 self.skip_blank = True
344 # Detect Commit-xxx tags
345 elif commit_tag_match:
346 name = commit_tag_match.group(1)
347 value = commit_tag_match.group(2)
349 self.AddToCommit(line, name, value)
350 self.skip_blank = True
351 elif name == 'changes':
352 self.in_change = 'Commit'
353 self.change_version = self.ParseVersion(value, line)
355 self.warn.append('Line %d: Ignoring Commit-%s' %
356 (self.linenum, name))
358 # Detect the start of a new commit
361 self.commit = commit.Commit(commit_match.group(1))
363 # Detect tags in the commit message
365 rtag_type, who = tag_match.groups()
366 self.AddCommitRtag(rtag_type, who)
367 # Remove Tested-by self, since few will take much notice
368 if (rtag_type == 'Tested-by' and
369 who.find(os.getenv('USER') + '@') != -1):
370 self.warn.append("Ignoring %s" % line)
371 elif rtag_type == 'Patch-cc':
372 self.commit.AddCc(who.split(','))
376 # Suppress duplicate signoffs
378 if (self.is_log or not self.commit or
379 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
382 # Well that means this is an ordinary line
384 # Look for space before tab
385 m = re_space_before_tab.match(line)
387 self.warn.append('Line %d/%d has space before tab' %
388 (self.linenum, m.start()))
390 # OK, we have a valid non-blank line
393 self.skip_blank = False
394 if self.state == STATE_DIFFS:
397 # If this is the start of the diffs section, emit our tags and
400 self.state = STATE_DIFFS
402 # Output the tags (signoff first), then change list
404 log = self.series.MakeChangeLog(self.commit)
407 out += self.commit.notes
409 elif self.found_test:
410 if not re_allowed_after_test.match(line):
411 self.lines_after_test += 1
416 """Close out processing of this patch stream"""
417 self.FinalizeChange()
419 if self.lines_after_test:
420 self.warn.append('Found %d lines after TEST=' %
421 self.lines_after_test)
423 def WriteMessageId(self, outfd):
424 """Write the Message-Id into the output.
426 This is based on the Change-Id in the original patch, the version,
430 outfd: Output stream file object
432 if not self.commit.change_id:
435 # If the count is -1 we're testing, so use a fixed time
436 if self.commit.count == -1:
437 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
439 time_now = datetime.datetime.now()
441 # In theory there is email.utils.make_msgid() which would be nice
442 # to use, but it already produces something way too long and thus
443 # will produce ugly commit lines if someone throws this into
444 # a "Link:" tag in the final commit. So (sigh) roll our own.
446 # Start with the time; presumably we wouldn't send the same series
447 # with the same Change-Id at the exact same second.
448 parts = [time_now.strftime("%Y%m%d%H%M%S")]
450 # These seem like they would be nice to include.
451 if 'prefix' in self.series:
452 parts.append(self.series['prefix'])
453 if 'version' in self.series:
454 parts.append("v%s" % self.series['version'])
456 parts.append(str(self.commit.count + 1))
458 # The Change-Id must be last, right before the @
459 parts.append(self.commit.change_id)
461 # Join parts together with "." and write it out.
462 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
464 def ProcessStream(self, infd, outfd):
465 """Copy a stream from infd to outfd, filtering out unwanting things.
467 This is used to process patch files one at a time.
470 infd: Input stream file object
471 outfd: Output stream file object
473 # Extract the filename from each diff, for nice warnings
476 re_fname = re.compile('diff --git a/(.*) b/.*')
478 self.WriteMessageId(outfd)
481 line = infd.readline()
484 out = self.ProcessLine(line)
486 # Try to detect blank lines at EOF
488 match = re_fname.match(line)
491 fname = match.group(1)
493 self.blank_count += 1
495 if self.blank_count and (line == '-- ' or match):
496 self.warn.append("Found possible blank line(s) at "
497 "end of file '%s'" % last_fname)
498 outfd.write('+\n' * self.blank_count)
499 outfd.write(line + '\n')
504 def GetMetaDataForList(commit_range, git_dir=None, count=None,
505 series = None, allow_overwrite=False):
506 """Reads out patch series metadata from the commits
508 This does a 'git log' on the relevant commits and pulls out the tags we
512 commit_range: Range of commits to count (e.g. 'HEAD..base')
513 git_dir: Path to git repositiory (None to use default)
514 count: Number of commits to list, or None for no limit
515 series: Series object to add information into. By default a new series
517 allow_overwrite: Allow tags to overwrite an existing tag
519 A Series object containing information about the commits.
523 series.allow_overwrite = allow_overwrite
524 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
526 stdout = command.RunPipe([params], capture=True).stdout
527 ps = PatchStream(series, is_log=True)
528 for line in stdout.splitlines():
533 def GetMetaData(branch, start, count):
534 """Reads out patch series metadata from the commits
536 This does a 'git log' on the relevant commits and pulls out the tags we
540 branch: Branch to use (None for current branch)
541 start: Commit to start from: 0=branch HEAD, 1=next one, etc.
542 count: Number of commits to list
544 return GetMetaDataForList('%s~%d' % (branch if branch else 'HEAD', start),
547 def GetMetaDataForTest(text):
548 """Process metadata from a file containing a git log. Used for tests
554 ps = PatchStream(series, is_log=True)
555 for line in text.splitlines():
560 def FixPatch(backup_dir, fname, series, commit):
561 """Fix up a patch file, by adding/removing as required.
563 We remove our tags from the patch file, insert changes lists, etc.
564 The patch file is processed in place, and overwritten.
566 A backup file is put into backup_dir (if not None).
569 fname: Filename to patch file to process
570 series: Series information about this patch set
571 commit: Commit object for this patch file
573 A list of errors, or [] if all ok.
575 handle, tmpname = tempfile.mkstemp()
576 outfd = os.fdopen(handle, 'w', encoding='utf-8')
577 infd = open(fname, 'r', encoding='utf-8')
578 ps = PatchStream(series)
580 ps.ProcessStream(infd, outfd)
584 # Create a backup file if required
586 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
587 shutil.move(tmpname, fname)
590 def FixPatches(series, fnames):
591 """Fix up a list of patches identified by filenames
593 The patch files are processed in place, and overwritten.
596 series: The series object
597 fnames: List of patch files to process
599 # Current workflow creates patches, so we shouldn't need a backup
600 backup_dir = None #tempfile.mkdtemp('clean-patch')
603 commit = series.commits[count]
606 result = FixPatch(backup_dir, fname, series, commit)
608 print('%d warnings for %s:' % (len(result), fname))
613 print('Cleaned %d patches' % count)
615 def InsertCoverLetter(fname, series, count):
616 """Inserts a cover letter with the required info into patch 0
619 fname: Input / output filename of the cover letter file
620 series: Series object
621 count: Number of patches in the series
623 fd = open(fname, 'r')
624 lines = fd.readlines()
627 fd = open(fname, 'w')
629 prefix = series.GetPatchPrefix()
631 if line.startswith('Subject:'):
632 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
633 zero_repeat = int(math.log10(count)) + 1
634 zero = '0' * zero_repeat
635 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
637 # Insert our cover letter
638 elif line.startswith('*** BLURB HERE ***'):
639 # First the blurb test
640 line = '\n'.join(text[1:]) + '\n'
641 if series.get('notes'):
642 line += '\n'.join(series.notes) + '\n'
644 # Now the change list
645 out = series.MakeChangeLog(None)
646 line += '\n' + '\n'.join(out)