1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2011 The Chromium OS Authors.
12 from patman import command
13 from patman import commit
14 from patman import gitutil
15 from patman.series import Series
17 # Tags that we detect and remove
18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
19 '|Reviewed-on:|Commit-\w*:')
21 # Lines which are allowed after a TEST= line
22 re_allowed_after_test = re.compile('^Signed-off-by:')
25 re_signoff = re.compile('^Signed-off-by: *(.*)')
28 re_cover = re.compile('^Cover-([a-z-]*): *(.*)')
31 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
33 # Change-Id will be used to generate the Message-Id and then be stripped
34 re_change_id = re.compile('^Change-Id: *(.*)')
37 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
39 # Commit tags that we want to collect and keep
40 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
42 # The start of a new commit in the git log
43 re_commit = re.compile('^commit ([0-9a-f]*)$')
45 # We detect these since checkpatch doesn't always do it
46 re_space_before_tab = re.compile('^[+].* \t')
48 # Match indented lines for changes
49 re_leading_whitespace = re.compile('^\s')
51 # States we can be in - can we use range() and still have comments?
52 STATE_MSG_HEADER = 0 # Still in the message header
53 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
54 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
55 STATE_DIFFS = 3 # In the diff part (past --- line)
58 """Class for detecting/injecting tags in a patch or series of patches
60 We support processing the output of 'git log' to read out the tags we
61 are interested in. We can also process a patch file in order to remove
62 unwanted tags or inject additional ones. These correspond to the two
65 def __init__(self, series, name=None, is_log=False):
66 self.skip_blank = False # True to skip a single blank line
67 self.found_test = False # Found a TEST= line
68 self.lines_after_test = 0 # Number of lines found after TEST=
69 self.warn = [] # List of warnings we have collected
70 self.linenum = 1 # Output line number we are up to
71 self.in_section = None # Name of start...END section we are in
72 self.notes = [] # Series notes
73 self.section = [] # The current section...END section
74 self.series = series # Info about the patch series
75 self.is_log = is_log # True if indent like git log
76 self.in_change = None # Name of the change list we are in
77 self.change_version = 0 # Non-zero if we are in a change list
78 self.change_lines = [] # Lines of the current change
79 self.blank_count = 0 # Number of blank lines stored up
80 self.state = STATE_MSG_HEADER # What state are we in?
81 self.signoff = [] # Contents of signoff line
82 self.commit = None # Current commit
84 def AddToSeries(self, line, name, value):
85 """Add a new Series-xxx tag.
87 When a Series-xxx tag is detected, we come here to record it, if we
88 are scanning a 'git log'.
91 line: Source line containing tag (useful for debug/error messages)
92 name: Tag name (part after 'Series-')
93 value: Tag value (part after 'Series-xxx: ')
96 self.in_section = name
97 self.skip_blank = False
99 self.series.AddTag(self.commit, line, name, value)
101 def AddToCommit(self, line, name, value):
102 """Add a new Commit-xxx tag.
104 When a Commit-xxx tag is detected, we come here to record it.
107 line: Source line containing tag (useful for debug/error messages)
108 name: Tag name (part after 'Commit-')
109 value: Tag value (part after 'Commit-xxx: ')
112 self.in_section = 'commit-' + name
113 self.skip_blank = False
115 def AddCommitRtag(self, rtag_type, who):
116 """Add a response tag to the current commit
119 key: rtag type (e.g. 'Reviewed-by')
120 who: Person who gave that rtag, e.g. 'Fred Bloggs <fred@bloggs.org>'
122 self.commit.AddRtag(rtag_type, who)
124 def CloseCommit(self):
125 """Save the current commit into our commit list, and reset our state"""
126 if self.commit and self.is_log:
127 self.series.AddCommit(self.commit)
129 # If 'END' is missing in a 'Cover-letter' section, and that section
130 # happens to show up at the very end of the commit message, this is
131 # the chance for us to fix it up.
132 if self.in_section == 'cover' and self.is_log:
133 self.series.cover = self.section
134 self.in_section = None
135 self.skip_blank = True
138 def ParseVersion(self, value, line):
139 """Parse a version from a *-changes tag
142 value: Tag value (part after 'xxx-changes: '
143 line: Source line containing tag
146 The version as an integer
150 except ValueError as str:
151 raise ValueError("%s: Cannot decode version info '%s'" %
152 (self.commit.hash, line))
154 def FinalizeChange(self):
155 """Finalize a (multi-line) change and add it to the series or commit"""
156 if not self.change_lines:
158 change = '\n'.join(self.change_lines)
160 if self.in_change == 'Series':
161 self.series.AddChange(self.change_version, self.commit, change)
162 elif self.in_change == 'Cover':
163 self.series.AddChange(self.change_version, None, change)
164 elif self.in_change == 'Commit':
165 self.commit.AddChange(self.change_version, change)
166 self.change_lines = []
168 def ProcessLine(self, line):
169 """Process a single line of a patch file or commit log
171 This process a line and returns a list of lines to output. The list
172 may be empty or may contain multiple output lines.
174 This is where all the complicated logic is located. The class's
175 state is used to move between different states and detect things
178 We can be in one of two modes:
179 self.is_log == True: This is 'git log' mode, where most output is
180 indented by 4 characters and we are scanning for tags
182 self.is_log == False: This is 'patch' mode, where we already have
183 all the tags, and are processing patches to remove junk we
184 don't want, and add things we think are required.
187 line: text line to process
190 list of output lines, or [] if nothing should be output
192 # Initially we have no output. Prepare the input line string
194 line = line.rstrip('\n')
196 commit_match = re_commit.match(line) if self.is_log else None
202 # Handle state transition and skipping blank lines
203 series_tag_match = re_series_tag.match(line)
204 change_id_match = re_change_id.match(line)
205 commit_tag_match = re_commit_tag.match(line)
206 cover_match = re_cover.match(line)
207 signoff_match = re_signoff.match(line)
208 leading_whitespace_match = re_leading_whitespace.match(line)
210 if self.state == STATE_PATCH_HEADER:
211 tag_match = re_tag.match(line)
212 is_blank = not line.strip()
214 if (self.state == STATE_MSG_HEADER
215 or self.state == STATE_PATCH_SUBJECT):
218 # We don't have a subject in the text stream of patch files
219 # It has its own line with a Subject: tag
220 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
223 self.state = STATE_MSG_HEADER
225 # If a tag is detected, or a new commit starts
226 if series_tag_match or commit_tag_match or change_id_match or \
227 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
228 # but we are already in a section, this means 'END' is missing
229 # for that section, fix it up.
231 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
232 if self.in_section == 'cover':
233 self.series.cover = self.section
234 elif self.in_section == 'notes':
236 self.series.notes += self.section
237 elif self.in_section == 'commit-notes':
239 self.commit.notes += self.section
241 self.warn.append("Unknown section '%s'" % self.in_section)
242 self.in_section = None
243 self.skip_blank = True
245 # but we are already in a change list, that means a blank line
246 # is missing, fix it up.
248 self.warn.append("Missing 'blank line' in section '%s-changes'" % self.in_change)
249 self.FinalizeChange()
250 self.in_change = None
251 self.change_version = 0
253 # If we are in a section, keep collecting lines until we see END
256 if self.in_section == 'cover':
257 self.series.cover = self.section
258 elif self.in_section == 'notes':
260 self.series.notes += self.section
261 elif self.in_section == 'commit-notes':
263 self.commit.notes += self.section
265 self.warn.append("Unknown section '%s'" % self.in_section)
266 self.in_section = None
267 self.skip_blank = True
270 self.section.append(line)
272 # Detect the commit subject
273 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
274 self.commit.subject = line
276 # Detect the tags we want to remove, and skip blank lines
277 elif re_remove.match(line) and not commit_tag_match:
278 self.skip_blank = True
280 # TEST= should be the last thing in the commit, so remove
281 # everything after it
282 if line.startswith('TEST='):
283 self.found_test = True
284 elif self.skip_blank and is_blank:
285 self.skip_blank = False
287 # Detect Cover-xxx tags
289 name = cover_match.group(1)
290 value = cover_match.group(2)
292 self.in_section = 'cover'
293 self.skip_blank = False
294 elif name == 'letter-cc':
295 self.AddToSeries(line, 'cover-cc', value)
296 elif name == 'changes':
297 self.in_change = 'Cover'
298 self.change_version = self.ParseVersion(value, line)
300 # If we are in a change list, key collected lines until a blank one
303 # Blank line ends this change list
304 self.FinalizeChange()
305 self.in_change = None
306 self.change_version = 0
308 self.FinalizeChange()
309 self.in_change = None
310 self.change_version = 0
311 out = self.ProcessLine(line)
313 if not leading_whitespace_match:
314 self.FinalizeChange()
315 self.change_lines.append(line)
316 self.skip_blank = False
318 # Detect Series-xxx tags
319 elif series_tag_match:
320 name = series_tag_match.group(1)
321 value = series_tag_match.group(2)
322 if name == 'changes':
323 # value is the version number: e.g. 1, or 2
324 self.in_change = 'Series'
325 self.change_version = self.ParseVersion(value, line)
327 self.AddToSeries(line, name, value)
328 self.skip_blank = True
330 # Detect Change-Id tags
331 elif change_id_match:
332 value = change_id_match.group(1)
334 if self.commit.change_id:
335 raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
336 (self.commit.hash, self.commit.change_id, value))
337 self.commit.change_id = value
338 self.skip_blank = True
340 # Detect Commit-xxx tags
341 elif commit_tag_match:
342 name = commit_tag_match.group(1)
343 value = commit_tag_match.group(2)
345 self.AddToCommit(line, name, value)
346 self.skip_blank = True
347 elif name == 'changes':
348 self.in_change = 'Commit'
349 self.change_version = self.ParseVersion(value, line)
351 # Detect the start of a new commit
354 self.commit = commit.Commit(commit_match.group(1))
356 # Detect tags in the commit message
358 rtag_type, who = tag_match.groups()
359 self.AddCommitRtag(rtag_type, who)
360 # Remove Tested-by self, since few will take much notice
361 if (rtag_type == 'Tested-by' and
362 who.find(os.getenv('USER') + '@') != -1):
363 self.warn.append("Ignoring %s" % line)
364 elif rtag_type == 'Patch-cc':
365 self.commit.AddCc(who.split(','))
369 # Suppress duplicate signoffs
371 if (self.is_log or not self.commit or
372 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
375 # Well that means this is an ordinary line
377 # Look for space before tab
378 m = re_space_before_tab.match(line)
380 self.warn.append('Line %d/%d has space before tab' %
381 (self.linenum, m.start()))
383 # OK, we have a valid non-blank line
386 self.skip_blank = False
387 if self.state == STATE_DIFFS:
390 # If this is the start of the diffs section, emit our tags and
393 self.state = STATE_DIFFS
395 # Output the tags (signoff first), then change list
397 log = self.series.MakeChangeLog(self.commit)
400 out += self.commit.notes
402 elif self.found_test:
403 if not re_allowed_after_test.match(line):
404 self.lines_after_test += 1
409 """Close out processing of this patch stream"""
410 self.FinalizeChange()
412 if self.lines_after_test:
413 self.warn.append('Found %d lines after TEST=' %
414 self.lines_after_test)
416 def WriteMessageId(self, outfd):
417 """Write the Message-Id into the output.
419 This is based on the Change-Id in the original patch, the version,
423 outfd: Output stream file object
425 if not self.commit.change_id:
428 # If the count is -1 we're testing, so use a fixed time
429 if self.commit.count == -1:
430 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
432 time_now = datetime.datetime.now()
434 # In theory there is email.utils.make_msgid() which would be nice
435 # to use, but it already produces something way too long and thus
436 # will produce ugly commit lines if someone throws this into
437 # a "Link:" tag in the final commit. So (sigh) roll our own.
439 # Start with the time; presumably we wouldn't send the same series
440 # with the same Change-Id at the exact same second.
441 parts = [time_now.strftime("%Y%m%d%H%M%S")]
443 # These seem like they would be nice to include.
444 if 'prefix' in self.series:
445 parts.append(self.series['prefix'])
446 if 'version' in self.series:
447 parts.append("v%s" % self.series['version'])
449 parts.append(str(self.commit.count + 1))
451 # The Change-Id must be last, right before the @
452 parts.append(self.commit.change_id)
454 # Join parts together with "." and write it out.
455 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
457 def ProcessStream(self, infd, outfd):
458 """Copy a stream from infd to outfd, filtering out unwanting things.
460 This is used to process patch files one at a time.
463 infd: Input stream file object
464 outfd: Output stream file object
466 # Extract the filename from each diff, for nice warnings
469 re_fname = re.compile('diff --git a/(.*) b/.*')
471 self.WriteMessageId(outfd)
474 line = infd.readline()
477 out = self.ProcessLine(line)
479 # Try to detect blank lines at EOF
481 match = re_fname.match(line)
484 fname = match.group(1)
486 self.blank_count += 1
488 if self.blank_count and (line == '-- ' or match):
489 self.warn.append("Found possible blank line(s) at "
490 "end of file '%s'" % last_fname)
491 outfd.write('+\n' * self.blank_count)
492 outfd.write(line + '\n')
497 def GetMetaDataForList(commit_range, git_dir=None, count=None,
498 series = None, allow_overwrite=False):
499 """Reads out patch series metadata from the commits
501 This does a 'git log' on the relevant commits and pulls out the tags we
505 commit_range: Range of commits to count (e.g. 'HEAD..base')
506 git_dir: Path to git repositiory (None to use default)
507 count: Number of commits to list, or None for no limit
508 series: Series object to add information into. By default a new series
510 allow_overwrite: Allow tags to overwrite an existing tag
512 A Series object containing information about the commits.
516 series.allow_overwrite = allow_overwrite
517 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
519 stdout = command.RunPipe([params], capture=True).stdout
520 ps = PatchStream(series, is_log=True)
521 for line in stdout.splitlines():
526 def GetMetaData(branch, start, count):
527 """Reads out patch series metadata from the commits
529 This does a 'git log' on the relevant commits and pulls out the tags we
533 branch: Branch to use (None for current branch)
534 start: Commit to start from: 0=branch HEAD, 1=next one, etc.
535 count: Number of commits to list
537 return GetMetaDataForList('%s~%d' % (branch if branch else 'HEAD', start),
540 def GetMetaDataForTest(text):
541 """Process metadata from a file containing a git log. Used for tests
547 ps = PatchStream(series, is_log=True)
548 for line in text.splitlines():
553 def FixPatch(backup_dir, fname, series, commit):
554 """Fix up a patch file, by adding/removing as required.
556 We remove our tags from the patch file, insert changes lists, etc.
557 The patch file is processed in place, and overwritten.
559 A backup file is put into backup_dir (if not None).
562 fname: Filename to patch file to process
563 series: Series information about this patch set
564 commit: Commit object for this patch file
566 A list of errors, or [] if all ok.
568 handle, tmpname = tempfile.mkstemp()
569 outfd = os.fdopen(handle, 'w', encoding='utf-8')
570 infd = open(fname, 'r', encoding='utf-8')
571 ps = PatchStream(series)
573 ps.ProcessStream(infd, outfd)
577 # Create a backup file if required
579 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
580 shutil.move(tmpname, fname)
583 def FixPatches(series, fnames):
584 """Fix up a list of patches identified by filenames
586 The patch files are processed in place, and overwritten.
589 series: The series object
590 fnames: List of patch files to process
592 # Current workflow creates patches, so we shouldn't need a backup
593 backup_dir = None #tempfile.mkdtemp('clean-patch')
596 commit = series.commits[count]
599 result = FixPatch(backup_dir, fname, series, commit)
601 print('%d warnings for %s:' % (len(result), fname))
606 print('Cleaned %d patches' % count)
608 def InsertCoverLetter(fname, series, count):
609 """Inserts a cover letter with the required info into patch 0
612 fname: Input / output filename of the cover letter file
613 series: Series object
614 count: Number of patches in the series
616 fd = open(fname, 'r')
617 lines = fd.readlines()
620 fd = open(fname, 'w')
622 prefix = series.GetPatchPrefix()
624 if line.startswith('Subject:'):
625 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
626 zero_repeat = int(math.log10(count)) + 1
627 zero = '0' * zero_repeat
628 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
630 # Insert our cover letter
631 elif line.startswith('*** BLURB HERE ***'):
632 # First the blurb test
633 line = '\n'.join(text[1:]) + '\n'
634 if series.get('notes'):
635 line += '\n'.join(series.notes) + '\n'
637 # Now the change list
638 out = series.MakeChangeLog(None)
639 line += '\n' + '\n'.join(out)