1 # Copyright (c) 2011 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
15 from series import Series
17 # Tags that we detect and remove
18 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
19 '|Reviewed-on:|Commit-\w*:')
21 # Lines which are allowed after a TEST= line
22 re_allowed_after_test = re.compile('^Signed-off-by:')
25 re_signoff = re.compile('^Signed-off-by: *(.*)')
27 # The start of the cover letter
28 re_cover = re.compile('^Cover-letter:')
31 re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
34 re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
37 re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
39 # Commit tags that we want to collect and keep
40 re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
42 # The start of a new commit in the git log
43 re_commit = re.compile('^commit ([0-9a-f]*)$')
45 # We detect these since checkpatch doesn't always do it
46 re_space_before_tab = re.compile('^[+].* \t')
48 # States we can be in - can we use range() and still have comments?
49 STATE_MSG_HEADER = 0 # Still in the message header
50 STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
51 STATE_PATCH_HEADER = 2 # In patch header (after the subject)
52 STATE_DIFFS = 3 # In the diff part (past --- line)
55 """Class for detecting/injecting tags in a patch or series of patches
57 We support processing the output of 'git log' to read out the tags we
58 are interested in. We can also process a patch file in order to remove
59 unwanted tags or inject additional ones. These correspond to the two
62 def __init__(self, series, name=None, is_log=False):
63 self.skip_blank = False # True to skip a single blank line
64 self.found_test = False # Found a TEST= line
65 self.lines_after_test = 0 # MNumber of lines found after TEST=
66 self.warn = [] # List of warnings we have collected
67 self.linenum = 1 # Output line number we are up to
68 self.in_section = None # Name of start...END section we are in
69 self.notes = [] # Series notes
70 self.section = [] # The current section...END section
71 self.series = series # Info about the patch series
72 self.is_log = is_log # True if indent like git log
73 self.in_change = 0 # Non-zero if we are in a change list
74 self.blank_count = 0 # Number of blank lines stored up
75 self.state = STATE_MSG_HEADER # What state are we in?
76 self.signoff = [] # Contents of signoff line
77 self.commit = None # Current commit
79 def AddToSeries(self, line, name, value):
80 """Add a new Series-xxx tag.
82 When a Series-xxx tag is detected, we come here to record it, if we
83 are scanning a 'git log'.
86 line: Source line containing tag (useful for debug/error messages)
87 name: Tag name (part after 'Series-')
88 value: Tag value (part after 'Series-xxx: ')
91 self.in_section = name
92 self.skip_blank = False
94 self.series.AddTag(self.commit, line, name, value)
96 def AddToCommit(self, line, name, value):
97 """Add a new Commit-xxx tag.
99 When a Commit-xxx tag is detected, we come here to record it.
102 line: Source line containing tag (useful for debug/error messages)
103 name: Tag name (part after 'Commit-')
104 value: Tag value (part after 'Commit-xxx: ')
107 self.in_section = 'commit-' + name
108 self.skip_blank = False
110 def CloseCommit(self):
111 """Save the current commit into our commit list, and reset our state"""
112 if self.commit and self.is_log:
113 self.series.AddCommit(self.commit)
116 def ProcessLine(self, line):
117 """Process a single line of a patch file or commit log
119 This process a line and returns a list of lines to output. The list
120 may be empty or may contain multiple output lines.
122 This is where all the complicated logic is located. The class's
123 state is used to move between different states and detect things
126 We can be in one of two modes:
127 self.is_log == True: This is 'git log' mode, where most output is
128 indented by 4 characters and we are scanning for tags
130 self.is_log == False: This is 'patch' mode, where we already have
131 all the tags, and are processing patches to remove junk we
132 don't want, and add things we think are required.
135 line: text line to process
138 list of output lines, or [] if nothing should be output
140 # Initially we have no output. Prepare the input line string
142 line = line.rstrip('\n')
144 commit_match = re_commit.match(line) if self.is_log else None
150 # Handle state transition and skipping blank lines
151 series_tag_match = re_series_tag.match(line)
152 commit_tag_match = re_commit_tag.match(line)
153 cover_match = re_cover.match(line)
154 cover_cc_match = re_cover_cc.match(line)
155 signoff_match = re_signoff.match(line)
157 if self.state == STATE_PATCH_HEADER:
158 tag_match = re_tag.match(line)
159 is_blank = not line.strip()
161 if (self.state == STATE_MSG_HEADER
162 or self.state == STATE_PATCH_SUBJECT):
165 # We don't have a subject in the text stream of patch files
166 # It has its own line with a Subject: tag
167 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
170 self.state = STATE_MSG_HEADER
172 # If a tag is detected, but we are already in a section,
173 # this means 'END' is missing for that section, fix it up.
174 if series_tag_match or commit_tag_match or \
175 cover_match or cover_cc_match or signoff_match:
177 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
178 if self.in_section == 'cover':
179 self.series.cover = self.section
180 elif self.in_section == 'notes':
182 self.series.notes += self.section
183 elif self.in_section == 'commit-notes':
185 self.commit.notes += self.section
187 self.warn.append("Unknown section '%s'" % self.in_section)
188 self.in_section = None
189 self.skip_blank = True
192 # If we are in a section, keep collecting lines until we see END
195 if self.in_section == 'cover':
196 self.series.cover = self.section
197 elif self.in_section == 'notes':
199 self.series.notes += self.section
200 elif self.in_section == 'commit-notes':
202 self.commit.notes += self.section
204 self.warn.append("Unknown section '%s'" % self.in_section)
205 self.in_section = None
206 self.skip_blank = True
209 self.section.append(line)
211 # Detect the commit subject
212 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
213 self.commit.subject = line
215 # Detect the tags we want to remove, and skip blank lines
216 elif re_remove.match(line) and not commit_tag_match:
217 self.skip_blank = True
219 # TEST= should be the last thing in the commit, so remove
220 # everything after it
221 if line.startswith('TEST='):
222 self.found_test = True
223 elif self.skip_blank and is_blank:
224 self.skip_blank = False
226 # Detect the start of a cover letter section
228 self.in_section = 'cover'
229 self.skip_blank = False
232 value = cover_cc_match.group(1)
233 self.AddToSeries(line, 'cover-cc', value)
235 # If we are in a change list, key collected lines until a blank one
238 # Blank line ends this change list
242 out = self.ProcessLine(line)
245 self.series.AddChange(self.in_change, self.commit, line)
246 self.skip_blank = False
248 # Detect Series-xxx tags
249 elif series_tag_match:
250 name = series_tag_match.group(1)
251 value = series_tag_match.group(2)
252 if name == 'changes':
253 # value is the version number: e.g. 1, or 2
256 except ValueError as str:
257 raise ValueError("%s: Cannot decode version info '%s'" %
258 (self.commit.hash, line))
259 self.in_change = int(value)
261 self.AddToSeries(line, name, value)
262 self.skip_blank = True
264 # Detect Commit-xxx tags
265 elif commit_tag_match:
266 name = commit_tag_match.group(1)
267 value = commit_tag_match.group(2)
269 self.AddToCommit(line, name, value)
270 self.skip_blank = True
272 # Detect the start of a new commit
275 self.commit = commit.Commit(commit_match.group(1))
277 # Detect tags in the commit message
279 # Remove Tested-by self, since few will take much notice
280 if (tag_match.group(1) == 'Tested-by' and
281 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
282 self.warn.append("Ignoring %s" % line)
283 elif tag_match.group(1) == 'Patch-cc':
284 self.commit.AddCc(tag_match.group(2).split(','))
288 # Suppress duplicate signoffs
290 if (self.is_log or not self.commit or
291 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
294 # Well that means this is an ordinary line
297 # Look for ugly ASCII characters
299 # TODO: Would be nicer to report source filename and line
301 self.warn.append("Line %d/%d ('%s') has funny ascii char" %
302 (self.linenum, pos, line))
305 # Look for space before tab
306 m = re_space_before_tab.match(line)
308 self.warn.append('Line %d/%d has space before tab' %
309 (self.linenum, m.start()))
311 # OK, we have a valid non-blank line
314 self.skip_blank = False
315 if self.state == STATE_DIFFS:
318 # If this is the start of the diffs section, emit our tags and
321 self.state = STATE_DIFFS
323 # Output the tags (signeoff first), then change list
325 log = self.series.MakeChangeLog(self.commit)
328 out += self.commit.notes
330 elif self.found_test:
331 if not re_allowed_after_test.match(line):
332 self.lines_after_test += 1
337 """Close out processing of this patch stream"""
339 if self.lines_after_test:
340 self.warn.append('Found %d lines after TEST=' %
341 self.lines_after_test)
343 def ProcessStream(self, infd, outfd):
344 """Copy a stream from infd to outfd, filtering out unwanting things.
346 This is used to process patch files one at a time.
349 infd: Input stream file object
350 outfd: Output stream file object
352 # Extract the filename from each diff, for nice warnings
355 re_fname = re.compile('diff --git a/(.*) b/.*')
357 line = infd.readline()
360 out = self.ProcessLine(line)
362 # Try to detect blank lines at EOF
364 match = re_fname.match(line)
367 fname = match.group(1)
369 self.blank_count += 1
371 if self.blank_count and (line == '-- ' or match):
372 self.warn.append("Found possible blank line(s) at "
373 "end of file '%s'" % last_fname)
374 outfd.write('+\n' * self.blank_count)
375 outfd.write(line + '\n')
380 def GetMetaDataForList(commit_range, git_dir=None, count=None,
381 series = None, allow_overwrite=False):
382 """Reads out patch series metadata from the commits
384 This does a 'git log' on the relevant commits and pulls out the tags we
388 commit_range: Range of commits to count (e.g. 'HEAD..base')
389 git_dir: Path to git repositiory (None to use default)
390 count: Number of commits to list, or None for no limit
391 series: Series object to add information into. By default a new series
393 allow_overwrite: Allow tags to overwrite an existing tag
395 A Series object containing information about the commits.
399 series.allow_overwrite = allow_overwrite
400 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
402 stdout = command.RunPipe([params], capture=True).stdout
403 ps = PatchStream(series, is_log=True)
404 for line in stdout.splitlines():
409 def GetMetaData(start, count):
410 """Reads out patch series metadata from the commits
412 This does a 'git log' on the relevant commits and pulls out the tags we
416 start: Commit to start from: 0=HEAD, 1=next one, etc.
417 count: Number of commits to list
419 return GetMetaDataForList('HEAD~%d' % start, None, count)
421 def FixPatch(backup_dir, fname, series, commit):
422 """Fix up a patch file, by adding/removing as required.
424 We remove our tags from the patch file, insert changes lists, etc.
425 The patch file is processed in place, and overwritten.
427 A backup file is put into backup_dir (if not None).
430 fname: Filename to patch file to process
431 series: Series information about this patch set
432 commit: Commit object for this patch file
434 A list of errors, or [] if all ok.
436 handle, tmpname = tempfile.mkstemp()
437 outfd = os.fdopen(handle, 'w')
438 infd = open(fname, 'r')
439 ps = PatchStream(series)
441 ps.ProcessStream(infd, outfd)
445 # Create a backup file if required
447 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
448 shutil.move(tmpname, fname)
451 def FixPatches(series, fnames):
452 """Fix up a list of patches identified by filenames
454 The patch files are processed in place, and overwritten.
457 series: The series object
458 fnames: List of patch files to process
460 # Current workflow creates patches, so we shouldn't need a backup
461 backup_dir = None #tempfile.mkdtemp('clean-patch')
464 commit = series.commits[count]
466 result = FixPatch(backup_dir, fname, series, commit)
468 print '%d warnings for %s:' % (len(result), fname)
473 print 'Cleaned %d patches' % count
476 def InsertCoverLetter(fname, series, count):
477 """Inserts a cover letter with the required info into patch 0
480 fname: Input / output filename of the cover letter file
481 series: Series object
482 count: Number of patches in the series
484 fd = open(fname, 'r')
485 lines = fd.readlines()
488 fd = open(fname, 'w')
490 prefix = series.GetPatchPrefix()
492 if line.startswith('Subject:'):
493 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
494 zero_repeat = int(math.log10(count)) + 1
495 zero = '0' * zero_repeat
496 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
498 # Insert our cover letter
499 elif line.startswith('*** BLURB HERE ***'):
500 # First the blurb test
501 line = '\n'.join(text[1:]) + '\n'
502 if series.get('notes'):
503 line += '\n'.join(series.notes) + '\n'
505 # Now the change list
506 out = series.MakeChangeLog(None)
507 line += '\n' + '\n'.join(out)