tools/mkowners/mkowners.py

   1 #!/usr/bin/env python3
   2 # Copyright 2017 gRPC authors.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import argparse
  17 import collections
  18 import operator
  19 import os
  20 import re
  21 import subprocess
  22
  23 #
  24 # Find the root of the git tree
  25 #
  26
  27 git_root = (subprocess.check_output(['git', 'rev-parse', '--show-toplevel'
  28                                     ]).decode('utf-8').strip())
  29
  30 #
  31 # Parse command line arguments
  32 #
  33
  34 default_out = os.path.join(git_root, '.github', 'CODEOWNERS')
  35
  36 argp = argparse.ArgumentParser('Generate .github/CODEOWNERS file')
  37 argp.add_argument('--out',
  38                   '-o',
  39                   type=str,
  40                   default=default_out,
  41                   help='Output file (default %s)' % default_out)
  42 args = argp.parse_args()
  43
  44 #
  45 # Walk git tree to locate all OWNERS files
  46 #
  47
  48 owners_files = [
  49     os.path.join(root, 'OWNERS')
  50     for root, dirs, files in os.walk(git_root)
  51     if 'OWNERS' in files
  52 ]
  53
  54 #
  55 # Parse owners files
  56 #
  57
  58 Owners = collections.namedtuple('Owners', 'parent directives dir')
  59 Directive = collections.namedtuple('Directive', 'who globs')
  60
  61
  62 def parse_owners(filename):
  63     with open(filename) as f:
  64         src = f.read().splitlines()
  65     parent = True
  66     directives = []
  67     for line in src:
  68         line = line.strip()
  69         # line := directive | comment
  70         if not line: continue
  71         if line[0] == '#': continue
  72         # it's a directive
  73         directive = None
  74         if line == 'set noparent':
  75             parent = False
  76         elif line == '*':
  77             directive = Directive(who='*', globs=[])
  78         elif ' ' in line:
  79             (who, globs) = line.split(' ', 1)
  80             globs_list = [glob for glob in globs.split(' ') if glob]
  81             directive = Directive(who=who, globs=globs_list)
  82         else:
  83             directive = Directive(who=line, globs=[])
  84         if directive:
  85             directives.append(directive)
  86     return Owners(parent=parent,
  87                   directives=directives,
  88                   dir=os.path.relpath(os.path.dirname(filename), git_root))
  89
  90
  91 owners_data = sorted([parse_owners(filename) for filename in owners_files],
  92                      key=operator.attrgetter('dir'))
  93
  94 #
  95 # Modify owners so that parented OWNERS files point to the actual
  96 # Owners tuple with their parent field
  97 #
  98
  99 new_owners_data = []
 100 for owners in owners_data:
 101     if owners.parent == True:
 102         best_parent = None
 103         best_parent_score = None
 104         for possible_parent in owners_data:
 105             if possible_parent is owners: continue
 106             rel = os.path.relpath(owners.dir, possible_parent.dir)
 107             # '..' ==> we had to walk up from possible_parent to get to owners
 108             #      ==> not a parent
 109             if '..' in rel: continue
 110             depth = len(rel.split(os.sep))
 111             if not best_parent or depth < best_parent_score:
 112                 best_parent = possible_parent
 113                 best_parent_score = depth
 114         if best_parent:
 115             owners = owners._replace(parent=best_parent.dir)
 116         else:
 117             owners = owners._replace(parent=None)
 118     new_owners_data.append(owners)
 119 owners_data = new_owners_data
 120
 121 #
 122 # In bottom to top order, process owners data structures to build up
 123 # a CODEOWNERS file for GitHub
 124 #
 125
 126
 127 def full_dir(rules_dir, sub_path):
 128     return os.path.join(rules_dir, sub_path) if rules_dir != '.' else sub_path
 129
 130
 131 # glob using git
 132 gg_cache = {}
 133
 134
 135 def git_glob(glob):
 136     global gg_cache
 137     if glob in gg_cache: return gg_cache[glob]
 138     r = set(
 139         subprocess.check_output([
 140             'git', 'ls-files', os.path.join(git_root, glob)
 141         ]).decode('utf-8').strip().splitlines())
 142     gg_cache[glob] = r
 143     return r
 144
 145
 146 def expand_directives(root, directives):
 147     globs = collections.OrderedDict()
 148     # build a table of glob --> owners
 149     for directive in directives:
 150         for glob in directive.globs or ['**']:
 151             if glob not in globs:
 152                 globs[glob] = []
 153             if directive.who not in globs[glob]:
 154                 globs[glob].append(directive.who)
 155     # expand owners for intersecting globs
 156     sorted_globs = sorted(globs.keys(),
 157                           key=lambda g: len(git_glob(full_dir(root, g))),
 158                           reverse=True)
 159     out_globs = collections.OrderedDict()
 160     for glob_add in sorted_globs:
 161         who_add = globs[glob_add]
 162         pre_items = [i for i in out_globs.items()]
 163         out_globs[glob_add] = who_add.copy()
 164         for glob_have, who_have in pre_items:
 165             files_add = git_glob(full_dir(root, glob_add))
 166             files_have = git_glob(full_dir(root, glob_have))
 167             intersect = files_have.intersection(files_add)
 168             if intersect:
 169                 for f in sorted(files_add):  # sorted to ensure merge stability
 170                     if f not in intersect:
 171                         out_globs[os.path.relpath(f, start=root)] = who_add
 172                 for who in who_have:
 173                     if who not in out_globs[glob_add]:
 174                         out_globs[glob_add].append(who)
 175     return out_globs
 176
 177
 178 def add_parent_to_globs(parent, globs, globs_dir):
 179     if not parent: return
 180     for owners in owners_data:
 181         if owners.dir == parent:
 182             owners_globs = expand_directives(owners.dir, owners.directives)
 183             for oglob, oglob_who in owners_globs.items():
 184                 for gglob, gglob_who in globs.items():
 185                     files_parent = git_glob(full_dir(owners.dir, oglob))
 186                     files_child = git_glob(full_dir(globs_dir, gglob))
 187                     intersect = files_parent.intersection(files_child)
 188                     gglob_who_orig = gglob_who.copy()
 189                     if intersect:
 190                         for f in sorted(files_child
 191                                        ):  # sorted to ensure merge stability
 192                             if f not in intersect:
 193                                 who = gglob_who_orig.copy()
 194                                 globs[os.path.relpath(f, start=globs_dir)] = who
 195                         for who in oglob_who:
 196                             if who not in gglob_who:
 197                                 gglob_who.append(who)
 198             add_parent_to_globs(owners.parent, globs, globs_dir)
 199             return
 200     assert (False)
 201
 202
 203 todo = owners_data.copy()
 204 done = set()
 205 with open(args.out, 'w') as out:
 206     out.write('# Auto-generated by the tools/mkowners/mkowners.py tool\n')
 207     out.write('# Uses OWNERS files in different modules throughout the\n')
 208     out.write('# repository as the source of truth for module ownership.\n')
 209     written_globs = []
 210     while todo:
 211         head, *todo = todo
 212         if head.parent and not head.parent in done:
 213             todo.append(head)
 214             continue
 215         globs = expand_directives(head.dir, head.directives)
 216         add_parent_to_globs(head.parent, globs, head.dir)
 217         for glob, owners in globs.items():
 218             skip = False
 219             for glob1, owners1, dir1 in reversed(written_globs):
 220                 files = git_glob(full_dir(head.dir, glob))
 221                 files1 = git_glob(full_dir(dir1, glob1))
 222                 intersect = files.intersection(files1)
 223                 if files == intersect:
 224                     if sorted(owners) == sorted(owners1):
 225                         skip = True  # nothing new in this rule
 226                         break
 227                 elif intersect:
 228                     # continuing would cause a semantic change since some files are
 229                     # affected differently by this rule and CODEOWNERS is order dependent
 230                     break
 231             if not skip:
 232                 out.write('/%s %s\n' %
 233                           (full_dir(head.dir, glob), ' '.join(owners)))
 234                 written_globs.append((glob, owners, head.dir))
 235         done.add(head.dir)