doc/tools/doxybuilder_funcs.py

   1 '''
   2   Copyright 2011 by the Massachusetts
   3   Institute of Technology.  All Rights Reserved.
   4
   5   Export of this software from the United States of America may
   6   require a specific license from the United States Government.
   7   It is the responsibility of any person or organization contemplating
   8   export to obtain such a license before exporting.
   9
  10   WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
  11   distribute this software and its documentation for any purpose and
  12   without fee is hereby granted, provided that the above copyright
  13   notice appear in all copies and that both that copyright notice and
  14   this permission notice appear in supporting documentation, and that
  15   the name of M.I.T. not be used in advertising or publicity pertaining
  16   to distribution of the software without specific, written prior
  17   permission.  Furthermore if you modify this software you must label
  18   your software as modified software and not distribute it in such a
  19   fashion that it might be confused with the original M.I.T. software.
  20   M.I.T. makes no representations about the suitability of
  21   this software for any purpose.  It is provided "as is" without express
  22   or implied warranty.
  23 '''
  24 import sys
  25 import re
  26
  27 from collections import defaultdict
  28 from xml.sax import make_parser
  29 from xml.sax.handler import ContentHandler
  30 from docmodel import *
  31
  32 exclude_funcs = ['krb5_free_octet_data']
  33
  34 class DocNode(object):
  35     """
  36     Represents the structure of xml node.
  37     """
  38     def __init__(self, name):
  39         """
  40         @param node: name - the name of a node.
  41         @param attributes: a dictionary populated with attributes of a node
  42         @param children: a dictionary with lists of children nodes. Nodes
  43             in lists are ordered as they appear in a document.
  44         @param content: a content of xml node represented as a list of
  45             tuples [(type,value)] with type = ['char'|'element'].
  46             If type is 'char' then the value is a character string otherwise
  47             it is a reference to a child node.
  48         """
  49         self.name = name
  50         self.content = list()
  51         self.attributes = dict()
  52         self.children = defaultdict(list)
  53
  54     def walk(self, decorators, sub_ws, stack=[]):
  55         result = list()
  56         decorator = decorators.get(self.name, decorators['default'])
  57         stack.append(decorators['default'])
  58         decorators['default'] = decorator
  59
  60         for (obj_type,obj) in self.content:
  61             if obj_type == 'char':
  62                 if obj != '':
  63                     result.append(obj)
  64             else:
  65                 partial = obj.walk(decorators,1, stack)
  66                 if partial is not None:
  67                     result.append(' %s ' % partial)
  68         decorators['default'] = stack.pop()
  69         result = decorator(self, ''.join(result))
  70         if result is not None:
  71             if sub_ws == 1:
  72                 result = re.sub(r'[ ]+', r' ', result)
  73             else:
  74                 result = result.strip()
  75
  76         return result
  77
  78     def getContent(self):
  79         decorators = {'default': lambda node,value: value}
  80         result = self.walk(decorators, 1)
  81         if len(result) == 0:
  82             result = None
  83
  84         return result
  85
  86     def __repr__(self):
  87         result = ['Content: %s' % self.content]
  88
  89         for (key,value) in self.attributes.iteritems():
  90             result.append('Attr: %s = %s' % (key,value))
  91         for (key,value) in self.children.iteritems():
  92             result.append('Child: %s,%i' % (key,len(value)))
  93
  94         return '\n'.join(result)
  95
  96 class DoxyContenHandler(object, ContentHandler):
  97     def __init__(self, builder):
  98         self.builder = builder
  99         self.counters = defaultdict(int)
 100         self._nodes = None
 101         self._current = None
 102
 103     def startDocument(self):
 104         pass
 105
 106     def endDocument(self):
 107         import sys
 108
 109     def startElement(self, name, attrs):
 110         if name == self.builder.toplevel:
 111             self._nodes = []
 112
 113         if name == 'memberdef':
 114             kind = attrs.get('kind')
 115             if kind is None:
 116                 raise ValueError('Kind is not defined')
 117             self.counters[kind] += 1
 118
 119         if self._nodes is None:
 120             return
 121
 122         node = DocNode(name)
 123         for (key,value) in attrs.items():
 124             node.attributes[key] = value
 125         if self._current is not None:
 126             self._current.children[name].append(node)
 127             self._nodes.append(self._current)
 128         self._current = node
 129
 130     def characters(self, content):
 131
 132         if self._current is not None:
 133             self._current.content.append(('char',content.strip()))
 134
 135     def endElement(self, name):
 136         if name == self.builder.toplevel:
 137             assert(len(self._nodes) == 0)
 138             self._nodes = None
 139             self.builder.document.append(self._current)
 140             self._current = None
 141         else:
 142             if self._nodes is not None:
 143                 node = self._current
 144                 self._current = self._nodes.pop()
 145                 self._current.content.append(('element',node))
 146
 147
 148 class XML2AST(object):
 149     """
 150     Translates XML document into Abstract Syntax Tree like representation
 151     The content of document is stored in self.document
 152     """
 153     def __init__(self, xmlpath, toplevel='doxygen'):
 154         self.document = list()
 155         self.toplevel = toplevel
 156         self.parser = make_parser()
 157         handler = DoxyContenHandler(self)
 158         self.parser.setContentHandler(handler)
 159         filename = 'krb5_8hin.xml'
 160         filepath = '%s/%s' % (xmlpath,filename)
 161         self.parser.parse(open(filepath,'r'))
 162
 163
 164 class DoxyFuncs(XML2AST):
 165     def __init__(self, path):
 166         super(DoxyFuncs, self).__init__(path,toplevel='memberdef')
 167         self.objects = list()
 168
 169     def run(self):
 170         for node in self.document:
 171             self.process(node)
 172
 173     def process(self, node):
 174         node_type = node.attributes['kind']
 175         if node_type == 'function':
 176             data = self._process_function_node(node)
 177         else:
 178             return
 179
 180         if 'name' in data and data['name'] in exclude_funcs:
 181             return
 182         self.objects.append(DocModel(**data))
 183
 184     def save(self, templates, target_dir):
 185         for obj in self.objects:
 186             template_path = templates[obj.category]
 187             outpath = '%s/%s.rst' % (target_dir,obj.name)
 188             obj.save(outpath, template_path)
 189
 190
 191     def _process_function_node(self, node):
 192         f_name = node.children['name'][0].getContent()
 193         f_Id = node.attributes['id']
 194         f_ret_type = self._process_type_node(node.children['type'][0])
 195         f_brief = node.children['briefdescription'][0].getContent()
 196         f_detailed = node.children['detaileddescription'][0]
 197         detailed_description = self._process_description_node(f_detailed)
 198         return_value_description = self._process_return_value_description(f_detailed)
 199         retval_description = self._process_retval_description(f_detailed)
 200         warning_description = self._process_warning_description(f_detailed)
 201         seealso_description = self._process_seealso_description(f_detailed)
 202         notes_description = self._process_notes_description(f_detailed)
 203         f_version = self._process_version_description(f_detailed)
 204         deprecated_description = self._process_deprecated_description(f_detailed)
 205         param_description_map = self.process_parameter_description(f_detailed)
 206         f_definition = node.children['definition'][0].getContent()
 207         f_argsstring = node.children['argsstring'][0].getContent()
 208
 209         function_descr = {'category': 'function',
 210                           'name': f_name,
 211                           'Id': f_Id,
 212                           'return_type': f_ret_type[1],
 213                           'return_description': return_value_description,
 214                           'retval_description': retval_description,
 215                           'sa_description': seealso_description,
 216                           'warn_description': warning_description,
 217                           'notes_description': notes_description,
 218                           'short_description': f_brief,
 219                           'version_num': f_version,
 220                           'long_description': detailed_description,
 221                           'deprecated_description': deprecated_description,
 222                           'parameters': list()}
 223
 224         parameters = function_descr['parameters']
 225         for (i,p) in enumerate(node.children['param']):
 226             type_node = p.children['type'][0]
 227             p_type = self._process_type_node(type_node)
 228             if p_type[1].find('...') > -1 :
 229                 p_name = ''
 230             else:
 231                 p_name = None
 232             p_name_node = p.children.get('declname')
 233             if p_name_node is not None:
 234                 p_name = p_name_node[0].getContent()
 235             (p_direction,p_descr) = param_description_map.get(p_name,(None,None))
 236
 237             param_descr = {'seqno': i,
 238                            'name': p_name,
 239                            'direction': p_direction,
 240                            'type': p_type[1],
 241                            'typeId': p_type[0],
 242                            'description': p_descr}
 243             parameters.append(param_descr)
 244         result = Function(**function_descr)
 245         print >> self.tmp, result
 246
 247         return function_descr
 248
 249     def _process_type_node(self, type_node):
 250         """
 251         Type node has form
 252             <type>type_string</type>
 253         for build in types and
 254             <type>
 255               <ref refid='reference',kindref='member|compound'>
 256                   'type_name'
 257               </ref></type>
 258               postfix (ex. *, **m, etc.)
 259             </type>
 260         for user defined types.
 261         """
 262         type_ref_node = type_node.children.get('ref')
 263         if type_ref_node is not None:
 264             p_type_id = type_ref_node[0].attributes['refid']
 265         else:
 266             p_type_id = None
 267         p_type = type_node.getContent()
 268         # remove some macros
 269         p_type = re.sub('KRB5_ATTR_DEPRECATED', '', p_type)
 270         p_type = re.sub('KRB5_CALLCONV_C', '', p_type)
 271         p_type = re.sub('KRB5_CALLCONV_WRONG', '', p_type)
 272         p_type = re.sub('KRB5_CALLCONV', '', p_type)
 273         p_type = p_type.strip()
 274
 275         return (p_type_id, p_type)
 276
 277     def _process_description_node(self, node):
 278         """
 279         Description node is comprised of <para>...</para> sections
 280         """
 281         para = node.children.get('para')
 282         result = list()
 283         if para is not None:
 284             decorators = {'default': self.paragraph_content_decorator}
 285             for e in para:
 286                 result.append(str(e.walk(decorators, 1)))
 287                 result.append('\n')
 288         result = '\n'.join(result)
 289
 290         return result
 291
 292     def return_value_description_decorator(self, node, value):
 293         if node.name == 'simplesect':
 294             if node.attributes['kind'] == 'return':
 295                 cont = set()
 296                 cont = node.getContent()
 297                 return  value
 298         else:
 299             return None
 300
 301     def paragraph_content_decorator(self, node, value):
 302         if node.name == 'para':
 303             return value + '\n'
 304         elif node.name == 'simplesect':
 305             if node.attributes['kind'] == 'return':
 306                 return None
 307         elif node.name == 'ref':
 308             if value.find('()') >= 0:
 309                 # functions
 310                 return ':c:func:' + '`' + value + '`'
 311             else:
 312                 # macro's
 313                 return ':data:' + '`' + value + '`'
 314         elif node.name == 'emphasis':
 315             return '*' + value + '*'
 316         elif node.name == 'itemizedlist':
 317             return '\n' + value
 318         elif node.name == 'listitem':
 319             return '\n\t - ' + value + '\n'
 320         elif node.name == 'computeroutput':
 321             return '**' + value + '**'
 322         else:
 323             return None
 324
 325     def parameter_name_decorator(self, node, value):
 326         if node.name == 'parametername':
 327             direction = node.attributes.get('direction')
 328             if direction is not None:
 329                 value = '%s:%s' % (value,direction)
 330             return value
 331
 332         elif node.name == 'parameterdescription':
 333             return None
 334         else:
 335             return value
 336
 337     def parameter_description_decorator(self, node, value):
 338         if node.name == 'parameterdescription':
 339             return value
 340         elif node.name == 'parametername':
 341             return None
 342         else:
 343             return value
 344
 345     def process_parameter_description(self, node):
 346         """
 347         Parameter descriptions reside inside detailed description section.
 348         """
 349         para = node.children.get('para')
 350         result = dict()
 351         if para is not None:
 352             for e in para:
 353
 354                 param_list = e.children.get('parameterlist')
 355                 if param_list is None:
 356                     continue
 357                 param_items = param_list[0].children.get('parameteritem')
 358                 if param_items is None:
 359                     continue
 360                 for it in param_items:
 361                     decorators = {'default': self.parameter_name_decorator}
 362                     direction = None
 363                     name = it.walk(decorators,0).split(':')
 364                     if len(name) == 2:
 365                         direction = name[1]
 366
 367                     decorators = {'default': self.parameter_description_decorator,
 368                                   'para': self.paragraph_content_decorator}
 369                     description = it.walk(decorators, 0)
 370                     result[name[0]] = (direction,description)
 371         return result
 372
 373
 374     def _process_return_value_description(self, node):
 375         result = None
 376         ret = list()
 377
 378         para = node.children.get('para')
 379         if para is not None:
 380             for p in para:
 381                 simplesect_list = p.children.get('simplesect')
 382                 if simplesect_list is None:
 383                     continue
 384                 for it in simplesect_list:
 385                     decorators = {'default': self.return_value_description_decorator,
 386                                   'para': self.parameter_name_decorator}
 387                     result = it.walk(decorators, 1)
 388                     if result is not None:
 389                         ret.append(result)
 390         return ret
 391
 392
 393     def _process_retval_description(self, node):
 394         """
 395         retval descriptions reside inside detailed description section.
 396         """
 397         para = node.children.get('para')
 398
 399         result = None
 400         ret = list()
 401         if para is not None:
 402
 403             for e in para:
 404                 param_list = e.children.get('parameterlist')
 405                 if param_list is None:
 406                     continue
 407                 for p in param_list:
 408                     kind = p.attributes['kind']
 409                     if kind == 'retval':
 410
 411                         param_items = p.children.get('parameteritem')
 412                         if param_items is None:
 413                             continue
 414
 415
 416                         for it in param_items:
 417                             param_descr = it.children.get('parameterdescription')
 418                             if param_descr is not None:
 419                                 val = param_descr[0].children.get('para')
 420
 421                                 if val is not None:
 422                                     val_descr = val[0].getContent()
 423
 424                                 else:
 425                                     val_descr =''
 426
 427                             decorators = {'default': self.parameter_name_decorator}
 428
 429                             name = it.walk(decorators, 1).split(':')
 430
 431                             val = name[0]
 432                             result = " %s  %s" % (val, val_descr)
 433                             ret.append (result)
 434         return ret
 435
 436     def return_warning_decorator(self, node, value):
 437         if node.name == 'simplesect':
 438             if node.attributes['kind'] == 'warning':
 439                 return value
 440         else:
 441             return None
 442
 443     def _process_warning_description(self, node):
 444         result = None
 445         para = node.children.get('para')
 446         if para is not None:
 447             for p in para:
 448                 simplesect_list = p.children.get('simplesect')
 449                 if simplesect_list is None:
 450                     continue
 451                 for it in simplesect_list:
 452                     decorators = {'default': self.return_warning_decorator,
 453                                   'para': self.paragraph_content_decorator}
 454                     result = it.walk(decorators, 1)
 455                     # Assuming that only one Warning per function
 456                     if result is not None:
 457                         return result
 458         return result
 459
 460     def return_seealso_decorator(self, node, value):
 461         if node.name == 'simplesect':
 462             if node.attributes['kind'] == 'see':
 463                 return value
 464         else:
 465             return None
 466
 467     def _process_seealso_description(self, node):
 468         result = None
 469         para = node.children.get('para')
 470         if para is not None:
 471             for p in para:
 472                 simplesect_list = p.children.get('simplesect')
 473                 if simplesect_list is None:
 474                     continue
 475                 for it in simplesect_list:
 476                     decorators = {'default': self.return_seealso_decorator,
 477                                   'para': self.paragraph_content_decorator}
 478                     result = it.walk(decorators, 1)
 479         return result
 480
 481     def return_version_decorator(self, node, value):
 482         if node.name == 'simplesect':
 483             if node.attributes['kind'] == 'version':
 484                 return value
 485         else:
 486             return None
 487
 488     def _process_version_description(self, node):
 489         result = None
 490         para = node.children.get('para')
 491         if para is not None:
 492             for p in para:
 493                 simplesect_list = p.children.get('simplesect')
 494                 if simplesect_list is None:
 495                     continue
 496                 for it in simplesect_list:
 497                     decorators = {'default': self.return_version_decorator,
 498                                   'para': self.paragraph_content_decorator}
 499                     result = it.walk(decorators, 1)
 500                     if result is not None:
 501                         return result
 502         return result
 503
 504     def return_notes_decorator(self, node, value):
 505         if node.name == 'simplesect':
 506             if node.attributes['kind'] == 'note':
 507                 # We indent notes with an extra tab.  Do it for all paragraphs.
 508                 return value.replace("\n  ", "\n\n\t  ");
 509         else:
 510             return None
 511
 512     def _process_notes_description(self, node):
 513         result = None
 514         para = node.children.get('para')
 515         if para is not None:
 516             for p in para:
 517                 simplesect_list = p.children.get('simplesect')
 518                 if simplesect_list is None:
 519                     continue
 520                 for it in simplesect_list:
 521                     decorators = {'default': self.return_notes_decorator,
 522                                   'para': self.paragraph_content_decorator}
 523                     result = it.walk(decorators, 1)
 524                     if result is not None:
 525                         return result
 526         return result
 527
 528     def return_deprecated_decorator(self, node, value):
 529         if node.name == 'xrefsect':
 530             if node.attributes['id'].find('deprecated_') > -1:
 531                 xreftitle = node.children.get('xreftitle')
 532                 if xreftitle[0] is not None:
 533                     xrefdescr = node.children.get('xrefdescription')
 534                     deprecated_descr = "DEPRECATED %s" % xrefdescr[0].getContent()
 535                     return deprecated_descr
 536         else:
 537             return None
 538
 539     def _process_deprecated_description(self, node):
 540         result = None
 541         para = node.children.get('para')
 542         if para is not None:
 543             for p in para:
 544                 xrefsect_list = p.children.get('xrefsect')
 545                 if xrefsect_list is None:
 546                     continue
 547                 for it in xrefsect_list:
 548                     decorators = {'default': self.return_deprecated_decorator,
 549                                   'para': self.paragraph_content_decorator}
 550                     result = it.walk(decorators, 1)
 551                     if result is not None:
 552                         return result
 553         return result
 554
 555     def break_into_lines(self, value, linelen=82):
 556         breaks = range(0,len(value),linelen) + [len(value)]
 557         result = list()
 558         for (start,end) in zip(breaks[:-1],breaks[1:]):
 559             result.append(value[start:end])
 560         result = '\n'.join(result)
 561
 562         return result
 563
 564     def _save(self, table, path = None):
 565         if path is None:
 566             f = sys.stdout
 567         else:
 568             f = open(path, 'w')
 569         for l in table:
 570             f.write('%s\n' % ','.join(l))
 571         if path is not None:
 572             f.close()
 573
 574
 575
 576 class DoxyBuilderFuncs(DoxyFuncs):
 577     def __init__(self, xmlpath, rstpath):
 578         super(DoxyBuilderFuncs,self).__init__(xmlpath)
 579         self.target_dir = rstpath
 580         outfile = '%s/%s' % (self.target_dir, 'out.txt')
 581         self.tmp = open(outfile, 'w')
 582
 583     def run_all(self):
 584         self.run()
 585         templates = {'function': 'func_document.tmpl'}
 586         self.save(templates, self.target_dir)
 587
 588     def test_run(self):
 589         self.run()
 590
 591 if __name__ == '__main__':
 592     builder = DoxyBuilderFuncs(xmlpath, rstpath)
 593     builder.run_all()
 594