From b6b5152f386ddae503674cc26200a547f3b4c8b0 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sat, 27 Jul 2013 15:31:42 +0200 Subject: [PATCH] properly handle expressions at the beginning of func/class/etc. blocks that start with a string literal but are not docstrings --- Cython/Compiler/ParseTreeTransforms.py | 9 +++- Cython/Compiler/Parsing.pxd | 4 +- Cython/Compiler/Parsing.py | 83 +++++++++++++++++++++++++--------- tests/run/r_docstrings.pyx | 48 +++++++++++++------- 4 files changed, 103 insertions(+), 41 deletions(-) diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index f0f4bbc..982c18a 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -121,13 +121,20 @@ class NormalizeTree(CythonTransform): def visit_CStructOrUnionDefNode(self, node): return self.visit_StatNode(node, True) - # Eliminate PassStatNode def visit_PassStatNode(self, node): + """Eliminate PassStatNode""" if not self.is_in_statlist: return Nodes.StatListNode(pos=node.pos, stats=[]) else: return [] + def visit_ExprStatNode(self, node): + """Eliminate useless string literals""" + if node.expr.is_string_literal: + return None + self.visitchildren(node) + return node + def visit_CDeclaratorNode(self, node): return node diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd index 6085dfd..73fb31c 100644 --- a/Cython/Compiler/Parsing.pxd +++ b/Cython/Compiler/Parsing.pxd @@ -125,7 +125,9 @@ cdef p_DEF_statement(PyrexScanner s) cdef p_IF_statement(PyrexScanner s, ctx) cdef p_statement(PyrexScanner s, ctx, bint first_statement = *) cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *) -cdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *) +cdef p_suite(PyrexScanner s, ctx = *) +cdef tuple p_suite_with_docstring(s, ctx, with_doc_only = *) +cdef tuple _extract_docstring(node) cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *) cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *) diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 5432265..7ce7bb9 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -1141,10 +1141,7 @@ def p_expression_or_assignment(s): rhs = p_testlist(s) return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs) expr = expr_list[0] - if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)): - return Nodes.PassStatNode(expr.pos) - else: - return Nodes.ExprStatNode(expr.pos, expr = expr) + return Nodes.ExprStatNode(expr.pos, expr=expr) rhs = expr_list[-1] if len(expr_list) == 2: @@ -1888,7 +1885,7 @@ def p_statement(s, ctx, first_statement = 0): elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': return p_property_decl(s) elif s.sy == 'pass' and ctx.level != 'property': - return p_pass_statement(s, with_newline = 1) + return p_pass_statement(s, with_newline=True) else: if ctx.level in ('c_class_pxd', 'property'): s.error("Executable statement not allowed here") @@ -1923,15 +1920,18 @@ def p_statement_list(s, ctx, first_statement = 0): else: return Nodes.StatListNode(pos, stats = stats) -def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0): - pos = s.position() + +def p_suite(s, ctx=Ctx()): + return p_suite_with_docstring(s, ctx, with_doc_only=False)[1] + + +def p_suite_with_docstring(s, ctx, with_doc_only=False): s.expect(':') doc = None - stmts = [] if s.sy == 'NEWLINE': s.next() s.expect_indent() - if with_doc or with_pseudo_doc: + if with_doc_only: doc = p_doc_string(s) body = p_statement_list(s, ctx) s.expect_dedent() @@ -1943,10 +1943,10 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0): else: body = p_pass_statement(s) s.expect_newline("Syntax error in declarations") - if with_doc: - return doc, body - else: - return body + if not with_doc_only: + doc, body = _extract_docstring(body) + return doc, body + def p_positional_and_keyword_args(s, end_sy_set, templates = None): """ @@ -2797,7 +2797,7 @@ def p_c_func_or_var_declaration(s, pos, ctx): if s.sy == ':': if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates: s.error("C function definition not allowed here") - doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1) + doc, suite = p_suite_with_docstring(s, Ctx(level='function')) result = Nodes.CFuncDefNode(pos, visibility = ctx.visibility, base_type = base_type, @@ -2884,7 +2884,7 @@ def p_def_statement(s, decorators=None): pos = s.position() s.next() name = EncodedString( p_ident(s) ) - s.expect('('); + s.expect('(') args, star_arg, starstar_arg = p_varargslist(s, terminator=')') s.expect(')') if p_nogil(s): @@ -2893,7 +2893,7 @@ def p_def_statement(s, decorators=None): if s.sy == '->': s.next() return_type_annotation = p_test(s) - doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1) + doc, body = p_suite_with_docstring(s, Ctx(level='function')) return Nodes.DefNode(pos, name = name, args = args, star_arg = star_arg, starstar_arg = starstar_arg, doc = doc, body = body, decorators = decorators, @@ -2944,8 +2944,8 @@ def p_class_statement(s, decorators): pos, positional_args, keyword_args, star_arg, None) if arg_tuple is None: # XXX: empty arg_tuple - arg_tuple = ExprNodes.TupleNode(pos, args = []) - doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1) + arg_tuple = ExprNodes.TupleNode(pos, args=[]) + doc, body = p_suite_with_docstring(s, Ctx(level='class')) return Nodes.PyClassDefNode(pos, name = class_name, bases = arg_tuple, @@ -2993,7 +2993,7 @@ def p_c_class_definition(s, pos, ctx): body_level = 'c_class_pxd' else: body_level = 'c_class' - doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1) + doc, body = p_suite_with_docstring(s, Ctx(level=body_level)) else: s.expect_newline("Syntax error in C class definition") doc = None @@ -3050,12 +3050,15 @@ def p_c_class_options(s): s.expect(']', "Expected 'object' or 'type'") return objstruct_name, typeobj_name + def p_property_decl(s): pos = s.position() - s.next() # 'property' + s.next() # 'property' name = p_ident(s) - doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1) - return Nodes.PropertyNode(pos, name = name, doc = doc, body = body) + doc, body = p_suite_with_docstring( + s, Ctx(level='property'), with_doc_only=True) + return Nodes.PropertyNode(pos, name=name, doc=doc, body=body) + def p_doc_string(s): if s.sy == 'BEGIN_STRING': @@ -3070,6 +3073,42 @@ def p_doc_string(s): else: return None + +def _extract_docstring(node): + """ + Extract a docstring from a statement or from the first statement + in a list. Remove the statement if found. Return a tuple + (plain-docstring or None, node). + """ + doc_node = None + if node is None: + pass + elif isinstance(node, Nodes.ExprStatNode): + if node.expr.is_string_literal: + doc_node = node.expr + node = Nodes.StatListNode(node.pos, stats=[]) + elif isinstance(node, Nodes.StatListNode) and node.stats: + stats = node.stats + if isinstance(stats[0], Nodes.ExprStatNode): + if stats[0].expr.is_string_literal: + doc_node = stats[0].expr + del stats[0] + + if doc_node is None: + doc = None + elif isinstance(doc_node, ExprNodes.BytesNode): + warning(node.pos, + "Python 3 requires docstrings to be unicode strings") + doc = doc_node.value + elif isinstance(doc_node, ExprNodes.StringNode): + doc = doc_node.unicode_value + if doc is None: + doc = doc_node.value + else: + doc = doc_node.value + return doc, node + + def p_code(s, level=None, ctx=Ctx): body = p_statement_list(s, ctx(level = level), first_statement = 1) if s.sy != 'EOF': diff --git a/tests/run/r_docstrings.pyx b/tests/run/r_docstrings.pyx index bcf0c9f..b01a8a4 100644 --- a/tests/run/r_docstrings.pyx +++ b/tests/run/r_docstrings.pyx @@ -3,7 +3,7 @@ # More comments -u'A module docstring' +'A module docstring' doctest = u"""# Python 3 gets all of these right ... >>> __doc__ @@ -28,48 +28,44 @@ doctest = u"""# Python 3 gets all of these right ... Compare with standard Python: >>> def Pyf(): - ... u''' + ... ''' ... This is a function docstring. ... ''' >>> Pyf.__doc__ - u'\\n This is a function docstring.\\n ' + '\\n This is a function docstring.\\n ' >>> class PyC: - ... u''' + ... ''' ... This is a class docstring. ... ''' >>> class PyCS(C): - ... u''' + ... ''' ... This is a subclass docstring. ... ''' >>> class PyCSS(CS): ... pass >>> PyC.__doc__ - u'\\n This is a class docstring.\\n ' + '\\n This is a class docstring.\\n ' >>> PyCS.__doc__ - u'\\n This is a subclass docstring.\\n ' + '\\n This is a subclass docstring.\\n ' >>> PyCSS.__doc__ """ -import sys -if sys.version_info[0] >= 3: - doctest = doctest.replace(u" u'", u" '") - -__test__ = {u"test_docstrings" : doctest} +__test__ = {"test_docstrings" : doctest} def f(): - u""" + """ This is a function docstring. """ class C: - u""" + """ This is a class docstring. """ class CS(C): - u""" + """ This is a subclass docstring. """ @@ -77,14 +73,32 @@ class CSS(CS): pass cdef class T: - u""" + """ This is an extension type docstring. """ cdef class TS(T): - u""" + """ This is an extension subtype docstring. """ cdef class TSS(TS): pass + + +def n(): + "This is not a docstring".lower() + +class PyN(object): + u"This is not a docstring".lower() + +cdef class CN(object): + b"This is not a docstring".lower() + + +def test_non_docstrings(): + """ + >>> n.__doc__ + >>> PyN.__doc__ + >>> CN.__doc__ + """ -- 2.7.4