doc/html/elementsoup.html

   1 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
   2 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   3 <head>
   4 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
   5 <meta name="generator" content="Docutils 0.5: http://docutils.sourceforge.net/" />
   6 <title>BeautifulSoup Parser</title>
   7 <link rel="stylesheet" href="style.css" type="text/css" />
   8 </head>
   9 <body>
  10 <div class="document" id="beautifulsoup-parser">
  11 <div class="sidemenu"><ul id="lxml-section"><li><span class="section title">lxml</span><ul class="menu foreign" id="index-menu"><li class="menu title"><a href="index.html">lxml</a><ul class="submenu"><li class="menu item"><a href="index.html#introduction">Introduction</a></li><li class="menu item"><a href="index.html#documentation">Documentation</a></li><li class="menu item"><a href="index.html#download">Download</a></li><li class="menu item"><a href="index.html#mailing-list">Mailing list</a></li><li class="menu item"><a href="index.html#bug-tracker">Bug tracker</a></li><li class="menu item"><a href="index.html#license">License</a></li><li class="menu item"><a href="index.html#old-versions">Old Versions</a></li></ul></li></ul><ul class="menu foreign" id="intro-menu"><li class="menu title"><a href="intro.html">Why lxml?</a><ul class="submenu"><li class="menu item"><a href="intro.html#motto">Motto</a></li><li class="menu item"><a href="intro.html#aims">Aims</a></li></ul></li></ul><ul class="menu foreign" id="installation-menu"><li class="menu title"><a href="installation.html">Installing lxml</a><ul class="submenu"><li class="menu item"><a href="installation.html#requirements">Requirements</a></li><li class="menu item"><a href="installation.html#installation">Installation</a></li><li class="menu item"><a href="installation.html#building-lxml-from-sources">Building lxml from sources</a></li><li class="menu item"><a href="installation.html#ms-windows">MS Windows</a></li><li class="menu item"><a href="installation.html#macos-x">MacOS-X</a></li></ul></li></ul><ul class="menu foreign" id="lxml2-menu"><li class="menu title"><a href="lxml2.html">What's new in lxml 2.0?</a><ul class="submenu"><li class="menu item"><a href="lxml2.html#changes-in-etree-and-objectify">Changes in etree and objectify</a></li><li class="menu item"><a href="lxml2.html#new-modules">New modules</a></li></ul></li></ul><ul class="menu foreign" id="performance-menu"><li class="menu title"><a href="performance.html">Benchmarks and Speed</a><ul class="submenu"><li class="menu item"><a href="performance.html#general-notes">General notes</a></li><li class="menu item"><a href="performance.html#how-to-read-the-timings">How to read the timings</a></li><li class="menu item"><a href="performance.html#parsing-and-serialising">Parsing and Serialising</a></li><li class="menu item"><a href="performance.html#the-elementtree-api">The ElementTree API</a></li><li class="menu item"><a href="performance.html#xpath">XPath</a></li><li class="menu item"><a href="performance.html#a-longer-example">A longer example</a></li><li class="menu item"><a href="performance.html#lxml-objectify">lxml.objectify</a></li></ul></li></ul><ul class="menu foreign" id="compatibility-menu"><li class="menu title"><a href="compatibility.html">ElementTree compatibility of lxml.etree</a></li></ul><ul class="menu foreign" id="FAQ-menu"><li class="menu title"><a href="FAQ.html">lxml FAQ - Frequently Asked Questions</a><ul class="submenu"><li class="menu item"><a href="FAQ.html#general-questions">General Questions</a></li><li class="menu item"><a href="FAQ.html#installation">Installation</a></li><li class="menu item"><a href="FAQ.html#contributing">Contributing</a></li><li class="menu item"><a href="FAQ.html#bugs">Bugs</a></li><li class="menu item"><a href="FAQ.html#threading">Threading</a></li><li class="menu item"><a href="FAQ.html#parsing-and-serialisation">Parsing and Serialisation</a></li><li class="menu item"><a href="FAQ.html#xpath-and-document-traversal">XPath and Document Traversal</a></li></ul></li></ul></li></ul><ul id="Developing with lxml-section"><li><span class="section title">Developing with lxml</span><ul class="menu foreign" id="tutorial-menu"><li class="menu title"><a href="tutorial.html">The lxml.etree Tutorial</a><ul class="submenu"><li class="menu item"><a href="tutorial.html#the-element-class">The Element class</a></li><li class="menu item"><a href="tutorial.html#the-elementtree-class">The ElementTree class</a></li><li class="menu item"><a href="tutorial.html#parsing-from-strings-and-files">Parsing from strings and files</a></li><li class="menu item"><a href="tutorial.html#namespaces">Namespaces</a></li><li class="menu item"><a href="tutorial.html#the-e-factory">The E-factory</a></li><li class="menu item"><a href="tutorial.html#elementpath">ElementPath</a></li></ul></li></ul><ul class="menu foreign" id="api index-menu"><li class="menu title"><a href="api/index.html">API reference</a></li></ul><ul class="menu foreign" id="api-menu"><li class="menu title"><a href="api.html">APIs specific to lxml.etree</a><ul class="submenu"><li class="menu item"><a href="api.html#lxml-etree">lxml.etree</a></li><li class="menu item"><a href="api.html#other-element-apis">Other Element APIs</a></li><li class="menu item"><a href="api.html#trees-and-documents">Trees and Documents</a></li><li class="menu item"><a href="api.html#iteration">Iteration</a></li><li class="menu item"><a href="api.html#error-handling-on-exceptions">Error handling on exceptions</a></li><li class="menu item"><a href="api.html#error-logging">Error logging</a></li><li class="menu item"><a href="api.html#serialisation">Serialisation</a></li><li class="menu item"><a href="api.html#cdata">CDATA</a></li><li class="menu item"><a href="api.html#xinclude-and-elementinclude">XInclude and ElementInclude</a></li><li class="menu item"><a href="api.html#write-c14n-on-elementtree">write_c14n on ElementTree</a></li></ul></li></ul><ul class="menu foreign" id="parsing-menu"><li class="menu title"><a href="parsing.html">Parsing XML and HTML with lxml</a><ul class="submenu"><li class="menu item"><a href="parsing.html#parsers">Parsers</a></li><li class="menu item"><a href="parsing.html#the-target-parser-interface">The target parser interface</a></li><li class="menu item"><a href="parsing.html#the-feed-parser-interface">The feed parser interface</a></li><li class="menu item"><a href="parsing.html#iterparse-and-iterwalk">iterparse and iterwalk</a></li><li class="menu item"><a href="parsing.html#python-unicode-strings">Python unicode strings</a></li></ul></li></ul><ul class="menu foreign" id="validation-menu"><li class="menu title"><a href="validation.html">Validation with lxml</a><ul class="submenu"><li class="menu item"><a href="validation.html#validation-at-parse-time">Validation at parse time</a></li><li class="menu item"><a href="validation.html#dtd">DTD</a></li><li class="menu item"><a href="validation.html#relaxng">RelaxNG</a></li><li class="menu item"><a href="validation.html#xmlschema">XMLSchema</a></li><li class="menu item"><a href="validation.html#schematron">Schematron</a></li></ul></li></ul><ul class="menu foreign" id="xpathxslt-menu"><li class="menu title"><a href="xpathxslt.html">XPath and XSLT with lxml</a><ul class="submenu"><li class="menu item"><a href="xpathxslt.html#xpath">XPath</a></li><li class="menu item"><a href="xpathxslt.html#xslt">XSLT</a></li></ul></li></ul><ul class="menu foreign" id="objectify-menu"><li class="menu title"><a href="objectify.html">lxml.objectify</a><ul class="submenu"><li class="menu item"><a href="objectify.html#the-lxml-objectify-api">The lxml.objectify API</a></li><li class="menu item"><a href="objectify.html#asserting-a-schema">Asserting a Schema</a></li><li class="menu item"><a href="objectify.html#objectpath">ObjectPath</a></li><li class="menu item"><a href="objectify.html#python-data-types">Python data types</a></li><li class="menu item"><a href="objectify.html#how-data-types-are-matched">How data types are matched</a></li><li class="menu item"><a href="objectify.html#what-is-different-from-lxml-etree">What is different from lxml.etree?</a></li></ul></li></ul><ul class="menu foreign" id="lxmlhtml-menu"><li class="menu title"><a href="lxmlhtml.html">lxml.html</a><ul class="submenu"><li class="menu item"><a href="lxmlhtml.html#parsing-html">Parsing HTML</a></li><li class="menu item"><a href="lxmlhtml.html#html-element-methods">HTML Element Methods</a></li><li class="menu item"><a href="lxmlhtml.html#running-html-doctests">Running HTML doctests</a></li><li class="menu item"><a href="lxmlhtml.html#creating-html-with-the-e-factory">Creating HTML with the E-factory</a></li><li class="menu item"><a href="lxmlhtml.html#working-with-links">Working with links</a></li><li class="menu item"><a href="lxmlhtml.html#forms">Forms</a></li><li class="menu item"><a href="lxmlhtml.html#cleaning-up-html">Cleaning up HTML</a></li><li class="menu item"><a href="lxmlhtml.html#html-diff">HTML Diff</a></li><li class="menu item"><a href="lxmlhtml.html#examples">Examples</a></li></ul></li></ul><ul class="menu foreign" id="cssselect-menu"><li class="menu title"><a href="cssselect.html">lxml.cssselect</a><ul class="submenu"><li class="menu item"><a href="cssselect.html#the-cssselector-class">The CSSSelector class</a></li><li class="menu item"><a href="cssselect.html#css-selectors">CSS Selectors</a></li><li class="menu item"><a href="cssselect.html#namespaces">Namespaces</a></li><li class="menu item"><a href="cssselect.html#limitations">Limitations</a></li></ul></li></ul><ul class="menu current" id="elementsoup-menu"><li class="menu title"><a href="elementsoup.html">BeautifulSoup Parser</a><ul class="submenu"><li class="menu item"><a href="elementsoup.html#parsing-with-the-soupparser">Parsing with the soupparser</a></li><li class="menu item"><a href="elementsoup.html#entity-handling">Entity handling</a></li><li class="menu item"><a href="elementsoup.html#using-soupparser-as-a-fallback">Using soupparser as a fallback</a></li><li class="menu item"><a href="elementsoup.html#using-only-the-encoding-detection">Using only the encoding detection</a></li></ul></li></ul><ul class="menu foreign" id="html5parser-menu"><li class="menu title"><a href="html5parser.html">html5lib Parser</a><ul class="submenu"><li class="menu item"><a href="html5parser.html#differences-to-regular-html-parsing">Differences to regular HTML parsing</a></li><li class="menu item"><a href="html5parser.html#function-reference">Function Reference</a></li></ul></li></ul></li></ul><ul id="Extending lxml-section"><li><span class="section title">Extending lxml</span><ul class="menu foreign" id="resolvers-menu"><li class="menu title"><a href="resolvers.html">Document loading and URL resolving</a><ul class="submenu"><li class="menu item"><a href="resolvers.html#uri-resolvers">URI Resolvers</a></li><li class="menu item"><a href="resolvers.html#document-loading-in-context">Document loading in context</a></li><li class="menu item"><a href="resolvers.html#i-o-access-control-in-xslt">I/O access control in XSLT</a></li></ul></li></ul><ul class="menu foreign" id="extensions-menu"><li class="menu title"><a href="extensions.html">Python extensions for XPath and XSLT</a><ul class="submenu"><li class="menu item"><a href="extensions.html#xpath-extension-functions">XPath Extension functions</a></li><li class="menu item"><a href="extensions.html#xslt-extension-elements">XSLT extension elements</a></li></ul></li></ul><ul class="menu foreign" id="element classes-menu"><li class="menu title"><a href="element_classes.html">Using custom Element classes in lxml</a><ul class="submenu"><li class="menu item"><a href="element_classes.html#background-on-element-proxies">Background on Element proxies</a></li><li class="menu item"><a href="element_classes.html#element-initialization">Element initialization</a></li><li class="menu item"><a href="element_classes.html#setting-up-a-class-lookup-scheme">Setting up a class lookup scheme</a></li><li class="menu item"><a href="element_classes.html#generating-xml-with-custom-classes">Generating XML with custom classes</a></li><li class="menu item"><a href="element_classes.html#implementing-namespaces">Implementing namespaces</a></li></ul></li></ul><ul class="menu foreign" id="sax-menu"><li class="menu title"><a href="sax.html">Sax support</a><ul class="submenu"><li class="menu item"><a href="sax.html#building-a-tree-from-sax-events">Building a tree from SAX events</a></li><li class="menu item"><a href="sax.html#producing-sax-events-from-an-elementtree-or-element">Producing SAX events from an ElementTree or Element</a></li><li class="menu item"><a href="sax.html#interfacing-with-pulldom-minidom">Interfacing with pulldom/minidom</a></li></ul></li></ul><ul class="menu foreign" id="capi-menu"><li class="menu title"><a href="capi.html">The public C-API of lxml.etree</a><ul class="submenu"><li class="menu item"><a href="capi.html#writing-external-modules-in-cython">Writing external modules in Cython</a></li><li class="menu item"><a href="capi.html#writing-external-modules-in-c">Writing external modules in C</a></li></ul></li></ul></li></ul><ul id="Developing lxml-section"><li><span class="section title">Developing lxml</span><ul class="menu foreign" id="build-menu"><li class="menu title"><a href="build.html">How to build lxml from source</a><ul class="submenu"><li class="menu item"><a href="build.html#cython">Cython</a></li><li class="menu item"><a href="build.html#subversion">Subversion</a></li><li class="menu item"><a href="build.html#setuptools">Setuptools</a></li><li class="menu item"><a href="build.html#running-the-tests-and-reporting-errors">Running the tests and reporting errors</a></li><li class="menu item"><a href="build.html#building-an-egg">Building an egg</a></li><li class="menu item"><a href="build.html#building-lxml-on-macos-x">Building lxml on MacOS-X</a></li><li class="menu item"><a href="build.html#static-linking-on-windows">Static linking on Windows</a></li><li class="menu item"><a href="build.html#building-debian-packages-from-svn-sources">Building Debian packages from SVN sources</a></li></ul></li></ul><ul class="menu foreign" id="lxml source howto-menu"><li class="menu title"><a href="lxml-source-howto.html">How to read the source of lxml</a><ul class="submenu"><li class="menu item"><a href="lxml-source-howto.html#what-is-cython">What is Cython?</a></li><li class="menu item"><a href="lxml-source-howto.html#where-to-start">Where to start?</a></li><li class="menu item"><a href="lxml-source-howto.html#lxml-etree">lxml.etree</a></li><li class="menu item"><a href="lxml-source-howto.html#python-modules">Python modules</a></li><li class="menu item"><a href="lxml-source-howto.html#lxml-objectify">lxml.objectify</a></li><li class="menu item"><a href="lxml-source-howto.html#lxml-html">lxml.html</a></li></ul></li></ul><ul class="menu foreign" id="changes 2 2 3-menu"><li class="menu title"><a href="changes-2.2.3.html">Release Changelog</a></li></ul><ul class="menu foreign" id="credits-menu"><li class="menu title"><a href="credits.html">Credits</a><ul class="submenu"><li class="menu item"><a href="credits.html#main-contributors">Main contributors</a></li><li class="menu item"><a href="credits.html#special-thanks-goes-to">Special thanks goes to:</a></li></ul></li></ul></li></ul></div><h1 class="title">BeautifulSoup Parser</h1>
  12
  13 <p><a class="reference external" href="http://www.crummy.com/software/BeautifulSoup/">BeautifulSoup</a> is a Python package that parses broken HTML, just like
  14 lxml supports it based on the parser of libxml2.  BeautifulSoup uses a
  15 different parsing approach.  It is not a real HTML parser but uses
  16 regular expressions to dive through tag soup.  It is therefore more
  17 forgiving in some cases and less good in others.  It is not uncommon
  18 that lxml/libxml2 parses and fixes broken HTML better, but
  19 BeautifulSoup has superiour <a class="reference external" href="http://www.crummy.com/software/BeautifulSoup/documentation.html#Beautiful%20Soup%20Gives%20You%20Unicode%2C%20Dammit">support for encoding detection</a>.  It
  20 very much depends on the input which parser works better.</p>
  21 <p>To prevent users from having to choose their parser library in
  22 advance, lxml can interface to the parsing capabilities of
  23 BeautifulSoup through the <tt class="docutils literal"><span class="pre">lxml.html.soupparser</span></tt> module.  It
  24 provides three main functions: <tt class="docutils literal"><span class="pre">fromstring()</span></tt> and <tt class="docutils literal"><span class="pre">parse()</span></tt> to
  25 parse a string or file using BeautifulSoup into an <tt class="docutils literal"><span class="pre">lxml.html</span></tt>
  26 document, and <tt class="docutils literal"><span class="pre">convert_tree()</span></tt> to convert an existing BeautifulSoup
  27 tree into a list of top-level Elements.</p>
  28 <div class="contents topic" id="contents">
  29 <p class="topic-title first">Contents</p>
  30 <ul class="simple">
  31 <li><a class="reference internal" href="#parsing-with-the-soupparser" id="id1">Parsing with the soupparser</a></li>
  32 <li><a class="reference internal" href="#entity-handling" id="id2">Entity handling</a></li>
  33 <li><a class="reference internal" href="#using-soupparser-as-a-fallback" id="id3">Using soupparser as a fallback</a></li>
  34 <li><a class="reference internal" href="#using-only-the-encoding-detection" id="id4">Using only the encoding detection</a></li>
  35 </ul>
  36 </div>
  37 <div class="section" id="parsing-with-the-soupparser">
  38 <h1>Parsing with the soupparser</h1>
  39 <p>The functions <tt class="docutils literal"><span class="pre">fromstring()</span></tt> and <tt class="docutils literal"><span class="pre">parse()</span></tt> behave as known from
  40 ElementTree.  The first returns a root Element, the latter returns an
  41 ElementTree.</p>
  42 <p>There is also a legacy module called <tt class="docutils literal"><span class="pre">lxml.html.ElementSoup</span></tt>, which
  43 mimics the interface provided by ElementTree's own <a class="reference external" href="http://effbot.org/zone/element-soup.htm">ElementSoup</a>
  44 module.  Note that the <tt class="docutils literal"><span class="pre">soupparser</span></tt> module was added in lxml 2.0.3.
  45 Previous versions of lxml 2.0.x only have the <tt class="docutils literal"><span class="pre">ElementSoup</span></tt> module.</p>
  46 <p>Here is a document full of tag soup, similar to, but not quite like, HTML:</p>
  47 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">tag_soup</span> <span class="o">=</span> <span class="s">'&lt;meta&gt;&lt;head&gt;&lt;title&gt;Hello&lt;/head&lt;body onload=crash()&gt;Hi all&lt;p&gt;'</span>
  48 </pre></div>
  49 <p>all you need to do is pass it to the <tt class="docutils literal"><span class="pre">fromstring()</span></tt> function:</p>
  50 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">lxml.html.soupparser</span> <span class="kn">import</span> <span class="n">fromstring</span>
  51 <span class="gp">&gt;&gt;&gt; </span><span class="n">root</span> <span class="o">=</span> <span class="n">fromstring</span><span class="p">(</span><span class="n">tag_soup</span><span class="p">)</span>
  52 </pre></div>
  53 <p>To see what we have here, you can serialise it:</p>
  54 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">lxml.etree</span> <span class="kn">import</span> <span class="n">tostring</span>
  55 <span class="gp">&gt;&gt;&gt; </span><span class="k">print</span> <span class="n">tostring</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">pretty_print</span><span class="o">=</span><span class="bp">True</span><span class="p">),</span>
  56 <span class="go">&lt;html&gt;</span>
  57 <span class="go">  &lt;meta/&gt;</span>
  58 <span class="go">  &lt;head&gt;</span>
  59 <span class="go">    &lt;title&gt;Hello&lt;/title&gt;</span>
  60 <span class="go">  &lt;/head&gt;</span>
  61 <span class="go">  &lt;body onload="crash()"&gt;Hi all&lt;p/&gt;&lt;/body&gt;</span>
  62 <span class="go">&lt;/html&gt;</span>
  63 </pre></div>
  64 <p>Not quite what you'd expect from an HTML page, but, well, it was broken
  65 already, right?  BeautifulSoup did its best, and so now it's a tree.</p>
  66 <p>To control which Element implementation is used, you can pass a
  67 <tt class="docutils literal"><span class="pre">makeelement</span></tt> factory function to <tt class="docutils literal"><span class="pre">parse()</span></tt> and <tt class="docutils literal"><span class="pre">fromstring()</span></tt>.
  68 By default, this is based on the HTML parser defined in <tt class="docutils literal"><span class="pre">lxml.html</span></tt>.</p>
  69 <p>For a quick comparison, libxml2 2.6.32 parses the same tag soup as
  70 follows.  The main difference is that libxml2 tries harder to adhere
  71 to the structure of an HTML document and moves misplaced tags where
  72 they (likely) belong.  Note, however, that the result can vary between
  73 parser versions.</p>
  74 <div class="syntax"><pre><span class="nt">&lt;html&gt;</span>
  75   <span class="nt">&lt;head&gt;</span>
  76     <span class="nt">&lt;meta/&gt;</span>
  77     <span class="nt">&lt;title&gt;</span>Hello<span class="nt">&lt;/title&gt;</span>
  78   <span class="nt">&lt;/head&gt;</span>
  79   <span class="nt">&lt;body&gt;</span>
  80     <span class="nt">&lt;p&gt;</span>Hi all<span class="nt">&lt;/p&gt;</span>
  81     <span class="nt">&lt;p/&gt;</span>
  82   <span class="nt">&lt;/body&gt;</span>
  83 <span class="nt">&lt;/html&gt;</span>
  84 </pre></div>
  85 </div>
  86 <div class="section" id="entity-handling">
  87 <h1>Entity handling</h1>
  88 <p>By default, the BeautifulSoup parser also replaces the entities it
  89 finds by their character equivalent.</p>
  90 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">tag_soup</span> <span class="o">=</span> <span class="s">'&lt;body&gt;&amp;copy;&amp;euro;&amp;#45;&amp;#245;&amp;#445;&lt;p&gt;'</span>
  91 <span class="gp">&gt;&gt;&gt; </span><span class="n">body</span> <span class="o">=</span> <span class="n">fromstring</span><span class="p">(</span><span class="n">tag_soup</span><span class="p">)</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'.//body'</span><span class="p">)</span>
  92 <span class="gp">&gt;&gt;&gt; </span><span class="n">body</span><span class="o">.</span><span class="n">text</span>
  93 <span class="go">u'\xa9\u20ac-\xf5\u01bd'</span>
  94 </pre></div>
  95 <p>If you want them back on the way out, you can just serialise with the
  96 default encoding, which is 'US-ASCII'.</p>
  97 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">)</span>
  98 <span class="go">'&lt;body&gt;&amp;#169;&amp;#8364;-&amp;#245;&amp;#445;&lt;p/&gt;&lt;/body&gt;'</span>
  99
 100 <span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">"html"</span><span class="p">)</span>
 101 <span class="go">'&lt;body&gt;&amp;#169;&amp;#8364;-&amp;#245;&amp;#445;&lt;p&gt;&lt;/p&gt;&lt;/body&gt;'</span>
 102 </pre></div>
 103 <p>Any other encoding will output the respective byte sequences.</p>
 104 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s">"utf-8"</span><span class="p">)</span>
 105 <span class="go">'&lt;body&gt;\xc2\xa9\xe2\x82\xac-\xc3\xb5\xc6\xbd&lt;p/&gt;&lt;/body&gt;'</span>
 106
 107 <span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">"html"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s">"utf-8"</span><span class="p">)</span>
 108 <span class="go">'&lt;body&gt;\xc2\xa9\xe2\x82\xac-\xc3\xb5\xc6\xbd&lt;p&gt;&lt;/p&gt;&lt;/body&gt;'</span>
 109
 110 <span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="nb">unicode</span><span class="p">)</span>
 111 <span class="go">u'&lt;body&gt;\xa9\u20ac-\xf5\u01bd&lt;p/&gt;&lt;/body&gt;'</span>
 112
 113 <span class="gp">&gt;&gt;&gt; </span><span class="n">tostring</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">"html"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="nb">unicode</span><span class="p">)</span>
 114 <span class="go">u'&lt;body&gt;\xa9\u20ac-\xf5\u01bd&lt;p&gt;&lt;/p&gt;&lt;/body&gt;'</span>
 115 </pre></div>
 116 </div>
 117 <div class="section" id="using-soupparser-as-a-fallback">
 118 <h1>Using soupparser as a fallback</h1>
 119 <p>The downside of using this parser is that it is <a class="reference external" href="http://blog.ianbicking.org/2008/03/30/python-html-parser-performance/">much slower</a> than
 120 the HTML parser of lxml.  So if performance matters, you might want to
 121 consider using <tt class="docutils literal"><span class="pre">soupparser</span></tt> only as a fallback for certain cases.</p>
 122 <p>One common problem of lxml's parser is that it might not get the
 123 encoding right in cases where the document contains a <tt class="docutils literal"><span class="pre">&lt;meta&gt;</span></tt> tag
 124 at the wrong place.  In this case, you can exploit the fact that lxml
 125 serialises much faster than most other HTML libraries for Python.
 126 Just serialise the document to unicode and if that gives you an
 127 exception, re-parse it with BeautifulSoup to see if that works
 128 better.</p>
 129 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="n">tag_soup</span> <span class="o">=</span> <span class="s">'''</span><span class="se">\</span>
 130 <span class="gp">... </span><span class="s">&lt;meta http-equiv="Content-Type"</span>
 131 <span class="gp">... </span><span class="s">      content="text/html;charset=utf-8" /&gt;</span>
 132 <span class="gp">... </span><span class="s">&lt;html&gt;</span>
 133 <span class="gp">... </span><span class="s">  &lt;head&gt;</span>
 134 <span class="gp">... </span><span class="s">    &lt;title&gt;Hello W</span><span class="se">\xc3\xb6</span><span class="s">rld!&lt;/title&gt;</span>
 135 <span class="gp">... </span><span class="s">  &lt;/head&gt;</span>
 136 <span class="gp">... </span><span class="s">  &lt;body&gt;Hi all&lt;/body&gt;</span>
 137 <span class="gp">... </span><span class="s">&lt;/html&gt;'''</span>
 138
 139 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">lxml.html</span>
 140 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">lxml.html.soupparser</span>
 141
 142 <span class="gp">&gt;&gt;&gt; </span><span class="n">root</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">tag_soup</span><span class="p">)</span>
 143 <span class="gp">&gt;&gt;&gt; </span><span class="k">try</span><span class="p">:</span>
 144 <span class="gp">... </span>    <span class="n">ignore</span> <span class="o">=</span> <span class="n">tostring</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="nb">unicode</span><span class="p">)</span>
 145 <span class="gp">... </span><span class="k">except</span> <span class="ne">UnicodeDecodeError</span><span class="p">:</span>
 146 <span class="gp">... </span>    <span class="n">root</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">soupparser</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">tag_soup</span><span class="p">)</span>
 147 </pre></div>
 148 </div>
 149 <div class="section" id="using-only-the-encoding-detection">
 150 <h1>Using only the encoding detection</h1>
 151 <p>If you prefer a 'real' (and fast) HTML parser instead of the regular
 152 expression based one in BeautifulSoup, you can still benefit from
 153 BeautifulSoup's <a class="reference external" href="http://www.crummy.com/software/BeautifulSoup/documentation.html#Beautiful%20Soup%20Gives%20You%20Unicode%2C%20Dammit">support for encoding detection</a> in the
 154 <tt class="docutils literal"><span class="pre">UnicodeDammit</span></tt> class.</p>
 155 <div class="syntax"><pre><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">BeautifulSoup</span> <span class="kn">import</span> <span class="n">UnicodeDammit</span>
 156
 157 <span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">decode_html</span><span class="p">(</span><span class="n">html_string</span><span class="p">):</span>
 158 <span class="gp">... </span>    <span class="n">converted</span> <span class="o">=</span> <span class="n">UnicodeDammit</span><span class="p">(</span><span class="n">html_string</span><span class="p">,</span> <span class="n">isHTML</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
 159 <span class="gp">... </span>    <span class="k">if</span> <span class="ow">not</span> <span class="n">converted</span><span class="o">.</span><span class="n">unicode</span><span class="p">:</span>
 160 <span class="gp">... </span>        <span class="k">raise</span> <span class="ne">UnicodeDecodeError</span><span class="p">(</span>
 161 <span class="gp">... </span>            <span class="s">"Failed to detect encoding, tried [</span><span class="si">%s</span><span class="s">]"</span><span class="p">,</span>
 162 <span class="gp">... </span>            <span class="s">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">converted</span><span class="o">.</span><span class="n">triedEncodings</span><span class="p">))</span>
 163 <span class="gp">... </span>    <span class="c"># print converted.originalEncoding</span>
 164 <span class="gp">... </span>    <span class="k">return</span> <span class="n">converted</span><span class="o">.</span><span class="n">unicode</span>
 165
 166 <span class="gp">&gt;&gt;&gt; </span><span class="n">root</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">decode_html</span><span class="p">(</span><span class="n">tag_soup</span><span class="p">))</span>
 167 </pre></div>
 168 </div>
 169 </div>
 170 <div class="footer">
 171 <hr class="footer" />
 172 Generated on: 2009-10-30.
 173
 174 </div>
 175 </body>
 176 </html>