doc/mkhtml.py

   1 from __future__ import absolute_import
   2
   3 from docstructure import SITE_STRUCTURE, HREF_MAP, BASENAME_MAP
   4 from lxml.etree import (parse, fromstring, ElementTree,
   5                         Element, SubElement, XPath, XML)
   6 import os
   7 import re
   8 import sys
   9 import copy
  10 import shutil
  11 import textwrap
  12 import subprocess
  13
  14 from io import open as open_file
  15
  16 RST2HTML_OPTIONS = " ".join([
  17     '--no-toc-backlinks',
  18     '--strip-comments',
  19     '--language en',
  20     '--date',
  21     ])
  22
  23 XHTML_NS = 'http://www.w3.org/1999/xhtml'
  24 htmlnsmap = {"h": XHTML_NS}
  25
  26 find_head = XPath("/h:html/h:head[1]", namespaces=htmlnsmap)
  27 find_body = XPath("/h:html/h:body[1]", namespaces=htmlnsmap)
  28 find_title = XPath("/h:html/h:head/h:title/text()", namespaces=htmlnsmap)
  29 find_title_tag = XPath("/h:html/h:head/h:title", namespaces=htmlnsmap)
  30 find_headings = XPath("//h:h1[not(@class)]//text()", namespaces=htmlnsmap)
  31 find_heading_tag = XPath("//h:h1[@class = 'title'][1]", namespaces=htmlnsmap)
  32 find_menu = XPath("//h:ul[@id=$name]", namespaces=htmlnsmap)
  33 find_page_end = XPath("/h:html/h:body/h:div[last()]", namespaces=htmlnsmap)
  34
  35 find_words = re.compile(r'(\w+)').findall
  36 replace_invalid = re.compile(r'[-_/.\s\\]').sub
  37
  38
  39 def make_menu_section_head(section, menuroot):
  40     section_id = section + '-section'
  41     section_head = menuroot.xpath("//ul[@id=$section]/li", section=section_id)
  42     if not section_head:
  43         ul = SubElement(menuroot, "ul", id=section_id)
  44         section_head = SubElement(ul, "li")
  45         title = SubElement(section_head, "span", {"class":"section title"})
  46         title.text = section
  47     else:
  48         section_head = section_head[0]
  49     return section_head
  50
  51
  52 def build_menu(tree, basename, section_head):
  53     page_title = find_title(tree)
  54     if page_title:
  55         page_title = page_title[0]
  56     else:
  57         page_title = replace_invalid('', basename.capitalize())
  58     build_menu_entry(page_title, basename+".html", section_head,
  59                      headings=find_headings(tree))
  60
  61
  62 def build_menu_entry(page_title, url, section_head, headings=None):
  63     page_id = replace_invalid(' ', os.path.splitext(url)[0]) + '-menu'
  64     ul = SubElement(section_head, "ul", {"class":"menu foreign", "id":page_id})
  65
  66     title = SubElement(ul, "li", {"class":"menu title"})
  67     a = SubElement(title, "a", href=url)
  68     a.text = page_title
  69
  70     if headings:
  71         subul = SubElement(title, "ul", {"class":"submenu"})
  72         for heading in headings:
  73             li = SubElement(subul, "li", {"class":"menu item"})
  74             try:
  75                 ref = heading.getparent().getparent().get('id')
  76             except AttributeError:
  77                 ref = None
  78             if ref is None:
  79                 ref = '-'.join(find_words(replace_invalid(' ', heading.lower())))
  80             a = SubElement(li, "a", href=url+'#'+ref)
  81             a.text = heading
  82
  83
  84 def merge_menu(tree, menu, name):
  85     menu_root = copy.deepcopy(menu)
  86     tree.getroot()[1][0].insert(0, menu_root)  # html->body->div[class=document]
  87     for el in menu_root.iter():
  88         tag = el.tag
  89         if tag[0] != '{':
  90             el.tag = "{http://www.w3.org/1999/xhtml}" + tag
  91     current_menu = find_menu(
  92         menu_root, name=replace_invalid(' ', name) + '-menu')
  93     if not current_menu:
  94         current_menu = find_menu(
  95             menu_root, name=replace_invalid('-', name) + '-menu')
  96     if current_menu:
  97         for submenu in current_menu:
  98             submenu.set("class", submenu.get("class", "").
  99                         replace("foreign", "current"))
 100     return tree
 101
 102
 103 def inject_flatter_button(tree):
 104     head = tree.xpath('h:head[1]', namespaces=htmlnsmap)[0]
 105     script = SubElement(head, '{%s}script' % XHTML_NS, type='text/javascript')
 106     script.text = """
 107     (function() {
 108         var s = document.createElement('script');
 109         var t = document.getElementsByTagName('script')[0];
 110         s.type = 'text/javascript';
 111         s.async = true;
 112         s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
 113         t.parentNode.insertBefore(s, t);
 114     })();
 115 """
 116     script.tail = '\n'
 117     intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]', namespaces=htmlnsmap)[0]
 118     intro_div.insert(-1, XML(
 119         '<p style="text-align: center;">Like working with lxml? '
 120         'Happy about the time that it just saved you? <br />'
 121         'Show your appreciation with <a href="http://flattr.com/thing/268156/lxml-The-Python-XML-Toolkit">Flattr</a>.<br />'
 122         '<a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="http://lxml.de/"></a>'
 123         '</p>'
 124         ))
 125
 126
 127 def inject_donate_buttons(lxml_path, rst2html_script, tree):
 128     command = ([sys.executable, rst2html_script]
 129                + RST2HTML_OPTIONS.split() + [os.path.join(lxml_path, 'README.rst')])
 130     rst2html = subprocess.Popen(command, stdout=subprocess.PIPE)
 131     stdout, _ = rst2html.communicate()
 132     readme = fromstring(stdout)
 133
 134     intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]',
 135                            namespaces=htmlnsmap)[0]
 136     support_div = readme.xpath('h:body//h:div[@id = "support-the-project"][1]',
 137                                namespaces=htmlnsmap)[0]
 138     intro_div.append(support_div)
 139
 140     finance_div = readme.xpath('h:body//h:div[@id = "project-income-report"][1]',
 141                                namespaces=htmlnsmap)[0]
 142     legal = readme.xpath('h:body//h:div[@id = "legal-notice-for-donations"][1]',
 143                          namespaces=htmlnsmap)[0]
 144     last_div = tree.xpath('h:body//h:div//h:div', namespaces=htmlnsmap)[-1]
 145     last_div.addnext(finance_div)
 146     finance_div.addnext(legal)
 147
 148
 149 def rest2html(script, source_path, dest_path, stylesheet_url):
 150     command = ('%s %s %s --stylesheet=%s --link-stylesheet %s > %s' %
 151                (sys.executable, script, RST2HTML_OPTIONS,
 152                 stylesheet_url, source_path, dest_path))
 153     subprocess.call(command, shell=True)
 154
 155
 156 def convert_changelog(lxml_path, changelog_file_path, rst2html_script, stylesheet_url):
 157     f = open_file(os.path.join(lxml_path, 'CHANGES.txt'), 'r', encoding='utf-8')
 158     try:
 159         content = f.read()
 160     finally:
 161         f.close()
 162
 163     links = dict(LP='`%s <https://bugs.launchpad.net/lxml/+bug/%s>`_',
 164                  GH='`%s <https://github.com/lxml/lxml/issues/%s>`_')
 165     replace_tracker_links = re.compile('((LP|GH)#([0-9]+))').sub
 166     def insert_link(match):
 167         text, ref_type, ref_id = match.groups()
 168         return links[ref_type] % (text, ref_id)
 169     content = replace_tracker_links(insert_link, content)
 170
 171     command = [sys.executable, rst2html_script] + RST2HTML_OPTIONS.split() + [
 172         '--link-stylesheet', '--stylesheet', stylesheet_url ]
 173     out_file = open(changelog_file_path, 'wb')
 174     try:
 175         rst2html = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out_file)
 176         rst2html.communicate(content.encode('utf8'))
 177     finally:
 178         out_file.close()
 179
 180
 181 def publish(dirname, lxml_path, release):
 182     if not os.path.exists(dirname):
 183         os.mkdir(dirname)
 184
 185     doc_dir = os.path.join(lxml_path, 'doc')
 186     script = os.path.join(doc_dir, 'rest2html.py')
 187     pubkey = os.path.join(doc_dir, 'pubkey.asc')
 188     stylesheet_url = 'style.css'
 189
 190     shutil.copy(pubkey, dirname)
 191
 192     href_map = HREF_MAP.copy()
 193     changelog_basename = 'changes-%s' % release
 194     href_map['Release Changelog'] = changelog_basename + '.html'
 195
 196     menu_js = textwrap.dedent('''
 197     function trigger_menu(event) {
 198         var sidemenu = document.getElementById("sidemenu");
 199         var classes = sidemenu.getAttribute("class");
 200         classes = (classes.indexOf(" visible") === -1) ? classes + " visible" : classes.replace(" visible", "");
 201         sidemenu.setAttribute("class", classes);
 202         event.preventDefault();
 203         event.stopPropagation();
 204     }
 205     function hide_menu() {
 206         var sidemenu = document.getElementById("sidemenu");
 207         var classes = sidemenu.getAttribute("class");
 208         if (classes.indexOf(" visible") !== -1) {
 209             sidemenu.setAttribute("class", classes.replace(" visible", ""));
 210         }
 211     }
 212     ''')
 213
 214     trees = {}
 215     menu = Element("div", {'class': 'sidemenu', 'id': 'sidemenu'})
 216     SubElement(menu, 'div', {'class': 'menutrigger', 'onclick': 'trigger_menu(event)'}).text = "Menu"
 217     menu_div = SubElement(menu, 'div', {'class': 'menu'})
 218     # build HTML pages and parse them back
 219     for section, text_files in SITE_STRUCTURE:
 220         section_head = make_menu_section_head(section, menu_div)
 221         for filename in text_files:
 222             if filename.startswith('@'):
 223                 # special menu entry
 224                 page_title = filename[1:]
 225                 url = href_map[page_title]
 226                 build_menu_entry(page_title, url, section_head)
 227             else:
 228                 path = os.path.join(doc_dir, filename)
 229                 basename = os.path.splitext(os.path.basename(filename))[0]
 230                 basename = BASENAME_MAP.get(basename, basename)
 231                 outname = basename + '.html'
 232                 outpath = os.path.join(dirname, outname)
 233
 234                 rest2html(script, path, outpath, stylesheet_url)
 235                 tree = parse(outpath)
 236
 237                 if filename == 'main.txt':
 238                     # inject donation buttons
 239                     #inject_flatter_button(tree)
 240                     inject_donate_buttons(lxml_path, script, tree)
 241
 242                 trees[filename] = (tree, basename, outpath)
 243                 build_menu(tree, basename, section_head)
 244
 245     # also convert CHANGES.txt
 246     convert_changelog(lxml_path, os.path.join(dirname, 'changes-%s.html' % release),
 247                       script, stylesheet_url)
 248
 249     # generate sitemap from menu
 250     sitemap = XML(textwrap.dedent('''\
 251     <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 252     <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 253       <head>
 254         <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 255         <title>Sitemap of lxml.de - Processing XML and HTML with Python</title>
 256         <meta content="lxml - the most feature-rich and easy-to-use library for processing XML and HTML in the Python language"
 257               name="description" />
 258         <meta content="Python XML, XML, XML processing, HTML, lxml, simple XML, ElementTree, etree, lxml.etree, objectify, XML parsing, XML validation, XPath, XSLT"
 259               name="keywords" />
 260       </head>
 261       <body>
 262         <h1>Sitemap of lxml.de - Processing XML and HTML with Python</h1>
 263       </body>
 264     </html>
 265     '''))
 266     sitemap_menu = copy.deepcopy(menu)
 267     SubElement(SubElement(sitemap_menu[-1], 'li'), 'a', href='http://lxml.de/files/').text = 'Download files'
 268     sitemap[-1].append(sitemap_menu)  # append to body
 269     ElementTree(sitemap).write(os.path.join(dirname, 'sitemap.html'))
 270
 271     # integrate sitemap into the menu
 272     SubElement(SubElement(menu_div[-1], 'li'), 'a', href='/sitemap.html').text = 'Sitemap'
 273
 274     # integrate menu into web pages
 275     for tree, basename, outpath in trees.itervalues():
 276         head = find_head(tree)[0]
 277         SubElement(head, 'script', type='text/javascript').text = menu_js
 278         SubElement(head, 'meta', name='viewport', content="width=device-width, initial-scale=1")
 279         find_body(tree)[0].set('onclick', 'hide_menu()')
 280
 281         new_tree = merge_menu(tree, menu, basename)
 282         title = find_title_tag(new_tree)
 283         if title and title[0].text == 'lxml':
 284             title[0].text = "lxml - Processing XML and HTML with Python"
 285             heading = find_heading_tag(new_tree)
 286             if heading:
 287                 heading[0].text = "lxml - XML and HTML with Python"
 288         new_tree.write(outpath)
 289
 290
 291 if __name__ == '__main__':
 292     publish(sys.argv[1], sys.argv[2], sys.argv[3])