1 import os, re, sys, subprocess
3 from distutils import log, version
4 from contextlib import closing
8 from urlparse import urljoin, unquote, urlparse
9 from urllib import urlretrieve, urlopen, urlcleanup
11 from urllib.parse import urljoin, unquote, urlparse
12 from urllib.request import urlretrieve, urlopen, urlcleanup
14 multi_make_options = []
16 import multiprocessing
17 cpus = multiprocessing.cpu_count()
21 multi_make_options = ['-j%d' % (cpus+1)]
26 # use pre-built libraries on Windows
28 def download_and_extract_windows_binaries(destdir):
29 url = "https://github.com/mhils/libxml2-win-binaries/releases"
30 filenames = list(_list_dir_urllib(url))
32 release_path = "/download/%s/" % find_max_version(
33 "library release", filenames, re.compile(r"/releases/tag/([0-9.]+[0-9])$"))
36 filename.rsplit('/', 1)[1]
37 for filename in filenames
38 if release_path in filename
41 arch = "win64" if sys.maxsize > 2**32 else "win32"
42 if sys.version_info < (3, 5):
43 arch = 'vs2008.' + arch
46 for libname in ['libxml2', 'libxslt', 'zlib', 'iconv']:
47 libs[libname] = "%s-%s.%s.zip" % (
49 find_max_version(libname, filenames),
53 if not os.path.exists(destdir):
56 for libname, libfn in libs.items():
57 srcfile = urljoin(url, libfn)
58 destfile = os.path.join(destdir, libfn)
59 if os.path.exists(destfile + ".keep"):
60 print('Using local copy of "{}"'.format(srcfile))
62 print('Retrieving "%s" to "%s"' % (srcfile, destfile))
63 urlcleanup() # work around FTP bug 27973 in Py2.7.12+
64 urlretrieve(srcfile, destfile)
65 d = unpack_zipfile(destfile, destdir)
71 def find_top_dir_of_zipfile(zipfile):
73 files = [f.filename for f in zipfile.filelist]
74 dirs = [d for d in files if d.endswith('/')]
78 topdir = topdir[:topdir.index("/")+1]
80 if not path.startswith(topdir):
84 "cannot determine single top-level directory in zip file %s" %
86 return topdir.rstrip('/')
89 def unpack_zipfile(zipfn, destdir):
90 assert zipfn.endswith('.zip')
92 print('Unpacking %s into %s' % (os.path.basename(zipfn), destdir))
93 f = zipfile.ZipFile(zipfn)
95 extracted_dir = os.path.join(destdir, find_top_dir_of_zipfile(f))
96 f.extractall(path=destdir)
99 assert os.path.exists(extracted_dir), 'missing: %s' % extracted_dir
103 def get_prebuilt_libxml2xslt(download_dir, static_include_dirs, static_library_dirs):
104 assert sys.platform.startswith('win')
105 libs = download_and_extract_windows_binaries(download_dir)
106 for libname, path in libs.items():
107 i = os.path.join(path, 'include')
108 l = os.path.join(path, 'lib')
109 assert os.path.exists(i), 'does not exist: %s' % i
110 assert os.path.exists(l), 'does not exist: %s' % l
111 static_include_dirs.append(i)
112 static_library_dirs.append(l)
115 ## Routines to download and build libxml2/xslt from sources:
117 LIBXML2_LOCATION = 'http://xmlsoft.org/sources/'
118 LIBICONV_LOCATION = 'https://ftp.gnu.org/pub/gnu/libiconv/'
119 ZLIB_LOCATION = 'https://zlib.net/'
120 match_libfile_version = re.compile('^[^-]*-([.0-9-]+)[.].*').match
123 def _find_content_encoding(response, default='iso8859-1'):
124 from email.message import Message
125 content_type = response.headers.get('Content-Type')
128 msg.add_header('Content-Type', content_type)
129 charset = msg.get_content_charset(default)
135 def remote_listdir(url):
137 return _list_dir_urllib(url)
139 assert url.lower().startswith('ftp://')
140 print("Requesting with urllib failed. Falling back to ftplib. "
141 "Proxy argument will be ignored for %s" % url)
142 return _list_dir_ftplib(url)
145 def _list_dir_ftplib(url):
146 parts = urlparse(url)
147 ftp = FTP(parts.netloc)
155 return parse_text_ftplist("\n".join(data))
158 def _list_dir_urllib(url):
159 with closing(urlopen(url)) as res:
160 charset = _find_content_encoding(res)
161 content_type = res.headers.get('Content-Type')
164 data = data.decode(charset)
165 if content_type and content_type.startswith('text/html'):
166 files = parse_html_filelist(data)
168 files = parse_text_ftplist(data)
172 def http_listfiles(url, re_pattern):
173 with closing(urlopen(url)) as res:
174 charset = _find_content_encoding(res)
176 files = re.findall(re_pattern, data.decode(charset))
180 def parse_text_ftplist(s):
181 for line in s.splitlines():
182 if not line.startswith('d'):
183 # -rw-r--r-- 1 ftp ftp 476 Sep 1 2011 md5sum.txt
184 # Last (9th) element is 'md5sum.txt' in the above example, but there
185 # may be variations, so we discard only the first 8 entries.
186 yield line.split(None, 8)[-1]
189 def parse_html_filelist(s):
190 re_href = re.compile(
191 r'<a\s+(?:[^>]*\s+)?href=["\']([^;?"\']+?)[;?"\']',
193 links = set(re_href.findall(s))
195 if not link.endswith('/'):
206 def download_libxml2(dest_dir, version=None):
207 """Downloads libxml2, returning the filename where the library was downloaded"""
208 #version_re = re.compile(r'LATEST_LIBXML2_IS_([0-9.]+[0-9](?:-[abrc0-9]+)?)')
209 version_re = re.compile(r'libxml2-([0-9.]+[0-9]).tar.gz')
210 filename = 'libxml2-%s.tar.gz'
211 return download_library(dest_dir, LIBXML2_LOCATION, 'libxml2',
212 version_re, filename, version=version)
215 def download_libxslt(dest_dir, version=None):
216 """Downloads libxslt, returning the filename where the library was downloaded"""
217 #version_re = re.compile(r'LATEST_LIBXSLT_IS_([0-9.]+[0-9](?:-[abrc0-9]+)?)')
218 version_re = re.compile(r'libxslt-([0-9.]+[0-9]).tar.gz')
219 filename = 'libxslt-%s.tar.gz'
220 return download_library(dest_dir, LIBXML2_LOCATION, 'libxslt',
221 version_re, filename, version=version)
224 def download_libiconv(dest_dir, version=None):
225 """Downloads libiconv, returning the filename where the library was downloaded"""
226 version_re = re.compile(r'libiconv-([0-9.]+[0-9]).tar.gz')
227 filename = 'libiconv-%s.tar.gz'
228 return download_library(dest_dir, LIBICONV_LOCATION, 'libiconv',
229 version_re, filename, version=version)
232 def download_zlib(dest_dir, version):
233 """Downloads zlib, returning the filename where the library was downloaded"""
234 version_re = re.compile(r'zlib-([0-9.]+[0-9]).tar.gz')
235 filename = 'zlib-%s.tar.gz'
236 return download_library(dest_dir, ZLIB_LOCATION, 'zlib',
237 version_re, filename, version=version)
240 def find_max_version(libname, filenames, version_re=None):
241 if version_re is None:
242 version_re = re.compile(r'%s-([0-9.]+[0-9](?:-[abrc0-9]+)?)' % libname)
245 match = version_re.search(fn)
247 version_string = match.group(1)
248 versions.append((tuple(map(tryint, version_string.split('.'))),
252 "Could not find the most current version of %s from the files: %s" % (
255 version_string = versions[-1][-1]
256 print('Latest version of %s is %s' % (libname, version_string))
257 return version_string
260 def download_library(dest_dir, location, name, version_re, filename, version=None):
263 if location.startswith('ftp://'):
264 fns = remote_listdir(location)
266 fns = http_listfiles(location, '(%s)' % filename.replace('%s', '(?:[0-9.]+[0-9])'))
267 version = find_max_version(name, fns, version_re)
269 # network failure - maybe we have the files already?
271 fns = os.listdir(dest_dir)
273 if fn.startswith(name+'-'):
274 match = match_libfile_version(fn)
276 version_tuple = tuple(map(tryint, match.group(1).split('.')))
277 if version_tuple > latest:
278 latest = version_tuple
281 if latest == (0,0,0):
284 filename = filename % version
285 full_url = urljoin(location, filename)
286 dest_filename = os.path.join(dest_dir, filename)
287 if os.path.exists(dest_filename):
288 print(('Using existing %s downloaded into %s '
289 '(delete this file if you want to re-download the package)') % (
290 name, dest_filename))
292 print('Downloading %s into %s from %s' % (name, dest_filename, full_url))
293 urlcleanup() # work around FTP bug 27973 in Py2.7.12
294 urlretrieve(full_url, dest_filename)
298 def unpack_tarball(tar_filename, dest):
299 print('Unpacking %s into %s' % (os.path.basename(tar_filename), dest))
300 tar = tarfile.open(tar_filename)
303 base_name = member.name.split('/')[0]
306 elif base_dir != base_name:
307 print('Unexpected path in %s: %s' % (tar_filename, base_name))
310 return os.path.join(dest, base_dir)
313 def call_subprocess(cmd, **kw):
315 cwd = kw.get('cwd', '.')
316 cmd_desc = ' '.join(cmd)
317 log.info('Running "%s" in %s' % (cmd_desc, cwd))
318 returncode = subprocess.call(cmd, **kw)
320 raise Exception('Command "%s" returned code %s' % (cmd_desc, returncode))
324 if not os.path.exists(dir):
328 def cmmi(configure_cmd, build_dir, multicore=None, **call_setup):
329 print('Starting build in %s' % build_dir)
330 call_subprocess(configure_cmd, cwd=build_dir, **call_setup)
332 make_jobs = multi_make_options
333 elif int(multicore) > 1:
334 make_jobs = ['-j%s' % multicore]
338 ['make'] + make_jobs,
339 cwd=build_dir, **call_setup)
341 ['make'] + make_jobs + ['install'],
342 cwd=build_dir, **call_setup)
345 def configure_darwin_env(env_setup):
347 # check target architectures on MacOS-X (ppc, i386, x86_64)
348 major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2]))
349 if major_version > 7:
350 # Check to see if ppc is supported (XCode4 drops ppc support)
352 if os.path.exists('/usr/bin/xcodebuild'):
353 pipe = subprocess.Popen(['/usr/bin/xcodebuild', '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
354 out, _ = pipe.communicate()
355 xcode_version = (out.decode('utf8').splitlines() or [''])[0]
356 # Also parse only first digit, because 3.2.1 can't be parsed nicely
357 if (xcode_version.startswith('Xcode') and
358 version.StrictVersion(xcode_version.split()[1]) >= version.StrictVersion('4.0')):
362 arch_string = "-arch ppc "
363 if minor_version < 6:
365 'CFLAGS': arch_string + "-arch i386 -isysroot /Developer/SDKs/MacOSX10.4u.sdk -O2",
366 'LDFLAGS': arch_string + "-arch i386 -isysroot /Developer/SDKs/MacOSX10.4u.sdk",
367 'MACOSX_DEPLOYMENT_TARGET': "10.3"
371 'CFLAGS': arch_string + "-arch i386 -arch x86_64 -O2",
372 'LDFLAGS': arch_string + "-arch i386 -arch x86_64",
373 'MACOSX_DEPLOYMENT_TARGET': "10.6"
375 env = os.environ.copy()
376 env_default.update(env)
377 env_setup['env'] = env_default
380 def build_libxml2xslt(download_dir, build_dir,
381 static_include_dirs, static_library_dirs,
382 static_cflags, static_binaries,
383 libxml2_version=None,
384 libxslt_version=None,
385 libiconv_version=None,
388 safe_mkdir(download_dir)
389 safe_mkdir(build_dir)
390 zlib_dir = unpack_tarball(download_zlib(download_dir, zlib_version), build_dir)
391 libiconv_dir = unpack_tarball(download_libiconv(download_dir, libiconv_version), build_dir)
392 libxml2_dir = unpack_tarball(download_libxml2(download_dir, libxml2_version), build_dir)
393 libxslt_dir = unpack_tarball(download_libxslt(download_dir, libxslt_version), build_dir)
394 prefix = os.path.join(os.path.abspath(build_dir), 'libxml2')
398 if sys.platform == 'darwin':
399 configure_darwin_env(call_setup)
401 configure_cmd = ['./configure',
402 '--disable-dependency-tracking',
404 '--prefix=%s' % prefix,
408 zlib_configure_cmd = [
410 '--prefix=%s' % prefix,
412 cmmi(zlib_configure_cmd, zlib_dir, multicore, **call_setup)
415 cmmi(configure_cmd, libiconv_dir, multicore, **call_setup)
418 libxml2_configure_cmd = configure_cmd + [
420 '--with-iconv=%s' % prefix,
421 '--with-zlib=%s' % prefix,
424 if not libxml2_version:
425 libxml2_version = os.path.basename(libxml2_dir).split('-', 1)[-1]
427 if tuple(map(tryint, libxml2_version.split('-', 1)[0].split('.'))) >= (2, 9, 5):
428 libxml2_configure_cmd.append('--without-lzma') # can't currently build that
431 if tuple(map(tryint, libxml2_version.split('-', 1)[0].split('.'))) >= (2, 7, 3):
432 libxml2_configure_cmd.append('--enable-rebuild-docs=no')
434 pass # this isn't required, so ignore any errors
435 cmmi(libxml2_configure_cmd, libxml2_dir, multicore, **call_setup)
438 libxslt_configure_cmd = configure_cmd + [
440 '--with-libxml-prefix=%s' % prefix,
443 cmmi(libxslt_configure_cmd, libxslt_dir, multicore, **call_setup)
445 # collect build setup for lxml
446 xslt_config = os.path.join(prefix, 'bin', 'xslt-config')
447 xml2_config = os.path.join(prefix, 'bin', 'xml2-config')
449 lib_dir = os.path.join(prefix, 'lib')
450 static_include_dirs.extend([
451 os.path.join(prefix, 'include'),
452 os.path.join(prefix, 'include', 'libxml2'),
453 os.path.join(prefix, 'include', 'libxslt'),
454 os.path.join(prefix, 'include', 'libexslt')])
455 static_library_dirs.append(lib_dir)
457 listdir = os.listdir(lib_dir)
458 static_binaries += [os.path.join(lib_dir, filename)
459 for lib in ['libxml2', 'libexslt', 'libxslt', 'iconv', 'libz']
460 for filename in listdir
461 if lib in filename and filename.endswith('.a')]
463 return xml2_config, xslt_config