Add script for fetching third-party sources

This change adds external/fetch_sources.py that will initialize libpng
and zlib source directories by fetching and extracting source packages.

The fetched packages are always verified by doing a SHA256 comparison to
avoid corrupted or malicious downloads.

Bug: 18329517
Change-Id: I0e32b411ff0e3691ab8fa33e0ed30259faee48c8
(cherry picked from commit 60d68b2463d1d6e1f8d55c55a923d304adcd1bf8)

diff --git a/external/fetch_sources.py b/external/fetch_sources.py
new file mode 100644 (file)
index 0000000..a957a3f
--- /dev/null
+++ b/external/fetch_sources.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import shutil
+import tarfile
+import urllib2
+import hashlib
+import argparse
+
+EXTERNAL_DIR   = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
+
+class SourcePackage:
+       def __init__(self, url, filename, checksum, dstDir, postExtract=None):
+               self.url                        = url
+               self.filename           = filename
+               self.checksum           = checksum
+               self.dstDir                     = dstDir
+               self.postExtract        = postExtract
+
+def computeChecksum (data):
+       return hashlib.sha256(data).hexdigest()
+
+def clean (pkg):
+       srcPath = os.path.join(EXTERNAL_DIR, pkg.dstDir)
+
+       for entry in os.listdir(srcPath):
+               if entry == "CMakeLists.txt":
+                       continue
+
+               fullPath = os.path.join(srcPath, entry)
+
+               if os.path.isfile(fullPath):
+                       os.unlink(fullPath)
+               elif os.path.isdir(fullPath):
+                       shutil.rmtree(fullPath, ignore_errors=False)
+
+def fetch (pkg):
+       print "Fetching %s" % pkg.url
+
+       req                     = urllib2.urlopen(pkg.url)
+       data            = req.read()
+       checksum        = computeChecksum(data)
+       dstPath         = os.path.join(EXTERNAL_DIR, pkg.filename)
+
+       if checksum != pkg.checksum:
+               raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (pkg.filename, pkg.checksum, checksum))
+
+       out = open(dstPath, 'wb')
+       out.write(data)
+       out.close()
+
+def extract (pkg):
+       print "Extracting %s to %s" % (pkg.filename, pkg.dstDir)
+
+       srcPath = os.path.join(EXTERNAL_DIR, pkg.filename)
+       tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % pkg.dstDir)
+       dstPath = os.path.join(EXTERNAL_DIR, pkg.dstDir)
+       archive = tarfile.open(srcPath)
+
+       if os.path.exists(tmpPath):
+               shutil.rmtree(tmpPath, ignore_errors=False)
+
+       os.mkdir(tmpPath)
+
+       archive.extractall(tmpPath)
+       archive.close()
+
+       extractedEntries = os.listdir(tmpPath)
+       if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
+               raise Exception("%s doesn't contain single top-level directory") % pkg.filename
+
+       topLevelPath = os.path.join(tmpPath, extractedEntries[0])
+
+       for entry in os.listdir(topLevelPath):
+               if os.path.exists(os.path.join(dstPath, entry)):
+                       print "  skipping %s" % entry
+                       continue
+
+               shutil.move(os.path.join(topLevelPath, entry), dstPath)
+
+       shutil.rmtree(tmpPath, ignore_errors=True)
+
+       if pkg.postExtract != None:
+               pkg.postExtract(dstPath)
+
+def postExtractLibpng (path):
+       shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
+                               os.path.join(path, "pnglibconf.h"))
+
+PACKAGES = [
+       SourcePackage("http://zlib.net/zlib-1.2.8.tar.gz",
+                                 "zlib-1.2.8.tar.gz",
+                                 "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
+                                 "zlib"),
+       SourcePackage("http://www.imagemagick.org/download/delegates/libpng-1.6.14.tar.gz",
+                                 "libpng-1.6.14.tar.gz",
+                                 "e6cab38f051bfc66929e766c1e67eba6fafac9e0f463ee3bbeb4ac16f173fe8e",
+                                 "libpng",
+                                 postExtract = postExtractLibpng),
+]
+
+def parseArgs ():
+       parser = argparse.ArgumentParser(description = "Fetch external sources")
+       parser.add_argument('--clean-only', dest='cleanOnly', action='store_true', default=False,
+                                               help='Clean only, do not fetch/extract')
+       parser.add_argument('--keep-archive', dest='keepArchive', action='store_true', default=False,
+                                               help='Keep archive after extracting')
+       return parser.parse_args()
+
+if __name__ == "__main__":
+       args = parseArgs()
+
+       for pkg in PACKAGES:
+               clean(pkg)
+
+               if args.cleanOnly:
+                       continue
+
+               fetch(pkg)
+               extract(pkg)
+
+               if not args.keepArchive:
+                       os.unlink(os.path.join(EXTERNAL_DIR, pkg.filename))

diff --git a/external/libpng/CMakeLists.txt b/external/libpng/CMakeLists.txt
index f3d6c65..838f892 100644 (file)
--- a/external/libpng/CMakeLists.txt
+++ b/external/libpng/CMakeLists.txt
@@ -4,35 +4,48 @@ if (NOT DE_DEFS)
        message(FATAL_ERROR "Include Defs.cmake")
 endif ()
 
-set(PNG_SRC_PATH "../../../libpng" CACHE STRING "Path to libpng source tree")
+if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/png.h")
+       set(DEFAULT_PNG_SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+else ()
+       # Assume build inside Android source tree
+       set(DEFAULT_PNG_SRC_PATH "../libpng")
+endif ()
+
+set(PNG_SRC_PATH ${DEFAULT_PNG_SRC_PATH} CACHE STRING "Path to libpng source tree")
+
+if (IS_ABSOLUTE ${PNG_SRC_PATH})
+       set(PNG_ABS_PATH ${PNG_SRC_PATH})
+else ()
+       set(PNG_ABS_PATH "${CMAKE_SOURCE_DIR}/${PNG_SRC_PATH}")
+endif ()
 
 set(PNG_SRCS
-       ${PNG_SRC_PATH}/png.h
-       ${PNG_SRC_PATH}/pngconf.h
-       ${PNG_SRC_PATH}/png.c
-       ${PNG_SRC_PATH}/pngset.c
-       ${PNG_SRC_PATH}/pngget.c
-       ${PNG_SRC_PATH}/pngrutil.c
-       ${PNG_SRC_PATH}/pngtrans.c
-       ${PNG_SRC_PATH}/pngwutil.c
-       ${PNG_SRC_PATH}/pngread.c
-       ${PNG_SRC_PATH}/pngrio.c
-       ${PNG_SRC_PATH}/pngwio.c
-       ${PNG_SRC_PATH}/pngwrite.c
-       ${PNG_SRC_PATH}/pngrtran.c
-       ${PNG_SRC_PATH}/pngwtran.c
-       ${PNG_SRC_PATH}/pngmem.c
-       ${PNG_SRC_PATH}/pngerror.c
-       ${PNG_SRC_PATH}/pngpread.c
+       ${PNG_ABS_PATH}/png.h
+       ${PNG_ABS_PATH}/pngconf.h
+       ${PNG_ABS_PATH}/png.c
+       ${PNG_ABS_PATH}/pngset.c
+       ${PNG_ABS_PATH}/pngget.c
+       ${PNG_ABS_PATH}/pngrutil.c
+       ${PNG_ABS_PATH}/pngtrans.c
+       ${PNG_ABS_PATH}/pngwutil.c
+       ${PNG_ABS_PATH}/pngread.c
+       ${PNG_ABS_PATH}/pngrio.c
+       ${PNG_ABS_PATH}/pngwio.c
+       ${PNG_ABS_PATH}/pngwrite.c
+       ${PNG_ABS_PATH}/pngrtran.c
+       ${PNG_ABS_PATH}/pngwtran.c
+       ${PNG_ABS_PATH}/pngmem.c
+       ${PNG_ABS_PATH}/pngerror.c
+       ${PNG_ABS_PATH}/pngpread.c
        )
 
 if (DE_CPU_IS_ARM_64)
        # armv8-a has always neon
        set(PNG_SRCS
                ${PNG_SRCS}
-               ${PNG_SRC_PATH}/arm/arm_init.c
-               ${PNG_SRC_PATH}/arm/filter_neon_intrinsics.c
-               ${PNG_SRC_PATH}/arm/filter_neon.S
+               ${PNG_ABS_PATH}/arm/arm_init.c
+               ${PNG_ABS_PATH}/arm/filter_neon_intrinsics.c
+               ${PNG_ABS_PATH}/arm/filter_neon.S
                )
 endif ()
 
@@ -50,10 +63,5 @@ set(CMAKE_C_FLAGS ${DE_3RD_PARTY_C_FLAGS})
 add_library(png STATIC ${PNG_SRCS})
 target_link_libraries(png ${ZLIB_LIBRARY})
 
-if (IS_ABSOLUTE ${PNG_SRC_PATH})
-       set(PNG_INCLUDE_PATH ${PNG_SRC_PATH} PARENT_SCOPE)
-else ()
-       set(PNG_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/${PNG_SRC_PATH}" PARENT_SCOPE)
-endif ()
-
-set(PNG_LIBRARY png PARENT_SCOPE)
+set(PNG_INCLUDE_PATH   ${PNG_ABS_PATH} PARENT_SCOPE)
+set(PNG_LIBRARY                        png                             PARENT_SCOPE)

diff --git a/external/zlib/CMakeLists.txt b/external/zlib/CMakeLists.txt
index f5d41b8..53c257a 100644 (file)
--- a/external/zlib/CMakeLists.txt
+++ b/external/zlib/CMakeLists.txt
@@ -4,35 +4,48 @@ if (NOT DE_DEFS)
        message(FATAL_ERROR "Include Defs.cmake")
 endif ()
 
-set(ZLIB_SRC_PATH "../../../zlib/src" CACHE STRING "Path to zlib source tree")
+if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/zlib.h")
+       set(DEFAULT_ZLIB_SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+else ()
+       # Assume build inside Android source tree
+       set(DEFAULT_ZLIB_SRC_PATH "../zlib/src")
+endif ()
+
+set(ZLIB_SRC_PATH ${DEFAULT_ZLIB_SRC_PATH} CACHE STRING "Path to zlib source tree")
+
+if (IS_ABSOLUTE ${ZLIB_SRC_PATH})
+       set(ZLIB_ABS_PATH ${ZLIB_SRC_PATH})
+else ()
+       set(ZLIB_ABS_PATH "${CMAKE_SOURCE_DIR}/${ZLIB_SRC_PATH}")
+endif ()
 
 set(ZLIB_SRCS
-       ${ZLIB_SRC_PATH}/adler32.c
-       ${ZLIB_SRC_PATH}/compress.c
-       ${ZLIB_SRC_PATH}/crc32.c
-       ${ZLIB_SRC_PATH}/crc32.h
-       ${ZLIB_SRC_PATH}/deflate.c
-       ${ZLIB_SRC_PATH}/deflate.h
-       ${ZLIB_SRC_PATH}/gzclose.c
-       ${ZLIB_SRC_PATH}/gzguts.h
-       ${ZLIB_SRC_PATH}/gzlib.c
-       ${ZLIB_SRC_PATH}/gzread.c
-       ${ZLIB_SRC_PATH}/gzwrite.c
-       ${ZLIB_SRC_PATH}/infback.c
-       ${ZLIB_SRC_PATH}/inffast.c
-       ${ZLIB_SRC_PATH}/inffast.h
-       ${ZLIB_SRC_PATH}/inffixed.h
-       ${ZLIB_SRC_PATH}/inflate.c
-       ${ZLIB_SRC_PATH}/inflate.h
-       ${ZLIB_SRC_PATH}/inftrees.c
-       ${ZLIB_SRC_PATH}/inftrees.h
-       ${ZLIB_SRC_PATH}/trees.c
-       ${ZLIB_SRC_PATH}/trees.h
-       ${ZLIB_SRC_PATH}/uncompr.c
-       ${ZLIB_SRC_PATH}/zconf.h
-       ${ZLIB_SRC_PATH}/zlib.h
-       ${ZLIB_SRC_PATH}/zutil.c
-       ${ZLIB_SRC_PATH}/zutil.h
+       ${ZLIB_ABS_PATH}/adler32.c
+       ${ZLIB_ABS_PATH}/compress.c
+       ${ZLIB_ABS_PATH}/crc32.c
+       ${ZLIB_ABS_PATH}/crc32.h
+       ${ZLIB_ABS_PATH}/deflate.c
+       ${ZLIB_ABS_PATH}/deflate.h
+       ${ZLIB_ABS_PATH}/gzclose.c
+       ${ZLIB_ABS_PATH}/gzguts.h
+       ${ZLIB_ABS_PATH}/gzlib.c
+       ${ZLIB_ABS_PATH}/gzread.c
+       ${ZLIB_ABS_PATH}/gzwrite.c
+       ${ZLIB_ABS_PATH}/infback.c
+       ${ZLIB_ABS_PATH}/inffast.c
+       ${ZLIB_ABS_PATH}/inffast.h
+       ${ZLIB_ABS_PATH}/inffixed.h
+       ${ZLIB_ABS_PATH}/inflate.c
+       ${ZLIB_ABS_PATH}/inflate.h
+       ${ZLIB_ABS_PATH}/inftrees.c
+       ${ZLIB_ABS_PATH}/inftrees.h
+       ${ZLIB_ABS_PATH}/trees.c
+       ${ZLIB_ABS_PATH}/trees.h
+       ${ZLIB_ABS_PATH}/uncompr.c
+       ${ZLIB_ABS_PATH}/zconf.h
+       ${ZLIB_ABS_PATH}/zlib.h
+       ${ZLIB_ABS_PATH}/zutil.c
+       ${ZLIB_ABS_PATH}/zutil.h
        )
 
 if (DE_COMPILER_IS_MSC)
@@ -41,6 +54,10 @@ if (DE_COMPILER_IS_MSC)
        # 4242, 4244: conversion from 'x' to 'y', possible loss of data
        # 4996: deprecated POSIX name
        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Wall /wd4127 /wd4131 /wd4242 /wd4244 /wd4996")
+
+elseif (DE_COMPILER_IS_GCC OR DE_COMPILER_IS_CLANG)
+       set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-implicit-function-declaration")
+
 endif ()
 
 if (DE_OS_IS_UNIX)
@@ -49,10 +66,5 @@ endif ()
 
 add_library(z STATIC ${ZLIB_SRCS})
 
-if (IS_ABSOLUTE ${ZLIB_SRC_PATH})
-       set(ZLIB_INCLUDE_PATH ${ZLIB_SRC_PATH} PARENT_SCOPE)
-else ()
-       set(ZLIB_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/${ZLIB_SRC_PATH}" PARENT_SCOPE)
-endif ()
-
-set(ZLIB_LIBRARY z PARENT_SCOPE)
+set(ZLIB_INCLUDE_PATH  ${ZLIB_ABS_PATH}        PARENT_SCOPE)
+set(ZLIB_LIBRARY               z                                       PARENT_SCOPE)