1 # -*- coding: utf-8 -*-
3 ############################################################################
6 # This file is a SCons (http://www.scons.org/) builder #
7 # Copyright (c) 2012-14, Philipp Kraus, <philipp.kraus@flashpixx.de> #
8 # Copyright 2014 Intel Mobile Communications GmbH All Rights Reserved. #
9 # This program is free software: you can redistribute it and/or modify #
10 # it under the terms of the GNU General Public License as #
11 # published by the Free Software Foundation, either version 3 of the #
12 # License, or (at your option) any later version. #
14 # This program is distributed in the hope that it will be useful, #
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
17 # GNU General Public License for more details. #
19 # You should have received a copy of the GNU General Public License #
20 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
21 ############################################################################
23 # This builder originated from work by Philipp Kraus and flashpixx project
24 # (see https://github.com/flashpixx). Based on the Unpack.py, it only
25 # contains changes to allow a complete unpacking of the archive.
26 # It is assumed that the target represents a file in the archive after it
29 # The Unpack Builder can be used for unpacking archives (eg Zip, TGZ, BZ, ... ).
30 # The emitter of the Builder reads the archive data and creates a returning
31 # file list the builder extract the archive. The environment variable
32 # stores a dictionary "UNPACK" for set different extractions (subdict "EXTRACTOR"):
34 # PRIORITY => a value for setting the extractor order (lower numbers = extractor is used earlier)
35 # SUFFIX => defines a list with file suffixes, which should be handled with this extractor
36 # EXTRACTSUFFIX => suffix of the extract command
37 # EXTRACTFLAGS => a string parameter for the RUN command for extracting the data
38 # EXTRACTCMD => full extract command of the builder
39 # RUN => the main program which will be started (if the parameter is empty, the extractor will be ignored)
40 # LISTCMD => the listing command for the emitter
41 # LISTFLAGS => the string options for the RUN command for showing a list of files
42 # LISTSUFFIX => suffix of the list command
43 # LISTEXTRACTOR => a optional Python function, that is called on each output line of the
44 # LISTCMD for extracting file & dir names, the function need two parameters (first line number,
45 # second line content) and must return a string with the file / dir path (other value types
48 # Other options in the UNPACK dictionary are:
49 # STOPONEMPTYFILE => bool variable for stoping if the file has empty size (default True)
50 # VIWEXTRACTOUTPUT => shows the output messages of the extraction command (default False)
51 # EXTRACTDIR => path in that the data will be extracted (default #)
53 # The file which is handled by the first suffix match of the extractor, the extractor list can be append for other files.
54 # The order of the extractor dictionary creates the listing & extractor command eg file extension .tar.gz should be
55 # before .gz, because the tar.gz is extract in one shoot.
57 # Under *nix system these tools are supported: tar, bzip2, gzip, unzip
58 # Under Windows only 7-Zip (http://www.7-zip.org/) is supported
62 import SCons.Errors, SCons.Warnings, SCons.Util
64 # enables Scons warning for this builder
65 class UnpackWarning(SCons.Warnings.Warning) :
68 SCons.Warnings.enableWarningClass(UnpackWarning)
70 # extractor function for Tar output
71 # @param env environment object
72 # @param count number of returning lines
73 # @param no number of the output line
74 # @param i line content
75 def __fileextractor_nix_tar( env, count, no, i ) :
78 # extractor function for GZip output,
79 # ignore the first line
80 # @param env environment object
81 # @param count number of returning lines
82 # @param no number of the output line
83 # @param i line content
84 def __fileextractor_nix_gzip( env, count, no, i ) :
89 # extractor function for Unzip output,
90 # ignore the first & last two lines
91 # @param env environment object
92 # @param count number of returning lines
93 # @param no number of the output line
94 # @param i line content
95 def __fileextractor_nix_unzip( env, count, no, i ) :
96 if no < 3 or no >= count - 2 :
100 # extractor function for 7-Zip
101 # @param env environment object
102 # @param count number of returning lines
103 # @param no number of the output line
104 # @param i line content
105 def __fileextractor_win_7zip( env, count, no, i ) :
107 if no > 8 and no < count - 2 :
113 # returns the extractor item for handling the source file
114 # @param source input source file
115 # @param env environment object
116 # @return extractor entry or None on non existing
117 def __getExtractor( source, env ) :
118 # we check each unpacker and get the correct list command first, run the command and
119 # replace the target filelist with the list values, we sorte the extractors by their priority
120 for unpackername, extractor in sorted(env["UNPACK"]["EXTRACTOR"].iteritems(), key = lambda (k,v) : (v["PRIORITY"],k)):
122 if not SCons.Util.is_String(extractor["RUN"]) :
123 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
124 if not len(extractor["RUN"]) :
125 raise SCons.Errors.StopError("run command of the unpack builder for [%s] archives is not set - can not extract files" % (unpackername))
128 if not SCons.Util.is_String(extractor["LISTFLAGS"]) :
129 raise SCons.Errors.StopError("list flags of the unpack builder for [%s] archives is not a string" % (unpackername))
130 if not SCons.Util.is_String(extractor["LISTCMD"]) :
131 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
133 if not SCons.Util.is_String(extractor["EXTRACTFLAGS"]) :
134 raise SCons.Errors.StopError("extract flags of the unpack builder for [%s] archives is not a string" % (unpackername))
135 if not SCons.Util.is_String(extractor["EXTRACTCMD"]) :
136 raise SCons.Errors.StopError("extract command of the unpack builder for [%s] archives is not a string" % (unpackername))
139 # check the source file suffix and if the first is found, run the list command
140 if not SCons.Util.is_List(extractor["SUFFIX"]) :
141 raise SCons.Errors.StopError("suffix list of the unpack builder for [%s] archives is not a list" % (unpackername))
143 for suffix in extractor["SUFFIX"] :
144 if str(source[0]).lower()[-len(suffix):] == suffix.lower() :
150 # creates the extracter output message
151 # @param s original message
152 # @param target target name
153 # @param source source name
154 # @param env environment object
155 def __message( s, target, source, env ) :
156 print "extract [%s] ..." % (source[0])
159 # action function for extracting of the data
160 # @param target target packed file
161 # @param source extracted files
162 # @param env environment object
163 def __action( target, source, env ) :
164 extractor = __getExtractor([File(source)], env)
166 raise SCons.Errors.StopError( "can not find any extractor value for the source file [%s]" % (source) )
168 extractor_cmd = extractor["EXTRACTCMD"]
170 # if the extract command is empty, we create an error
171 if len(extractor_cmd) == 0 :
172 raise SCons.Errors.StopError( "the extractor command for the source file [%s] is empty" % (source) )
174 # build it now (we need the shell, because some programs need it)
177 cmd = env.subst(extractor_cmd, source=source, target=target)
178 cwd = os.path.realpath('.')
180 if env["UNPACK"]["VIWEXTRACTOUTPUT"] :
181 handle = subprocess.Popen( cmd, shell=True )
183 devnull = open(os.devnull, "wb")
184 handle = subprocess.Popen(cmd, shell=True, stdout=devnull, cwd = cwd)
186 if handle.wait() <> 0 :
187 print '''******************************* Error *****************************************
189 * Fail to unpack (%s). It should be due to it isn't downloaded completely.
190 * Please download it manually or delete it and let the script auto
193 *******************************************************************************
195 raise SCons.Errors.BuildError( "error running extractor [%s] on the source [%s]" % (cmd, source) )
197 # emitter function for getting the files
199 # @param target target packed file
200 # @param source extracted files
201 # @param env environment object
202 def __emitter( target, source, env ) :
203 return target, source
205 def __unpack_all(env, target, source) :
206 if os.path.exists(target):
209 print "Unpacking %s ..." % source
210 __action(target, source, env)
212 # generate function, that adds the builder to the environment
213 # @param env environment object
214 def generate( env ) :
215 # setup environment variable
217 "STOPONEMPTYFILE" : True,
218 "VIWEXTRACTOUTPUT" : False,
219 "EXTRACTDIR" : os.curdir,
223 "SUFFIX" : [".tar.gz", ".tgz", ".tar.gzip"],
224 "EXTRACTSUFFIX" : "",
226 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTSUFFIX']}",
228 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['LISTSUFFIX']}",
231 "LISTEXTRACTOR" : None
236 "SUFFIX" : [".tar.bz", ".tbz", ".tar.bz2", ".tar.bzip2", ".tar.bzip"],
237 "EXTRACTSUFFIX" : "",
239 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTSUFFIX']}",
241 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['LISTSUFFIX']}",
244 "LISTEXTRACTOR" : None
249 "SUFFIX" : [".bz", "bzip", ".bz2", ".bzip2"],
250 "EXTRACTSUFFIX" : "",
252 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTSUFFIX']}",
254 "LISTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['LISTSUFFIX']}",
257 "LISTEXTRACTOR" : None
262 "SUFFIX" : [".gz", ".gzip"],
263 "EXTRACTSUFFIX" : "",
265 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTSUFFIX']}",
267 "LISTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['LISTSUFFIX']}",
270 "LISTEXTRACTOR" : None
276 "EXTRACTSUFFIX" : "",
278 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['EXTRACTSUFFIX']}",
280 "LISTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['LISTSUFFIX']}",
283 "LISTEXTRACTOR" : None
289 "EXTRACTSUFFIX" : "",
291 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTSUFFIX']}",
293 "LISTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['LISTSUFFIX']}",
296 "LISTEXTRACTOR" : None
301 # read tools for Windows system
302 if env["PLATFORM"] <> "darwin" and "win" in env["PLATFORM"] :
304 if env.WhereIs("7z") :
305 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "7z"
306 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
307 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "x"
308 toolset["EXTRACTOR"]["TARGZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
309 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "x"
310 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
312 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "7z"
313 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
314 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "x"
315 toolset["EXTRACTOR"]["TARBZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
316 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "x"
317 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
319 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "7z"
320 toolset["EXTRACTOR"]["BZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
321 toolset["EXTRACTOR"]["BZIP"]["LISTFLAGS"] = "l"
322 toolset["EXTRACTOR"]["BZIP"]["LISTSUFFIX"] = "-y -so"
323 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "x"
324 toolset["EXTRACTOR"]["BZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
326 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "7z"
327 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
328 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "l"
329 toolset["EXTRACTOR"]["GZIP"]["LISTSUFFIX"] = "-y -so"
330 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "x"
331 toolset["EXTRACTOR"]["GZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
333 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "7z"
334 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
335 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "l"
336 toolset["EXTRACTOR"]["ZIP"]["LISTSUFFIX"] = "-y -so"
337 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "x"
338 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
340 toolset["EXTRACTOR"]["TAR"]["RUN"] = "7z"
341 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
342 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "l"
343 toolset["EXTRACTOR"]["TAR"]["LISTSUFFIX"] = "-y -ttar -so"
344 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "x"
345 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-y -ttar -oc:${UNPACK['EXTRACTDIR']}"
347 # here can add some other Windows tools, that can handle the archive files
348 # but I don't know which ones can handle all file types
352 # read the tools on *nix systems and sets the default parameters
353 elif env["PLATFORM"] in ["darwin", "linux", "posix"] :
355 if env.WhereIs("unzip") :
356 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "unzip"
357 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_unzip
358 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "-l"
359 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "-oqq"
360 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-d ${UNPACK['EXTRACTDIR']}"
362 if env.WhereIs("tar") :
363 toolset["EXTRACTOR"]["TAR"]["RUN"] = "tar"
364 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
365 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "tvf"
366 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "xf"
367 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
369 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "tar"
370 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
371 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "xfz"
372 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "tvfz"
373 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
375 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "tar"
376 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
377 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "xfj"
378 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "tvfj"
379 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
381 if env.WhereIs("bzip2") :
382 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "bzip2"
383 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "-df"
385 if env.WhereIs("gzip") :
386 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "gzip"
387 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_gzip
388 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "-l"
389 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "-df"
392 raise SCons.Errors.StopError("Unpack tool detection on this platform [%s] unkown" % (env["PLATFORM"]))
394 # the target_factory must be a "Entry", because the target list can be files and dirs, so we can not specified the targetfactory explicite
395 env.Replace(UNPACK = toolset)
396 env.AddMethod(__unpack_all, 'UnpackAll')
398 # env["BUILDERS"]["UnpackAll"] = SCons.Builder.Builder( action = __action, emitter = __emitter, target_factory = SCons.Node.FS.Entry, source_factory = SCons.Node.FS.File, single_source = True, PRINT_CMD_LINE_FUNC = __message )
401 # existing function of the builder
402 # @param env environment object