1 # -*- coding: utf-8 -*-
3 ############################################################################
6 # This file is a SCons (http://www.scons.org/) builder #
7 # Copyright (c) 2012-14, Philipp Kraus, <philipp.kraus@flashpixx.de> #
8 # Copyright 2014 Intel Mobile Communications GmbH All Rights Reserved. #
9 # This program is free software: you can redistribute it and/or modify #
10 # it under the terms of the GNU General Public License as #
11 # published by the Free Software Foundation, either version 3 of the #
12 # License, or (at your option) any later version. #
14 # This program is distributed in the hope that it will be useful, #
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
17 # GNU General Public License for more details. #
19 # You should have received a copy of the GNU General Public License #
20 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
21 ############################################################################
23 # This builder originated from work by Philipp Kraus and flashpixx project
24 # (see https://github.com/flashpixx). Based on the Unpack.py, it only
25 # contains changes to allow a complete unpacking of the archive.
26 # It is assumed that the target represents a file in the archive after it
29 # The Unpack Builder can be used for unpacking archives (eg Zip, TGZ, BZ, ... ).
30 # The emitter of the Builder reads the archive data and creates a returning
31 # file list the builder extract the archive. The environment variable
32 # stores a dictionary "UNPACK" for set different extractions (subdict "EXTRACTOR"):
34 # PRIORITY => a value for setting the extractor order (lower numbers = extractor is used earlier)
35 # SUFFIX => defines a list with file suffixes, which should be handled with this extractor
36 # EXTRACTSUFFIX => suffix of the extract command
37 # EXTRACTFLAGS => a string parameter for the RUN command for extracting the data
38 # EXTRACTCMD => full extract command of the builder
39 # RUN => the main program which will be started (if the parameter is empty, the extractor will be ignored)
40 # LISTCMD => the listing command for the emitter
41 # LISTFLAGS => the string options for the RUN command for showing a list of files
42 # LISTSUFFIX => suffix of the list command
43 # LISTEXTRACTOR => a optional Python function, that is called on each output line of the
44 # LISTCMD for extracting file & dir names, the function need two parameters (first line number,
45 # second line content) and must return a string with the file / dir path (other value types
48 # Other options in the UNPACK dictionary are:
49 # STOPONEMPTYFILE => bool variable for stoping if the file has empty size (default True)
50 # VIWEXTRACTOUTPUT => shows the output messages of the extraction command (default False)
51 # EXTRACTDIR => path in that the data will be extracted (default #)
53 # The file which is handled by the first suffix match of the extractor, the extractor list can be append for other files.
54 # The order of the extractor dictionary creates the listing & extractor command eg file extension .tar.gz should be
55 # before .gz, because the tar.gz is extract in one shoot.
57 # Under *nix system these tools are supported: tar, bzip2, gzip, unzip
58 # Under Windows only 7-Zip (http://www.7-zip.org/) is supported
62 import SCons.Errors, SCons.Warnings, SCons.Util
64 # enables Scons warning for this builder
65 class UnpackWarning(SCons.Warnings.Warning) :
68 SCons.Warnings.enableWarningClass(UnpackWarning)
70 # extractor function for Tar output
71 # @param env environment object
72 # @param count number of returning lines
73 # @param no number of the output line
74 # @param i line content
75 def __fileextractor_nix_tar( env, count, no, i ) :
78 # extractor function for GZip output,
79 # ignore the first line
80 # @param env environment object
81 # @param count number of returning lines
82 # @param no number of the output line
83 # @param i line content
84 def __fileextractor_nix_gzip( env, count, no, i ) :
89 # extractor function for Unzip output,
90 # ignore the first & last two lines
91 # @param env environment object
92 # @param count number of returning lines
93 # @param no number of the output line
94 # @param i line content
95 def __fileextractor_nix_unzip( env, count, no, i ) :
96 if no < 3 or no >= count - 2 :
100 # extractor function for 7-Zip
101 # @param env environment object
102 # @param count number of returning lines
103 # @param no number of the output line
104 # @param i line content
105 def __fileextractor_win_7zip( env, count, no, i ) :
107 if no > 8 and no < count - 2 :
113 # returns the extractor item for handling the source file
114 # @param source input source file
115 # @param env environment object
116 # @return extractor entry or None on non existing
117 def __getExtractor( source, env ) :
118 # we check each unpacker and get the correct list command first, run the command and
119 # replace the target filelist with the list values, we sorte the extractors by their priority
120 for unpackername, extractor in sorted(env["UNPACK"]["EXTRACTOR"].iteritems(), key = lambda (k,v) : (v["PRIORITY"],k)):
122 if not SCons.Util.is_String(extractor["RUN"]) :
123 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
124 if not len(extractor["RUN"]) :
125 raise SCons.Errors.StopError("run command of the unpack builder for [%s] archives is not set - can not extract files" % (unpackername))
128 if not SCons.Util.is_String(extractor["LISTFLAGS"]) :
129 raise SCons.Errors.StopError("list flags of the unpack builder for [%s] archives is not a string" % (unpackername))
130 if not SCons.Util.is_String(extractor["LISTCMD"]) :
131 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
133 if not SCons.Util.is_String(extractor["EXTRACTFLAGS"]) :
134 raise SCons.Errors.StopError("extract flags of the unpack builder for [%s] archives is not a string" % (unpackername))
135 if not SCons.Util.is_String(extractor["EXTRACTCMD"]) :
136 raise SCons.Errors.StopError("extract command of the unpack builder for [%s] archives is not a string" % (unpackername))
139 # check the source file suffix and if the first is found, run the list command
140 if not SCons.Util.is_List(extractor["SUFFIX"]) :
141 raise SCons.Errors.StopError("suffix list of the unpack builder for [%s] archives is not a list" % (unpackername))
143 for suffix in extractor["SUFFIX"] :
144 if str(source[0]).lower()[-len(suffix):] == suffix.lower() :
150 # creates the extracter output message
151 # @param s original message
152 # @param target target name
153 # @param source source name
154 # @param env environment object
155 def __message( s, target, source, env ) :
156 print "extract [%s] ..." % (source[0])
159 # action function for extracting of the data
160 # @param target target packed file
161 # @param source extracted files
162 # @param env environment object
163 def __action( target, source, env ) :
164 extractor = __getExtractor(source, env)
166 raise SCons.Errors.StopError( "can not find any extractor value for the source file [%s]" % (source[0]) )
168 extractor_cmd = extractor["EXTRACTCMD"]
170 # if the extract command is empty, we create an error
171 if len(extractor_cmd) == 0 :
172 raise SCons.Errors.StopError( "the extractor command for the source file [%s] is empty" % (source[0]) )
174 # build it now (we need the shell, because some programs need it)
177 source_path = os.path.realpath(source[0].path)
178 target_path = os.path.realpath(target[0].path)
180 cmd = env.subst(extractor_cmd, source=source_path, target=target)
181 cwd = os.path.dirname(source_path)
183 if env["UNPACK"]["VIWEXTRACTOUTPUT"] :
184 handle = subprocess.Popen( cmd, shell=True )
186 devnull = open(os.devnull, "wb")
187 handle = subprocess.Popen( cmd, shell=True, stdout=devnull, cwd=cwd)
189 if handle.wait() <> 0 :
190 raise SCons.Errors.BuildError( "error running extractor [%s] on the source [%s]" % (cmd, source[0]) )
192 fhandle = open(target_path, 'a')
194 os.utime(target_path, None)
199 # emitter function for getting the files
201 # @param target target packed file
202 # @param source extracted files
203 # @param env environment object
204 def __emitter( target, source, env ) :
205 return target, source
208 # generate function, that adds the builder to the environment
209 # @param env environment object
210 def generate( env ) :
211 # setup environment variable
213 "STOPONEMPTYFILE" : True,
214 "VIWEXTRACTOUTPUT" : False,
215 "EXTRACTDIR" : os.curdir,
219 "SUFFIX" : [".tar.gz", ".tgz", ".tar.gzip"],
220 "EXTRACTSUFFIX" : "",
222 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTSUFFIX']}",
224 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['LISTSUFFIX']}",
227 "LISTEXTRACTOR" : None
232 "SUFFIX" : [".tar.bz", ".tbz", ".tar.bz2", ".tar.bzip2", ".tar.bzip"],
233 "EXTRACTSUFFIX" : "",
235 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTSUFFIX']}",
237 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['LISTSUFFIX']}",
240 "LISTEXTRACTOR" : None
245 "SUFFIX" : [".bz", "bzip", ".bz2", ".bzip2"],
246 "EXTRACTSUFFIX" : "",
248 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTSUFFIX']}",
250 "LISTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['LISTSUFFIX']}",
253 "LISTEXTRACTOR" : None
258 "SUFFIX" : [".gz", ".gzip"],
259 "EXTRACTSUFFIX" : "",
261 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTSUFFIX']}",
263 "LISTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['LISTSUFFIX']}",
266 "LISTEXTRACTOR" : None
272 "EXTRACTSUFFIX" : "",
274 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['EXTRACTSUFFIX']}",
276 "LISTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['LISTSUFFIX']}",
279 "LISTEXTRACTOR" : None
285 "EXTRACTSUFFIX" : "",
287 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTSUFFIX']}",
289 "LISTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['LISTSUFFIX']}",
292 "LISTEXTRACTOR" : None
297 # read tools for Windows system
298 if env["PLATFORM"] <> "darwin" and "win" in env["PLATFORM"] :
300 if env.WhereIs("7z") :
301 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "7z"
302 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
303 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "x"
304 toolset["EXTRACTOR"]["TARGZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
305 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "x"
306 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
308 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "7z"
309 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
310 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "x"
311 toolset["EXTRACTOR"]["TARBZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
312 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "x"
313 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
315 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "7z"
316 toolset["EXTRACTOR"]["BZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
317 toolset["EXTRACTOR"]["BZIP"]["LISTFLAGS"] = "l"
318 toolset["EXTRACTOR"]["BZIP"]["LISTSUFFIX"] = "-y -so"
319 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "x"
320 toolset["EXTRACTOR"]["BZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
322 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "7z"
323 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
324 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "l"
325 toolset["EXTRACTOR"]["GZIP"]["LISTSUFFIX"] = "-y -so"
326 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "x"
327 toolset["EXTRACTOR"]["GZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
329 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "7z"
330 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
331 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "l"
332 toolset["EXTRACTOR"]["ZIP"]["LISTSUFFIX"] = "-y -so"
333 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "x"
334 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
336 toolset["EXTRACTOR"]["TAR"]["RUN"] = "7z"
337 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
338 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "l"
339 toolset["EXTRACTOR"]["TAR"]["LISTSUFFIX"] = "-y -ttar -so"
340 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "x"
341 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-y -ttar -oc:${UNPACK['EXTRACTDIR']}"
343 # here can add some other Windows tools, that can handle the archive files
344 # but I don't know which ones can handle all file types
348 # read the tools on *nix systems and sets the default parameters
349 elif env["PLATFORM"] in ["darwin", "linux", "posix"] :
351 if env.WhereIs("unzip") :
352 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "unzip"
353 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_unzip
354 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "-l"
355 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "-oqq"
356 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-d ${UNPACK['EXTRACTDIR']}"
358 if env.WhereIs("tar") :
359 toolset["EXTRACTOR"]["TAR"]["RUN"] = "tar"
360 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
361 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "tvf"
362 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "xf"
363 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
365 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "tar"
366 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
367 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "xfz"
368 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "tvfz"
369 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
371 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "tar"
372 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
373 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "xfj"
374 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "tvfj"
375 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
377 if env.WhereIs("bzip2") :
378 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "bzip2"
379 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "-df"
381 if env.WhereIs("gzip") :
382 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "gzip"
383 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_gzip
384 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "-l"
385 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "-df"
388 raise SCons.Errors.StopError("Unpack tool detection on this platform [%s] unkown" % (env["PLATFORM"]))
390 # the target_factory must be a "Entry", because the target list can be files and dirs, so we can not specified the targetfactory explicite
391 env.Replace(UNPACK = toolset)
392 env["BUILDERS"]["UnpackAll"] = SCons.Builder.Builder( action = __action, emitter = __emitter, target_factory = SCons.Node.FS.Entry, source_factory = SCons.Node.FS.File, single_source = True, PRINT_CMD_LINE_FUNC = __message )
395 # existing function of the builder
396 # @param env environment object