1 # -*- coding: utf-8 -*-
3 # -- Dual Licence ----------------------------------------------------------
5 ############################################################################
8 # This file is a SCons (http://www.scons.org/) builder #
9 # Copyright (c) 2012-14, Philipp Kraus, <philipp.kraus@flashpixx.de> #
10 # This program is free software: you can redistribute it and/or modify #
11 # it under the terms of the GNU General Public License as #
12 # published by the Free Software Foundation, either version 3 of the #
13 # License, or (at your option) any later version. #
15 # This program is distributed in the hope that it will be useful, #
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
18 # GNU General Public License for more details. #
20 # You should have received a copy of the GNU General Public License #
21 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
22 ############################################################################
24 # --------------------------------------------------------------------------
26 ############################################################################
27 # BSD 3-Clause License #
29 # This file is a SCons (http://www.scons.org/) builder #
30 # Copyright (c) 2012-14, Philipp Kraus, <philipp.kraus@flashpixx.de> #
31 # All rights reserved. #
33 # Redistribution and use in source and binary forms, with or without #
34 # modification, are permitted provided that the following conditions are #
37 # 1. Redistributions of source code must retain the above copyright #
38 # notice, this list of conditions and the following disclaimer. #
40 # 2. Redistributions in binary form must reproduce the above copyright #
41 # notice, this list of conditions and the following disclaimer in the #
42 # documentation and/or other materials provided with the distribution. #
44 # 3. Neither the name of the copyright holder nor the names of its #
45 # contributors may be used to endorse or promote products derived from #
46 # this software without specific prior written permission. #
48 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS #
49 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
50 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
51 # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
52 # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
53 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED #
54 # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR #
55 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF #
56 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING #
57 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS #
58 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
59 ############################################################################
63 # The Unpack Builder can be used for unpacking archives (eg Zip, TGZ, BZ, ... ).
64 # The emitter of the Builder reads the archive data and creates a returning file list
65 # the builder extract the archive. The environment variable stores a dictionary "UNPACK"
66 # for set different extractions (subdict "EXTRACTOR"):
68 # PRIORITY => a value for setting the extractor order (lower numbers = extractor is used earlier)
69 # SUFFIX => defines a list with file suffixes, which should be handled with this extractor
70 # EXTRACTSUFFIX => suffix of the extract command
71 # EXTRACTFLAGS => a string parameter for the RUN command for extracting the data
72 # EXTRACTCMD => full extract command of the builder
73 # RUN => the main program which will be started (if the parameter is empty, the extractor will be ignored)
74 # LISTCMD => the listing command for the emitter
75 # LISTFLAGS => the string options for the RUN command for showing a list of files
76 # LISTSUFFIX => suffix of the list command
77 # LISTEXTRACTOR => a optional Python function, that is called on each output line of the
78 # LISTCMD for extracting file & dir names, the function need two parameters (first line number,
79 # second line content) and must return a string with the file / dir path (other value types
82 # Other options in the UNPACK dictionary are:
83 # STOPONEMPTYFILE => bool variable for stoping if the file has empty size (default True)
84 # VIWEXTRACTOUTPUT => shows the output messages of the extraction command (default False)
85 # EXTRACTDIR => path in that the data will be extracted (default #)
87 # The file which is handled by the first suffix match of the extractor, the extractor list can be append for other files.
88 # The order of the extractor dictionary creates the listing & extractor command eg file extension .tar.gz should be
89 # before .gz, because the tar.gz is extract in one shoot.
91 # Under *nix system these tools are supported: tar, bzip2, gzip, unzip
92 # Under Windows only 7-Zip (http://www.7-zip.org/) is supported
97 import SCons.Errors, SCons.Warnings
101 # enables Scons warning for this builder
102 class UnpackWarning(SCons.Warnings.Warning) :
105 SCons.Warnings.enableWarningClass(UnpackWarning)
109 # extractor function for Tar output
110 # @param env environment object
111 # @param count number of returning lines
112 # @param no number of the output line
113 # @param i line content
114 def __fileextractor_nix_tar( env, count, no, i ) :
117 # extractor function for GZip output,
118 # ignore the first line
119 # @param env environment object
120 # @param count number of returning lines
121 # @param no number of the output line
122 # @param i line content
123 def __fileextractor_nix_gzip( env, count, no, i ) :
128 # extractor function for Unzip output,
129 # ignore the first & last two lines
130 # @param env environment object
131 # @param count number of returning lines
132 # @param no number of the output line
133 # @param i line content
134 def __fileextractor_nix_unzip( env, count, no, i ) :
135 if no < 3 or no >= count - 2 :
139 # extractor function for 7-Zip
140 # @param env environment object
141 # @param count number of returning lines
142 # @param no number of the output line
143 # @param i line content
144 def __fileextractor_win_7zip( env, count, no, i ) :
146 if no > 8 and no < count - 2 :
152 # returns the extractor item for handling the source file
153 # @param source input source file
154 # @param env environment object
155 # @return extractor entry or None on non existing
156 def __getExtractor( source, env ) :
157 # we check each unpacker and get the correc list command first, run the command and
158 # replace the target filelist with the list values, we sorte the extractors by their priority
159 for unpackername, extractor in sorted(env["UNPACK"]["EXTRACTOR"].iteritems(), key = lambda (k,v) : (v["PRIORITY"],k)):
161 # if the run command not set, we continue the extractor search, otherwise we check the extractor parameters
162 if not SCons.Util.is_String(extractor["RUN"]) :
163 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
164 if not len(extractor["RUN"]) :
165 raise SCons.Errors.StopError("run command of the unpack builder for [%s] archives is not set - can not extract files" % (unpackername))
168 if not SCons.Util.is_String(extractor["LISTFLAGS"]) :
169 raise SCons.Errors.StopError("list flags of the unpack builder for [%s] archives is not a string" % (unpackername))
170 if not SCons.Util.is_String(extractor["LISTCMD"]) :
171 raise SCons.Errors.StopError("list command of the unpack builder for [%s] archives is not a string" % (unpackername))
173 if not SCons.Util.is_String(extractor["EXTRACTFLAGS"]) :
174 raise SCons.Errors.StopError("extract flags of the unpack builder for [%s] archives is not a string" % (unpackername))
175 if not SCons.Util.is_String(extractor["EXTRACTCMD"]) :
176 raise SCons.Errors.StopError("extract command of the unpack builder for [%s] archives is not a string" % (unpackername))
179 # check the source file suffix and if the first is found, run the list command
180 if not SCons.Util.is_List(extractor["SUFFIX"]) :
181 raise SCons.Errors.StopError("suffix list of the unpack builder for [%s] archives is not a list" % (unpackername))
183 for suffix in extractor["SUFFIX"] :
184 if str(source[0]).lower()[-len(suffix):] == suffix.lower() :
190 # creates the extracter output message
191 # @param s original message
192 # @param target target name
193 # @param source source name
194 # @param env environment object
195 def __message( s, target, source, env ) :
196 print "extract [%s] ..." % (source[0])
199 # action function for extracting of the data
200 # @param target target packed file
201 # @param source extracted files
202 # @env environment object
203 def __action( target, source, env ) :
204 extractor = __getExtractor(source, env)
206 raise SCons.Errors.StopError( "can not find any extractor value for the source file [%s]" % (source[0]) )
209 # if the extract command is empty, we create an error
210 if len(extractor["EXTRACTCMD"]) == 0 :
211 raise SCons.Errors.StopError( "the extractor command for the source file [%s] is empty" % (source[0]) )
213 # build it now (we need the shell, because some programs need it)
216 source_path = os.path.realpath(source[0].path)
217 target_path = os.path.realpath(target[0].path)
219 cmd = env.subst(extractor["EXTRACTCMD"], source=source_path, target=target)
220 cwd = os.path.dirname(source_path)
222 if env["UNPACK"]["VIWEXTRACTOUTPUT"] :
223 handle = subprocess.Popen( cmd, shell=True )
225 devnull = open(os.devnull, "wb")
226 handle = subprocess.Popen( cmd, shell=True, stdout=devnull, cwd=cwd)
228 if handle.wait() <> 0 :
229 raise SCons.Errors.BuildError( "error running extractor [%s] on the source [%s]" % (cmd, source[0]) )
231 fhandle = open(target_path, 'a')
233 os.utime(target_path, None)
238 # emitter function for getting the files
240 # @param target target packed file
241 # @param source extracted files
242 # @env environment object
243 def __emitter( target, source, env ) :
244 return target, source
249 # generate function, that adds the builder to the environment
250 # @env environment object
251 def generate( env ) :
252 # setup environment variable
254 "STOPONEMPTYFILE" : True,
255 "VIWEXTRACTOUTPUT" : False,
256 "EXTRACTDIR" : os.curdir,
260 "SUFFIX" : [".tar.gz", ".tgz", ".tar.gzip"],
261 "EXTRACTSUFFIX" : "",
263 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['EXTRACTSUFFIX']}",
265 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARGZ']['RUN']} ${UNPACK['EXTRACTOR']['TARGZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARGZ']['LISTSUFFIX']}",
268 "LISTEXTRACTOR" : None
273 "SUFFIX" : [".tar.bz", ".tbz", ".tar.bz2", ".tar.bzip2", ".tar.bzip"],
274 "EXTRACTSUFFIX" : "",
276 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['EXTRACTSUFFIX']}",
278 "LISTCMD" : "${UNPACK['EXTRACTOR']['TARBZ']['RUN']} ${UNPACK['EXTRACTOR']['TARBZ']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TARBZ']['LISTSUFFIX']}",
281 "LISTEXTRACTOR" : None
286 "SUFFIX" : [".bz", "bzip", ".bz2", ".bzip2"],
287 "EXTRACTSUFFIX" : "",
289 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['EXTRACTSUFFIX']}",
291 "LISTCMD" : "${UNPACK['EXTRACTOR']['BZIP']['RUN']} ${UNPACK['EXTRACTOR']['BZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['BZIP']['LISTSUFFIX']}",
294 "LISTEXTRACTOR" : None
299 "SUFFIX" : [".gz", ".gzip"],
300 "EXTRACTSUFFIX" : "",
302 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['EXTRACTSUFFIX']}",
304 "LISTCMD" : "${UNPACK['EXTRACTOR']['GZIP']['RUN']} ${UNPACK['EXTRACTOR']['GZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['GZIP']['LISTSUFFIX']}",
307 "LISTEXTRACTOR" : None
313 "EXTRACTSUFFIX" : "",
315 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['EXTRACTSUFFIX']}",
317 "LISTCMD" : "${UNPACK['EXTRACTOR']['TAR']['RUN']} ${UNPACK['EXTRACTOR']['TAR']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['TAR']['LISTSUFFIX']}",
320 "LISTEXTRACTOR" : None
326 "EXTRACTSUFFIX" : "",
328 "EXTRACTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['EXTRACTSUFFIX']}",
330 "LISTCMD" : "${UNPACK['EXTRACTOR']['ZIP']['RUN']} ${UNPACK['EXTRACTOR']['ZIP']['LISTFLAGS']} $SOURCE ${UNPACK['EXTRACTOR']['ZIP']['LISTSUFFIX']}",
333 "LISTEXTRACTOR" : None
338 # read tools for Windows system
339 if env["PLATFORM"] <> "darwin" and "win" in env["PLATFORM"] :
341 if env.WhereIs("7z"):
342 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "7z"
343 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
344 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "x"
345 toolset["EXTRACTOR"]["TARGZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
346 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "x"
347 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
349 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "7z"
350 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
351 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "x"
352 toolset["EXTRACTOR"]["TARBZ"]["LISTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} l -sii -ttar -y -so"
353 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "x"
354 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-so -y | ${UNPACK['EXTRACTOR']['TARGZ']['RUN']} x -sii -ttar -y -oc:${UNPACK['EXTRACTDIR']}"
356 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "7z"
357 toolset["EXTRACTOR"]["BZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
358 toolset["EXTRACTOR"]["BZIP"]["LISTFLAGS"] = "l"
359 toolset["EXTRACTOR"]["BZIP"]["LISTSUFFIX"] = "-y -so"
360 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "x"
361 toolset["EXTRACTOR"]["BZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
363 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "7z"
364 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
365 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "l"
366 toolset["EXTRACTOR"]["GZIP"]["LISTSUFFIX"] = "-y -so"
367 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "x"
368 toolset["EXTRACTOR"]["GZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
370 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "7z"
371 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
372 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "l"
373 toolset["EXTRACTOR"]["ZIP"]["LISTSUFFIX"] = "-y -so"
374 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "x"
375 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-y -oc:${UNPACK['EXTRACTDIR']}"
377 toolset["EXTRACTOR"]["TAR"]["RUN"] = "7z"
378 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_win_7zip
379 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "l"
380 toolset["EXTRACTOR"]["TAR"]["LISTSUFFIX"] = "-y -ttar -so"
381 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "x"
382 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-y -ttar -oc:${UNPACK['EXTRACTDIR']}"
384 # here can add some other Windows tools, that can handle the archive files
385 # but I don't know which ones can handle all file types
389 # read the tools on *nix systems and sets the default parameters
390 elif env["PLATFORM"] in ["darwin", "linux", "posix", "msys"] :
392 if env.WhereIs("unzip") :
393 toolset["EXTRACTOR"]["ZIP"]["RUN"] = "unzip"
394 toolset["EXTRACTOR"]["ZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_unzip
395 toolset["EXTRACTOR"]["ZIP"]["LISTFLAGS"] = "-l"
396 toolset["EXTRACTOR"]["ZIP"]["EXTRACTFLAGS"] = "-oqq"
397 toolset["EXTRACTOR"]["ZIP"]["EXTRACTSUFFIX"] = "-d ${UNPACK['EXTRACTDIR']}"
399 if env.WhereIs("tar") :
400 toolset["EXTRACTOR"]["TAR"]["RUN"] = "tar"
401 toolset["EXTRACTOR"]["TAR"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
402 toolset["EXTRACTOR"]["TAR"]["LISTFLAGS"] = "tvf"
403 toolset["EXTRACTOR"]["TAR"]["EXTRACTFLAGS"] = "xf"
404 toolset["EXTRACTOR"]["TAR"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
406 toolset["EXTRACTOR"]["TARGZ"]["RUN"] = "tar"
407 toolset["EXTRACTOR"]["TARGZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
408 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTFLAGS"] = "xfz"
409 toolset["EXTRACTOR"]["TARGZ"]["LISTFLAGS"] = "tvfz"
410 toolset["EXTRACTOR"]["TARGZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
412 toolset["EXTRACTOR"]["TARBZ"]["RUN"] = "tar"
413 toolset["EXTRACTOR"]["TARBZ"]["LISTEXTRACTOR"] = __fileextractor_nix_tar
414 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTFLAGS"] = "xfj"
415 toolset["EXTRACTOR"]["TARBZ"]["LISTFLAGS"] = "tvfj"
416 toolset["EXTRACTOR"]["TARBZ"]["EXTRACTSUFFIX"] = "-C ${UNPACK['EXTRACTDIR']}"
418 if env.WhereIs("bzip2") :
419 toolset["EXTRACTOR"]["BZIP"]["RUN"] = "bzip2"
420 toolset["EXTRACTOR"]["BZIP"]["EXTRACTFLAGS"] = "-df"
422 if env.WhereIs("gzip") :
423 toolset["EXTRACTOR"]["GZIP"]["RUN"] = "gzip"
424 toolset["EXTRACTOR"]["GZIP"]["LISTEXTRACTOR"] = __fileextractor_nix_gzip
425 toolset["EXTRACTOR"]["GZIP"]["LISTFLAGS"] = "-l"
426 toolset["EXTRACTOR"]["GZIP"]["EXTRACTFLAGS"] = "-df"
429 raise SCons.Errors.StopError("Unpack tool detection on this platform [%s] unkown" % (env["PLATFORM"]))
431 # the target_factory must be a "Entry", because the target list can be files and dirs, so we can not specified the targetfactory explicite
432 env.Replace(UNPACK = toolset)
433 env["BUILDERS"]["UnpackAll"] = SCons.Builder.Builder( action = __action, emitter = __emitter, target_factory = SCons.Node.FS.Entry, source_factory = SCons.Node.FS.File, single_source = True, PRINT_CMD_LINE_FUNC = __message )
436 # existing function of the builder
437 # @param env environment object