Manual Strip Mode for generating Debug package & Stripped package
[scm/meta/abs.git] / abs
1 #!/usr/bin/env python
2 # vim: ai ts=4 sts=4 et sw=4
3 #
4 # Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd.
5 #
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the Free
8 # Software Foundation; version 2 of the License
9 #
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13 # for more details.
14 #
15
16 import sys
17 import os
18 import subprocess
19 import re
20 import argparse
21 from argparse import ArgumentParser
22 import ConfigParser
23 import glob
24 import fnmatch
25 import shutil
26 import zipfile
27 import errno
28
29 g_home = os.path.dirname(os.path.realpath(__file__))
30
31 class LocalError(Exception):
32     """Local error exception."""
33
34     pass
35
36 class Executor(object):
37     """Subprocess wrapper"""
38
39     def __init__(self, checker=None):
40         self.stdout = subprocess.PIPE
41         self.stderr = subprocess.STDOUT
42         self.checker = checker
43
44     def run(self, cmdline_or_args, show=False, checker=False):
45         """Execute external command"""
46
47         out = ''
48         try:
49             process = subprocess.Popen(cmdline_or_args, \
50                                        stdout=self.stdout, \
51                                        stderr=self.stderr, \
52                                        shell=True)
53             while True:
54                 line = process.stdout.readline()
55                 if show:
56                     print line.rstrip()
57                 out = out + '\n' + line.rstrip()
58                 if not line:
59                     break
60         except:
61             raise LocalError('Running process failed')
62
63         return out
64
65 def list_files(path, ext=None):
66
67     f_list = []
68     for root, dirnames, filenames in os.walk(path):
69         if ext is None:
70             for filename in filenames:
71                 f_list.append(os.path.join(root, filename))
72         else:
73             for filename in fnmatch.filter(filenames, '*.'+ext):
74                 f_list.append(os.path.join(root, filename))
75     return f_list
76
77 class FakeSecHead(object):
78
79     def __init__(self, fp):
80
81         self.fp = fp
82         self.sechead = '[ascection]\n'
83
84     def readline(self):
85
86         if self.sechead:
87             try:
88                 return self.sechead
89             finally:
90                 self.sechead = None
91         else:
92             return self.fp.readline()
93
94 class ErrorParser(object):
95     """Inspect specific error string"""
96
97     parsers = []
98
99     def __init__(self):
100
101         ErrorParser = {'GNU_LINKER':['(.*?):?\(\.\w+\+.*\): (.*)', \
102                                      '(.*[/\\\])?ld(\.exe)?: (.*)'], \
103                        'GNU_GCC':['(.*?):(\d+):(\d+:)? [Ee]rror: ([`\'"](.*)[\'"] undeclared .*)', \
104                                   '(.*?):(\d+):(\d+:)? [Ee]rror: (conflicting types for .*[`\'"](.*)[\'"].*)', \
105                                   '(.*?):(\d+):(\d+:)? (parse error before.*[`\'"](.*)[\'"].*)', \
106                                   '(.*?):(\d+):(\d+:)?\s*(([Ee]rror)|(ERROR)): (.*)'], \
107 #                                  '(.*?):(\d+):(\d+:)? (.*)'], \
108                        'GNU_GMAKE':['(.*):(\d*): (\*\*\* .*)', \
109                                     '.*make.*: \*\*\* .*', \
110                                     '.*make.*: Target (.*) not remade because of errors.', \
111                                     '.*[Cc]ommand not found.*', \
112                                     'Error:\s*(.*)'], \
113                        'TIZEN_NATIVE':['.*ninja: build stopped.*', \
114                                        'edje_cc: Error..(.*):(\d).*', \
115                                        'edje_cc: Error.*']}
116
117         for parser in ErrorParser:
118             parser_env = os.getenv('SDK_ERROR_'+parser)
119             if parser_env:
120                 self.parsers.append(parser_env)
121             else:
122                 for msg in ErrorParser[parser]:
123                     self.parsers.append(msg)
124
125     def check(self, full_log):
126         """Check error string line by line"""
127
128         #snipset_text = full_log[:full_log.rfind('PLATFORM_VER\t')].split('\n')
129         for line in full_log.split('\n'):
130             errline = re.search('|'.join(self.parsers), line[:1024])
131             if errline:
132                 return errline.string #Errors
133         return None #No error
134
135 class _Rootstrap(object):
136     """Tizen SDK rootstrap info.
137        Used only in Sdk class"""
138
139     rootstrap_list = None
140     sdk_path = None
141
142     def __init__(self, sdk_path=None, config=None):
143
144         self.tizen = sdk_path
145         self.list_rootstrap()
146         self.config_file = config
147
148     def list_rootstrap(self):
149         """List all the rootstraps"""
150
151         if self.rootstrap_list != None:
152             return self.rootstrap_list
153
154         cmdline = self.tizen + ' list rootstrap'
155         ret = Executor().run(cmdline, show=False)
156         for x in ret.splitlines():
157             if re.search('(mobile|wearable)-(2.4|3.0)-(device|emulator).core.*', x):
158                 if self.rootstrap_list == None:
159                     self.rootstrap_list = []
160                 self.rootstrap_list.append(x.split(' ')[0])
161         return self.rootstrap_list
162
163     def check_rootstrap(self, rootstrap, show=True):
164         """Specific rootstrap is in the SDK
165            Otherwise use default"""
166
167         if rootstrap == None:
168             rootstrap = 'mobile-3.0-emulator.core' #default
169         if rootstrap in self.rootstrap_list:
170             return rootstrap
171         else:
172             if show == True:
173                 print '  ERROR: Rootstrap [%s] does not exist' % rootstrap
174                 print '  Update your rootstrap or use one of:\n    %s' \
175                        % '\n    '.join(self.list_rootstrap())
176             return None
177
178 class Sdk(object):
179     """Tizen SDK related job"""
180
181     rs = None #_Rootstrap class instance
182     rootstrap_list = None
183     sdk_to_search = ['tizen-sdk/tools/ide/bin/tizen', \
184                      'tizen-sdk-ux/tools/ide/bin/tizen', \
185                      'tizen-sdk-cli/tools/ide/bin/tizen']
186
187     def __init__(self, sdkpath=None):
188
189         self.error_parser = ErrorParser()
190         self.runtool = Executor(checker=self.error_parser)
191
192         self.home = os.getenv('HOME')
193         self.config_file = os.path.join(g_home, '.abs')
194
195         if sdkpath is None:
196             self.tizen = self.get_user_root()
197             if self.tizen is None or self.tizen == '':
198                 for i in self.sdk_to_search:
199                     if os.path.isfile(os.path.join(self.home, i)):
200                         self.tizen = os.path.join(self.home, i)
201                         break
202         else:
203             self.tizen = os.path.join(sdkpath, 'tools/ide/bin/tizen')
204             self.update_user_root(self.tizen)
205
206         if not os.path.isfile(self.tizen):
207             print 'Cannot locate cli tool'
208             raise LocalError('Fail to locate cli tool')
209
210         self.rs = _Rootstrap(sdk_path=self.tizen, config=self.config_file)
211
212     def get_user_root(self):
213
214         if os.path.isfile(self.config_file):
215             config = ConfigParser.RawConfigParser()
216             config.read(self.config_file)
217             return config.get('Global', 'tizen')
218         return None
219
220     def update_user_root(self, path):
221
222         if not os.path.isfile(self.config_file):
223             with open(self.config_file, 'w') as f:
224                 f.write('[Global]\n')
225                 f.write('tizen = %s\n' % path)
226             return
227
228         config = ConfigParser.RawConfigParser()
229         config.read(self.config_file)
230         config.set('Global', 'tizen', path)
231         with open(self.config_file, 'wb') as cf:
232             config.write(cf)
233
234     def list_rootstrap(self):
235         return self.rs.list_rootstrap()
236
237     def check_rootstrap(self, rootstrap):
238         return self.rs.check_rootstrap(rootstrap)
239
240     def _run(self, command, args, show=True, checker=False):
241         """Run a tizen command"""
242
243         cmd = [self.tizen, command] + args
244         print '\nRunning command:\n    %s' % ' '.join(cmd)
245         return self.runtool.run(' '.join(cmd), show=show, checker=checker)
246
247     def copytree2(self, src, dst, symlinks=False, ignore=None):
248         """Copy with Ignore & Overwrite"""
249         names = os.listdir(src)
250         if ignore is not None:
251             ignored_names = ignore(src, names)
252         else:
253             ignored_names = set()
254
255         try:
256             os.makedirs(dst)
257         except:
258             pass
259
260         errors = []
261         for name in names:
262             if name in ignored_names:
263                 continue
264             srcname = os.path.join(src, name)
265             dstname = os.path.join(dst, name)
266             try:
267                 if symlinks and os.path.islink(srcname):
268                     linkto = os.readlink(srcname)
269                     os.symlink(linkto, dstname)
270                 elif os.path.isdir(srcname):
271                     self.copytree2(srcname, dstname, symlinks, ignore)
272                 else:
273                     # Will raise a SpecialFileError for unsupported file types
274                     shutil.copy2(srcname, dstname)
275             # catch the Error from the recursive copytree so that we can
276             # continue with other files
277             except shutil.Error, err:
278                 errors.extend(err.args[0])
279             except EnvironmentError, why:
280                 errors.append((srcname, dstname, str(why)))
281         try:
282             shutil.copystat(src, dst)
283         except OSError, why:
284             if WindowsError is not None and isinstance(why, WindowsError):
285                 # Copying file access times may fail on Windows
286                 pass
287             else:
288                 errors.append((src, dst, str(why)))
289         #if errors:
290          #   raise shutil.Error, errors
291
292     def _copy_build_output(self, src, dst):
293         if not os.path.isdir(src) :
294             return
295         try:
296             self.copytree2(src, dst, ignore=shutil.ignore_patterns('*.edc', '*.po', 'objs', '*.info', '*.so', 'CMakeLists.txt', '*.h', '*.c'))
297         except OSError as exc:
298             # File already exist
299             if exc.errno == errno.EEXIST:
300                 shutil.copy(src, dst)
301             if exc.errno == errno.ENOENT:
302                 shutil.copy(src, dst)
303             else:
304                 raise
305
306     def _package_sharedlib(self, project_path, conf, app_name):
307         """If -r option used for packaging, make zip file from copied files"""
308         #project_path=project['path']
309         project_build_output_path=os.path.join(project_path, conf)
310         package_path=os.path.join(project_build_output_path, '.pkg')
311
312         if os.path.isdir(package_path):
313             shutil.rmtree(package_path)
314         os.makedirs(package_path)
315         os.makedirs(os.path.join(package_path, 'lib'))
316
317         #Copy project resource
318         self._copy_build_output(os.path.join(project_path, 'lib'), os.path.join(package_path, 'lib'))
319         self._copy_build_output(os.path.join(project_path, 'res'), os.path.join(package_path, 'res'))
320
321         #Copy built res resource
322         self._copy_build_output(os.path.join(project_build_output_path, 'res'), os.path.join(package_path, 'res'))
323
324         #Copy so library file
325         for filename in list_files(project_build_output_path, 'so'):
326             shutil.copy(filename, os.path.join(package_path, 'lib'))
327
328         # Copy so library file
329         zipname=app_name + '.zip'
330         rsrc_zip = os.path.join(project_build_output_path, zipname)
331         myZipFile = zipfile.ZipFile(rsrc_zip, 'w')
332         for filename in list_files(package_path):
333             try:
334                 myZipFile.write(filename, filename.replace(package_path, ''))
335             except Exception, e:
336                 print str(e)
337         myZipFile.close()
338         return rsrc_zip
339
340     def build_native(self, source, rootstrap=None, arch=None, conf='Debug'):
341         """SDK CLI build command"""
342
343         _rootstrap = self.check_rootstrap(rootstrap)
344         if _rootstrap == None:
345             raise LocalError('Rootstrap %s not exist' % rootstrap)
346
347         if rootstrap is None and arch is None:
348             rootstrap = _rootstrap
349             self.arch = 'x86'
350         elif arch is None:
351             if 'emulator' in rootstrap: self.arch = 'x86'
352             elif 'device' in rootstrap: self.arch = 'arm'
353         elif rootstrap is None:
354             if arch not in ['x86', 'arm']:
355                 raise LocalError('Architecture and rootstrap mismatch')
356             rootstrap = _rootstrap
357             if arch == 'arm': rootstrap = rootstrap.replace('emulator', 'device')
358
359         for x in source.project_list:
360             out = self._run('build-native', ['-r', rootstrap, '-a', self.arch, '-C', conf, '--' , x['path']], checker=True)
361             logpath = os.path.join(source.output_dir, \
362                                   'build_%s_%s' % (rootstrap, os.path.basename(x['path'])))
363             if not os.path.isdir(source.output_dir):
364                 os.makedirs(source.output_dir)
365             with open(logpath, 'w') as lf:
366                 lf.write(out)
367             ret = self.error_parser.check(out)
368             if ret:
369                 with open(logpath+'.log', 'w') as lf:
370                     lf.write(out)
371                 raise LocalError(ret)
372
373     def package(self, source, cert=None, pkg_type=None, conf=None):
374         """SDK CLI package command"""
375
376         if cert is None: cert = 'ABS'
377         if pkg_type is None: pkg_type = 'tpk'
378         if conf is None: conf = 'Debug'
379
380         final_app = ''
381         main_args = ['-t', pkg_type, '-s', cert]
382         out = '' #logfile
383
384         if conf == 'Release' :
385             main_args.extend(['--strip', 'on'])
386
387         for i, x in enumerate(source.project_list):
388             if x['type'] == 'app':
389                 out = '%s\n%s' % (out, \
390                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
391                 try:
392                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
393                 except:
394                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
395                 x['out_package'] = final_app
396             elif x['type'] == 'sharedLib':
397                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
398                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
399             else:
400                 raise LocalError('Not supported project type %s' % x['type'])
401
402         if source.b_multi == True:
403             extra_args=[]
404             print 'THIS IS MULTI PROJECT'
405             for i, x in enumerate(source.project_list):
406                 if x['out_package'] != final_app and x['type'] == 'app':
407                     extra_args.extend(['-r', x['out_package']])
408                 elif x['type'] == 'sharedLib':
409                     extra_args.extend(['-r', x['out_package']])
410
411             extra_args.extend(['--', final_app])
412             out = self._run('package', main_args + extra_args)
413
414         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
415         print 'Packaging final step again!'
416         out = self._run('package', main_args + ['--', final_app])
417
418         #Copy tpk to output directory
419         shutil.copy(final_app, source.output_dir)
420
421     def package_new(self, source, cert=None, pkg_type=None, conf=None, manual_strip=False):
422         """SDK CLI package command
423             IF Debug + Manual Strip off then generate package-name-debug.tpk
424             IF Debug + Manual Strip on then generate package-name.tpk with custom strip
425             IF Release then generate package-name.tpk with strip option
426         """
427
428         if cert is None: cert = 'ABS'
429         if pkg_type is None: pkg_type = 'tpk'
430         if conf is None: conf = 'Debug'
431
432         final_app = ''
433         main_args = ['-t', pkg_type, '-s', cert]
434         out = '' #logfile
435
436         # remove tpk or zip file on project path
437         package_list = []
438         for i, x in enumerate(source.project_list):
439             package_list.extend(list_files(os.path.join(x['path'], conf), ext='tpk'))
440             package_list.extend(list_files(os.path.join(x['path'], conf), ext='zip'))
441
442         for k in package_list :
443             print ' package list ' + k;
444             os.remove(k)
445
446         # Manual strip
447         if manual_strip == True :
448             strip_cmd='';
449             if self.arch == None:
450                 raise LocalError('Architecture is Noen')
451             elif self.arch == 'x86' :
452                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../i386-linux-gnueabi-gcc-4.9/bin/i386-linux-gnueabi-strip')
453             elif self.arch == 'arm' :
454                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../arm-linux-gnueabi-gcc-4.9/bin/arm-linux-gnueabi-strip')
455
456             print strip_cmd
457
458             for i, x in enumerate(source.project_list):
459                 dir = os.path.join(x['path'], conf)
460                 files = [os.path.join(dir,f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir,f))]
461                 for k in files:
462                     cmdline = strip_cmd + ' ' + k;
463                     #print 'my command line ' + cmdline;
464                     Executor().run(cmdline, show=False)
465         elif conf == 'Release':
466             main_args.extend(['--strip', 'on'])
467
468         for i, x in enumerate(source.project_list):
469             if x['type'] == 'app':
470                 out = '%s\n%s' % (out, \
471                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
472                 try:
473                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
474                 except:
475                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
476                 x['out_package'] = final_app
477             elif x['type'] == 'sharedLib':
478                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
479                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
480             else:
481                 raise LocalError('Not supported project type %s' % x['type'])
482
483         if source.b_multi == True:
484             extra_args=[]
485             print 'THIS IS MULTI PROJECT'
486             for i, x in enumerate(source.project_list):
487                 if x['out_package'] != final_app and x['type'] == 'app':
488                     extra_args.extend(['-r', x['out_package']])
489                 elif x['type'] == 'sharedLib':
490                     extra_args.extend(['-r', x['out_package']])
491
492             extra_args.extend(['--', final_app])
493             out = self._run('package', main_args + extra_args)
494
495         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
496         print 'Packaging final step again!'
497         out = self._run('package', main_args + ['--', final_app])
498
499         #Copy tpk to output directory
500         if conf == 'Debug' and manual_strip == False :
501             basename = os.path.splitext(final_app)[0]
502             newname = basename +'-debug.tpk'
503             os.rename(final_app, newname)
504             shutil.copy(newname, source.output_dir)
505         else :
506             shutil.copy(final_app, source.output_dir)
507
508     def clean(self, source):
509         """SDK CLI clean command"""
510
511         if os.path.isdir(source.multizip_path):
512             shutil.rmtree(source.multizip_path)
513
514         if os.path.isfile(os.path.join(source.multizip_path, '.zip')):
515             os.remove(os.path.join(source.multizip_path, '.zip'))
516
517         for x in source.project_list:
518             self._run('clean', ['--', x['path']], show=False)
519
520 class Source(object):
521     """Project source related job"""
522
523     workspace = '' #Project root directory
524     project_list = []
525     b_multi = False
526     multi_conf_file = 'WORKSPACE' #Assume multi-project if this file exist.
527     multizip_path = '' #For multi-project packaging -r option
528     property_dict = {}
529     output_dir = '_abs_out_'
530
531     def __init__(self, src=None):
532
533         if src == None:
534             self.workspace = os.getcwd()
535         else:
536             self.workspace = os.path.abspath(src)
537         self.output_dir = os.path.join(self.workspace, self.output_dir)
538
539         os.environ['workspace_loc']=str(os.path.realpath(self.workspace))
540
541         self.multizip_path = os.path.join(self.workspace, 'multizip')
542         self.pre_process()
543
544     def set_properties(self, path):
545         """Fetch all properties from project_def.prop"""
546
547         mydict = {}
548         cp = ConfigParser.SafeConfigParser()
549         cp.optionxform = str
550         cp.readfp(FakeSecHead(open(os.path.join(path, 'project_def.prop'))))
551         for x in cp.items('ascection'):
552             mydict[x[0]] = x[1]
553         mydict['path'] = path
554         return mydict
555
556     def pre_process(self):
557
558         if os.path.isfile(os.path.join(self.workspace, self.multi_conf_file)):
559             self.b_multi = True
560             with open(os.path.join(self.workspace, self.multi_conf_file)) as f:
561                 for line in f:
562                     file_path = os.path.join(self.workspace, line.rstrip())
563                     self.project_list.append(self.set_properties(file_path))
564         else:
565             self.b_multi = False
566             file_path = os.path.join(self.workspace)
567             self.project_list.append(self.set_properties(file_path))
568
569 def argument_parsing(argv):
570     """Any arguments passed from user"""
571
572     parser = argparse.ArgumentParser(description='ABS command line interface')
573
574     subparsers = parser.add_subparsers(dest='subcommands')
575
576     #### [subcommand - BUILD] ####
577     build = subparsers.add_parser('build')
578     build.add_argument('-w', '--workspace', action='store', dest='workspace', \
579                         help='source directory')
580     build.add_argument('-r', '--rootstrap', action='store', dest='rootstrap', \
581                         help='(ex, mobile-3.0-device.core) rootstrap name')
582     build.add_argument('-a', '--arch', action='store', dest='arch', \
583                         help='(x86|arm) Architecture to build')
584     build.add_argument('-t', '--type', action='store', dest='type', \
585                         help='(tpk|wgt) Packaging type')
586     build.add_argument('-s', '--cert', action='store', dest='cert', \
587                         help='(ex, ABS) Certificate profile name')
588     build.add_argument('-c', '--conf', action='store',default='Release', dest='conf', \
589                         help='(ex, Debug|Release) Build Configuration')
590     build.add_argument('--sdkpath', action='store', dest='sdkpath', \
591                         help='Specify Tizen SDK installation root (one time init).' \
592                              ' ex) /home/yours/tizen-sdk/')
593
594     return parser.parse_args(argv[1:])
595
596 def build_main(args):
597     """Command [build] entry point."""
598
599     my_source = Source(src=args.workspace)
600     my_sdk = Sdk(sdkpath=args.sdkpath)
601     my_sdk.clean(my_source)
602     my_sdk.build_native(my_source, rootstrap=args.rootstrap, arch=args.arch, conf=args.conf)
603     if args.conf == 'Debug' :
604         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
605         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf, manual_strip=True)
606     else :
607         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
608
609 def main(argv):
610     """Script entry point."""
611
612     print 'ABS SCRIPT FROM GIT'
613
614     args = argument_parsing(argv)
615
616     if args.subcommands == 'build':
617         return build_main(args)
618     else:
619         print 'Unsupported command %s' % args.subcommands
620         raise LocalError('Command %s not supported' % args.subcommands)
621
622 if __name__ == '__main__':
623
624     try:
625         sys.exit(main(sys.argv))
626     except Exception, e:
627         print 'Exception %s' % str(e)
628         sys.exit(1)
629