Support Arch (x84_64, aarch64)
[scm/meta/abs.git] / abs
1 #!/usr/bin/env python
2 # vim: ai ts=4 sts=4 et sw=4
3 #
4 # Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd.
5 #
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the Free
8 # Software Foundation; version 2 of the License
9 #
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13 # for more details.
14 #
15
16 import sys
17 import os
18 import subprocess
19 import re
20 import argparse
21 from argparse import ArgumentParser
22 import ConfigParser
23 import glob
24 import fnmatch
25 import shutil
26 import zipfile
27 import errno
28
29 g_home = os.path.dirname(os.path.realpath(__file__))
30
31 class LocalError(Exception):
32     """Local error exception."""
33
34     pass
35
36 class Executor(object):
37     """Subprocess wrapper"""
38
39     def __init__(self, checker=None):
40         self.stdout = subprocess.PIPE
41         self.stderr = subprocess.STDOUT
42         self.checker = checker
43
44     def run(self, cmdline_or_args, show=False, checker=False):
45         """Execute external command"""
46
47         out = ''
48         try:
49             process = subprocess.Popen(cmdline_or_args, \
50                                        stdout=self.stdout, \
51                                        stderr=self.stderr, \
52                                        shell=True)
53             while True:
54                 line = process.stdout.readline()
55                 if show:
56                     print line.rstrip()
57                 out = out + '\n' + line.rstrip()
58                 if not line:
59                     break
60         except:
61             raise LocalError('Running process failed')
62
63         return out
64
65 def list_files(path, ext=None):
66
67     f_list = []
68     for root, dirnames, filenames in os.walk(path):
69         if ext is None:
70             for filename in filenames:
71                 f_list.append(os.path.join(root, filename))
72         else:
73             for filename in fnmatch.filter(filenames, '*.'+ext):
74                 f_list.append(os.path.join(root, filename))
75     return f_list
76
77 class FakeSecHead(object):
78
79     def __init__(self, fp):
80
81         self.fp = fp
82         self.sechead = '[ascection]\n'
83
84     def readline(self):
85
86         if self.sechead:
87             try:
88                 return self.sechead
89             finally:
90                 self.sechead = None
91         else:
92             return self.fp.readline()
93
94 class ErrorParser(object):
95     """Inspect specific error string"""
96
97     parsers = []
98
99     def __init__(self):
100
101         ErrorParser = {'GNU_LINKER':['(.*?):?\(\.\w+\+.*\): (.*)', \
102                                      '(.*[/\\\])?ld(\.exe)?: (.*)'], \
103                        'GNU_GCC':['(.*?):(\d+):(\d+:)? [Ee]rror: ([`\'"](.*)[\'"] undeclared .*)', \
104                                   '(.*?):(\d+):(\d+:)? [Ee]rror: (conflicting types for .*[`\'"](.*)[\'"].*)', \
105                                   '(.*?):(\d+):(\d+:)? (parse error before.*[`\'"](.*)[\'"].*)', \
106                                   '(.*?):(\d+):(\d+:)?\s*(([Ee]rror)|(ERROR)): (.*)'], \
107 #                                  '(.*?):(\d+):(\d+:)? (.*)'], \
108                        'GNU_GMAKE':['(.*):(\d*): (\*\*\* .*)', \
109                                     '.*make.*: \*\*\* .*', \
110                                     '.*make.*: Target (.*) not remade because of errors.', \
111                                     '.*[Cc]ommand not found.*', \
112                                     'Error:\s*(.*)'], \
113                        'TIZEN_NATIVE':['.*ninja: build stopped.*', \
114                                        'edje_cc: Error..(.*):(\d).*', \
115                                        'edje_cc: Error.*']}
116
117         for parser in ErrorParser:
118             parser_env = os.getenv('SDK_ERROR_'+parser)
119             if parser_env:
120                 self.parsers.append(parser_env)
121             else:
122                 for msg in ErrorParser[parser]:
123                     self.parsers.append(msg)
124
125     def check(self, full_log):
126         """Check error string line by line"""
127
128         #snipset_text = full_log[:full_log.rfind('PLATFORM_VER\t')].split('\n')
129         for line in full_log.split('\n'):
130             errline = re.search('|'.join(self.parsers), line[:1024])
131             if errline:
132                 return errline.string #Errors
133         return None #No error
134
135 class _Rootstrap(object):
136     """Tizen SDK rootstrap info.
137        Used only in Sdk class"""
138
139     rootstrap_list = None
140     sdk_path = None
141
142     def __init__(self, sdk_path=None, config=None):
143
144         self.tizen = sdk_path
145         self.list_rootstrap()
146         self.config_file = config
147
148     def list_rootstrap(self):
149         """List all the rootstraps"""
150
151         if self.rootstrap_list != None:
152             return self.rootstrap_list
153
154         cmdline = self.tizen + ' list rootstrap'
155         ret = Executor().run(cmdline, show=False)
156         for x in ret.splitlines():
157             if re.search('(mobile|wearable)-(2.4|3.0)-(device|emulator|device64|emulator64).core.*', x):
158                 if self.rootstrap_list == None:
159                     self.rootstrap_list = []
160                 self.rootstrap_list.append(x.split(' ')[0])
161         return self.rootstrap_list
162
163     def check_rootstrap(self, rootstrap, show=True):
164         """Specific rootstrap is in the SDK
165            Otherwise use default"""
166
167         if rootstrap == None:
168             rootstrap = 'mobile-3.0-emulator.core' #default
169         if rootstrap in self.rootstrap_list:
170             return rootstrap
171         else:
172             if show == True:
173                 print '  ERROR: Rootstrap [%s] does not exist' % rootstrap
174                 print '  Update your rootstrap or use one of:\n    %s' \
175                        % '\n    '.join(self.list_rootstrap())
176             return None
177
178 class Sdk(object):
179     """Tizen SDK related job"""
180
181     rs = None #_Rootstrap class instance
182     rootstrap_list = None
183     sdk_to_search = ['tizen-sdk/tools/ide/bin/tizen', \
184                      'tizen-sdk-ux/tools/ide/bin/tizen', \
185                      'tizen-sdk-cli/tools/ide/bin/tizen']
186
187     def __init__(self, sdkpath=None):
188
189         self.error_parser = ErrorParser()
190         self.runtool = Executor(checker=self.error_parser)
191
192         self.home = os.getenv('HOME')
193         self.config_file = os.path.join(g_home, '.abs')
194
195         if sdkpath is None:
196             self.tizen = self.get_user_root()
197             if self.tizen is None or self.tizen == '':
198                 for i in self.sdk_to_search:
199                     if os.path.isfile(os.path.join(self.home, i)):
200                         self.tizen = os.path.join(self.home, i)
201                         break
202         else:
203             self.tizen = os.path.join(sdkpath, 'tools/ide/bin/tizen')
204             self.update_user_root(self.tizen)
205
206         if not os.path.isfile(self.tizen):
207             print 'Cannot locate cli tool'
208             raise LocalError('Fail to locate cli tool')
209
210         self.rs = _Rootstrap(sdk_path=self.tizen, config=self.config_file)
211
212     def get_user_root(self):
213
214         if os.path.isfile(self.config_file):
215             config = ConfigParser.RawConfigParser()
216             config.read(self.config_file)
217             return config.get('Global', 'tizen')
218         return None
219
220     def update_user_root(self, path):
221
222         if not os.path.isfile(self.config_file):
223             with open(self.config_file, 'w') as f:
224                 f.write('[Global]\n')
225                 f.write('tizen = %s\n' % path)
226             return
227
228         config = ConfigParser.RawConfigParser()
229         config.read(self.config_file)
230         config.set('Global', 'tizen', path)
231         with open(self.config_file, 'wb') as cf:
232             config.write(cf)
233
234     def list_rootstrap(self):
235         return self.rs.list_rootstrap()
236
237     def check_rootstrap(self, rootstrap):
238         return self.rs.check_rootstrap(rootstrap)
239
240     def _run(self, command, args, show=True, checker=False):
241         """Run a tizen command"""
242
243         cmd = [self.tizen, command] + args
244         print '\nRunning command:\n    %s' % ' '.join(cmd)
245         return self.runtool.run(' '.join(cmd), show=show, checker=checker)
246
247     def copytree2(self, src, dst, symlinks=False, ignore=None):
248         """Copy with Ignore & Overwrite"""
249         names = os.listdir(src)
250         if ignore is not None:
251             ignored_names = ignore(src, names)
252         else:
253             ignored_names = set()
254
255         try:
256             os.makedirs(dst)
257         except:
258             pass
259
260         errors = []
261         for name in names:
262             if name in ignored_names:
263                 continue
264             srcname = os.path.join(src, name)
265             dstname = os.path.join(dst, name)
266             try:
267                 if symlinks and os.path.islink(srcname):
268                     linkto = os.readlink(srcname)
269                     os.symlink(linkto, dstname)
270                 elif os.path.isdir(srcname):
271                     self.copytree2(srcname, dstname, symlinks, ignore)
272                 else:
273                     # Will raise a SpecialFileError for unsupported file types
274                     shutil.copy2(srcname, dstname)
275             # catch the Error from the recursive copytree so that we can
276             # continue with other files
277             except shutil.Error, err:
278                 errors.extend(err.args[0])
279             except EnvironmentError, why:
280                 errors.append((srcname, dstname, str(why)))
281         try:
282             shutil.copystat(src, dst)
283         except OSError, why:
284             if WindowsError is not None and isinstance(why, WindowsError):
285                 # Copying file access times may fail on Windows
286                 pass
287             else:
288                 errors.append((src, dst, str(why)))
289         #if errors:
290          #   raise shutil.Error, errors
291
292     def _copy_build_output(self, src, dst):
293         if not os.path.isdir(src) :
294             return
295         try:
296             self.copytree2(src, dst, ignore=shutil.ignore_patterns('*.edc', '*.po', 'objs', '*.info', '*.so', 'CMakeLists.txt', '*.h', '*.c'))
297         except OSError as exc:
298             # File already exist
299             if exc.errno == errno.EEXIST:
300                 shutil.copy(src, dst)
301             if exc.errno == errno.ENOENT:
302                 shutil.copy(src, dst)
303             else:
304                 raise
305
306     def _package_sharedlib(self, project_path, conf, app_name):
307         """If -r option used for packaging, make zip file from copied files"""
308         #project_path=project['path']
309         project_build_output_path=os.path.join(project_path, conf)
310         package_path=os.path.join(project_build_output_path, '.pkg')
311
312         if os.path.isdir(package_path):
313             shutil.rmtree(package_path)
314         os.makedirs(package_path)
315         os.makedirs(os.path.join(package_path, 'lib'))
316
317         #Copy project resource
318         self._copy_build_output(os.path.join(project_path, 'lib'), os.path.join(package_path, 'lib'))
319         self._copy_build_output(os.path.join(project_path, 'res'), os.path.join(package_path, 'res'))
320
321         #Copy built res resource
322         self._copy_build_output(os.path.join(project_build_output_path, 'res'), os.path.join(package_path, 'res'))
323
324         #Copy so library file
325         for filename in list_files(project_build_output_path, 'so'):
326             shutil.copy(filename, os.path.join(package_path, 'lib'))
327
328         # Copy so library file
329         zipname=app_name + '.zip'
330         rsrc_zip = os.path.join(project_build_output_path, zipname)
331         myZipFile = zipfile.ZipFile(rsrc_zip, 'w')
332         for filename in list_files(package_path):
333             try:
334                 myZipFile.write(filename, filename.replace(package_path, ''))
335             except Exception, e:
336                 print str(e)
337         myZipFile.close()
338         return rsrc_zip
339
340     def build_native(self, source, rootstrap=None, arch=None, conf='Debug'):
341         """SDK CLI build command"""
342
343         _rootstrap = self.check_rootstrap(rootstrap)
344         if _rootstrap == None:
345             raise LocalError('Rootstrap %s not exist' % rootstrap)
346
347         if rootstrap is None and arch is None:
348             rootstrap = _rootstrap
349             self.arch = 'x86'
350         elif arch is None:
351             if 'emulator64' in rootstrap: self.arch = 'x86_64'
352             elif 'device64' in rootstrap: self.arch = 'aarch64'
353             elif 'emulator' in rootstrap: self.arch = 'x86'
354             elif 'device' in rootstrap: self.arch = 'arm'
355         elif rootstrap is None:
356             if arch not in ['x86', 'arm', 'aarch64', 'x86_64']:
357                 raise LocalError('Architecture and rootstrap mismatch')
358
359             rootstrap = _rootstrap
360             if arch == 'x86_64': rootstrap = rootstrap.replace('emulator', 'emulator64')
361             elif arch == 'aarch64': rootstrap = rootstrap.replace('emulator', 'device64')
362             elif arch == 'arm': rootstrap = rootstrap.replace('emulator', 'device')
363
364         for x in source.project_list:
365             out = self._run('build-native', ['-r', rootstrap, '-a', self.arch, '-C', conf, '-c', 'gcc', '--' , x['path']], checker=True)
366             logpath = os.path.join(source.output_dir, \
367                                   'build_%s_%s' % (rootstrap, os.path.basename(x['path'])))
368             if not os.path.isdir(source.output_dir):
369                 os.makedirs(source.output_dir)
370             with open(logpath, 'w') as lf:
371                 lf.write(out)
372             ret = self.error_parser.check(out)
373             if ret:
374                 with open(logpath+'.log', 'w') as lf:
375                     lf.write(out)
376                 raise LocalError(ret)
377
378     def package(self, source, cert=None, pkg_type=None, conf=None):
379         """SDK CLI package command"""
380
381         if cert is None: cert = 'ABS'
382         if pkg_type is None: pkg_type = 'tpk'
383         if conf is None: conf = 'Debug'
384
385         final_app = ''
386         main_args = ['-t', pkg_type, '-s', cert]
387         out = '' #logfile
388
389         if conf == 'Release' :
390             main_args.extend(['--strip', 'on'])
391
392         for i, x in enumerate(source.project_list):
393             if x['type'] == 'app':
394                 out = '%s\n%s' % (out, \
395                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
396                 try:
397                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
398                 except:
399                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
400                 x['out_package'] = final_app
401             elif x['type'] == 'sharedLib':
402                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
403                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
404             else:
405                 raise LocalError('Not supported project type %s' % x['type'])
406
407         if source.b_multi == True:
408             extra_args=[]
409             print 'THIS IS MULTI PROJECT'
410             for i, x in enumerate(source.project_list):
411                 if x['out_package'] != final_app and x['type'] == 'app':
412                     extra_args.extend(['-r', x['out_package']])
413                 elif x['type'] == 'sharedLib':
414                     extra_args.extend(['-r', x['out_package']])
415
416             extra_args.extend(['--', final_app])
417             out = self._run('package', main_args + extra_args)
418
419         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
420         print 'Packaging final step again!'
421         out = self._run('package', main_args + ['--', final_app])
422
423         #Copy tpk to output directory
424         shutil.copy(final_app, source.output_dir)
425
426     def package_new(self, source, cert=None, pkg_type=None, conf=None, manual_strip=False):
427         """SDK CLI package command
428             IF Debug + Manual Strip off then generate package-name-debug.tpk
429             IF Debug + Manual Strip on then generate package-name.tpk with custom strip
430             IF Release then generate package-name.tpk with strip option
431         """
432
433         if cert is None: cert = 'ABS'
434         if pkg_type is None: pkg_type = 'tpk'
435         if conf is None: conf = 'Debug'
436
437         final_app = ''
438         main_args = ['-t', pkg_type, '-s', cert]
439         out = '' #logfile
440
441         # remove tpk or zip file on project path
442         package_list = []
443         for i, x in enumerate(source.project_list):
444             package_list.extend(list_files(os.path.join(x['path'], conf), ext='tpk'))
445             package_list.extend(list_files(os.path.join(x['path'], conf), ext='zip'))
446
447         for k in package_list :
448             print ' package list ' + k;
449             os.remove(k)
450
451         # Manual strip
452         if manual_strip == True :
453             strip_cmd='';
454             if self.arch == None:
455                 raise LocalError('Architecture is Noen')
456             elif self.arch == 'x86' :
457                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../i386-linux-gnueabi-gcc-4.9/bin/i386-linux-gnueabi-strip')
458             elif self.arch == 'arm' :
459                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../arm-linux-gnueabi-gcc-4.9/bin/arm-linux-gnueabi-strip')
460             elif self.arch == 'x86_64' :
461                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../x86_64-linux-gnu-gcc-4.9/bin/x86_64-linux-gnu-strip')
462             elif self.arch == 'aarch64' :
463                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../aarch64-linux-gnu-gcc-4.9/bin/aarch64-linux-gnu-strip')
464
465             print strip_cmd
466
467             for i, x in enumerate(source.project_list):
468                 dir = os.path.join(x['path'], conf)
469                 files = [os.path.join(dir,f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir,f))]
470                 for k in files:
471                     cmdline = strip_cmd + ' ' + k;
472                     #print 'my command line ' + cmdline;
473                     Executor().run(cmdline, show=False)
474         elif conf == 'Release':
475             main_args.extend(['--strip', 'on'])
476
477         for i, x in enumerate(source.project_list):
478             if x['type'] == 'app':
479                 out = '%s\n%s' % (out, \
480                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
481                 try:
482                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
483                 except:
484                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
485                 x['out_package'] = final_app
486             elif x['type'] == 'sharedLib':
487                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
488                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
489             else:
490                 raise LocalError('Not supported project type %s' % x['type'])
491
492         if source.b_multi == True:
493             extra_args=[]
494             print 'THIS IS MULTI PROJECT'
495             for i, x in enumerate(source.project_list):
496                 if x['out_package'] != final_app and x['type'] == 'app':
497                     extra_args.extend(['-r', x['out_package']])
498                 elif x['type'] == 'sharedLib':
499                     extra_args.extend(['-r', x['out_package']])
500
501             extra_args.extend(['--', final_app])
502             out = self._run('package', main_args + extra_args)
503
504         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
505         print 'Packaging final step again!'
506         out = self._run('package', main_args + ['--', final_app])
507
508         #Copy tpk to output directory
509         if conf == 'Debug' and manual_strip == False :
510             basename = os.path.splitext(final_app)[0]
511             newname = basename +'-debug.tpk'
512             os.rename(final_app, newname)
513             shutil.copy(newname, source.output_dir)
514         else :
515             shutil.copy(final_app, source.output_dir)
516
517     def clean(self, source):
518         """SDK CLI clean command"""
519
520         if os.path.isdir(source.multizip_path):
521             shutil.rmtree(source.multizip_path)
522
523         if os.path.isfile(os.path.join(source.multizip_path, '.zip')):
524             os.remove(os.path.join(source.multizip_path, '.zip'))
525
526         for x in source.project_list:
527             self._run('clean', ['--', x['path']], show=False)
528
529 class Source(object):
530     """Project source related job"""
531
532     workspace = '' #Project root directory
533     project_list = []
534     b_multi = False
535     multi_conf_file = 'WORKSPACE' #Assume multi-project if this file exist.
536     multizip_path = '' #For multi-project packaging -r option
537     property_dict = {}
538     output_dir = '_abs_out_'
539
540     def __init__(self, src=None):
541
542         if src == None:
543             self.workspace = os.getcwd()
544         else:
545             self.workspace = os.path.abspath(src)
546         self.output_dir = os.path.join(self.workspace, self.output_dir)
547
548         os.environ['workspace_loc']=str(os.path.realpath(self.workspace))
549
550         self.multizip_path = os.path.join(self.workspace, 'multizip')
551         self.pre_process()
552
553     def set_properties(self, path):
554         """Fetch all properties from project_def.prop"""
555
556         mydict = {}
557         cp = ConfigParser.SafeConfigParser()
558         cp.optionxform = str
559         cp.readfp(FakeSecHead(open(os.path.join(path, 'project_def.prop'))))
560         for x in cp.items('ascection'):
561             mydict[x[0]] = x[1]
562         mydict['path'] = path
563         return mydict
564
565     def pre_process(self):
566
567         if os.path.isfile(os.path.join(self.workspace, self.multi_conf_file)):
568             self.b_multi = True
569             with open(os.path.join(self.workspace, self.multi_conf_file)) as f:
570                 for line in f:
571                     file_path = os.path.join(self.workspace, line.rstrip())
572                     self.project_list.append(self.set_properties(file_path))
573         else:
574             self.b_multi = False
575             file_path = os.path.join(self.workspace)
576             self.project_list.append(self.set_properties(file_path))
577
578 def argument_parsing(argv):
579     """Any arguments passed from user"""
580
581     parser = argparse.ArgumentParser(description='ABS command line interface')
582
583     subparsers = parser.add_subparsers(dest='subcommands')
584
585     #### [subcommand - BUILD] ####
586     build = subparsers.add_parser('build')
587     build.add_argument('-w', '--workspace', action='store', dest='workspace', \
588                         help='source directory')
589     build.add_argument('-r', '--rootstrap', action='store', dest='rootstrap', \
590                         help='(ex, mobile-3.0-device.core) rootstrap name')
591     build.add_argument('-a', '--arch', action='store', dest='arch', \
592                         help='(x86|arm|x86_64|aarch64) Architecture to build')
593     build.add_argument('-t', '--type', action='store', dest='type', \
594                         help='(tpk|wgt) Packaging type')
595     build.add_argument('-s', '--cert', action='store', dest='cert', \
596                         help='(ex, ABS) Certificate profile name')
597     build.add_argument('-c', '--conf', action='store',default='Release', dest='conf', \
598                         help='(ex, Debug|Release) Build Configuration')
599     build.add_argument('--sdkpath', action='store', dest='sdkpath', \
600                         help='Specify Tizen SDK installation root (one time init).' \
601                              ' ex) /home/yours/tizen-sdk/')
602
603     return parser.parse_args(argv[1:])
604
605 def build_main(args):
606     """Command [build] entry point."""
607
608     my_source = Source(src=args.workspace)
609     my_sdk = Sdk(sdkpath=args.sdkpath)
610     my_sdk.clean(my_source)
611     my_sdk.build_native(my_source, rootstrap=args.rootstrap, arch=args.arch, conf=args.conf)
612     if args.conf == 'Debug' :
613         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
614         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf, manual_strip=True)
615     else :
616         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
617
618 def main(argv):
619     """Script entry point."""
620
621     print 'ABS SCRIPT FROM GIT'
622
623     args = argument_parsing(argv)
624
625     if args.subcommands == 'build':
626         return build_main(args)
627     else:
628         print 'Unsupported command %s' % args.subcommands
629         raise LocalError('Command %s not supported' % args.subcommands)
630
631 if __name__ == '__main__':
632
633     try:
634         sys.exit(main(sys.argv))
635     except Exception, e:
636         print 'Exception %s' % str(e)
637         sys.exit(1)
638