Merge "Support Arch (x84_64, aarch64)"
[scm/meta/abs.git] / abs
1 #!/usr/bin/env python
2 # vim: ai ts=4 sts=4 et sw=4
3 #
4 # Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd.
5 #
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the Free
8 # Software Foundation; version 2 of the License
9 #
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13 # for more details.
14 #
15
16 import sys
17 import os
18 import subprocess
19 import re
20 import argparse
21 from argparse import ArgumentParser
22 import ConfigParser
23 import glob
24 import fnmatch
25 import shutil
26 import zipfile
27 import errno
28
29 g_home = os.path.dirname(os.path.realpath(__file__))
30
31 class LocalError(Exception):
32     """Local error exception."""
33
34     pass
35
36 class Executor(object):
37     """Subprocess wrapper"""
38
39     def __init__(self, checker=None):
40         self.stdout = subprocess.PIPE
41         self.stderr = subprocess.STDOUT
42         self.checker = checker
43
44     def run(self, cmdline_or_args, show=False, checker=False):
45         """Execute external command"""
46
47         out = ''
48         try:
49             process = subprocess.Popen(cmdline_or_args, \
50                                        stdout=self.stdout, \
51                                        stderr=self.stderr, \
52                                        shell=True)
53             while True:
54                 line = process.stdout.readline()
55                 if show:
56                     print line.rstrip()
57                 out = out + '\n' + line.rstrip()
58                 if not line:
59                     break
60         except:
61             raise LocalError('Running process failed')
62
63         return out
64
65 def list_files(path, ext=None):
66
67     f_list = []
68     for root, dirnames, filenames in os.walk(path):
69         if ext is None:
70             for filename in filenames:
71                 f_list.append(os.path.join(root, filename))
72         else:
73             for filename in fnmatch.filter(filenames, '*.'+ext):
74                 f_list.append(os.path.join(root, filename))
75     return f_list
76
77 class FakeSecHead(object):
78
79     def __init__(self, fp):
80
81         self.fp = fp
82         self.sechead = '[ascection]\n'
83
84     def readline(self):
85
86         if self.sechead:
87             try:
88                 return self.sechead
89             finally:
90                 self.sechead = None
91         else:
92             return self.fp.readline()
93
94 class ErrorParser(object):
95     """Inspect specific error string"""
96
97     parsers = []
98
99     def __init__(self):
100
101         ErrorParser = {'GNU_LINKER':['(.*?):?\(\.\w+\+.*\): (.*)', \
102                                      '(.*[/\\\])?ld(\.exe)?: (.*)'], \
103                        'GNU_GCC':['(.*?):(\d+):(\d+:)? [Ee]rror: ([`\'"](.*)[\'"] undeclared .*)', \
104                                   '(.*?):(\d+):(\d+:)? [Ee]rror: (conflicting types for .*[`\'"](.*)[\'"].*)', \
105                                   '(.*?):(\d+):(\d+:)? (parse error before.*[`\'"](.*)[\'"].*)', \
106                                   '(.*?):(\d+):(\d+:)?\s*(([Ee]rror)|(ERROR)): (.*)'], \
107 #                                  '(.*?):(\d+):(\d+:)? (.*)'], \
108                        'GNU_GMAKE':['(.*):(\d*): (\*\*\* .*)', \
109                                     '.*make.*: \*\*\* .*', \
110                                     '.*make.*: Target (.*) not remade because of errors.', \
111                                     '.*[Cc]ommand not found.*', \
112                                     'Error:\s*(.*)'], \
113                        'TIZEN_NATIVE':['.*ninja: build stopped.*', \
114                                        'edje_cc: Error..(.*):(\d).*', \
115                                        'edje_cc: Error.*']}
116
117         for parser in ErrorParser:
118             parser_env = os.getenv('SDK_ERROR_'+parser)
119             if parser_env:
120                 self.parsers.append(parser_env)
121             else:
122                 for msg in ErrorParser[parser]:
123                     self.parsers.append(msg)
124
125     def check(self, full_log):
126         """Check error string line by line"""
127
128         #snipset_text = full_log[:full_log.rfind('PLATFORM_VER\t')].split('\n')
129         for line in full_log.split('\n'):
130             errline = re.search('|'.join(self.parsers), line[:1024])
131             if errline:
132                 return errline.string #Errors
133         return None #No error
134
135 class _Rootstrap(object):
136     """Tizen SDK rootstrap info.
137        Used only in Sdk class"""
138
139     rootstrap_list = None
140     sdk_path = None
141
142     def __init__(self, sdk_path=None, config=None):
143
144         self.tizen = sdk_path
145         self.list_rootstrap()
146         self.config_file = config
147
148     def list_rootstrap(self):
149         """List all the rootstraps"""
150
151         if self.rootstrap_list != None:
152             return self.rootstrap_list
153
154         cmdline = self.tizen + ' list rootstrap'
155         ret = Executor().run(cmdline, show=False)
156         for x in ret.splitlines():
157             if re.search('(mobile|wearable)-(2.4|3.0)-(device|emulator|device64|emulator64).core.*', x):
158                 if self.rootstrap_list == None:
159                     self.rootstrap_list = []
160                 self.rootstrap_list.append(x.split(' ')[0])
161         return self.rootstrap_list
162
163     def check_rootstrap(self, rootstrap, show=True):
164         """Specific rootstrap is in the SDK
165            Otherwise use default"""
166
167         if rootstrap == None:
168             rootstrap = 'mobile-3.0-emulator.core' #default
169         if rootstrap in self.rootstrap_list:
170             return rootstrap
171         else:
172             if show == True:
173                 print '  ERROR: Rootstrap [%s] does not exist' % rootstrap
174                 print '  Update your rootstrap or use one of:\n    %s' \
175                        % '\n    '.join(self.list_rootstrap())
176             return None
177
178 class Sdk(object):
179     """Tizen SDK related job"""
180
181     rs = None #_Rootstrap class instance
182     rootstrap_list = None
183     sdk_to_search = ['tizen-studio/tools/ide/bin/tizen', \
184                      'tizen-sdk/tools/ide/bin/tizen', \
185                      'tizen-sdk-ux/tools/ide/bin/tizen', \
186                      'tizen-sdk-cli/tools/ide/bin/tizen']
187
188     def __init__(self, sdkpath=None):
189
190         self.error_parser = ErrorParser()
191         self.runtool = Executor(checker=self.error_parser)
192
193         self.home = os.getenv('HOME')
194         self.config_file = os.path.join(g_home, '.abs')
195
196         if sdkpath is None:
197             self.tizen = self.get_user_root()
198             if self.tizen is None or self.tizen == '':
199                 for i in self.sdk_to_search:
200                     if os.path.isfile(os.path.join(self.home, i)):
201                         self.tizen = os.path.join(self.home, i)
202                         break
203         else:
204             self.tizen = os.path.join(sdkpath, 'tools/ide/bin/tizen')
205             self.update_user_root(self.tizen)
206
207         if not os.path.isfile(self.tizen):
208             print 'Cannot locate cli tool'
209             raise LocalError('Fail to locate cli tool')
210
211         self.rs = _Rootstrap(sdk_path=self.tizen, config=self.config_file)
212
213     def get_user_root(self):
214
215         if os.path.isfile(self.config_file):
216             config = ConfigParser.RawConfigParser()
217             config.read(self.config_file)
218             return config.get('Global', 'tizen')
219         return None
220
221     def update_user_root(self, path):
222
223         if not os.path.isfile(self.config_file):
224             with open(self.config_file, 'w') as f:
225                 f.write('[Global]\n')
226                 f.write('tizen = %s\n' % path)
227             return
228
229         config = ConfigParser.RawConfigParser()
230         config.read(self.config_file)
231         config.set('Global', 'tizen', path)
232         with open(self.config_file, 'wb') as cf:
233             config.write(cf)
234
235     def list_rootstrap(self):
236         return self.rs.list_rootstrap()
237
238     def check_rootstrap(self, rootstrap):
239         return self.rs.check_rootstrap(rootstrap)
240
241     def _run(self, command, args, show=True, checker=False):
242         """Run a tizen command"""
243
244         cmd = [self.tizen, command] + args
245         print '\nRunning command:\n    %s' % ' '.join(cmd)
246         return self.runtool.run(' '.join(cmd), show=show, checker=checker)
247
248     def copytree2(self, src, dst, symlinks=False, ignore=None):
249         """Copy with Ignore & Overwrite"""
250         names = os.listdir(src)
251         if ignore is not None:
252             ignored_names = ignore(src, names)
253         else:
254             ignored_names = set()
255
256         try:
257             os.makedirs(dst)
258         except:
259             pass
260
261         errors = []
262         for name in names:
263             if name in ignored_names:
264                 continue
265             srcname = os.path.join(src, name)
266             dstname = os.path.join(dst, name)
267             try:
268                 if symlinks and os.path.islink(srcname):
269                     linkto = os.readlink(srcname)
270                     os.symlink(linkto, dstname)
271                 elif os.path.isdir(srcname):
272                     self.copytree2(srcname, dstname, symlinks, ignore)
273                 else:
274                     # Will raise a SpecialFileError for unsupported file types
275                     shutil.copy2(srcname, dstname)
276             # catch the Error from the recursive copytree so that we can
277             # continue with other files
278             except shutil.Error, err:
279                 errors.extend(err.args[0])
280             except EnvironmentError, why:
281                 errors.append((srcname, dstname, str(why)))
282         try:
283             shutil.copystat(src, dst)
284         except OSError, why:
285             if WindowsError is not None and isinstance(why, WindowsError):
286                 # Copying file access times may fail on Windows
287                 pass
288             else:
289                 errors.append((src, dst, str(why)))
290         #if errors:
291          #   raise shutil.Error, errors
292
293     def _copy_build_output(self, src, dst):
294         if not os.path.isdir(src) :
295             return
296         try:
297             self.copytree2(src, dst, ignore=shutil.ignore_patterns('*.edc', '*.po', 'objs', '*.info', '*.so', 'CMakeLists.txt', '*.h', '*.c'))
298         except OSError as exc:
299             # File already exist
300             if exc.errno == errno.EEXIST:
301                 shutil.copy(src, dst)
302             if exc.errno == errno.ENOENT:
303                 shutil.copy(src, dst)
304             else:
305                 raise
306
307     def _package_sharedlib(self, project_path, conf, app_name):
308         """If -r option used for packaging, make zip file from copied files"""
309         #project_path=project['path']
310         project_build_output_path=os.path.join(project_path, conf)
311         package_path=os.path.join(project_build_output_path, '.pkg')
312
313         if os.path.isdir(package_path):
314             shutil.rmtree(package_path)
315         os.makedirs(package_path)
316         os.makedirs(os.path.join(package_path, 'lib'))
317
318         #Copy project resource
319         self._copy_build_output(os.path.join(project_path, 'lib'), os.path.join(package_path, 'lib'))
320         self._copy_build_output(os.path.join(project_path, 'res'), os.path.join(package_path, 'res'))
321
322         #Copy built res resource
323         self._copy_build_output(os.path.join(project_build_output_path, 'res'), os.path.join(package_path, 'res'))
324
325         #Copy so library file
326         for filename in list_files(project_build_output_path, 'so'):
327             shutil.copy(filename, os.path.join(package_path, 'lib'))
328
329         # Copy so library file
330         zipname=app_name + '.zip'
331         rsrc_zip = os.path.join(project_build_output_path, zipname)
332         myZipFile = zipfile.ZipFile(rsrc_zip, 'w')
333         for filename in list_files(package_path):
334             try:
335                 myZipFile.write(filename, filename.replace(package_path, ''))
336             except Exception, e:
337                 print str(e)
338         myZipFile.close()
339         return rsrc_zip
340
341     def build_native(self, source, rootstrap=None, arch=None, conf='Debug'):
342         """SDK CLI build command"""
343
344         _rootstrap = self.check_rootstrap(rootstrap)
345         if _rootstrap == None:
346             raise LocalError('Rootstrap %s not exist' % rootstrap)
347
348         if rootstrap is None and arch is None:
349             rootstrap = _rootstrap
350             self.arch = 'x86'
351         elif arch is None:
352             if 'emulator64' in rootstrap: self.arch = 'x86_64'
353             elif 'device64' in rootstrap: self.arch = 'aarch64'
354             elif 'emulator' in rootstrap: self.arch = 'x86'
355             elif 'device' in rootstrap: self.arch = 'arm'
356         elif rootstrap is None:
357             if arch not in ['x86', 'arm', 'aarch64', 'x86_64']:
358                 raise LocalError('Architecture and rootstrap mismatch')
359
360             rootstrap = _rootstrap
361             if arch == 'x86_64': rootstrap = rootstrap.replace('emulator', 'emulator64')
362             elif arch == 'aarch64': rootstrap = rootstrap.replace('emulator', 'device64')
363             elif arch == 'arm': rootstrap = rootstrap.replace('emulator', 'device')
364
365         for x in source.project_list:
366             out = self._run('build-native', ['-r', rootstrap, '-a', self.arch, '-C', conf, '-c', 'gcc', '--' , x['path']], checker=True)
367             logpath = os.path.join(source.output_dir, \
368                                   'build_%s_%s' % (rootstrap, os.path.basename(x['path'])))
369             if not os.path.isdir(source.output_dir):
370                 os.makedirs(source.output_dir)
371             with open(logpath, 'w') as lf:
372                 lf.write(out)
373             ret = self.error_parser.check(out)
374             if ret:
375                 with open(logpath+'.log', 'w') as lf:
376                     lf.write(out)
377                 raise LocalError(ret)
378
379     def package(self, source, cert=None, pkg_type=None, conf=None):
380         """SDK CLI package command"""
381
382         if cert is None: cert = 'ABS'
383         if pkg_type is None: pkg_type = 'tpk'
384         if conf is None: conf = 'Debug'
385
386         final_app = ''
387         main_args = ['-t', pkg_type, '-s', cert]
388         out = '' #logfile
389
390         if conf == 'Release' :
391             main_args.extend(['--strip', 'on'])
392
393         for i, x in enumerate(source.project_list):
394             if x['type'] == 'app':
395                 out = '%s\n%s' % (out, \
396                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
397                 try:
398                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
399                 except:
400                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
401                 x['out_package'] = final_app
402             elif x['type'] == 'sharedLib':
403                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
404                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
405             else:
406                 raise LocalError('Not supported project type %s' % x['type'])
407
408         if source.b_multi == True:
409             extra_args=[]
410             print 'THIS IS MULTI PROJECT'
411             for i, x in enumerate(source.project_list):
412                 if x['out_package'] != final_app and x['type'] == 'app':
413                     extra_args.extend(['-r', x['out_package']])
414                 elif x['type'] == 'sharedLib':
415                     extra_args.extend(['-r', x['out_package']])
416
417             extra_args.extend(['--', final_app])
418             out = self._run('package', main_args + extra_args)
419
420         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
421         print 'Packaging final step again!'
422         out = self._run('package', main_args + ['--', final_app])
423
424         #Copy tpk to output directory
425         shutil.copy(final_app, source.output_dir)
426
427     def package_new(self, source, cert=None, pkg_type=None, conf=None, manual_strip=False):
428         """SDK CLI package command
429             IF Debug + Manual Strip off then generate package-name-debug.tpk
430             IF Debug + Manual Strip on then generate package-name.tpk with custom strip
431             IF Release then generate package-name.tpk with strip option
432         """
433
434         if cert is None: cert = 'ABS'
435         if pkg_type is None: pkg_type = 'tpk'
436         if conf is None: conf = 'Debug'
437
438         final_app = ''
439         main_args = ['-t', pkg_type, '-s', cert]
440         out = '' #logfile
441
442         # remove tpk or zip file on project path
443         package_list = []
444         for i, x in enumerate(source.project_list):
445             package_list.extend(list_files(os.path.join(x['path'], conf), ext='tpk'))
446             package_list.extend(list_files(os.path.join(x['path'], conf), ext='zip'))
447
448         for k in package_list :
449             print ' package list ' + k;
450             os.remove(k)
451
452         # Manual strip
453         if manual_strip == True :
454             strip_cmd='';
455             if self.arch == None:
456                 raise LocalError('Architecture is Noen')
457             elif self.arch == 'x86' :
458                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../i386-linux-gnueabi-gcc-4.9/bin/i386-linux-gnueabi-strip')
459             elif self.arch == 'arm' :
460                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../arm-linux-gnueabi-gcc-4.9/bin/arm-linux-gnueabi-strip')
461             elif self.arch == 'x86_64' :
462                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../x86_64-linux-gnu-gcc-4.9/bin/x86_64-linux-gnu-strip')
463             elif self.arch == 'aarch64' :
464                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../aarch64-linux-gnu-gcc-4.9/bin/aarch64-linux-gnu-strip')
465
466             print strip_cmd
467
468             for i, x in enumerate(source.project_list):
469                 dir = os.path.join(x['path'], conf)
470                 files = [os.path.join(dir,f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir,f))]
471                 for k in files:
472                     cmdline = strip_cmd + ' ' + k;
473                     #print 'my command line ' + cmdline;
474                     Executor().run(cmdline, show=False)
475         elif conf == 'Release':
476             main_args.extend(['--strip', 'on'])
477
478         for i, x in enumerate(source.project_list):
479             if x['type'] == 'app':
480                 out = '%s\n%s' % (out, \
481                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
482                 try:
483                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
484                 except:
485                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
486                 x['out_package'] = final_app
487             elif x['type'] == 'sharedLib':
488                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
489                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
490             else:
491                 raise LocalError('Not supported project type %s' % x['type'])
492
493         if source.b_multi == True:
494             extra_args=[]
495             print 'THIS IS MULTI PROJECT'
496             for i, x in enumerate(source.project_list):
497                 if x['out_package'] != final_app and x['type'] == 'app':
498                     extra_args.extend(['-r', x['out_package']])
499                 elif x['type'] == 'sharedLib':
500                     extra_args.extend(['-r', x['out_package']])
501
502             extra_args.extend(['--', final_app])
503             out = self._run('package', main_args + extra_args)
504
505         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
506         print 'Packaging final step again!'
507         out = self._run('package', main_args + ['--', final_app])
508
509         #Copy tpk to output directory
510         if conf == 'Debug' and manual_strip == False :
511             basename = os.path.splitext(final_app)[0]
512             newname = basename +'-debug.tpk'
513             os.rename(final_app, newname)
514             shutil.copy(newname, source.output_dir)
515         else :
516             shutil.copy(final_app, source.output_dir)
517
518     def clean(self, source):
519         """SDK CLI clean command"""
520
521         if os.path.isdir(source.multizip_path):
522             shutil.rmtree(source.multizip_path)
523
524         if os.path.isfile(os.path.join(source.multizip_path, '.zip')):
525             os.remove(os.path.join(source.multizip_path, '.zip'))
526
527         for x in source.project_list:
528             self._run('clean', ['--', x['path']], show=False)
529
530 class Source(object):
531     """Project source related job"""
532
533     workspace = '' #Project root directory
534     project_list = []
535     b_multi = False
536     multi_conf_file = 'WORKSPACE' #Assume multi-project if this file exist.
537     multizip_path = '' #For multi-project packaging -r option
538     property_dict = {}
539     output_dir = '_abs_out_'
540
541     def __init__(self, src=None):
542
543         if src == None:
544             self.workspace = os.getcwd()
545         else:
546             self.workspace = os.path.abspath(src)
547         self.output_dir = os.path.join(self.workspace, self.output_dir)
548
549         os.environ['workspace_loc']=str(os.path.realpath(self.workspace))
550
551         self.multizip_path = os.path.join(self.workspace, 'multizip')
552         self.pre_process()
553
554     def set_properties(self, path):
555         """Fetch all properties from project_def.prop"""
556
557         mydict = {}
558         cp = ConfigParser.SafeConfigParser()
559         cp.optionxform = str
560         cp.readfp(FakeSecHead(open(os.path.join(path, 'project_def.prop'))))
561         for x in cp.items('ascection'):
562             mydict[x[0]] = x[1]
563         mydict['path'] = path
564         return mydict
565
566     def pre_process(self):
567
568         if os.path.isfile(os.path.join(self.workspace, self.multi_conf_file)):
569             self.b_multi = True
570             with open(os.path.join(self.workspace, self.multi_conf_file)) as f:
571                 for line in f:
572                     file_path = os.path.join(self.workspace, line.rstrip())
573                     self.project_list.append(self.set_properties(file_path))
574         else:
575             self.b_multi = False
576             file_path = os.path.join(self.workspace)
577             self.project_list.append(self.set_properties(file_path))
578
579 def argument_parsing(argv):
580     """Any arguments passed from user"""
581
582     parser = argparse.ArgumentParser(description='ABS command line interface')
583
584     subparsers = parser.add_subparsers(dest='subcommands')
585
586     #### [subcommand - BUILD] ####
587     build = subparsers.add_parser('build')
588     build.add_argument('-w', '--workspace', action='store', dest='workspace', \
589                         help='source directory')
590     build.add_argument('-r', '--rootstrap', action='store', dest='rootstrap', \
591                         help='(ex, mobile-3.0-device.core) rootstrap name')
592     build.add_argument('-a', '--arch', action='store', dest='arch', \
593                         help='(x86|arm|x86_64|aarch64) Architecture to build')
594     build.add_argument('-t', '--type', action='store', dest='type', \
595                         help='(tpk|wgt) Packaging type')
596     build.add_argument('-s', '--cert', action='store', dest='cert', \
597                         help='(ex, ABS) Certificate profile name')
598     build.add_argument('-c', '--conf', action='store',default='Release', dest='conf', \
599                         help='(ex, Debug|Release) Build Configuration')
600     build.add_argument('--sdkpath', action='store', dest='sdkpath', \
601                         help='Specify Tizen SDK installation root (one time init).' \
602                              ' ex) /home/yours/tizen-sdk/')
603
604     return parser.parse_args(argv[1:])
605
606 def build_main(args):
607     """Command [build] entry point."""
608
609     my_source = Source(src=args.workspace)
610     my_sdk = Sdk(sdkpath=args.sdkpath)
611     my_sdk.clean(my_source)
612     my_sdk.build_native(my_source, rootstrap=args.rootstrap, arch=args.arch, conf=args.conf)
613     if args.conf == 'Debug' :
614         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
615         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf, manual_strip=True)
616     else :
617         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
618
619 def main(argv):
620     """Script entry point."""
621
622     print 'ABS SCRIPT FROM GIT'
623
624     args = argument_parsing(argv)
625
626     if args.subcommands == 'build':
627         return build_main(args)
628     else:
629         print 'Unsupported command %s' % args.subcommands
630         raise LocalError('Command %s not supported' % args.subcommands)
631
632 if __name__ == '__main__':
633
634     try:
635         sys.exit(main(sys.argv))
636     except Exception, e:
637         print 'Exception %s' % str(e)
638         sys.exit(1)