Add tizen-studio path as default
[scm/meta/abs.git] / abs
1 #!/usr/bin/env python
2 # vim: ai ts=4 sts=4 et sw=4
3 #
4 # Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd.
5 #
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the Free
8 # Software Foundation; version 2 of the License
9 #
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13 # for more details.
14 #
15
16 import sys
17 import os
18 import subprocess
19 import re
20 import argparse
21 from argparse import ArgumentParser
22 import ConfigParser
23 import glob
24 import fnmatch
25 import shutil
26 import zipfile
27 import errno
28
29 g_home = os.path.dirname(os.path.realpath(__file__))
30
31 class LocalError(Exception):
32     """Local error exception."""
33
34     pass
35
36 class Executor(object):
37     """Subprocess wrapper"""
38
39     def __init__(self, checker=None):
40         self.stdout = subprocess.PIPE
41         self.stderr = subprocess.STDOUT
42         self.checker = checker
43
44     def run(self, cmdline_or_args, show=False, checker=False):
45         """Execute external command"""
46
47         out = ''
48         try:
49             process = subprocess.Popen(cmdline_or_args, \
50                                        stdout=self.stdout, \
51                                        stderr=self.stderr, \
52                                        shell=True)
53             while True:
54                 line = process.stdout.readline()
55                 if show:
56                     print line.rstrip()
57                 out = out + '\n' + line.rstrip()
58                 if not line:
59                     break
60         except:
61             raise LocalError('Running process failed')
62
63         return out
64
65 def list_files(path, ext=None):
66
67     f_list = []
68     for root, dirnames, filenames in os.walk(path):
69         if ext is None:
70             for filename in filenames:
71                 f_list.append(os.path.join(root, filename))
72         else:
73             for filename in fnmatch.filter(filenames, '*.'+ext):
74                 f_list.append(os.path.join(root, filename))
75     return f_list
76
77 class FakeSecHead(object):
78
79     def __init__(self, fp):
80
81         self.fp = fp
82         self.sechead = '[ascection]\n'
83
84     def readline(self):
85
86         if self.sechead:
87             try:
88                 return self.sechead
89             finally:
90                 self.sechead = None
91         else:
92             return self.fp.readline()
93
94 class ErrorParser(object):
95     """Inspect specific error string"""
96
97     parsers = []
98
99     def __init__(self):
100
101         ErrorParser = {'GNU_LINKER':['(.*?):?\(\.\w+\+.*\): (.*)', \
102                                      '(.*[/\\\])?ld(\.exe)?: (.*)'], \
103                        'GNU_GCC':['(.*?):(\d+):(\d+:)? [Ee]rror: ([`\'"](.*)[\'"] undeclared .*)', \
104                                   '(.*?):(\d+):(\d+:)? [Ee]rror: (conflicting types for .*[`\'"](.*)[\'"].*)', \
105                                   '(.*?):(\d+):(\d+:)? (parse error before.*[`\'"](.*)[\'"].*)', \
106                                   '(.*?):(\d+):(\d+:)?\s*(([Ee]rror)|(ERROR)): (.*)'], \
107 #                                  '(.*?):(\d+):(\d+:)? (.*)'], \
108                        'GNU_GMAKE':['(.*):(\d*): (\*\*\* .*)', \
109                                     '.*make.*: \*\*\* .*', \
110                                     '.*make.*: Target (.*) not remade because of errors.', \
111                                     '.*[Cc]ommand not found.*', \
112                                     'Error:\s*(.*)'], \
113                        'TIZEN_NATIVE':['.*ninja: build stopped.*', \
114                                        'edje_cc: Error..(.*):(\d).*', \
115                                        'edje_cc: Error.*']}
116
117         for parser in ErrorParser:
118             parser_env = os.getenv('SDK_ERROR_'+parser)
119             if parser_env:
120                 self.parsers.append(parser_env)
121             else:
122                 for msg in ErrorParser[parser]:
123                     self.parsers.append(msg)
124
125     def check(self, full_log):
126         """Check error string line by line"""
127
128         #snipset_text = full_log[:full_log.rfind('PLATFORM_VER\t')].split('\n')
129         for line in full_log.split('\n'):
130             errline = re.search('|'.join(self.parsers), line[:1024])
131             if errline:
132                 return errline.string #Errors
133         return None #No error
134
135 class _Rootstrap(object):
136     """Tizen SDK rootstrap info.
137        Used only in Sdk class"""
138
139     rootstrap_list = None
140     sdk_path = None
141
142     def __init__(self, sdk_path=None, config=None):
143
144         self.tizen = sdk_path
145         self.list_rootstrap()
146         self.config_file = config
147
148     def list_rootstrap(self):
149         """List all the rootstraps"""
150
151         if self.rootstrap_list != None:
152             return self.rootstrap_list
153
154         cmdline = self.tizen + ' list rootstrap'
155         ret = Executor().run(cmdline, show=False)
156         for x in ret.splitlines():
157             if re.search('(mobile|wearable)-(2.4|3.0)-(device|emulator).core.*', x):
158                 if self.rootstrap_list == None:
159                     self.rootstrap_list = []
160                 self.rootstrap_list.append(x.split(' ')[0])
161         return self.rootstrap_list
162
163     def check_rootstrap(self, rootstrap, show=True):
164         """Specific rootstrap is in the SDK
165            Otherwise use default"""
166
167         if rootstrap == None:
168             rootstrap = 'mobile-3.0-emulator.core' #default
169         if rootstrap in self.rootstrap_list:
170             return rootstrap
171         else:
172             if show == True:
173                 print '  ERROR: Rootstrap [%s] does not exist' % rootstrap
174                 print '  Update your rootstrap or use one of:\n    %s' \
175                        % '\n    '.join(self.list_rootstrap())
176             return None
177
178 class Sdk(object):
179     """Tizen SDK related job"""
180
181     rs = None #_Rootstrap class instance
182     rootstrap_list = None
183     sdk_to_search = ['tizen-studio/tools/ide/bin/tizen', \
184                      'tizen-sdk/tools/ide/bin/tizen', \
185                      'tizen-sdk-ux/tools/ide/bin/tizen', \
186                      'tizen-sdk-cli/tools/ide/bin/tizen']
187
188     def __init__(self, sdkpath=None):
189
190         self.error_parser = ErrorParser()
191         self.runtool = Executor(checker=self.error_parser)
192
193         self.home = os.getenv('HOME')
194         self.config_file = os.path.join(g_home, '.abs')
195
196         if sdkpath is None:
197             self.tizen = self.get_user_root()
198             if self.tizen is None or self.tizen == '':
199                 for i in self.sdk_to_search:
200                     if os.path.isfile(os.path.join(self.home, i)):
201                         self.tizen = os.path.join(self.home, i)
202                         break
203         else:
204             self.tizen = os.path.join(sdkpath, 'tools/ide/bin/tizen')
205             self.update_user_root(self.tizen)
206
207         if not os.path.isfile(self.tizen):
208             print 'Cannot locate cli tool'
209             raise LocalError('Fail to locate cli tool')
210
211         self.rs = _Rootstrap(sdk_path=self.tizen, config=self.config_file)
212
213     def get_user_root(self):
214
215         if os.path.isfile(self.config_file):
216             config = ConfigParser.RawConfigParser()
217             config.read(self.config_file)
218             return config.get('Global', 'tizen')
219         return None
220
221     def update_user_root(self, path):
222
223         if not os.path.isfile(self.config_file):
224             with open(self.config_file, 'w') as f:
225                 f.write('[Global]\n')
226                 f.write('tizen = %s\n' % path)
227             return
228
229         config = ConfigParser.RawConfigParser()
230         config.read(self.config_file)
231         config.set('Global', 'tizen', path)
232         with open(self.config_file, 'wb') as cf:
233             config.write(cf)
234
235     def list_rootstrap(self):
236         return self.rs.list_rootstrap()
237
238     def check_rootstrap(self, rootstrap):
239         return self.rs.check_rootstrap(rootstrap)
240
241     def _run(self, command, args, show=True, checker=False):
242         """Run a tizen command"""
243
244         cmd = [self.tizen, command] + args
245         print '\nRunning command:\n    %s' % ' '.join(cmd)
246         return self.runtool.run(' '.join(cmd), show=show, checker=checker)
247
248     def copytree2(self, src, dst, symlinks=False, ignore=None):
249         """Copy with Ignore & Overwrite"""
250         names = os.listdir(src)
251         if ignore is not None:
252             ignored_names = ignore(src, names)
253         else:
254             ignored_names = set()
255
256         try:
257             os.makedirs(dst)
258         except:
259             pass
260
261         errors = []
262         for name in names:
263             if name in ignored_names:
264                 continue
265             srcname = os.path.join(src, name)
266             dstname = os.path.join(dst, name)
267             try:
268                 if symlinks and os.path.islink(srcname):
269                     linkto = os.readlink(srcname)
270                     os.symlink(linkto, dstname)
271                 elif os.path.isdir(srcname):
272                     self.copytree2(srcname, dstname, symlinks, ignore)
273                 else:
274                     # Will raise a SpecialFileError for unsupported file types
275                     shutil.copy2(srcname, dstname)
276             # catch the Error from the recursive copytree so that we can
277             # continue with other files
278             except shutil.Error, err:
279                 errors.extend(err.args[0])
280             except EnvironmentError, why:
281                 errors.append((srcname, dstname, str(why)))
282         try:
283             shutil.copystat(src, dst)
284         except OSError, why:
285             if WindowsError is not None and isinstance(why, WindowsError):
286                 # Copying file access times may fail on Windows
287                 pass
288             else:
289                 errors.append((src, dst, str(why)))
290         #if errors:
291          #   raise shutil.Error, errors
292
293     def _copy_build_output(self, src, dst):
294         if not os.path.isdir(src) :
295             return
296         try:
297             self.copytree2(src, dst, ignore=shutil.ignore_patterns('*.edc', '*.po', 'objs', '*.info', '*.so', 'CMakeLists.txt', '*.h', '*.c'))
298         except OSError as exc:
299             # File already exist
300             if exc.errno == errno.EEXIST:
301                 shutil.copy(src, dst)
302             if exc.errno == errno.ENOENT:
303                 shutil.copy(src, dst)
304             else:
305                 raise
306
307     def _package_sharedlib(self, project_path, conf, app_name):
308         """If -r option used for packaging, make zip file from copied files"""
309         #project_path=project['path']
310         project_build_output_path=os.path.join(project_path, conf)
311         package_path=os.path.join(project_build_output_path, '.pkg')
312
313         if os.path.isdir(package_path):
314             shutil.rmtree(package_path)
315         os.makedirs(package_path)
316         os.makedirs(os.path.join(package_path, 'lib'))
317
318         #Copy project resource
319         self._copy_build_output(os.path.join(project_path, 'lib'), os.path.join(package_path, 'lib'))
320         self._copy_build_output(os.path.join(project_path, 'res'), os.path.join(package_path, 'res'))
321
322         #Copy built res resource
323         self._copy_build_output(os.path.join(project_build_output_path, 'res'), os.path.join(package_path, 'res'))
324
325         #Copy so library file
326         for filename in list_files(project_build_output_path, 'so'):
327             shutil.copy(filename, os.path.join(package_path, 'lib'))
328
329         # Copy so library file
330         zipname=app_name + '.zip'
331         rsrc_zip = os.path.join(project_build_output_path, zipname)
332         myZipFile = zipfile.ZipFile(rsrc_zip, 'w')
333         for filename in list_files(package_path):
334             try:
335                 myZipFile.write(filename, filename.replace(package_path, ''))
336             except Exception, e:
337                 print str(e)
338         myZipFile.close()
339         return rsrc_zip
340
341     def build_native(self, source, rootstrap=None, arch=None, conf='Debug'):
342         """SDK CLI build command"""
343
344         _rootstrap = self.check_rootstrap(rootstrap)
345         if _rootstrap == None:
346             raise LocalError('Rootstrap %s not exist' % rootstrap)
347
348         if rootstrap is None and arch is None:
349             rootstrap = _rootstrap
350             self.arch = 'x86'
351         elif arch is None:
352             if 'emulator' in rootstrap: self.arch = 'x86'
353             elif 'device' in rootstrap: self.arch = 'arm'
354         elif rootstrap is None:
355             if arch not in ['x86', 'arm']:
356                 raise LocalError('Architecture and rootstrap mismatch')
357             rootstrap = _rootstrap
358             if arch == 'arm': rootstrap = rootstrap.replace('emulator', 'device')
359
360         for x in source.project_list:
361             out = self._run('build-native', ['-r', rootstrap, '-a', self.arch, '-C', conf, '--' , x['path']], checker=True)
362             logpath = os.path.join(source.output_dir, \
363                                   'build_%s_%s' % (rootstrap, os.path.basename(x['path'])))
364             if not os.path.isdir(source.output_dir):
365                 os.makedirs(source.output_dir)
366             with open(logpath, 'w') as lf:
367                 lf.write(out)
368             ret = self.error_parser.check(out)
369             if ret:
370                 with open(logpath+'.log', 'w') as lf:
371                     lf.write(out)
372                 raise LocalError(ret)
373
374     def package(self, source, cert=None, pkg_type=None, conf=None):
375         """SDK CLI package command"""
376
377         if cert is None: cert = 'ABS'
378         if pkg_type is None: pkg_type = 'tpk'
379         if conf is None: conf = 'Debug'
380
381         final_app = ''
382         main_args = ['-t', pkg_type, '-s', cert]
383         out = '' #logfile
384
385         if conf == 'Release' :
386             main_args.extend(['--strip', 'on'])
387
388         for i, x in enumerate(source.project_list):
389             if x['type'] == 'app':
390                 out = '%s\n%s' % (out, \
391                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
392                 try:
393                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
394                 except:
395                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
396                 x['out_package'] = final_app
397             elif x['type'] == 'sharedLib':
398                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
399                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
400             else:
401                 raise LocalError('Not supported project type %s' % x['type'])
402
403         if source.b_multi == True:
404             extra_args=[]
405             print 'THIS IS MULTI PROJECT'
406             for i, x in enumerate(source.project_list):
407                 if x['out_package'] != final_app and x['type'] == 'app':
408                     extra_args.extend(['-r', x['out_package']])
409                 elif x['type'] == 'sharedLib':
410                     extra_args.extend(['-r', x['out_package']])
411
412             extra_args.extend(['--', final_app])
413             out = self._run('package', main_args + extra_args)
414
415         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
416         print 'Packaging final step again!'
417         out = self._run('package', main_args + ['--', final_app])
418
419         #Copy tpk to output directory
420         shutil.copy(final_app, source.output_dir)
421
422     def package_new(self, source, cert=None, pkg_type=None, conf=None, manual_strip=False):
423         """SDK CLI package command
424             IF Debug + Manual Strip off then generate package-name-debug.tpk
425             IF Debug + Manual Strip on then generate package-name.tpk with custom strip
426             IF Release then generate package-name.tpk with strip option
427         """
428
429         if cert is None: cert = 'ABS'
430         if pkg_type is None: pkg_type = 'tpk'
431         if conf is None: conf = 'Debug'
432
433         final_app = ''
434         main_args = ['-t', pkg_type, '-s', cert]
435         out = '' #logfile
436
437         # remove tpk or zip file on project path
438         package_list = []
439         for i, x in enumerate(source.project_list):
440             package_list.extend(list_files(os.path.join(x['path'], conf), ext='tpk'))
441             package_list.extend(list_files(os.path.join(x['path'], conf), ext='zip'))
442
443         for k in package_list :
444             print ' package list ' + k;
445             os.remove(k)
446
447         # Manual strip
448         if manual_strip == True :
449             strip_cmd='';
450             if self.arch == None:
451                 raise LocalError('Architecture is Noen')
452             elif self.arch == 'x86' :
453                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../i386-linux-gnueabi-gcc-4.9/bin/i386-linux-gnueabi-strip')
454             elif self.arch == 'arm' :
455                 strip_cmd = os.path.join(os.path.dirname(self.tizen), '../../arm-linux-gnueabi-gcc-4.9/bin/arm-linux-gnueabi-strip')
456
457             print strip_cmd
458
459             for i, x in enumerate(source.project_list):
460                 dir = os.path.join(x['path'], conf)
461                 files = [os.path.join(dir,f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir,f))]
462                 for k in files:
463                     cmdline = strip_cmd + ' ' + k;
464                     #print 'my command line ' + cmdline;
465                     Executor().run(cmdline, show=False)
466         elif conf == 'Release':
467             main_args.extend(['--strip', 'on'])
468
469         for i, x in enumerate(source.project_list):
470             if x['type'] == 'app':
471                 out = '%s\n%s' % (out, \
472                       self._run('package', main_args + ['--',os.path.join(x['path'],conf)]))
473                 try:
474                     final_app = list_files(os.path.join(x['path'], conf), ext='tpk')[0]
475                 except:
476                     raise LocalError('TPK file not generated for %s.' % x['APPNAME'])
477                 x['out_package'] = final_app
478             elif x['type'] == 'sharedLib':
479                 self._package_sharedlib(x['path'], conf, x['APPNAME'])
480                 x['out_package'] = list_files(os.path.join(x['path'], conf), ext='zip')[0]
481             else:
482                 raise LocalError('Not supported project type %s' % x['type'])
483
484         if source.b_multi == True:
485             extra_args=[]
486             print 'THIS IS MULTI PROJECT'
487             for i, x in enumerate(source.project_list):
488                 if x['out_package'] != final_app and x['type'] == 'app':
489                     extra_args.extend(['-r', x['out_package']])
490                 elif x['type'] == 'sharedLib':
491                     extra_args.extend(['-r', x['out_package']])
492
493             extra_args.extend(['--', final_app])
494             out = self._run('package', main_args + extra_args)
495
496         #TODO: signature validation check failed : Invalid file reference. An unsigned file was found.
497         print 'Packaging final step again!'
498         out = self._run('package', main_args + ['--', final_app])
499
500         #Copy tpk to output directory
501         if conf == 'Debug' and manual_strip == False :
502             basename = os.path.splitext(final_app)[0]
503             newname = basename +'-debug.tpk'
504             os.rename(final_app, newname)
505             shutil.copy(newname, source.output_dir)
506         else :
507             shutil.copy(final_app, source.output_dir)
508
509     def clean(self, source):
510         """SDK CLI clean command"""
511
512         if os.path.isdir(source.multizip_path):
513             shutil.rmtree(source.multizip_path)
514
515         if os.path.isfile(os.path.join(source.multizip_path, '.zip')):
516             os.remove(os.path.join(source.multizip_path, '.zip'))
517
518         for x in source.project_list:
519             self._run('clean', ['--', x['path']], show=False)
520
521 class Source(object):
522     """Project source related job"""
523
524     workspace = '' #Project root directory
525     project_list = []
526     b_multi = False
527     multi_conf_file = 'WORKSPACE' #Assume multi-project if this file exist.
528     multizip_path = '' #For multi-project packaging -r option
529     property_dict = {}
530     output_dir = '_abs_out_'
531
532     def __init__(self, src=None):
533
534         if src == None:
535             self.workspace = os.getcwd()
536         else:
537             self.workspace = os.path.abspath(src)
538         self.output_dir = os.path.join(self.workspace, self.output_dir)
539
540         os.environ['workspace_loc']=str(os.path.realpath(self.workspace))
541
542         self.multizip_path = os.path.join(self.workspace, 'multizip')
543         self.pre_process()
544
545     def set_properties(self, path):
546         """Fetch all properties from project_def.prop"""
547
548         mydict = {}
549         cp = ConfigParser.SafeConfigParser()
550         cp.optionxform = str
551         cp.readfp(FakeSecHead(open(os.path.join(path, 'project_def.prop'))))
552         for x in cp.items('ascection'):
553             mydict[x[0]] = x[1]
554         mydict['path'] = path
555         return mydict
556
557     def pre_process(self):
558
559         if os.path.isfile(os.path.join(self.workspace, self.multi_conf_file)):
560             self.b_multi = True
561             with open(os.path.join(self.workspace, self.multi_conf_file)) as f:
562                 for line in f:
563                     file_path = os.path.join(self.workspace, line.rstrip())
564                     self.project_list.append(self.set_properties(file_path))
565         else:
566             self.b_multi = False
567             file_path = os.path.join(self.workspace)
568             self.project_list.append(self.set_properties(file_path))
569
570 def argument_parsing(argv):
571     """Any arguments passed from user"""
572
573     parser = argparse.ArgumentParser(description='ABS command line interface')
574
575     subparsers = parser.add_subparsers(dest='subcommands')
576
577     #### [subcommand - BUILD] ####
578     build = subparsers.add_parser('build')
579     build.add_argument('-w', '--workspace', action='store', dest='workspace', \
580                         help='source directory')
581     build.add_argument('-r', '--rootstrap', action='store', dest='rootstrap', \
582                         help='(ex, mobile-3.0-device.core) rootstrap name')
583     build.add_argument('-a', '--arch', action='store', dest='arch', \
584                         help='(x86|arm) Architecture to build')
585     build.add_argument('-t', '--type', action='store', dest='type', \
586                         help='(tpk|wgt) Packaging type')
587     build.add_argument('-s', '--cert', action='store', dest='cert', \
588                         help='(ex, ABS) Certificate profile name')
589     build.add_argument('-c', '--conf', action='store',default='Release', dest='conf', \
590                         help='(ex, Debug|Release) Build Configuration')
591     build.add_argument('--sdkpath', action='store', dest='sdkpath', \
592                         help='Specify Tizen SDK installation root (one time init).' \
593                              ' ex) /home/yours/tizen-sdk/')
594
595     return parser.parse_args(argv[1:])
596
597 def build_main(args):
598     """Command [build] entry point."""
599
600     my_source = Source(src=args.workspace)
601     my_sdk = Sdk(sdkpath=args.sdkpath)
602     my_sdk.clean(my_source)
603     my_sdk.build_native(my_source, rootstrap=args.rootstrap, arch=args.arch, conf=args.conf)
604     if args.conf == 'Debug' :
605         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
606         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf, manual_strip=True)
607     else :
608         my_sdk.package_new(my_source, pkg_type=args.type, cert=args.cert, conf=args.conf)
609
610 def main(argv):
611     """Script entry point."""
612
613     print 'ABS SCRIPT FROM GIT'
614
615     args = argument_parsing(argv)
616
617     if args.subcommands == 'build':
618         return build_main(args)
619     else:
620         print 'Unsupported command %s' % args.subcommands
621         raise LocalError('Command %s not supported' % args.subcommands)
622
623 if __name__ == '__main__':
624
625     try:
626         sys.exit(main(sys.argv))
627     except Exception, e:
628         print 'Exception %s' % str(e)
629         sys.exit(1)