X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Ftools%2Fswarming_client%2Fisolate.py;h=c3b66988205fc6056782895def7798cc63021915;hb=ff3e2503a20db9193d323c1d19c38c68004dec4a;hp=3ff305e5787209f984582b87e63c36ceee07957e;hpb=7338fba38ba696536d1cc9d389afd716a6ab2fe6;p=platform%2Fframework%2Fweb%2Fcrosswalk.git

diff --git a/src/tools/swarming_client/isolate.py b/src/tools/swarming_client/isolate.py
index 3ff305e5..c3b6698 100755
--- a/src/tools/swarming_client/isolate.py
+++ b/src/tools/swarming_client/isolate.py
@@ -22,6 +22,7 @@ import re
 import subprocess
 import sys
 
+import auth
 import isolate_format
 import isolateserver
 import run_isolated
@@ -38,7 +39,7 @@ from utils import file_path
 from utils import tools
 
 
-__version__ = '0.2'
+__version__ = '0.3.1'
 
 
 class ExecutionError(Exception):
@@ -615,11 +616,17 @@ class SavedState(Flattenable):
       raise isolateserver.ConfigError(
           'Unsupported version \'%s\'' % out.version)
 
-    # The .isolate file must be valid. It could be absolute on Windows if the
-    # drive containing the .isolate and the drive containing the .isolated files
-    # differ.
-    assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
-    assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
+    # The .isolate file must be valid. If it is not present anymore, zap the
+    # value as if it was not noted, so .isolate_file can safely be overriden
+    # later.
+    if out.isolate_file and not os.path.isfile(out.isolate_filepath):
+      out.isolate_file = None
+    if out.isolate_file:
+      # It could be absolute on Windows if the drive containing the .isolate and
+      # the drive containing the .isolated files differ, .e.g .isolate is on
+      # C:\\ and .isolated is on D:\\   .
+      assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
+      assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
     return out
 
   def flatten(self):
@@ -981,19 +988,6 @@ def merge(complete_state, trace_blacklist):
         exceptions[0][2]
 
 
-def get_remap_dir(root_dir, isolated, outdir):
-  """If necessary, creates a directory aside the root directory."""
-  if outdir:
-    if not os.path.isdir(outdir):
-      os.makedirs(outdir)
-    return outdir
-
-  if not os.path.isabs(root_dir):
-    root_dir = os.path.join(os.path.dirname(isolated), root_dir)
-  return run_isolated.make_temp_dir(
-      'isolate-%s' % datetime.date.today(), root_dir)
-
-
 def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
   """Creates a isolated tree usable for test execution.
 
@@ -1024,16 +1018,40 @@ def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
   return cwd
 
 
-### Commands.
+def prepare_for_archival(options, cwd):
+  """Loads the isolated file and create 'infiles' for archival."""
+  complete_state = load_complete_state(
+      options, cwd, options.subdir, False)
+  # Make sure that complete_state isn't modified until save_files() is
+  # called, because any changes made to it here will propagate to the files
+  # created (which is probably not intended).
+  complete_state.save_files()
 
+  infiles = complete_state.saved_state.files
+  # Add all the .isolated files.
+  isolated_hash = []
+  isolated_files = [
+    options.isolated,
+  ] + complete_state.saved_state.child_isolated_files
+  for item in isolated_files:
+    item_path = os.path.join(
+        os.path.dirname(complete_state.isolated_filepath), item)
+    # Do not use isolateserver.hash_file() here because the file is
+    # likely smallish (under 500kb) and its file size is needed.
+    with open(item_path, 'rb') as f:
+      content = f.read()
+    isolated_hash.append(
+        complete_state.saved_state.algo(content).hexdigest())
+    isolated_metadata = {
+      'h': isolated_hash[-1],
+      's': len(content),
+      'priority': '0'
+    }
+    infiles[item_path] = isolated_metadata
+  return complete_state, infiles, isolated_hash
 
-def add_subdir_flag(parser):
-  parser.add_option(
-      '--subdir',
-      help='Filters to a subdirectory. Its behavior changes depending if it '
-           'is a relative path as a string or as a path variable. Path '
-           'variables are always keyed from the directory containing the '
-           '.isolate file. Anything else is keyed on the root directory.')
+
+### Commands.
 
 
 def CMDarchive(parser, args):
@@ -1042,63 +1060,28 @@ def CMDarchive(parser, args):
   All the files listed in the .isolated file are put in the isolate server
   cache via isolateserver.py.
   """
-  add_subdir_flag(parser)
+  add_subdir_option(parser)
+  isolateserver.add_isolate_server_options(parser, False)
+  auth.add_auth_options(parser)
   options, args = parser.parse_args(args)
+  auth.process_auth_options(parser, options)
+  isolateserver.process_isolate_server_options(parser, options)
   if args:
     parser.error('Unsupported argument: %s' % args)
-
+  cwd = os.getcwd()
   with tools.Profiler('GenerateHashtable'):
     success = False
     try:
-      complete_state = load_complete_state(
-          options, os.getcwd(), options.subdir, False)
-      if not options.outdir:
-        options.outdir = os.path.join(
-            os.path.dirname(complete_state.isolated_filepath), 'hashtable')
-      # Make sure that complete_state isn't modified until save_files() is
-      # called, because any changes made to it here will propagate to the files
-      # created (which is probably not intended).
-      complete_state.save_files()
-
-      infiles = complete_state.saved_state.files
-      # Add all the .isolated files.
-      isolated_hash = []
-      isolated_files = [
-        options.isolated,
-      ] + complete_state.saved_state.child_isolated_files
-      for item in isolated_files:
-        item_path = os.path.join(
-            os.path.dirname(complete_state.isolated_filepath), item)
-        # Do not use isolateserver.hash_file() here because the file is
-        # likely smallish (under 500kb) and its file size is needed.
-        with open(item_path, 'rb') as f:
-          content = f.read()
-        isolated_hash.append(
-            complete_state.saved_state.algo(content).hexdigest())
-        isolated_metadata = {
-          'h': isolated_hash[-1],
-          's': len(content),
-          'priority': '0'
-        }
-        infiles[item_path] = isolated_metadata
-
+      complete_state, infiles, isolated_hash = prepare_for_archival(
+          options, cwd)
       logging.info('Creating content addressed object store with %d item',
                    len(infiles))
 
-      if file_path.is_url(options.outdir):
-        isolateserver.upload_tree(
-            base_url=options.outdir,
-            indir=complete_state.root_dir,
-            infiles=infiles,
-            namespace='default-gzip')
-      else:
-        recreate_tree(
-            outdir=options.outdir,
-            indir=complete_state.root_dir,
-            infiles=infiles,
-            action=run_isolated.HARDLINK_WITH_FALLBACK,
-            as_hash=True)
-        # TODO(maruel): Make the files read-only?
+      isolateserver.upload_tree(
+          base_url=options.isolate_server,
+          indir=complete_state.root_dir,
+          infiles=infiles,
+          namespace=options.namespace)
       success = True
       print('%s  %s' % (isolated_hash[0], os.path.basename(options.isolated)))
     finally:
@@ -1106,12 +1089,12 @@ def CMDarchive(parser, args):
       # important so no stale swarm job is executed.
       if not success and os.path.isfile(options.isolated):
         os.remove(options.isolated)
-  return not success
+  return int(not success)
 
 
 def CMDcheck(parser, args):
   """Checks that all the inputs are present and generates .isolated."""
-  add_subdir_flag(parser)
+  add_subdir_option(parser)
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
@@ -1124,13 +1107,49 @@ def CMDcheck(parser, args):
   return 0
 
 
-CMDhashtable = CMDarchive
+def CMDhashtable(parser, args):
+  """Creates a .isolated file and stores the contains in a directory.
+
+  All the files listed in the .isolated file are put in the directory with their
+  sha-1 as their file name. When using an NFS/CIFS server, the files can then be
+  shared accross slaves without an isolate server.
+  """
+  add_subdir_option(parser)
+  isolateserver.add_outdir_options(parser)
+  add_skip_refresh_option(parser)
+  options, args = parser.parse_args(args)
+  if args:
+    parser.error('Unsupported argument: %s' % args)
+  cwd = os.getcwd()
+  isolateserver.process_outdir_options(parser, options, cwd)
+
+  success = False
+  try:
+    complete_state, infiles, isolated_hash = prepare_for_archival(options, cwd)
+    logging.info('Creating content addressed object store with %d item',
+                  len(infiles))
+    if not os.path.isdir(options.outdir):
+      os.makedirs(options.outdir)
+
+    # TODO(maruel): Make the files read-only?
+    recreate_tree(
+        outdir=options.outdir,
+        indir=complete_state.root_dir,
+        infiles=infiles,
+        action=run_isolated.HARDLINK_WITH_FALLBACK,
+        as_hash=True)
+    success = True
+    print('%s  %s' % (isolated_hash[0], os.path.basename(options.isolated)))
+  finally:
+    # If the command failed, delete the .isolated file if it exists. This is
+    # important so no stale swarm job is executed.
+    if not success and os.path.isfile(options.isolated):
+      os.remove(options.isolated)
+  return int(not success)
 
 
 def CMDmerge(parser, args):
   """Reads and merges the data from the trace back into the original .isolate.
-
-  Ignores --outdir.
   """
   parser.require_isolated = False
   add_trace_option(parser)
@@ -1145,16 +1164,10 @@ def CMDmerge(parser, args):
 
 
 def CMDread(parser, args):
-  """Reads the trace file generated with command 'trace'.
-
-  Ignores --outdir.
-  """
+  """Reads the trace file generated with command 'trace'."""
   parser.require_isolated = False
   add_trace_option(parser)
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
+  add_skip_refresh_option(parser)
   parser.add_option(
       '-m', '--merge', action='store_true',
       help='merge the results back in the .isolate file instead of printing')
@@ -1187,28 +1200,23 @@ def CMDremap(parser, args):
   run.
   """
   parser.require_isolated = False
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
+  isolateserver.add_outdir_options(parser)
+  add_skip_refresh_option(parser)
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
-  if options.outdir and file_path.is_url(options.outdir):
-    parser.error('Can\'t use url for --outdir with mode remap.')
-
-  complete_state = load_complete_state(
-      options, os.getcwd(), None, options.skip_refresh)
-
-  outdir = get_remap_dir(
-      complete_state.root_dir, options.isolated, options.outdir)
-
-  print('Remapping into %s' % outdir)
-  if len(os.listdir(outdir)):
+  cwd = os.getcwd()
+  isolateserver.process_outdir_options(parser, options, cwd)
+  complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
+
+  if not os.path.isdir(options.outdir):
+    os.makedirs(options.outdir)
+  print('Remapping into %s' % options.outdir)
+  if os.listdir(options.outdir):
     raise ExecutionError('Can\'t remap in a non-empty directory')
 
   create_isolate_tree(
-      outdir, complete_state.root_dir, complete_state.saved_state.files,
+      options.outdir, complete_state.root_dir, complete_state.saved_state.files,
       complete_state.saved_state.relative_cwd,
       complete_state.saved_state.read_only)
   if complete_state.isolated_filepath:
@@ -1250,21 +1258,15 @@ def CMDrun(parser, args):
   """Runs the test executable in an isolated (temporary) directory.
 
   All the dependencies are mapped into the temporary directory and the
-  directory is cleaned up after the target exits. Warning: if --outdir is
-  specified, it is deleted upon exit.
+  directory is cleaned up after the target exits.
 
   Argument processing stops at -- and these arguments are appended to the
   command line of the target to run. For example, use:
     isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
   """
   parser.require_isolated = False
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
+  add_skip_refresh_option(parser)
   options, args = parser.parse_args(args)
-  if options.outdir and file_path.is_url(options.outdir):
-    parser.error('Can\'t use url for --outdir with mode run.')
 
   complete_state = load_complete_state(
       options, os.getcwd(), None, options.skip_refresh)
@@ -1273,9 +1275,10 @@ def CMDrun(parser, args):
     raise ExecutionError('No command to run.')
   cmd = tools.fix_python_path(cmd)
 
+  outdir = run_isolated.make_temp_dir(
+      'isolate-%s' % datetime.date.today(),
+      os.path.dirname(complete_state.root_dir))
   try:
-    outdir = get_remap_dir(
-        complete_state.root_dir, options.isolated, options.outdir)
     # TODO(maruel): Use run_isolated.run_tha_test().
     cwd = create_isolate_tree(
         outdir, complete_state.root_dir, complete_state.saved_state.files,
@@ -1284,8 +1287,7 @@ def CMDrun(parser, args):
     logging.info('Running %s, cwd=%s' % (cmd, cwd))
     result = subprocess.call(cmd, cwd=cwd)
   finally:
-    if options.outdir:
-      run_isolated.rmtree(options.outdir)
+    run_isolated.rmtree(outdir)
 
   if complete_state.isolated_filepath:
     complete_state.save_files()
@@ -1309,10 +1311,7 @@ def CMDtrace(parser, args):
   parser.add_option(
       '-m', '--merge', action='store_true',
       help='After tracing, merge the results back in the .isolate file')
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
+  add_skip_refresh_option(parser)
   options, args = parser.parse_args(args)
 
   complete_state = load_complete_state(
@@ -1429,6 +1428,15 @@ def add_variable_option(parser):
            'paths in the .isolate file but are not considered relative paths.')
 
 
+def add_subdir_option(parser):
+  parser.add_option(
+      '--subdir',
+      help='Filters to a subdirectory. Its behavior changes depending if it '
+           'is a relative path as a string or as a path variable. Path '
+           'variables are always keyed from the directory containing the '
+           '.isolate file. Anything else is keyed on the root directory.')
+
+
 def add_trace_option(parser):
   """Adds --trace-blacklist to the parser."""
   parser.add_option(
@@ -1439,6 +1447,13 @@ def add_trace_option(parser):
            'test case.')
 
 
+def add_skip_refresh_option(parser):
+  parser.add_option(
+      '--skip-refresh', action='store_true',
+      help='Skip reading .isolate file and do not refresh the hash of '
+           'dependencies')
+
+
 def parse_isolated_option(parser, options, cwd, require_isolated):
   """Processes --isolated."""
   if options.isolated:
@@ -1485,11 +1500,6 @@ class OptionParserIsolate(tools.OptionParserWithLogging):
         help='.isolate file to load the dependency data from')
     add_variable_option(group)
     group.add_option(
-        '-o', '--outdir', metavar='DIR',
-        help='Directory used to recreate the tree or store the hash table. '
-             'Defaults: run|remap: a /tmp subdirectory, others: '
-             'defaults to the directory containing --isolated')
-    group.add_option(
         '--ignore_broken_items', action='store_true',
         default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
         help='Indicates that invalid entries in the isolated file to be '
@@ -1518,12 +1528,6 @@ class OptionParserIsolate(tools.OptionParserWithLogging):
       options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
       options.isolate = file_path.get_native_path_case(options.isolate)
 
-    if options.outdir and not file_path.is_url(options.outdir):
-      options.outdir = unicode(options.outdir).replace('/', os.path.sep)
-      # outdir doesn't need native path case since tracing is never done from
-      # there.
-      options.outdir = os.path.normpath(os.path.join(cwd, options.outdir))
-
     return options, args