-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+# Copyright 2013 The Swarming Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0 that
+# can be found in the LICENSE file.
-"""Provides functions: get_native_path_case(), isabs() and safe_join()."""
+"""Provides functions: get_native_path_case(), isabs() and safe_join().
+This module assumes that filesystem is not changing while current process
+is running and thus it caches results of functions that depend on FS state.
+"""
+
+import ctypes
import logging
import os
+import posixpath
import re
+import shlex
+import shutil
+import stat
import sys
import unicodedata
+import time
+
+from utils import tools
+
+
+# Types of action accepted by link_file().
+HARDLINK, HARDLINK_WITH_FALLBACK, SYMLINK, COPY = range(1, 5)
+
## OS-specific imports
return os.path.basename(get_native_path_case(os.path.join(root, item)))
+ @tools.profile
+ @tools.cached
def get_native_path_case(p):
"""Returns the native path case for an existing file.
return out[0].upper() + out[1:] + suffix
+ def enum_processes_win():
+ """Returns all processes on the system that are accessible to this process.
+
+ Returns:
+ Win32_Process COM objects. See
+ http://msdn.microsoft.com/library/aa394372.aspx for more details.
+ """
+ import win32com.client # pylint: disable=F0401
+ wmi_service = win32com.client.Dispatch('WbemScripting.SWbemLocator')
+ wbem = wmi_service.ConnectServer('.', 'root\\cimv2')
+ return [proc for proc in wbem.ExecQuery('SELECT * FROM Win32_Process')]
+
+
+ def filter_processes_dir_win(processes, root_dir):
+ """Returns all processes which has their main executable located inside
+ root_dir.
+ """
+ def normalize_path(filename):
+ try:
+ return GetLongPathName(unicode(filename)).lower()
+ except: # pylint: disable=W0702
+ return unicode(filename).lower()
+
+ root_dir = normalize_path(root_dir)
+
+ def process_name(proc):
+ if proc.ExecutablePath:
+ return normalize_path(proc.ExecutablePath)
+ # proc.ExecutablePath may be empty if the process hasn't finished
+ # initializing, but the command line may be valid.
+ if proc.CommandLine is None:
+ return None
+ parsed_line = shlex.split(proc.CommandLine)
+ if len(parsed_line) >= 1 and os.path.isabs(parsed_line[0]):
+ return normalize_path(parsed_line[0])
+ return None
+
+ long_names = ((process_name(proc), proc) for proc in processes)
+
+ return [
+ proc for name, proc in long_names
+ if name is not None and name.startswith(root_dir)
+ ]
+
+
+ def filter_processes_tree_win(processes):
+ """Returns all the processes under the current process."""
+ # Convert to dict.
+ processes = {p.ProcessId: p for p in processes}
+ root_pid = os.getpid()
+ out = {root_pid: processes[root_pid]}
+ while True:
+ found = set()
+ for pid in out:
+ found.update(
+ p.ProcessId for p in processes.itervalues()
+ if p.ParentProcessId == pid)
+ found -= set(out)
+ if not found:
+ break
+ out.update((p, processes[p]) for p in found)
+ return out.values()
+
elif sys.platform == 'darwin':
return item
item = item.lower()
- for element in os.listdir(root_path):
+ for element in listdir(root_path):
if element.lower() == item:
return element
+ @tools.profile
+ @tools.cached
def get_native_path_case(path):
"""Returns the native path case for an existing file.
# There was a symlink, process it.
base, symlink, rest = _split_at_symlink_native(None, path)
- assert symlink, (path, base, symlink, rest, resolved)
+ if not symlink:
+ # TODO(maruel): This can happen on OSX because we use stale APIs on OSX.
+ # Fixing the APIs usage will likely fix this bug. The bug occurs due to
+ # hardlinked files, where the API may return one file path or the other
+ # depending on how it feels.
+ return base
prev = base
base = safe_join(_native_case(base), symlink)
assert len(base) > len(prev)
return os.path.basename(get_native_path_case(os.path.join(root, item)))
+ @tools.profile
+ @tools.cached
def get_native_path_case(path):
"""Returns the native path case for an existing file.
# OS so this needs to be done here to be coherent between OSes.
out = os.path.normpath(path)
if path.endswith(os.path.sep) and not out.endswith(os.path.sep):
- return out + os.path.sep
- return out
+ out = out + os.path.sep
+ # In 99.99% of cases on Linux out == path. Since a return value is cached
+ # forever, reuse (also cached) |path| object. It safes approx 7MB of ram
+ # when isolating Chromium tests. It's important on memory constrained
+ # systems running ARM.
+ return path if out == path else out
if sys.platform != 'win32': # All non-Windows OSes.
return out
+ @tools.profile
def split_at_symlink(base_dir, relfile):
"""Scans each component of relfile and cut the string at the symlink if
there is any.
break
index += 1
return relfile, None, None
+
+
+@tools.profile
+@tools.cached
+def listdir(abspath):
+ """Lists a directory given an absolute path to it."""
+ if not isabs(abspath):
+ raise ValueError(
+ 'list_dir(%r): Require an absolute path' % abspath, abspath)
+ return os.listdir(abspath)
+
+
+def relpath(path, root):
+ """os.path.relpath() that keeps trailing os.path.sep."""
+ out = os.path.relpath(path, root)
+ if path.endswith(os.path.sep):
+ out += os.path.sep
+ return out
+
+
+def safe_relpath(filepath, basepath):
+ """Do not throw on Windows when filepath and basepath are on different drives.
+
+ Different than relpath() above since this one doesn't keep the trailing
+ os.path.sep and it swallows exceptions on Windows and return the original
+ absolute path in the case of different drives.
+ """
+ try:
+ return os.path.relpath(filepath, basepath)
+ except ValueError:
+ assert sys.platform == 'win32'
+ return filepath
+
+
+def normpath(path):
+ """os.path.normpath() that keeps trailing os.path.sep."""
+ out = os.path.normpath(path)
+ if path.endswith(os.path.sep):
+ out += os.path.sep
+ return out
+
+
+def posix_relpath(path, root):
+ """posix.relpath() that keeps trailing slash.
+
+ It is different from relpath() since it can be used on Windows.
+ """
+ out = posixpath.relpath(path, root)
+ if path.endswith('/'):
+ out += '/'
+ return out
+
+
+def cleanup_path(x):
+ """Cleans up a relative path. Converts any os.path.sep to '/' on Windows."""
+ if x:
+ x = x.rstrip(os.path.sep).replace(os.path.sep, '/')
+ if x == '.':
+ x = ''
+ if x:
+ x += '/'
+ return x
+
+
+def is_url(path):
+ """Returns True if it looks like an HTTP url instead of a file path."""
+ return bool(re.match(r'^https?://.+$', path))
+
+
+def path_starts_with(prefix, path):
+ """Returns true if the components of the path |prefix| are the same as the
+ initial components of |path| (or all of the components of |path|). The paths
+ must be absolute.
+ """
+ assert os.path.isabs(prefix) and os.path.isabs(path)
+ prefix = os.path.normpath(prefix)
+ path = os.path.normpath(path)
+ assert prefix == get_native_path_case(prefix), prefix
+ assert path == get_native_path_case(path), path
+ prefix = prefix.rstrip(os.path.sep) + os.path.sep
+ path = path.rstrip(os.path.sep) + os.path.sep
+ return path.startswith(prefix)
+
+
+@tools.profile
+def fix_native_path_case(root, path):
+ """Ensures that each component of |path| has the proper native case.
+
+ It does so by iterating slowly over the directory elements of |path|. The file
+ must exist.
+ """
+ native_case_path = root
+ for raw_part in path.split(os.sep):
+ if not raw_part or raw_part == '.':
+ break
+
+ part = find_item_native_case(native_case_path, raw_part)
+ if not part:
+ raise OSError(
+ 'File %s doesn\'t exist' %
+ os.path.join(native_case_path, raw_part))
+ native_case_path = os.path.join(native_case_path, part)
+
+ return os.path.normpath(native_case_path)
+
+
+def ensure_command_has_abs_path(command, cwd):
+ """Ensures that an isolate command uses absolute path.
+
+ This is needed since isolate can specify a command relative to 'cwd' and
+ subprocess.call doesn't consider 'cwd' when searching for executable.
+ """
+ if not os.path.isabs(command[0]):
+ command[0] = os.path.abspath(os.path.join(cwd, command[0]))
+
+
+def is_same_filesystem(path1, path2):
+ """Returns True if both paths are on the same filesystem.
+
+ This is required to enable the use of hardlinks.
+ """
+ assert os.path.isabs(path1), path1
+ assert os.path.isabs(path2), path2
+ if sys.platform == 'win32':
+ # If the drive letter mismatches, assume it's a separate partition.
+ # TODO(maruel): It should look at the underlying drive, a drive letter could
+ # be a mount point to a directory on another drive.
+ assert re.match(r'^[a-zA-Z]\:\\.*', path1), path1
+ assert re.match(r'^[a-zA-Z]\:\\.*', path2), path2
+ if path1[0].lower() != path2[0].lower():
+ return False
+ return os.stat(path1).st_dev == os.stat(path2).st_dev
+
+
+def get_free_space(path):
+ """Returns the number of free bytes."""
+ if sys.platform == 'win32':
+ free_bytes = ctypes.c_ulonglong(0)
+ ctypes.windll.kernel32.GetDiskFreeSpaceExW(
+ ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
+ return free_bytes.value
+ # For OSes other than Windows.
+ f = os.statvfs(path) # pylint: disable=E1101
+ return f.f_bfree * f.f_frsize
+
+
+### Write file functions.
+
+
+def hardlink(source, link_name):
+ """Hardlinks a file.
+
+ Add support for os.link() on Windows.
+ """
+ if sys.platform == 'win32':
+ if not ctypes.windll.kernel32.CreateHardLinkW(
+ unicode(link_name), unicode(source), 0):
+ raise OSError()
+ else:
+ os.link(source, link_name)
+
+
+def readable_copy(outfile, infile):
+ """Makes a copy of the file that is readable by everyone."""
+ shutil.copy2(infile, outfile)
+ read_enabled_mode = (os.stat(outfile).st_mode | stat.S_IRUSR |
+ stat.S_IRGRP | stat.S_IROTH)
+ os.chmod(outfile, read_enabled_mode)
+
+
+def set_read_only(path, read_only):
+ """Sets or resets the write bit on a file or directory.
+
+ Zaps out access to 'group' and 'others'.
+ """
+ assert isinstance(read_only, bool), read_only
+ mode = os.lstat(path).st_mode
+ # TODO(maruel): Stop removing GO bits.
+ if read_only:
+ mode = mode & 0500
+ else:
+ mode = mode | 0200
+ if hasattr(os, 'lchmod'):
+ os.lchmod(path, mode) # pylint: disable=E1101
+ else:
+ if stat.S_ISLNK(mode):
+ # Skip symlink without lchmod() support.
+ logging.debug(
+ 'Can\'t change %sw bit on symlink %s',
+ '-' if read_only else '+', path)
+ return
+
+ # TODO(maruel): Implement proper DACL modification on Windows.
+ os.chmod(path, mode)
+
+
+def try_remove(filepath):
+ """Removes a file without crashing even if it doesn't exist."""
+ try:
+ # TODO(maruel): Not do it unless necessary since it slows this function
+ # down.
+ if sys.platform == 'win32':
+ # Deleting a read-only file will fail if it is read-only.
+ set_read_only(filepath, False)
+ else:
+ # Deleting a read-only file will fail if the directory is read-only.
+ set_read_only(os.path.dirname(filepath), False)
+ os.remove(filepath)
+ except OSError:
+ pass
+
+
+def link_file(outfile, infile, action):
+ """Links a file. The type of link depends on |action|."""
+ if action not in (HARDLINK, HARDLINK_WITH_FALLBACK, SYMLINK, COPY):
+ raise ValueError('Unknown mapping action %s' % action)
+ if not os.path.isfile(infile):
+ raise OSError('%s is missing' % infile)
+ if os.path.isfile(outfile):
+ raise OSError(
+ '%s already exist; insize:%d; outsize:%d' %
+ (outfile, os.stat(infile).st_size, os.stat(outfile).st_size))
+
+ if action == COPY:
+ readable_copy(outfile, infile)
+ elif action == SYMLINK and sys.platform != 'win32':
+ # On windows, symlink are converted to hardlink and fails over to copy.
+ os.symlink(infile, outfile) # pylint: disable=E1101
+ else:
+ # HARDLINK or HARDLINK_WITH_FALLBACK.
+ try:
+ hardlink(infile, outfile)
+ except OSError as e:
+ if action == HARDLINK:
+ raise OSError('Failed to hardlink %s to %s: %s' % (infile, outfile, e))
+ # Probably a different file system.
+ logging.warning(
+ 'Failed to hardlink, failing back to copy %s to %s' % (
+ infile, outfile))
+ readable_copy(outfile, infile)
+
+
+### Write directory functions.
+
+
+def make_tree_read_only(root):
+ """Makes all the files in the directories read only.
+
+ Also makes the directories read only, only if it makes sense on the platform.
+
+ This means no file can be created or deleted.
+ """
+ logging.debug('make_tree_read_only(%s)', root)
+ assert os.path.isabs(root), root
+ for dirpath, dirnames, filenames in os.walk(root, topdown=True):
+ for filename in filenames:
+ set_read_only(os.path.join(dirpath, filename), True)
+ if sys.platform != 'win32':
+ # It must not be done on Windows.
+ for dirname in dirnames:
+ set_read_only(os.path.join(dirpath, dirname), True)
+ if sys.platform != 'win32':
+ set_read_only(root, True)
+
+
+def make_tree_files_read_only(root):
+ """Makes all the files in the directories read only but not the directories
+ themselves.
+
+ This means files can be created or deleted.
+ """
+ logging.debug('make_tree_files_read_only(%s)', root)
+ assert os.path.isabs(root), root
+ if sys.platform != 'win32':
+ set_read_only(root, False)
+ for dirpath, dirnames, filenames in os.walk(root, topdown=True):
+ for filename in filenames:
+ set_read_only(os.path.join(dirpath, filename), True)
+ if sys.platform != 'win32':
+ # It must not be done on Windows.
+ for dirname in dirnames:
+ set_read_only(os.path.join(dirpath, dirname), False)
+
+
+def make_tree_writeable(root):
+ """Makes all the files in the directories writeable.
+
+ Also makes the directories writeable, only if it makes sense on the platform.
+
+ It is different from make_tree_deleteable() because it unconditionally affects
+ the files.
+ """
+ logging.debug('make_tree_writeable(%s)', root)
+ assert os.path.isabs(root), root
+ if sys.platform != 'win32':
+ set_read_only(root, False)
+ for dirpath, dirnames, filenames in os.walk(root, topdown=True):
+ for filename in filenames:
+ set_read_only(os.path.join(dirpath, filename), False)
+ if sys.platform != 'win32':
+ # It must not be done on Windows.
+ for dirname in dirnames:
+ set_read_only(os.path.join(dirpath, dirname), False)
+
+
+def make_tree_deleteable(root):
+ """Changes the appropriate permissions so the files in the directories can be
+ deleted.
+
+ On Windows, the files are modified. On other platforms, modify the directory.
+ It only does the minimum so the files can be deleted safely.
+
+ Warning on Windows: since file permission is modified, the file node is
+ modified. This means that for hard-linked files, every directory entry for the
+ file node has its file permission modified.
+ """
+ logging.debug('make_tree_deleteable(%s)', root)
+ assert os.path.isabs(root), root
+ if sys.platform != 'win32':
+ set_read_only(root, False)
+ for dirpath, dirnames, filenames in os.walk(root, topdown=True):
+ if sys.platform == 'win32':
+ for filename in filenames:
+ set_read_only(os.path.join(dirpath, filename), False)
+ else:
+ for dirname in dirnames:
+ set_read_only(os.path.join(dirpath, dirname), False)
+
+
+def rmtree(root):
+ """Wrapper around shutil.rmtree() to retry automatically on Windows.
+
+ On Windows, forcibly kills processes that are found to interfere with the
+ deletion.
+
+ Returns:
+ True on normal execution, False if berserk techniques (like killing
+ processes) had to be used.
+ """
+ make_tree_deleteable(root)
+ logging.info('rmtree(%s)', root)
+ if sys.platform != 'win32':
+ shutil.rmtree(root)
+ return True
+
+ # Windows is more 'challenging'. First tries the soft way: tries 3 times to
+ # delete and sleep a bit in between.
+ max_tries = 3
+ for i in xrange(max_tries):
+ # errors is a list of tuple(function, path, excinfo).
+ errors = []
+ shutil.rmtree(root, onerror=lambda *args: errors.append(args))
+ if not errors:
+ return True
+ if i == max_tries - 1:
+ sys.stderr.write(
+ 'Failed to delete %s. The following files remain:\n' % root)
+ for _, path, _ in errors:
+ sys.stderr.write('- %s\n' % path)
+ else:
+ delay = (i+1)*2
+ sys.stderr.write(
+ 'Failed to delete %s (%d files remaining).\n'
+ ' Maybe the test has a subprocess outliving it.\n'
+ ' Sleeping %d seconds.\n' %
+ (root, len(errors), delay))
+ time.sleep(delay)
+
+ # The soft way was not good enough. Try the hard way. Enumerates both:
+ # - all child processes from this process.
+ # - processes where the main executable in inside 'root'. The reason is that
+ # the ancestry may be broken so stray grand-children processes could be
+ # undetected by the first technique.
+ # This technique is not fool-proof but gets mostly there.
+ def get_processes():
+ processes = enum_processes_win()
+ tree_processes = filter_processes_tree_win(processes)
+ dir_processes = filter_processes_dir_win(processes, root)
+ # Convert to dict to remove duplicates.
+ processes = {p.ProcessId: p for p in tree_processes}
+ processes.update((p.ProcessId, p) for p in dir_processes)
+ processes.pop(os.getpid())
+ return processes
+
+ for i in xrange(3):
+ sys.stderr.write('Enumerating processes:\n')
+ processes = get_processes()
+ if not processes:
+ break
+ for _, proc in sorted(processes.iteritems()):
+ sys.stderr.write(
+ '- pid %d; Handles: %d; Exe: %s; Cmd: %s\n' % (
+ proc.ProcessId,
+ proc.HandleCount,
+ proc.ExecutablePath,
+ proc.CommandLine))
+ sys.stderr.write('Terminating %d processes.\n' % len(processes))
+ for pid in sorted(processes):
+ try:
+ # Killing is asynchronous.
+ os.kill(pid, 9)
+ sys.stderr.write('- %d killed\n' % pid)
+ except OSError:
+ sys.stderr.write('- failed to kill %s\n' % pid)
+ if i < 2:
+ time.sleep((i+1)*2)
+ else:
+ processes = get_processes()
+ if processes:
+ sys.stderr.write('Failed to terminate processes.\n')
+ raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
+
+ # Now that annoying processes in root are evicted, try again.
+ errors = []
+ shutil.rmtree(root, onerror=lambda *args: errors.append(args))
+ if errors:
+ # There's no hope.
+ sys.stderr.write(
+ 'Failed to delete %s. The following files remain:\n' % root)
+ for _, path, _ in errors:
+ sys.stderr.write('- %s\n' % path)
+ raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
+ return False
projects / platform / framework / web / crosswalk.git / blobdiff