src/native_client/build/package_version/package_info.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2014 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """A package is a JSON file describing a list of package archives."""
   7
   8 import json
   9 import os
  10 import posixpath
  11 import sys
  12
  13 sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
  14 import pynacl.file_tools
  15 import pynacl.gsd_storage
  16
  17 import archive_info
  18 import error
  19
  20 PACKAGE_KEY_ARCHIVES = 'archives'
  21 PACKAGE_KEY_VERSION = 'version'
  22
  23 CURRENT_PACKAGE_VERSION = 1
  24
  25
  26 def ReadPackageFile(package_file):
  27   """Returns a PackageInfoTuple representation of a JSON package file."""
  28   with open(package_file, 'rt') as f:
  29     json_value = json.load(f)
  30
  31     # TODO(dyen): Support old format temporarily when it was a list of archives.
  32     if isinstance(json_value, list):
  33       return { PACKAGE_KEY_ARCHIVES: json_value, PACKAGE_KEY_VERSION: 0 }
  34     else:
  35       return json_value
  36
  37
  38 def GetFileBaseName(filename):
  39   """Removes all extensions from a file.
  40
  41   (Note: os.path.splitext() only removes the last extension).
  42   """
  43   first_ext = filename.find('.')
  44   if first_ext != -1:
  45     filename = filename[:first_ext]
  46   return filename
  47
  48
  49 def GetLocalPackageName(local_package_file):
  50   """Returns the package name given a local package file."""
  51   package_basename = os.path.basename(local_package_file)
  52   return GetFileBaseName(package_basename)
  53
  54
  55 def GetRemotePackageName(remote_package_file):
  56   """Returns the package name given a remote posix based package file."""
  57   package_basename = posixpath.basename(remote_package_file)
  58   return GetFileBaseName(package_basename)
  59
  60
  61 def DownloadPackageInfoFiles(local_package_file, remote_package_file,
  62                              downloader=None):
  63   """Downloads all package info files from a downloader.
  64
  65   Downloads a package file from the cloud along with all of the archive
  66   info files. Archive info files are expected to be in a directory with the
  67   name of the package along side the package file. Files will be downloaded
  68   in the same structure.
  69
  70   Args:
  71     local_package_file: Local package file where root file will live.
  72     remote_package_file: Remote package URL to download from.
  73     downloader: Optional downloader if standard HTTP one should not be used.
  74   """
  75   if downloader is None:
  76     downloader = pynacl.gsd_storage.HttpDownload
  77
  78   pynacl.file_tools.MakeParentDirectoryIfAbsent(local_package_file)
  79   downloader(remote_package_file, local_package_file)
  80   if not os.path.isfile(local_package_file):
  81     raise error.Error('Could not download package file: %s.' %
  82                       remote_package_file)
  83
  84   package_data = ReadPackageFile(local_package_file)
  85   archive_list = package_data[PACKAGE_KEY_ARCHIVES]
  86   local_package_name = GetLocalPackageName(local_package_file)
  87   remote_package_name = GetRemotePackageName(remote_package_file)
  88
  89   local_archive_dir = os.path.join(os.path.dirname(local_package_file),
  90                                    local_package_name)
  91   remote_archive_dir = posixpath.join(posixpath.dirname(remote_package_file),
  92                                       remote_package_name)
  93
  94   pynacl.file_tools.MakeDirectoryIfAbsent(local_archive_dir)
  95   for archive in archive_list:
  96     archive_file = archive + '.json'
  97     local_archive_file = os.path.join(local_archive_dir, archive_file)
  98     remote_archive_file = posixpath.join(remote_archive_dir, archive_file)
  99     downloader(remote_archive_file, local_archive_file)
 100     if not os.path.isfile(local_archive_file):
 101       raise error.Error('Could not download archive file: %s.' %
 102                         remote_archive_file)
 103
 104 def UploadPackageInfoFiles(storage, package_target, package_name,
 105                            remote_package_file, local_package_file,
 106                            skip_missing=False, annotate=False):
 107   """Uploads all package info files from a downloader.
 108
 109   Uploads a package file to the cloud along with all of the archive info
 110   files. Archive info files are expected to be in a directory with the
 111   name of the package along side the package file. Files will be uploaded
 112   using the same file structure.
 113
 114   Args:
 115     storage: Cloud storage object to store the files to.
 116     remote_package_file: Remote package URL to upload to.
 117     local_package_file: Local package file where root file lives.
 118     skip_missing: Whether to skip missing archive files or error.
 119     annotate: Whether to annotate build bot links.
 120   Returns:
 121     The URL where the root package file is located.
 122   """
 123   package_data = ReadPackageFile(local_package_file)
 124   archive_list = package_data[PACKAGE_KEY_ARCHIVES]
 125   local_package_name = GetLocalPackageName(local_package_file)
 126   remote_package_name = GetRemotePackageName(remote_package_file)
 127
 128   local_archive_dir = os.path.join(os.path.dirname(local_package_file),
 129                                    local_package_name)
 130   remote_archive_dir = posixpath.join(posixpath.dirname(remote_package_file),
 131                                       remote_package_name)
 132
 133   num_archives = len(archive_list)
 134   for index, archive in enumerate(archive_list):
 135     archive_file = archive + '.json'
 136     local_archive_file = os.path.join(local_archive_dir, archive_file)
 137     remote_archive_file = posixpath.join(remote_archive_dir, archive_file)
 138     if skip_missing and not os.path.isfile(local_archive_file):
 139       continue
 140
 141     archive_url = storage.PutFile(local_archive_file, remote_archive_file)
 142     if annotate:
 143       print ('@@@STEP_LINK@download (%s/%s/%s.json [%d/%d])@%s@@@' %
 144              (package_target, package_name, archive, index+1, num_archives,
 145               archive_url))
 146
 147   package_url = storage.PutFile(local_package_file, remote_package_file)
 148   if annotate:
 149     print ('@@@STEP_LINK@download (%s/%s.json)@%s@@@' %
 150            (package_target, package_name, package_url))
 151   return package_url
 152
 153
 154 class PackageInfo(object):
 155   """A package file is a list of package archives (usually .tar or .tgz files).
 156
 157   PackageInfo will contain a list of ArchiveInfo objects, ArchiveInfo will
 158   contain all the necessary information for an archive (name, URL, hash...etc.).
 159   """
 160   def __init__(self, package_file=None, skip_missing=False):
 161     self._archive_list = []
 162     self._package_version = CURRENT_PACKAGE_VERSION
 163
 164     if package_file is not None:
 165       self.LoadPackageFile(package_file, skip_missing)
 166
 167   def __eq__(self, other):
 168     if type(self) != type(other):
 169       return False
 170     elif self.GetPackageVersion() != other.GetPackageVersion():
 171       return False
 172
 173     archives1 = [archive.GetArchiveData() for archive in self.GetArchiveList()]
 174     archives2 = [archive.GetArchiveData() for archive in other.GetArchiveList()]
 175     return set(archives1) == set(archives2)
 176
 177   def __repr__(self):
 178     return 'PackageInfo(%s)' % self.DumpPackageJson()
 179
 180   def LoadPackageFile(self, package_file, skip_missing=False):
 181     """Loads a package file into this object.
 182
 183     Args:
 184       package_file: Filename or JSON dictionary.
 185     """
 186     archive_names = None
 187     self._archive_list = []
 188
 189     # TODO(dyen): Support old format temporarily when it was a list of archives.
 190     if isinstance(package_file, list) or isinstance(package_file, dict):
 191       if isinstance(package_file, list):
 192         self._package_version = 0
 193         archive_list = package_file
 194       else:
 195         self._package_version = package_file[PACKAGE_KEY_VERSION]
 196         archive_list = package_file[PACKAGE_KEY_ARCHIVES]
 197
 198       if archive_list:
 199         if isinstance(archive_list[0], archive_info.ArchiveInfo):
 200           # Setting a list of ArchiveInfo objects, no need to interpret JSON.
 201           self._archive_list = archive_list
 202         else:
 203           # Assume to be JSON.
 204           for archive_json in archive_list:
 205             archive = archive_info.ArchiveInfo(archive_info_file=archive_json)
 206             self._archive_list.append(archive)
 207
 208     elif isinstance(package_file, basestring):
 209       package_data = ReadPackageFile(package_file)
 210       self._package_version = package_data[PACKAGE_KEY_VERSION]
 211       archive_names = package_data[PACKAGE_KEY_ARCHIVES]
 212
 213       package_name = GetLocalPackageName(package_file)
 214       archive_dir = os.path.join(os.path.dirname(package_file), package_name)
 215       for archive in archive_names:
 216         arch_file = archive + '.json'
 217         arch_path = os.path.join(archive_dir, arch_file)
 218         if not os.path.isfile(arch_path):
 219           if not skip_missing:
 220             raise package_version.Error(
 221                 'Package (%s) points to invalid archive file (%s).' %
 222                 (package_file, arch_path))
 223           archive_desc = archive_info.ArchiveInfo(name=archive)
 224         else:
 225           archive_desc = archive_info.ArchiveInfo(archive_info_file=arch_path)
 226         self._archive_list.append(archive_desc)
 227     else:
 228       raise package_version.Error('Invalid load package file type (%s): %s',
 229                          (type(package_file), package_file))
 230
 231   def SavePackageFile(self, package_file):
 232     """Saves this object as a serialized JSON file.
 233
 234     Args:
 235       package_file: File path where JSON file will be saved.
 236     """
 237     package_name = GetLocalPackageName(package_file)
 238     archive_dir = os.path.join(os.path.dirname(package_file), package_name)
 239     pynacl.file_tools.MakeDirectoryIfAbsent(archive_dir)
 240
 241     archive_list = []
 242
 243     for archive in self.GetArchiveList():
 244       archive_data = archive.GetArchiveData()
 245       archive_list.append(archive_data.name)
 246
 247       archive_file = archive_data.name + '.json'
 248       archive_path = os.path.join(archive_dir, archive_file)
 249       archive.SaveArchiveInfoFile(archive_path)
 250
 251     package_json = {
 252         PACKAGE_KEY_ARCHIVES: archive_list,
 253         PACKAGE_KEY_VERSION: self._package_version
 254         }
 255
 256     with open(package_file, 'wt') as f:
 257       json.dump(package_json, f, sort_keys=True,
 258                 indent=2, separators=(',', ': '))
 259
 260   def DumpPackageJson(self):
 261     """Returns a dictionary representation of the JSON of this object."""
 262     archives = [archive.DumpArchiveJson() for archive in self.GetArchiveList()]
 263     return {
 264         PACKAGE_KEY_ARCHIVES: archives,
 265         PACKAGE_KEY_VERSION: self._package_version
 266         }
 267
 268   def ClearArchiveList(self):
 269     """Clears this object so it represents no archives."""
 270     self._archive_list = []
 271
 272   def AppendArchive(self, archive_info):
 273     """Append a package archive into this object"""
 274     self._archive_list.append(archive_info)
 275
 276   def GetArchiveList(self):
 277     """Returns the sorted list of ARCHIVE_INFOs this object represents."""
 278     return sorted(self._archive_list,
 279                   key=lambda archive : archive.GetArchiveData().name)
 280
 281   def GetPackageVersion(self):
 282     """Returns the version of this package."""
 283     return self._package_version