src/native_client/build/package_version/package_version_test.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2014 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Tests for package version."""
   7
   8 import os
   9 import random
  10 import sys
  11 import tarfile
  12 import unittest
  13
  14 sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
  15 import pynacl.fake_downloader
  16 import pynacl.fake_storage
  17 import pynacl.file_tools
  18 import pynacl.platform
  19 import pynacl.working_directory
  20
  21 import archive_info
  22 import package_info
  23 import package_locations
  24 import package_version
  25
  26 class TestPackageVersion(unittest.TestCase):
  27
  28   def setUp(self):
  29     self._fake_storage = pynacl.fake_storage.FakeStorage()
  30     self._fake_downloader = pynacl.fake_downloader.FakeDownloader()
  31
  32   def GenerateMockFile(self, work_dir, mock_dir = 'mock_dir',
  33                        mock_file='mockfile.txt', contents='mock contents'):
  34     """Generates a file with random content in it.
  35
  36     Args:
  37       work_dir: Root working directory where mock directory will live.
  38       mock_dir: Mock directory under root directory where mock file will live.
  39       mock_file: Mock filename which generated file will be generated under.
  40       contents: Base content for the mock file.
  41     Returns:
  42       File path for the mock file with the mock contents with random content.
  43     """
  44     full_mock_dir = os.path.join(work_dir, mock_dir)
  45     if not os.path.isdir(full_mock_dir):
  46       os.makedirs(full_mock_dir)
  47
  48     full_mock_file = os.path.join(full_mock_dir, mock_file)
  49     with open(full_mock_file, 'wt') as f:
  50       f.write(contents)
  51       f.write(str(random.random()))
  52
  53     return full_mock_file
  54
  55   def GeneratePackageInfo(self, archive_list, name_dict={},
  56                           url_dict={}, src_dir_dict={}, dir_dict={}):
  57     """Generates a package_info.PackageInfo object for list of archives."
  58
  59     Args:
  60       archive_list: List of file paths where package archives sit.
  61       name_dict: optional dict of archive to names, otherwise use filename.
  62       url_dict: dict of archive file path to URL if url exists.
  63       src_dir_dict: dict of archive file path to source tar dir if exists.
  64       dir_dict: dict of archive file path to root dir if exists.
  65     """
  66     package_desc = package_info.PackageInfo()
  67     for archive_file in archive_list:
  68       archive_name = name_dict.get(archive_file, os.path.basename(archive_file))
  69
  70       if os.path.isfile(archive_file):
  71         archive_hash = archive_info.GetArchiveHash(archive_file)
  72       else:
  73         archive_hash = 'invalid'
  74
  75       archive_url = url_dict.get(archive_file, None)
  76       archive_src_tar_dir = src_dir_dict.get(archive_file, '')
  77       archive_dir = dir_dict.get(archive_file, '')
  78       archive_desc = archive_info.ArchiveInfo(archive_name,
  79                                               archive_hash,
  80                                               url=archive_url,
  81                                               tar_src_dir=archive_src_tar_dir,
  82                                               extract_dir=archive_dir)
  83       package_desc.AppendArchive(archive_desc)
  84
  85     return package_desc
  86
  87   def test_DownloadArchive(self):
  88     # Check that we can download a package archive correctly.
  89     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
  90       mock_tar = self.GenerateMockFile(work_dir)
  91
  92       fake_url = 'http://www.fake.com/archive.tar'
  93       self._fake_downloader.StoreURL(fake_url, mock_tar)
  94
  95       package_desc = self.GeneratePackageInfo(
  96           [mock_tar],
  97           url_dict={mock_tar: fake_url}
  98       )
  99
 100       tar_dir = os.path.join(work_dir, 'tar_dir')
 101       package_target = 'archive_target'
 102       package_name = 'archive_name'
 103       package_version.DownloadPackageArchives(
 104           tar_dir,
 105           package_target,
 106           package_name,
 107           package_desc,
 108           downloader=self._fake_downloader.Download
 109       )
 110       self.assertEqual(self._fake_downloader.GetDownloadCount(), 1,
 111                        "Expected a single archive to have been downloaded.")
 112
 113       mock_name = os.path.basename(mock_tar)
 114       local_archive_file = package_locations.GetLocalPackageArchiveFile(
 115           tar_dir,
 116           package_target,
 117           package_name,
 118           mock_name
 119       )
 120
 121       self.assertEqual(
 122           archive_info.GetArchiveHash(local_archive_file),
 123           archive_info.GetArchiveHash(mock_tar)
 124        )
 125
 126   def test_DownloadArchiveMissingURLFails(self):
 127     # Checks that we fail when the archive has no URL set.
 128     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 129       package_desc = package_info.PackageInfo()
 130       archive_desc = archive_info.ArchiveInfo('missing_name.tar',
 131                                               'missing_hash',
 132                                               url=None)
 133       package_desc.AppendArchive(archive_desc)
 134
 135       tar_dir = os.path.join(work_dir, 'tar_dir')
 136       self.assertRaises(
 137           IOError,
 138           package_version.DownloadPackageArchives,
 139           tar_dir,
 140           'missing_target',
 141           'missing_name',
 142           package_desc,
 143           downloader=self._fake_downloader.Download
 144       )
 145
 146   def test_DownloadArchiveMismatchFails(self):
 147     # Check download archive fails when the hash does not match expected hash.
 148     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 149       mock_tar = self.GenerateMockFile(work_dir)
 150       fake_url = 'http://www.fake.com/archive.tar'
 151       self._fake_downloader.StoreURL(fake_url, mock_tar)
 152
 153       package_desc = package_info.PackageInfo()
 154       archive_desc = archive_info.ArchiveInfo('invalid_name.tar',
 155                                               'invalid_hash',
 156                                               url=fake_url)
 157       package_desc.AppendArchive(archive_desc)
 158
 159       tar_dir = os.path.join(work_dir, 'tar_dir')
 160       self.assertRaises(
 161           IOError,
 162           package_version.DownloadPackageArchives,
 163           tar_dir,
 164           'mismatch_target',
 165           'mismatch_name',
 166           package_desc,
 167           downloader=self._fake_downloader.Download
 168       )
 169
 170   def test_ArchivePackageArchives(self):
 171     # Check if we can archive a list of archives to the tar directory.
 172     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 173       mock_tar1 = self.GenerateMockFile(
 174           work_dir,
 175           mock_file='file1.tar',
 176           contents='mock contents 1'
 177       )
 178       mock_tar2 = self.GenerateMockFile(
 179           work_dir,
 180           mock_file='file2.tar',
 181           contents='mock contents 2'
 182       )
 183
 184       tar_dir = os.path.join(work_dir, 'tar_dir')
 185       package_target = 'test_package_archives'
 186       package_name = 'package_archives'
 187       package_version.ArchivePackageArchives(
 188           tar_dir,
 189           package_target,
 190           package_name,
 191           [mock_tar1, mock_tar2]
 192       )
 193
 194       package_file = package_locations.GetLocalPackageFile(
 195           tar_dir,
 196           package_target,
 197           package_name
 198       )
 199       expected_package_desc = self.GeneratePackageInfo([mock_tar1, mock_tar2])
 200       package_desc = package_info.PackageInfo(package_file)
 201
 202       self.assertEqual(expected_package_desc, package_desc)
 203                       # "Archived package does not match mock package.")
 204
 205   def test_PackageUpload(self):
 206     # Check if we can properly upload a package file from the tar directory.
 207     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 208       tar_dir = os.path.join(work_dir, 'tar_dir')
 209       package_target = 'test_package_archives'
 210       package_name = 'package_archives'
 211       package_revision = 10
 212       package_version.ArchivePackageArchives(
 213           tar_dir,
 214           package_target,
 215           package_name,
 216           []
 217       )
 218
 219       package_version.UploadPackage(
 220           self._fake_storage,
 221           package_revision,
 222           tar_dir,
 223           package_target,
 224           package_name,
 225           False
 226       )
 227       self.assertEqual(
 228           self._fake_storage.WriteCount(),
 229           1,
 230           "Package did not get properly uploaded"
 231       )
 232
 233       remote_package_key = package_locations.GetRemotePackageKey(
 234           False,
 235           package_revision,
 236           package_target,
 237           package_name
 238       )
 239       downloaded_package = os.path.join(work_dir, 'download_package.json')
 240       package_info.DownloadPackageInfoFiles(
 241           downloaded_package,
 242           remote_package_key,
 243           downloader=self._fake_storage.GetFile)
 244       downloaded_package_desc = package_info.PackageInfo(downloaded_package)
 245
 246       original_package_file = package_locations.GetLocalPackageFile(
 247           tar_dir,
 248           package_target,
 249           package_name
 250       )
 251       original_package_desc = package_info.PackageInfo(original_package_file)
 252
 253       self.assertEqual(downloaded_package_desc, original_package_desc)
 254
 255   def test_CustomPackageUpload(self):
 256     # Check if we can upload a package file from a custom location.
 257     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 258       custom_package_file = os.path.join(work_dir, 'custom_package.json')
 259       package_desc = self.GeneratePackageInfo([])
 260       package_desc.SavePackageFile(custom_package_file)
 261
 262       tar_dir = os.path.join(work_dir, 'tar_dir')
 263       package_target = 'custom_package_target'
 264       package_name = 'custom_package'
 265       package_revision = 10
 266
 267       package_version.UploadPackage(
 268           self._fake_storage,
 269           package_revision,
 270           tar_dir,
 271           package_target,
 272           package_name,
 273           False,
 274           custom_package_file=custom_package_file
 275       )
 276       self.assertEqual(
 277           self._fake_storage.WriteCount(),
 278           1,
 279           "Package did not get properly uploaded"
 280       )
 281
 282       remote_package_key = package_locations.GetRemotePackageKey(
 283           False,
 284           package_revision,
 285           package_target,
 286           package_name
 287       )
 288       downloaded_package = os.path.join(work_dir, 'download_package.json')
 289       package_info.DownloadPackageInfoFiles(
 290           downloaded_package,
 291           remote_package_key,
 292           downloader=self._fake_storage.GetFile)
 293       downloaded_package_desc = package_info.PackageInfo(downloaded_package)
 294
 295       original_package_desc = package_info.PackageInfo(custom_package_file)
 296
 297       self.assertEqual(downloaded_package_desc, original_package_desc)
 298
 299   def test_UploadKeepsArchiveURL(self):
 300     # Checks if the archive URL is kept after a package upload.
 301     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 302       mock_tar = self.GenerateMockFile(work_dir)
 303       mock_url = 'http://www.mock.com/mock.tar'
 304       package_desc = self.GeneratePackageInfo(
 305           [mock_tar],
 306           url_dict={mock_tar: mock_url}
 307       )
 308
 309       package_file = os.path.join(work_dir, 'package_file.json')
 310       package_desc.SavePackageFile(package_file)
 311
 312       tar_dir = os.path.join(work_dir, 'tar_dir')
 313       package_target = 'custom_package_target'
 314       package_name = 'custom_package'
 315       package_revision = 10
 316
 317       package_version.UploadPackage(
 318           self._fake_storage,
 319           package_revision,
 320           tar_dir,
 321           package_target,
 322           package_name,
 323           False,
 324           custom_package_file=package_file
 325       )
 326       self.assertEqual(
 327           self._fake_storage.WriteCount(),
 328           2,
 329           "Package did not get properly uploaded"
 330       )
 331
 332       remote_package_key = package_locations.GetRemotePackageKey(
 333           False,
 334           package_revision,
 335           package_target,
 336           package_name
 337       )
 338       downloaded_package = os.path.join(work_dir, 'download_package.json')
 339       package_info.DownloadPackageInfoFiles(
 340           downloaded_package,
 341           remote_package_key,
 342           downloader=self._fake_storage.GetFile)
 343       downloaded_package_desc = package_info.PackageInfo(downloaded_package)
 344
 345       # Verify everything (including URL) still matches.
 346       self.assertEqual(downloaded_package_desc, package_desc)
 347
 348   def test_NoArchiveURLDoesUpload(self):
 349     # Checks when uploading package with no archive URL, archive is uploaded.
 350     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 351       tar_dir = os.path.join(work_dir, 'tar_dir')
 352       package_target = 'custom_package_target'
 353       package_name = 'custom_package'
 354       package_revision = 10
 355
 356       mock_file = self.GenerateMockFile(work_dir)
 357       mock_tar = package_locations.GetLocalPackageArchiveFile(
 358           tar_dir,
 359           package_target,
 360           package_name,
 361           'archive_name.tar'
 362       )
 363       os.makedirs(os.path.dirname(mock_tar))
 364       with tarfile.TarFile(mock_tar, 'w') as f:
 365         f.add(mock_file)
 366
 367       package_desc = self.GeneratePackageInfo([mock_tar])
 368
 369       package_file = os.path.join(work_dir, 'package_file.json')
 370       package_desc.SavePackageFile(package_file)
 371
 372       package_version.UploadPackage(
 373           self._fake_storage,
 374           package_revision,
 375           tar_dir,
 376           package_target,
 377           package_name,
 378           False,
 379           custom_package_file=package_file
 380       )
 381       self.assertEqual(
 382           self._fake_storage.WriteCount(),
 383           3,
 384           "3 files (package, archive_info, archive) should have been uploaded."
 385       )
 386
 387       remote_package_key = package_locations.GetRemotePackageKey(
 388           False,
 389           package_revision,
 390           package_target,
 391           package_name
 392       )
 393       downloaded_package = os.path.join(work_dir, 'download_package.json')
 394       package_info.DownloadPackageInfoFiles(
 395           downloaded_package,
 396           remote_package_key,
 397           downloader=self._fake_storage.GetFile)
 398       downloaded_package_desc = package_info.PackageInfo(downloaded_package)
 399
 400       archive_list = downloaded_package_desc.GetArchiveList()
 401       self.assertEqual(len(archive_list), 1,
 402                        "The downloaded package does not have 1 archive.")
 403       self.assertTrue(archive_list[0].GetArchiveData().url,
 404                       "The downloaded archive still does not have a proper URL")
 405
 406   def test_ExtractPackageTargets(self):
 407     # Tests that we can extract package targets from the tar directory properly.
 408     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 409       mock_file1 = self.GenerateMockFile(work_dir, mock_file='mockfile1.txt')
 410       mock_file2 = self.GenerateMockFile(work_dir, mock_file='mockfile2.txt')
 411       mock_file3 = self.GenerateMockFile(work_dir, mock_file='mockfile3.txt')
 412
 413       tar_dir = os.path.join(work_dir, 'tar_dir')
 414       dest_dir = os.path.join(work_dir, 'dest_dir')
 415       package_target = 'custom_package_target'
 416       package_name = 'custom_package'
 417       package_revision = 10
 418
 419       mock_tar1 = package_locations.GetLocalPackageArchiveFile(
 420           tar_dir,
 421           package_target,
 422           package_name,
 423           'archive_name1.tar'
 424       )
 425       os.makedirs(os.path.dirname(mock_tar1))
 426       with tarfile.TarFile(mock_tar1, 'w') as f:
 427         f.add(mock_file1, arcname=os.path.basename(mock_file1))
 428
 429       mock_tar2 = package_locations.GetLocalPackageArchiveFile(
 430           tar_dir,
 431           package_target,
 432           package_name,
 433           'archive_name2.tar'
 434       )
 435       with tarfile.TarFile(mock_tar2, 'w') as f:
 436         f.add(mock_file2, arcname=os.path.basename(mock_file2))
 437
 438       mock_tar3 = package_locations.GetLocalPackageArchiveFile(
 439           tar_dir,
 440           package_target,
 441           package_name,
 442           'archive_name3.tar'
 443       )
 444       with tarfile.TarFile(mock_tar3, 'w') as f:
 445         arcname = os.path.join('rel_dir', os.path.basename(mock_file3))
 446         f.add(mock_file3, arcname=arcname)
 447
 448       package_desc = self.GeneratePackageInfo(
 449         [mock_tar1, mock_tar2, mock_tar3],
 450         dir_dict={mock_tar2: 'tar2_dir'},
 451         src_dir_dict={mock_tar3: 'rel_dir'},
 452       )
 453       package_file = package_locations.GetLocalPackageFile(
 454           tar_dir,
 455           package_target,
 456           package_name
 457       )
 458       package_desc.SavePackageFile(package_file)
 459
 460       package_version.ExtractPackageTargets(
 461           [(package_target, package_name)],
 462           tar_dir,
 463           dest_dir,
 464           downloader=self._fake_downloader.Download
 465       )
 466       self.assertEqual(self._fake_downloader.GetDownloadCount(), 0,
 467                        "Extracting a package should not download anything.")
 468
 469       full_dest_dir = package_locations.GetFullDestDir(
 470           dest_dir,
 471           package_target,
 472           package_name
 473       )
 474       dest_mock_file1 = os.path.join(
 475           full_dest_dir,
 476           os.path.basename(mock_file1)
 477       )
 478       dest_mock_file2 = os.path.join(
 479           full_dest_dir,
 480           'tar2_dir',
 481           os.path.basename(mock_file2)
 482       )
 483       dest_mock_file3 = os.path.join(
 484           full_dest_dir,
 485           os.path.basename(mock_file3)
 486       )
 487
 488       with open(mock_file1, 'rb') as f:
 489         mock_contents1 = f.read()
 490       with open(mock_file2, 'rb') as f:
 491         mock_contents2 = f.read()
 492       with open(mock_file3, 'rb') as f:
 493         mock_contents3 = f.read()
 494       with open(dest_mock_file1, 'rb') as f:
 495         dest_mock_contents1 = f.read()
 496       with open(dest_mock_file2, 'rb') as f:
 497         dest_mock_contents2 = f.read()
 498       with open(dest_mock_file3, 'rb') as f:
 499         dest_mock_contents3 = f.read()
 500
 501       self.assertEqual(mock_contents1, dest_mock_contents1)
 502       self.assertEqual(mock_contents2, dest_mock_contents2)
 503       self.assertEqual(mock_contents3, dest_mock_contents3)
 504
 505   def test_DownloadMismatchArchiveUponExtraction(self):
 506     # Tests that mismatching archive files are downloaded upon extraction.
 507     with pynacl.working_directory.TemporaryWorkingDirectory() as work_dir:
 508       mock_file1 = self.GenerateMockFile(work_dir, mock_file='mockfile1.txt')
 509       mock_file2 = self.GenerateMockFile(work_dir, mock_file='mockfile2.txt')
 510
 511       tar_dir = os.path.join(work_dir, 'tar_dir')
 512       dest_dir = os.path.join(work_dir, 'dest_dir')
 513       mock_tars_dir = os.path.join(work_dir, 'mock_tars')
 514       package_target = 'custom_package_target'
 515       package_name = 'custom_package'
 516       package_revision = 10
 517
 518       # Create mock tars and mock URLS where the tars can be downloaded from.
 519       os.makedirs(mock_tars_dir)
 520       mock_tar1 = os.path.join(mock_tars_dir, 'mock1.tar')
 521       mock_url1 = 'https://www.mock.com/tar1.tar'
 522       with tarfile.TarFile(mock_tar1, 'w') as f:
 523         f.add(mock_file1, arcname=os.path.basename(mock_file1))
 524       self._fake_downloader.StoreURL(mock_url1, mock_tar1)
 525
 526       mock_tar2 = os.path.join(mock_tars_dir, 'mock2.tar')
 527       mock_url2 = 'https://www.mock.com/tar2.tar'
 528       with tarfile.TarFile(mock_tar2, 'w') as f:
 529         f.add(mock_file2, arcname=os.path.basename(mock_file2))
 530       self._fake_downloader.StoreURL(mock_url2, mock_tar2)
 531
 532       # Have tar1 be missing, have tar2 be a file with invalid data.
 533       mismatch_tar2 = package_locations.GetLocalPackageArchiveFile(
 534           tar_dir,
 535           package_target,
 536           package_name,
 537           'mock2.tar'
 538       )
 539       os.makedirs(os.path.dirname(mismatch_tar2))
 540       with open(mismatch_tar2, 'wb') as f:
 541         f.write('mismatch tar')
 542
 543       package_desc = self.GeneratePackageInfo(
 544         [mock_tar1, mock_tar2],
 545         url_dict= {mock_tar1: mock_url1, mock_tar2: mock_url2}
 546       )
 547       package_file = package_locations.GetLocalPackageFile(
 548           tar_dir,
 549           package_target,
 550           package_name
 551       )
 552       package_desc.SavePackageFile(package_file)
 553
 554       package_version.ExtractPackageTargets(
 555           [(package_target, package_name)],
 556           tar_dir,
 557           dest_dir,
 558           downloader=self._fake_downloader.Download
 559       )
 560       self.assertEqual(self._fake_downloader.GetDownloadCount(), 2,
 561                        "Expected to download exactly 2 mismatched archives.")
 562
 563       full_dest_dir = package_locations.GetFullDestDir(
 564           dest_dir,
 565           package_target,
 566           package_name
 567       )
 568
 569       dest_mock_file2 = os.path.join(
 570           full_dest_dir,
 571           os.path.basename(mock_file2)
 572       )
 573       dest_mock_file1 = os.path.join(
 574           full_dest_dir,
 575           os.path.basename(mock_file1)
 576       )
 577
 578       with open(mock_file1, 'rb') as f:
 579         mock_contents1 = f.read()
 580       with open(mock_file2, 'rb') as f:
 581         mock_contents2 = f.read()
 582       with open(dest_mock_file1, 'rb') as f:
 583         dest_mock_contents1 = f.read()
 584       with open(dest_mock_file2, 'rb') as f:
 585         dest_mock_contents2 = f.read()
 586
 587       self.assertEqual(mock_contents1, dest_mock_contents1)
 588       self.assertEqual(mock_contents2, dest_mock_contents2)
 589
 590
 591 if __name__ == '__main__':
 592   unittest.main()