logger = gbplog.getLogger('source_service')
logger.setLevel(gbplog.INFO)
+
+class MirrorGitRepository(GitRepository): # pylint: disable=R0904
+ """Special Git repository to enable a mirrored clone with a working copy"""
+
+ def set_config(self, name, value, replace=False):
+ """Add a config value"""
+ args = ['--replace-all'] if replace else ['--add']
+ args.extend([name, value])
+ stderr, ret = self._git_inout('config', args)[1:]
+ if ret:
+ raise GitRepositoryError('Failed to set config %s=%s (%s)' %
+ (name, value, stderr))
+
+ def force_fetch(self):
+ """Fetch with specific arguments"""
+ # Update all refs
+ self._git_command('fetch', ['-q', '-u', '-p', 'origin'])
+ try:
+ # Fetch remote HEAD separately
+ self._git_command('fetch', ['-q', '-u', 'origin', 'HEAD'])
+ except GitRepositoryError:
+ # If remote HEAD is invalid, invalidate FETCH_HEAD, too
+ with open(os.path.join(self.git_dir, 'FETCH_HEAD'), 'w') as fhead:
+ fhead.write('0000000000000000000000000000000000000000\n')
+
+ @classmethod
+ def clone(cls, path, url, bare=False):
+ """Create a mirrored clone"""
+ if bare:
+ return super(MirrorGitRepository, cls).clone(path, url,
+ mirror=bare,
+ auto_name=False)
+ else:
+ logger.debug('Initializing non-bare mirrored repo')
+ repo = cls.create(path)
+ repo.add_remote_repo('origin', url)
+ repo.set_config('remote.origin.fetch', '+refs/*:refs/*', True)
+ repo.force_fetch()
+ return repo
+
+
class CachedRepoError(Exception):
"""Repository cache errors"""
pass
class CachedRepo(object):
"""Object representing a cached repository"""
- def __init__(self, url):
+ def __init__(self, url, bare=False):
self.basedir = '/var/cache/obs/git-buildpackage-repos/'
self.repodir = None
self.repo = None
self.lock = None
- # Check repository cache base
+ self._init_cache_base()
+ self._init_git_repo(url, bare)
+
+ def _init_cache_base(self):
+ """Check and initialize repository cache base directory"""
if 'CACHEDIR' in os.environ:
self.basedir = os.environ['CACHEDIR']
logger.debug("Using cache basedir '%s'" % self.basedir)
raise CachedRepoError('Failed to create cache base dir: %s' %
str(err))
- # Safe dir name
- reponame = url.split('/')[-1].split(':')[-1]
- postfix = hashlib.sha1(url).hexdigest()
- reponame = reponame + '_' + postfix
- self.repodir = os.path.join(self.basedir, reponame)
-
- # Acquire lock
+ def _acquire_lock(self, repodir):
+ """Acquire the repository lock"""
logger.debug("Acquiring repository lock")
try:
- self.lock = open(self.repodir + '.lock', 'w')
+ self.lock = open(repodir + '.lock', 'w')
except IOError as err:
raise CachedRepoError('Unable to open repo lock file: %s' % err)
fcntl.flock(self.lock, fcntl.LOCK_EX)
logger.debug("Repository lock acquired")
- # Update repo cache
+ def _release_lock(self):
+ """Release the repository lock"""
+ if self.lock:
+ fcntl.flock(self.lock, fcntl.LOCK_UN)
+ self.lock = None
+
+ def _init_git_repo(self, url, bare):
+ """Clone / update a remote git repository"""
+ # Safe repo dir name
+ reponame = url.split('/')[-1].split(':')[-1]
+ postfix = hashlib.sha1(url).hexdigest() # pylint: disable=E1101
+ reponame = reponame + '_' + postfix
+ self.repodir = os.path.join(self.basedir, reponame)
+
+ # Acquire repository lock
+ self._acquire_lock(self.repodir)
+
if os.path.exists(self.repodir):
try:
- self.repo = GitRepository(self.repodir)
+ self.repo = MirrorGitRepository(self.repodir)
except GitRepositoryError:
pass
- if not self.repo:
+ if not self.repo or self.repo.bare != bare:
logger.info('Removing corrupted repo cache %s' % self.repodir)
try:
+ self.repo = None
shutil.rmtree(self.repodir)
except OSError as err:
raise CachedRepoError('Failed to remove repo cache dir: %s'
% str(err))
else:
logger.info('Fetching from remote')
- self.repo.fetch()
-
+ try:
+ self.repo.force_fetch()
+ except GitRepositoryError as err:
+ raise CachedRepoError('Failed to fetch from remote: %s' %
+ err)
if not self.repo:
logger.info('Cloning from %s' % url)
try:
- self.repo = GitRepository.clone(self.repodir, url,
- mirror=True, auto_name=False)
+ self.repo = MirrorGitRepository.clone(self.repodir, url,
+ bare=bare)
except GitRepositoryError as err:
- raise CachedRepoError('Failed to clone: %s' % str(err))
-
- def _release_lock(self):
- """Release the repository lock"""
- if self.lock:
- fcntl.flock(self.lock, fcntl.LOCK_UN)
+ raise CachedRepoError('Failed to clone: %s' % err)
def __del__(self):
self._release_lock()
+ def update_working_copy(self, commitish='HEAD'):
+ """Reset HEAD to the given commit-ish"""
+ if self.repo.bare:
+ raise CachedRepoError('Cannot update working copy of a bare repo')
+
+ # Update HEAD from FETCH_HEAD, so that HEAD points to remote HEAD.
+ # We do it this way because FETCH_HEAD may point to an invalid object
+ # and we don't wont to update the working copy at this point.
+ shutil.copyfile(os.path.join(self.repo.git_dir, 'FETCH_HEAD'),
+ os.path.join(self.repo.git_dir, 'HEAD'))
+ # Resolve commit-ish to sha-1 and set HEAD (and working copy) to it
+ try:
+ sha = self.repo.rev_parse(commitish)
+ self.repo.set_branch(sha)
+ except GitRepositoryError as err:
+ raise CachedRepoError("Unknown ref '%s': %s" % (commitish, err))
+ self.repo.force_head(sha, hard=True)
+ self.repo.update_submodules(init=True, recursive=True, fetch=True)
+ return sha
+
import shutil
import stat
import tempfile
-from nose.tools import assert_raises
+from nose.tools import assert_raises # pylint: disable=E0611
from gbp.git.repository import GitRepository, GitRepositoryError
-from obs_service_gbp import CachedRepo, CachedRepoError
+from obs_service_gbp import MirrorGitRepository, CachedRepo, CachedRepoError
from obs_service_gbp.command import main as service
orig_repo.commit_dir(TEST_DATA_DIR, 'Initial version', 'master',
create_missing_branch=True)
orig_repo.force_head('master', hard=True)
+ # Make new commit
+ cls.update_repository_file(orig_repo, 'foo.txt', 'new data\n')
return orig_repo
@classmethod
cls.cachedir = os.path.join(cls.workdir, 'cache')
os.environ['CACHEDIR'] = cls.cachedir
# Create an orig repo for testing
- cls.orig_repo = cls.create_orig_repo('orig').path
+ cls.orig_repo = cls.create_orig_repo('orig')
@classmethod
def teardown_class(cls):
if not 'DEBUG_NOSETESTS' in os.environ:
shutil.rmtree(cls.workdir)
+ @staticmethod
+ def update_repository_file(repo, filename, data):
+ """Append data to file in git repository and commit changes"""
+ with open(os.path.join(repo.path, filename), 'a') as filep:
+ filep.write(data)
+ repo.add_files(filename)
+ repo.commit_files(filename, "Update %s" % filename)
+
class TestBasicFunctionality(UnitTestsBase):
"""Base class for unit tests"""
+ def __init__(self):
+ self.tmpdir = None
+ super(TestBasicFunctionality, self).__init__()
+
def setup(self):
"""Test case setup"""
# Change to a temporary directory
def test_basic_export(self):
"""Test that export works"""
- assert service(['--url', self.orig_repo]) == 0
+ assert service(['--url', self.orig_repo.path]) == 0
+
+ def test_gbp_failure(self):
+ """Test git-buildpackage failure"""
+ assert service(['--url', self.orig_repo.path, '--outdir=foo/bar']) == 2
def test_options_outdir(self):
"""Test the --outdir option"""
outdir = os.path.join(self.tmpdir, 'outdir')
- assert service(['--url', self.orig_repo, '--outdir=%s' % outdir]) == 0
+ args = ['--url', self.orig_repo.path, '--outdir=%s' % outdir]
+ assert service(args) == 0
assert os.path.isdir(outdir)
def test_options_revision(self):
"""Test the --revision option"""
- assert service(['--url', self.orig_repo, '--revision=master']) == 0
- assert service(['--url', self.orig_repo, '--revision=foobar']) == 1
+ assert service(['--url', self.orig_repo.path, '--revision=master']) == 0
+ assert service(['--url', self.orig_repo.path, '--revision=foobar']) == 1
def test_options_verbose(self):
"""Test the --verbose option"""
- assert service(['--url', self.orig_repo, '--verbose=yes']) == 0
+ assert service(['--url', self.orig_repo.path, '--verbose=yes']) == 0
with assert_raises(SystemExit):
- service(['--url', self.orig_repo, '--verbose=foob'])
+ service(['--url', self.orig_repo.path, '--verbose=foob'])
def test_options_spec_vcs_tag(self):
"""Test the --spec-vcs-tag option"""
- assert service(['--url', self.orig_repo,
+ assert service(['--url', self.orig_repo.path,
'--spec-vcs-tag=orig/%(tagname)s']) == 0
+class TestObsRepoGitRepository(UnitTestsBase):
+ """Test the special GitRepository class"""
+
+ def test_set_config(self):
+ """Test the set config functionality"""
+ repo = MirrorGitRepository.create('testrepo')
+ with assert_raises(GitRepositoryError):
+ repo.set_config('foo', 'bar')
+ repo.set_config('foo.bar', 'baz')
+ repo.set_config('foo.bar', 'bax', replace=True)
+ assert repo.get_config('foo.bar') == 'bax'
+
class TestCachedRepo(UnitTestsBase):
"""Test CachedRepo class"""
"""Test invalid url"""
with assert_raises(CachedRepoError):
CachedRepo('foo/bar.git')
+ with assert_raises(CachedRepoError):
+ CachedRepo('foo/baz.git', bare=True)
+
+ # Try updating from non-existing repo
+ repo = CachedRepo(self.orig_repo.path)
+ del repo
+ shutil.move(self.orig_repo.path, self.orig_repo.path + '.tmp')
+ with assert_raises(CachedRepoError):
+ repo = CachedRepo(self.orig_repo.path)
+ shutil.move(self.orig_repo.path + '.tmp', self.orig_repo.path)
def test_clone_and_fetch(self):
"""Basic test for cloning and fetching"""
# Clone
- repo = CachedRepo(self.orig_repo)
+ repo = CachedRepo(self.orig_repo.path)
assert repo
- assert repo.repo.bare
- repo._release_lock()
+ assert repo.repo.bare is not True
+ sha = repo.repo.rev_parse('master')
+ path = repo.repo.path
+ del repo
+ # Make new commit in "upstream"
+ self.update_repository_file(self.orig_repo, 'foo.txt', 'more data\n')
# Fetch
- repo2 = CachedRepo(self.orig_repo)
- assert repo2
- assert repo.repo.path == repo2.repo.path
+ repo = CachedRepo(self.orig_repo.path)
+ assert repo
+ assert path == repo.repo.path
+ assert sha != repo.repo.rev_parse('master')
+
+ def test_update_working_copy(self):
+ """Test update functionality"""
+ repo = CachedRepo(self.orig_repo.path)
+ # Check that the refs are mapped correctly
+ sha = repo.update_working_copy('HEAD~1')
+ assert sha == self.orig_repo.rev_parse('HEAD~1')
+ sha = self.orig_repo.rev_parse('HEAD')
+ assert sha == repo.update_working_copy('HEAD')
+ assert sha == repo.update_working_copy(sha)
+
+ with assert_raises(CachedRepoError):
+ sha = repo.update_working_copy('foo/bar')
+
+ def test_update_bare(self):
+ """Test update for bare repository"""
+ repo = CachedRepo(self.orig_repo.path, bare=True)
+ with assert_raises(CachedRepoError):
+ repo.update_working_copy('HEAD')
+
+ def test_invalid_remote_head(self):
+ """Test clone/update from remote whose HEAD is invalid"""
+ repo = CachedRepo(self.orig_repo.path)
+ del repo
+
+ # Make remote HEAD point to a non-existent branch
+ orig_branch = self.orig_repo.get_branch()
+ with open(os.path.join(self.orig_repo.git_dir, 'HEAD'), 'w') as head:
+ head.write('ref: refs/heads/non-existent-branch\n')
+
+ repo = CachedRepo(self.orig_repo.path)
+ # Local HEAD should be invalid, now
+ with assert_raises(CachedRepoError):
+ repo.update_working_copy('HEAD')
+ # Test valid refs, too
+ assert repo.update_working_copy('master')
+
+ # Reset orig repo to original state
+ self.orig_repo.set_branch(orig_branch)
def test_corrupted_cache(self):
"""Test recovering from corrupted cache"""
# Clone
- repo = CachedRepo(self.orig_repo)
+ repo = CachedRepo(self.orig_repo.path)
# Corrupt repo
- shutil.rmtree(os.path.join(repo.repo.path, 'refs'))
+ shutil.rmtree(os.path.join(repo.repo.path, '.git/refs'))
with assert_raises(GitRepositoryError):
repo.repo.rev_parse('HEAD')
- repo._release_lock()
+ del repo
# Update and check status
- repo = CachedRepo(self.orig_repo)
+ repo = CachedRepo(self.orig_repo.path)
assert repo.repo.rev_parse('HEAD')
+ def test_changing_repotype(self):
+ """Test changing repo type from bare -> normal"""
+ # Clone
+ repo = CachedRepo(self.orig_repo.path, bare=True)
+ assert repo.repo.bare == True
+ del repo
+ repo = CachedRepo(self.orig_repo.path, bare=False)
+ assert repo.repo.bare == False
+
def test_cache_access_error(self):
"""Test cached directory with invalid permissions"""
# Check base cachedir creation access error
os.chmod(self.workdir, 0)
with assert_raises(CachedRepoError):
- repo = CachedRepo(self.orig_repo)
+ repo = CachedRepo(self.orig_repo.path)
os.chmod(self.workdir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
- repo = CachedRepo(self.orig_repo)
- repo._release_lock()
+ repo = CachedRepo(self.orig_repo.path)
+ del repo
- # Check cache access error
+ # Check cache base dir access error
os.chmod(self.cachedir, 0)
with assert_raises(CachedRepoError):
- repo = CachedRepo(self.orig_repo)
+ repo = CachedRepo(self.orig_repo.path)
+ os.chmod(self.cachedir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+ repo = CachedRepo(self.orig_repo.path)
+ del repo
+
+ # Check repodir delete eror
+ os.chmod(self.cachedir, stat.S_IREAD | stat.S_IEXEC)
+ with assert_raises(CachedRepoError):
+ # Change repo type -> tries to delete
+ repo = CachedRepo(self.orig_repo.path, bare=True)
os.chmod(self.cachedir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
- repo = CachedRepo(self.orig_repo)
projects / services / obs-service-git-buildpackage.git / commitdiff