From: Markus Lehtonen Date: Thu, 10 Jul 2014 09:17:33 +0000 (+0300) Subject: Implement repocache-adm command line tool X-Git-Tag: submit/devel/20190730.075437~22 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b8548b7c5bc5aa7683c391a43bb27707787550c0;p=services%2Fobs-service-git-buildpackage.git Implement repocache-adm command line tool This command line tool is intended for managing a gbp repository cache. It will contain functionality for e.g. checking, cleaning up and repairing the repository cache. The tool is supposed to be runnable on a live system. This initial implementation only contains a rudimentary 'stat' command for printing repocache status/statistics. Change-Id: I8607fa2c60241e02ef7d8c04c8223f6136ac0943 Signed-off-by: Markus Lehtonen --- diff --git a/.coveragerc b/.coveragerc index 921c9ef..7767f95 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -include = obs_service_gbp/*, obs_service_gbp_utils/*, gbp_repocache/* +include = obs_service_gbp/*, obs_service_gbp_utils/*, gbp_repocache/*, repocache_adm/* diff --git a/packaging/obs-service-git-buildpackage.spec b/packaging/obs-service-git-buildpackage.spec index fad2b0a..9f0e4f7 100644 --- a/packaging/obs-service-git-buildpackage.spec +++ b/packaging/obs-service-git-buildpackage.spec @@ -93,4 +93,6 @@ rm -rf %{buildroot}%{python_sitelib}/*info %files -n gbp-repocache %defattr(-,root,root,-) %doc COPYING +%{_bindir}/repocache-adm %{python_sitelib}/gbp_repocache +%{python_sitelib}/repocache_adm diff --git a/repocache_adm/__init__.py b/repocache_adm/__init__.py new file mode 100644 index 0000000..1f64207 --- /dev/null +++ b/repocache_adm/__init__.py @@ -0,0 +1,20 @@ +# vim:fileencoding=utf-8:et:ts=4:sw=4:sts=4 +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +"""Module containing the repocache-adm tool""" + diff --git a/repocache_adm/adm.py b/repocache_adm/adm.py new file mode 100755 index 0000000..5d93815 --- /dev/null +++ b/repocache_adm/adm.py @@ -0,0 +1,59 @@ +#!/usr/bin/python -u +# vim:fileencoding=utf-8:et:ts=4:sw=4:sts=4 +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +"""The repocache-adm tool""" + +import logging +import sys + +from argparse import ArgumentParser + +from repocache_adm.cmd_stat import Stat + + +def parse_args(argv): + """Command line argument parser""" + + parser = ArgumentParser() + parser.add_argument('-c', '--cache-dir', required=True, + help='Repocache base directory') + parser.add_argument('-d', '--debug', action='store_true', + help='Debug output') + subparsers = parser.add_subparsers() + + # Add subcommands + for subcommand in (Stat,): + subcommand.add_subparser(subparsers) + + return parser.parse_args(argv) + + +def main(argv=None): + """Main entry point for the command line tool""" + logging.basicConfig() + args = parse_args(argv) + if args.debug: + logging.root.setLevel(logging.DEBUG) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/repocache_adm/cmd_stat.py b/repocache_adm/cmd_stat.py new file mode 100644 index 0000000..e723bd9 --- /dev/null +++ b/repocache_adm/cmd_stat.py @@ -0,0 +1,73 @@ +#!/usr/bin/python -u +# vim:fileencoding=utf-8:et:ts=4:sw=4:sts=4 +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +"""The stat subcommand""" + +import logging +import os +import subprocess + +from repocache_adm.common import SubcommandBase, pprint_sz + +class Stat(SubcommandBase): + """Subcommand for checking the repo cache""" + + name = 'stat' + description = 'Display repocache status' + help_msg = None + + @classmethod + def main(cls, args): + """Entry point for 'check' subcommand""" + + log = logging.getLogger(cls.name) + + path = os.path.abspath(args.cache_dir) + if not os.path.isdir(args.cache_dir): + log.error("repocache basedir '%s' not found", path) + return 1 + + log.info("Checking repository cache in '%s'", path) + + popen = subprocess.Popen(['du', '-d2', '-B1', '-0'], cwd=path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = popen.communicate() + if popen.returncode: + log.error("Failed to run 'du': %s", err) + return 1 + + total_sz = -1 + num_repos = 0 + for line in out.split('\0'): + if not line: + continue + size, name = line.split() + if name == '.': + total_sz = int(size) + else: + base = os.path.split(name)[0] + if base != '.': + # This is a repository + num_repos += 1 + + pretty_sz = " (%s)" % pprint_sz(total_sz) if total_sz >= 1024 else "" + print "Status of %s:" % path + print "Total of %d repos taking %d bytes%s of disk space" % \ + (num_repos, total_sz, pretty_sz) + return 0 diff --git a/repocache_adm/common.py b/repocache_adm/common.py new file mode 100644 index 0000000..197828b --- /dev/null +++ b/repocache_adm/common.py @@ -0,0 +1,71 @@ +#!/usr/bin/python -u +# vim:fileencoding=utf-8:et:ts=4:sw=4:sts=4 +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +"""Common functionality of the adm module""" + + +def pprint_sz(size): + """Pretty print file size in human readable format + + >>> pprint_sz(0) + '0 bytes' + >>> pprint_sz(1023) + '1023 bytes' + >>> pprint_sz(1024*1024) + '1.0 MB' + >>> pprint_sz(1024*1024*1024*(1024 + 512)) + '1.5 TB' + """ + if size < 1024: + return "%d bytes" % size + + units = ['kB', 'MB', 'GB', 'TB'] + power = unit = None + for power, unit in enumerate(units, 2): + if size < pow(1024, power): + break + return "%.1f %s" % (float(size) / pow(1024, power - 1), unit) + + +class SubcommandBase(object): + """Base class / API for subcommand implementations""" + + name = None + description = None + help_msg = None + + @classmethod + def add_subparser(cls, subparsers): + """Add and initialize argparse subparser for the subcommand""" + parser = subparsers.add_parser(cls.name, + description=cls.description, + help=cls.help_msg) + cls.add_arguments(parser) + parser.set_defaults(func=cls.main) + + @classmethod + def add_arguments(cls, parser): + """Prototype method for adding subcommand specific arguments""" + pass + + @classmethod + def main(cls, args): + """Prototype entry point for subcommands""" + raise NotImplementedError("Command %s not implemented" % cls.__name__) + diff --git a/setup.cfg b/setup.cfg index 0e06169..f4abaa9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [nosetests] with-coverage=1 -cover-package=obs_service_gbp,obs_service_gbp_utils,gbp_repocache +cover-package=obs_service_gbp,obs_service_gbp_utils,gbp_repocache,repocache_adm with-xunit=1 with-doctest=1 diff --git a/setup.py b/setup.py index 2399355..1f16160 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,12 @@ setup(name='obs_service_gbp', author_email='markus.lehtonen@linux.intel.com', url=tag_from_spec('URL'), license=tag_from_spec('License'), - packages=['obs_service_gbp', 'obs_service_gbp_utils', 'gbp_repocache'], + packages=['obs_service_gbp', 'obs_service_gbp_utils', 'gbp_repocache', + 'repocache_adm'], data_files=[('/usr/lib/obs/service', ['service/git-buildpackage', 'service/git-buildpackage.service']), ('/etc/obs/services', ['config/git-buildpackage'])], + entry_points={ + 'console_scripts': ['repocache-adm = repocache_adm.adm:main'] + } ) diff --git a/tests/test_repocache_adm.py b/tests/test_repocache_adm.py new file mode 100644 index 0000000..309db6e --- /dev/null +++ b/tests/test_repocache_adm.py @@ -0,0 +1,96 @@ +# vim:fileencoding=utf-8:et:ts=4:sw=4:sts=4 +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +"""Unit tests for the repocache-adm command line tool""" + +import mock +import os +import shutil +from nose.tools import assert_raises, eq_ # pylint: disable=E0611 + +from gbp_repocache import CachedRepo +from repocache_adm.adm import main as adm +from repocache_adm.common import SubcommandBase +from tests import UnitTestsBase + +# Disable "Method could be a function" +# pylint: disable=R0201 +# Disable "Method 'main' is abstract in class 'XYZ' but is not overridden" +# pylint: disable=W0223 + + +class BadSubcommand(SubcommandBase): + """Broken subcommand""" + name = 'stat' + +class TestRepocacheAdm(UnitTestsBase): + """Test repocache-adm command line tool""" + + @classmethod + def setup_class(cls): + """Test class setup""" + super(TestRepocacheAdm, cls).setup_class() + + # Create another orig repo for testing + cls._template_repo2 = cls.create_orig_repo('orig2') + + # Create a reference cache + cls._template_cache = os.path.abspath('cache') + # Create cached repos - need to del instances to release the repo lock + _cached = CachedRepo(cls._template_cache, cls._template_repo.path, + bare=False) + del _cached + _cached = CachedRepo(cls._template_cache, cls._template_repo2.path, + bare=True) + del _cached + + def setup(self): + """Test case setup""" + super(TestRepocacheAdm, self).setup() + + # Create test-case specific cache + shutil.copytree(self._template_cache, self.cachedir) + + @mock.patch('repocache_adm.adm.Stat', BadSubcommand) + def test_not_implemented(self): + """Test a badly written subcommand""" + with assert_raises(NotImplementedError): + adm(['-c', self.cachedir, 'stat']) + + def test_invalid_args(self): + """Test invalid command line args""" + # Non-existing option + with assert_raises(SystemExit): + adm(['--foo']) + # Option without argument + with assert_raises(SystemExit): + adm(['-c']) + # Unknown subcommand + with assert_raises(SystemExit): + adm(['foocmd']) + + def test_stat(self): + """Basic test for the 'stat' subcommand""" + # With debug + eq_(adm(['-d', '-c', self.cachedir, 'stat']), 0) + + def test_stat_fail(self): + """Failure cases for the 'stat' subcommand""" + # Non-existent cache dir + eq_(adm(['-c', 'non-existent', 'stat']), 1) +