From cb309c59635b2faee63a0d2ef061e29317b7a28d Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 17 Jun 2013 15:50:53 +0300 Subject: [PATCH] Added support of complex data types to BackendDB It's implemented using json. When some key or list item or string is marked as jsoned BackendDB will try to encode it to json before storing in Redis and decode it when retriving. Change-Id: Ifdba0453fa9bbef173cef301e587ded536c98be3 Signed-off-by: Ed Bartosh --- common/backenddb.py | 42 +++++++++++++++++----- tests/test_backenddb.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/common/backenddb.py b/common/backenddb.py index 79c9f01..45ca892 100644 --- a/common/backenddb.py +++ b/common/backenddb.py @@ -17,6 +17,8 @@ Example of usage: import redis import yaml +import json +from copy import deepcopy class BackendDBError(Exception): """BackendDB custom exception.""" @@ -37,9 +39,20 @@ class Entity(object): repos['Myrepo'] = {'name': 'Myreponame', 'latest_snapshot': '123.4'} """ - def __init__(self, dbobj, prefix): + def __init__(self, dbobj, prefix, jsoned=None): + """__init__ + Args: + dbobj - Redis database object + prefix - Uniq prefix for this entity + jsoned - list of jsoned keys(for dicts) + or indexes (for iterables) or bool (for strings). + It's used to convert data to json + format when storing in db and convert it back when + it's retrived. + """ self._db = dbobj self._prefix = prefix + self.jsoned = jsoned or [] def keys(self): """Get set of keys started with the self._prefix.""" @@ -51,21 +64,32 @@ class Entity(object): def __getitem__(self, key): key = "%s%s" % (self._prefix, key) - if self._db.type(key) == 'hash': - return self._db.hgetall(key) + value = self._db.hgetall(key) + for field in self.jsoned: + value[field] = json.loads(value[field]) elif self._db.type(key) == 'list': - return self._db.lrange(key, 0, -1) + value = self._db.lrange(key, 0, -1) + for i in self.jsoned: + value[i] = json.loads(value[i]) elif self._db.type(key) == 'string': - return self._db.get(key) + value = self._db.get(key) + if self.jsoned: + value = json.loads(value) + return value def __setitem__(self, key, value): key = "%s%s" % (self._prefix, key) - if isinstance(value, dict): - self._db.hmset(key, value) + mvalue = deepcopy(value) + for field in self.jsoned: + mvalue[field] = json.dumps(value[field]) + self._db.hmset(key, mvalue) elif isinstance(value, list): - for item in value: + mvalue = deepcopy(value) + for i in self.jsoned: + mvalue[i] = json.dumps(mvalue[i]) + for item in mvalue: self._db.rpush(key, item) else: self._db.set(key, value) @@ -108,7 +132,7 @@ class BackendDB(): def get_repos(self): """Return repos entity object.""" - return Entity(self._redis, "repo:") + return Entity(self._redis, "repo:", ["Architectures"]) def read_repos(self, yamlobj): """ diff --git a/tests/test_backenddb.py b/tests/test_backenddb.py index 9648a52..9f09b7e 100644 --- a/tests/test_backenddb.py +++ b/tests/test_backenddb.py @@ -20,12 +20,14 @@ import unittest import re +import json + from mock import patch from StringIO import StringIO import redis -from common.backenddb import BackendDB, BackendDBError +from common.backenddb import BackendDB, Entity, BackendDBError REPOS = """ Repositories: @@ -108,6 +110,21 @@ class RedisMock(object): """Get all the fields and values in a hash.""" return self._dict.get(key) + def rpush(self, key, item): + """Add item to the list.""" + if key not in self._dict: + self._dict[key] = [] + self._dict[key].append(item) + + def lrange(self, key, starti, endi): + """Get items from the list.""" + length = len(self._dict[key]) + if starti < length: + starti = 0 + if endi == -1 or endi > length: + endi = length + 1 + return self._dict[key][starti:endi] + def type(self, key): """Returns the type of key ``key``""" if isinstance(self._dict[key], dict): @@ -162,7 +179,7 @@ class BackendDBTest(unittest.TestCase): 'TopLevel': '/srv/trunk', 'SnapshotDir': '/srv/snapshots/trunk', 'Link': 'latest', - 'Architectures': "['ia32', 'armv7l']", + 'Architectures': ['ia32', 'armv7l'], 'Release': 'release', 'PartOf': 'trunk', 'Location': '/srv/trunk/repos/repo1'} repos = bdb.get_repos() @@ -170,7 +187,7 @@ class BackendDBTest(unittest.TestCase): def test_set_repo(self): """Setting repo data.""" - data = {'field1': 'value1', 'field2': 'value2'} + data = {'field1': 'value1', 'Architectures': ['arm']} bdb = BackendDB() repos = bdb.get_repos() repos['Test'] = data @@ -189,7 +206,7 @@ class BackendDBTest(unittest.TestCase): bdb.read_repos(REPOS) repos = bdb.get_repos() repos.delete('Repo1') - repos['Test'] = {'key': 'value'} + repos['Test'] = {'key': 'value', 'Architectures': ['arm', 'i385']} self.assertEqual(repos.keys(), set(['Repo2', 'Repo3', 'Test'])) bdb.read_repos(REPOS) self.assertEqual(repos.keys(), set(['Repo1', 'Repo2', 'Repo3'])) @@ -216,3 +233,71 @@ class BackendDBTest(unittest.TestCase): bdb.read_repos(REPOS) repos = bdb.get_repos() self.assertEqual(len(repos), 3) + + def test_get_release_ids(self): + """Test getting release ids.""" + bdb = BackendDB() + ids = bdb.get_release_ids() + ids[0] = 'release1' + self.assertEqual(ids[0], 'release1') + + +@patch('redis.Redis', RedisMock) # pylint: disable=R0904 +class EntityTest(unittest.TestCase): + '''Tests Entity functionality.''' + + def test_list_value(self): + """Test using list as value.""" + entity = Entity(redis.Redis(), 'prefix:') + value = [1, 2, 3] + entity['list'] = value + self.assertEqual(entity['list'], value) + + def test_dict_value(self): + """Test using dict as value.""" + entity = Entity(redis.Redis(), 'prefix:') + value = {'field1': 'val1', 'field2': 'val2'} + entity['dict'] = value + self.assertEqual(entity['dict'], value) + + def test_string_value(self): + """Test using string as value.""" + entity = Entity(redis.Redis(), 'prefix:') + value = 'data' + entity['string'] = value + self.assertEqual(entity['string'], value) + + def test_jsoned_dict_fields(self): + """ + Test using dict as a value and list as a value of that dict. + list should be automatically jsoned and unjsoned when storing/retriving + + """ + entity = Entity(redis.Redis(), 'prefix:', ['field']) + value = {'field': [1, 2, 3]} + entity['dict'] = value + self.assertEqual(entity['dict'], value) + + def test_jsoned_list_items(self): + """ + Test using nested list as a value. + 2nd level list should be automatically jsoned and unjsoned + when storing/retriving + + """ + entity = Entity(redis.Redis(), 'prefix:', [0]) + value = [[1, 2, 3], 2, 3] + entity['list'] = value + self.assertEqual(entity['list'], value) + + def test_jsoned_string_value(self): + """ + Test using jsoned string as a value. + value should be automatically unjsoned when retriving + + """ + entity = Entity(redis.Redis(), 'prefix:', True) + value = [0, [1, 2, 3], {'4': 5}, 'end'] + entity['string'] = json.dumps(value) + self.assertEqual(entity['string'], value) + -- 2.7.4