virtual bool commit() = 0;
virtual void close() = 0;
+ virtual void keys(vector<key_type>* keys) = 0;
+
Database() { }
virtual ~Database() { }
iterator(const iterator& other)
: parent_(other.parent_),
- state_(other.state_->clone()) { }
+ state_(other.state_ ? other.state_->clone()
+ : shared_ptr<DatabaseState>()) { }
iterator& operator=(iterator copy) {
copy.swap(*this);
}
iterator& operator++() {
- parent_->increment(state_);
+ parent_->increment(&state_);
return *this;
}
iterator operator++(int) {
iterator copy(*this);
- parent_->increment(state_);
+ parent_->increment(&state_);
return copy;
}
virtual bool equal(shared_ptr<DatabaseState> state1,
shared_ptr<DatabaseState> state2) const = 0;
- virtual void increment(shared_ptr<DatabaseState> state) const = 0;
+ virtual void increment(shared_ptr<DatabaseState>* state) const = 0;
virtual KV& dereference(
shared_ptr<DatabaseState> state) const = 0;
};
#include <string>
#include <utility>
+#include <vector>
#include "caffe/common.hpp"
#include "caffe/database.hpp"
bool commit();
void close();
+ void keys(vector<key_type>* keys);
+
const_iterator begin() const;
const_iterator cbegin() const;
const_iterator end() const;
bool equal(shared_ptr<DatabaseState> state1,
shared_ptr<DatabaseState> state2) const;
- void increment(shared_ptr<DatabaseState> state) const;
+ void increment(shared_ptr<DatabaseState>* state) const;
Database::KV& dereference(shared_ptr<DatabaseState> state) const;
shared_ptr<leveldb::DB> db_;
#include <string>
#include <utility>
+#include <vector>
#include "lmdb.h"
bool commit();
void close();
+ void keys(vector<key_type>* keys);
+
const_iterator begin() const;
const_iterator cbegin() const;
const_iterator end() const;
bool equal(shared_ptr<DatabaseState> state1,
shared_ptr<DatabaseState> state2) const;
- void increment(shared_ptr<DatabaseState> state) const;
+ void increment(shared_ptr<DatabaseState>* state) const;
Database::KV& dereference(shared_ptr<DatabaseState> state) const;
MDB_env* env_;
#include <string>
#include <utility>
+#include <vector>
#include "caffe/leveldb_database.hpp"
db_.reset();
}
+void LeveldbDatabase::keys(vector<key_type>* keys) {
+ LOG(INFO) << "LevelDB: Keys";
+
+ keys->clear();
+ for (Database::const_iterator iter = begin(); iter != end(); ++iter) {
+ LOG(INFO) << "KEY";
+ keys->push_back(iter->key);
+ }
+}
+
LeveldbDatabase::const_iterator LeveldbDatabase::begin() const {
CHECK_NOTNULL(db_.get());
shared_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
if (!iter->Valid()) {
iter.reset();
}
- shared_ptr<DatabaseState> state(new LeveldbState(db_, iter));
+
+ shared_ptr<DatabaseState> state;
+ if (iter) {
+ state.reset(new LeveldbState(db_, iter));
+ }
return const_iterator(this, state);
}
LeveldbDatabase::const_iterator LeveldbDatabase::end() const {
- shared_ptr<leveldb::Iterator> iter;
- shared_ptr<DatabaseState> state(new LeveldbState(db_, iter));
+ shared_ptr<DatabaseState> state;
return const_iterator(this, state);
}
shared_ptr<LeveldbState> leveldb_state1 =
boost::dynamic_pointer_cast<LeveldbState>(state1);
- CHECK_NOTNULL(leveldb_state1.get());
-
shared_ptr<LeveldbState> leveldb_state2 =
boost::dynamic_pointer_cast<LeveldbState>(state2);
- CHECK_NOTNULL(leveldb_state2.get());
-
- CHECK(!leveldb_state1->iter_ || leveldb_state1->iter_->Valid());
- CHECK(!leveldb_state2->iter_ || leveldb_state2->iter_->Valid());
+ LOG(INFO) << leveldb_state1 << " " << leveldb_state2;
// The KV store doesn't really have any sort of ordering,
// so while we can do a sequential scan over the collection,
// we can't really use subranges.
- return !leveldb_state1->iter_ && !leveldb_state2->iter_;
+ return !leveldb_state1 && !leveldb_state2;
}
-void LeveldbDatabase::increment(shared_ptr<DatabaseState> state) const {
+void LeveldbDatabase::increment(shared_ptr<DatabaseState>* state) const {
shared_ptr<LeveldbState> leveldb_state =
- boost::dynamic_pointer_cast<LeveldbState>(state);
+ boost::dynamic_pointer_cast<LeveldbState>(*state);
CHECK_NOTNULL(leveldb_state.get());
iter->Next();
if (!iter->Valid()) {
- iter.reset();
+ state->reset();
}
}
#include <string>
#include <utility>
+#include <vector>
#include "caffe/lmdb_database.hpp"
}
}
+void LmdbDatabase::keys(vector<key_type>* keys) {
+ LOG(INFO) << "LMDB: Keys";
+
+ keys->clear();
+ for (Database::const_iterator iter = begin(); iter != end(); ++iter) {
+ keys->push_back(iter->key);
+ }
+}
+
LmdbDatabase::const_iterator LmdbDatabase::begin() const {
- MDB_cursor* cursor;
int retval;
- retval = mdb_cursor_open(txn_, dbi_, &cursor);
+
+ MDB_txn* iter_txn;
+
+ retval = mdb_txn_begin(env_, NULL, MDB_RDONLY, &iter_txn);
+ CHECK_EQ(MDB_SUCCESS, retval) << "mdb_txn_begin failed "
+ << mdb_strerror(retval);
+
+ MDB_cursor* cursor;
+ retval = mdb_cursor_open(iter_txn, dbi_, &cursor);
CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
MDB_val key;
MDB_val val;
retval = mdb_cursor_get(cursor, &key, &val, MDB_FIRST);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- shared_ptr<DatabaseState> state(new LmdbState(cursor, txn_, &dbi_));
+ CHECK(MDB_SUCCESS == retval || MDB_NOTFOUND == retval)
+ << mdb_strerror(retval);
+
+ shared_ptr<DatabaseState> state;
+ if (MDB_SUCCESS == retval) {
+ state.reset(new LmdbState(cursor, iter_txn, &dbi_));
+ }
return const_iterator(this, state);
}
LmdbDatabase::const_iterator LmdbDatabase::end() const {
- shared_ptr<DatabaseState> state(new LmdbState(NULL, txn_, &dbi_));
+ shared_ptr<DatabaseState> state;
return const_iterator(this, state);
}
shared_ptr<LmdbState> lmdb_state1 =
boost::dynamic_pointer_cast<LmdbState>(state1);
- CHECK_NOTNULL(lmdb_state1.get());
-
shared_ptr<LmdbState> lmdb_state2 =
boost::dynamic_pointer_cast<LmdbState>(state2);
- CHECK_NOTNULL(lmdb_state2.get());
-
// The KV store doesn't really have any sort of ordering,
// so while we can do a sequential scan over the collection,
// we can't really use subranges.
- return !lmdb_state1->cursor_ && !lmdb_state2->cursor_;
+ return !lmdb_state1 && !lmdb_state2;
}
-void LmdbDatabase::increment(shared_ptr<DatabaseState> state) const {
+void LmdbDatabase::increment(shared_ptr<DatabaseState>* state) const {
shared_ptr<LmdbState> lmdb_state =
- boost::dynamic_pointer_cast<LmdbState>(state);
+ boost::dynamic_pointer_cast<LmdbState>(*state);
CHECK_NOTNULL(lmdb_state.get());
int retval = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
if (MDB_NOTFOUND == retval) {
mdb_cursor_close(cursor);
- cursor = NULL;
+ state->reset();
} else {
CHECK_EQ(MDB_SUCCESS, retval) << mdb_strerror(retval);
}
database->close();
}
+TYPED_TEST(DatabaseTest, TestKeysLevelDB) {
+ string name = this->DBName();
+ shared_ptr<Database> database = DatabaseFactory("leveldb");
+ EXPECT_TRUE(database->open(name, Database::New));
+
+ Database::key_type key1 = this->TestKey();
+ Database::value_type value1 = this->TestValue();
+
+ EXPECT_TRUE(database->put(key1, value1));
+
+ Database::key_type key2 = this->TestAltKey();
+ Database::value_type value2 = this->TestAltValue();
+
+ EXPECT_TRUE(database->put(key2, value2));
+
+ EXPECT_TRUE(database->commit());
+
+ vector<Database::key_type> keys;
+ database->keys(&keys);
+
+ EXPECT_EQ(2, keys.size());
+
+ EXPECT_TRUE(this->BufferEq(keys.at(0), key1) ||
+ this->BufferEq(keys.at(0), key2));
+ EXPECT_TRUE(this->BufferEq(keys.at(1), key1) ||
+ this->BufferEq(keys.at(2), key2));
+ EXPECT_FALSE(this->BufferEq(keys.at(0), keys.at(1)));
+}
+
+TYPED_TEST(DatabaseTest, TestKeysNoCommitLevelDB) {
+ string name = this->DBName();
+ shared_ptr<Database> database = DatabaseFactory("leveldb");
+ EXPECT_TRUE(database->open(name, Database::New));
+
+ Database::key_type key1 = this->TestKey();
+ Database::value_type value1 = this->TestValue();
+
+ EXPECT_TRUE(database->put(key1, value1));
+
+ Database::key_type key2 = this->TestAltKey();
+ Database::value_type value2 = this->TestAltValue();
+
+ EXPECT_TRUE(database->put(key2, value2));
+
+ vector<Database::key_type> keys;
+ database->keys(&keys);
+
+ EXPECT_EQ(0, keys.size());
+}
+
TYPED_TEST(DatabaseTest, TestIteratorsLevelDB) {
string name = this->DBName();
shared_ptr<Database> database = DatabaseFactory("leveldb");
database->close();
}
+TYPED_TEST(DatabaseTest, TestKeysLMDB) {
+ string name = this->DBName();
+ shared_ptr<Database> database = DatabaseFactory("lmdb");
+ EXPECT_TRUE(database->open(name, Database::New));
+
+ Database::key_type key1 = this->TestKey();
+ Database::value_type value1 = this->TestValue();
+
+ EXPECT_TRUE(database->put(key1, value1));
+
+ Database::key_type key2 = this->TestAltKey();
+ Database::value_type value2 = this->TestAltValue();
+
+ EXPECT_TRUE(database->put(key2, value2));
+
+ EXPECT_TRUE(database->commit());
+
+ vector<Database::key_type> keys;
+ database->keys(&keys);
+
+ EXPECT_EQ(2, keys.size());
+
+ EXPECT_TRUE(this->BufferEq(keys.at(0), key1) ||
+ this->BufferEq(keys.at(0), key2));
+ EXPECT_TRUE(this->BufferEq(keys.at(1), key1) ||
+ this->BufferEq(keys.at(2), key2));
+ EXPECT_FALSE(this->BufferEq(keys.at(0), keys.at(1)));
+}
+
+TYPED_TEST(DatabaseTest, TestKeysNoCommitLMDB) {
+ string name = this->DBName();
+ shared_ptr<Database> database = DatabaseFactory("lmdb");
+ EXPECT_TRUE(database->open(name, Database::New));
+
+ Database::key_type key1 = this->TestKey();
+ Database::value_type value1 = this->TestValue();
+
+ EXPECT_TRUE(database->put(key1, value1));
+
+ Database::key_type key2 = this->TestAltKey();
+ Database::value_type value2 = this->TestAltValue();
+
+ EXPECT_TRUE(database->put(key2, value2));
+
+ vector<Database::key_type> keys;
+ database->keys(&keys);
+
+ EXPECT_EQ(0, keys.size());
+}
+
+
TYPED_TEST(DatabaseTest, TestIteratorsLMDB) {
string name = this->DBName();
shared_ptr<Database> database = DatabaseFactory("lmdb");