+++ /dev/null
-#ifndef CAFFE_DATASET_H_
-#define CAFFE_DATASET_H_
-
-#include <google/protobuf/message.h>
-
-#include <algorithm>
-#include <iterator>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/common.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-namespace caffe {
-
-namespace dataset_internal {
-
-using google::protobuf::Message;
-
-template<bool condition>
-struct static_assertion {};
-template<>
-struct static_assertion<true> {
- enum {
- DEFAULT_CODER_NOT_AVAILABLE
- };
-};
-
-template <typename T>
-struct DefaultCoder {
- using static_assertion<sizeof(T) == 0>::DEFAULT_CODER_NOT_AVAILABLE;
- static bool serialize(const T& obj, string* serialized);
- static bool serialize(const T& obj, vector<char>* serialized);
- static bool deserialize(const string& serialized, T* obj);
- static bool deserialize(const char* data, size_t size, T* obj);
-};
-
-template <>
-struct DefaultCoder<Message> {
- static bool serialize(const Message& obj, string* serialized) {
- return obj.SerializeToString(serialized);
- }
-
- static bool serialize(const Message& obj, vector<char>* serialized) {
- serialized->resize(obj.ByteSize());
- return obj.SerializeWithCachedSizesToArray(
- reinterpret_cast<unsigned char*>(serialized->data()));
- }
-
- static bool deserialize(const string& serialized, Message* obj) {
- return obj->ParseFromString(serialized);
- }
-
- static bool deserialize(const char* data, size_t size, Message* obj) {
- return obj->ParseFromArray(data, size);
- }
-};
-
-template <>
-struct DefaultCoder<caffe::Datum> : public DefaultCoder<Message> { };
-
-template <>
-struct DefaultCoder<string> {
- static bool serialize(string obj, string* serialized) {
- *serialized = obj;
- return true;
- }
-
- static bool serialize(const string& obj, vector<char>* serialized) {
- vector<char> temp(obj.data(), obj.data() + obj.size());
- serialized->swap(temp);
- return true;
- }
-
- static bool deserialize(const string& serialized, string* obj) {
- *obj = serialized;
- return true;
- }
-
- static bool deserialize(const char* data, size_t size, string* obj) {
- string temp_string(data, size);
- obj->swap(temp_string);
- return true;
- }
-};
-
-template <>
-struct DefaultCoder<vector<char> > {
- static bool serialize(vector<char> obj, string* serialized) {
- string tmp(obj.data(), obj.size());
- serialized->swap(tmp);
- return true;
- }
-
- static bool serialize(const vector<char>& obj, vector<char>* serialized) {
- *serialized = obj;
- return true;
- }
-
- static bool deserialize(const string& serialized, vector<char>* obj) {
- vector<char> tmp(serialized.data(), serialized.data() + serialized.size());
- obj->swap(tmp);
- return true;
- }
-
- static bool deserialize(const char* data, size_t size, vector<char>* obj) {
- vector<char> tmp(data, data + size);
- obj->swap(tmp);
- return true;
- }
-};
-
-} // namespace dataset_internal
-
-template <typename K, typename V,
- typename KCoder = dataset_internal::DefaultCoder<K>,
- typename VCoder = dataset_internal::DefaultCoder<V> >
-class Dataset {
- public:
- enum Mode {
- New,
- ReadWrite,
- ReadOnly
- };
-
- typedef K key_type;
- typedef V value_type;
-
- struct KV {
- K key;
- V value;
- };
-
- virtual bool open(const string& filename, Mode mode) = 0;
- virtual bool put(const K& key, const V& value) = 0;
- virtual bool get(const K& key, V* value) = 0;
- virtual bool first_key(K* key) = 0;
- virtual bool last_key(K* key) = 0;
- virtual bool commit() = 0;
- virtual void close() = 0;
-
- virtual void keys(vector<K>* keys) = 0;
-
- Dataset() { }
- virtual ~Dataset() { }
-
- class iterator;
- typedef iterator const_iterator;
-
- virtual const_iterator begin() const = 0;
- virtual const_iterator cbegin() const = 0;
- virtual const_iterator end() const = 0;
- virtual const_iterator cend() const = 0;
-
- protected:
- class DatasetState;
-
- public:
- class iterator : public std::iterator<std::forward_iterator_tag, KV> {
- public:
- typedef KV T;
- typedef T value_type;
- typedef T& reference_type;
- typedef T* pointer_type;
-
- iterator()
- : parent_(NULL) { }
-
- iterator(const Dataset* parent, shared_ptr<DatasetState> state)
- : parent_(parent),
- state_(state) { }
-
- iterator(const iterator& other)
- : parent_(other.parent_),
- state_(other.state_ ? other.state_->clone()
- : shared_ptr<DatasetState>()) { }
-
- iterator& operator=(iterator copy) {
- copy.swap(*this);
- return *this;
- }
-
- void swap(iterator& other) throw() {
- std::swap(this->parent_, other.parent_);
- std::swap(this->state_, other.state_);
- }
-
- bool operator==(const iterator& other) const {
- return parent_->equal(state_, other.state_);
- }
-
- bool operator!=(const iterator& other) const {
- return !(*this == other);
- }
-
- iterator& operator++() {
- parent_->increment(&state_);
- return *this;
- }
- iterator operator++(int) {
- iterator copy(*this);
- parent_->increment(&state_);
- return copy;
- }
-
- reference_type operator*() const {
- return parent_->dereference(state_);
- }
-
- pointer_type operator->() const {
- return &parent_->dereference(state_);
- }
-
- protected:
- const Dataset* parent_;
- shared_ptr<DatasetState> state_;
- };
-
- protected:
- class DatasetState {
- public:
- virtual ~DatasetState() { }
- virtual shared_ptr<DatasetState> clone() = 0;
- };
-
- virtual bool equal(shared_ptr<DatasetState> state1,
- shared_ptr<DatasetState> state2) const = 0;
- virtual void increment(shared_ptr<DatasetState>* state) const = 0;
- virtual KV& dereference(
- shared_ptr<DatasetState> state) const = 0;
-};
-
-} // namespace caffe
-
-#define INSTANTIATE_DATASET(type) \
- template class type<string, string>; \
- template class type<string, vector<char> >; \
- template class type<string, caffe::Datum>;
-
-#endif // CAFFE_DATASET_H_
+++ /dev/null
-#ifndef CAFFE_DATASET_FACTORY_H_
-#define CAFFE_DATASET_FACTORY_H_
-
-#include <string>
-
-#include "caffe/common.hpp"
-#include "caffe/dataset.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-namespace caffe {
-
-template <typename K, typename V>
-shared_ptr<Dataset<K, V> > DatasetFactory(const DataParameter_DB& type);
-
-template <typename K, typename V>
-shared_ptr<Dataset<K, V> > DatasetFactory(const string& type);
-
-} // namespace caffe
-
-#endif // CAFFE_DATASET_FACTORY_H_
+++ /dev/null
-#ifndef CAFFE_LEVELDB_DATASET_H_
-#define CAFFE_LEVELDB_DATASET_H_
-
-#include <leveldb/db.h>
-#include <leveldb/write_batch.h>
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/common.hpp"
-#include "caffe/dataset.hpp"
-
-namespace caffe {
-
-template <typename K, typename V,
- typename KCoder = dataset_internal::DefaultCoder<K>,
- typename VCoder = dataset_internal::DefaultCoder<V> >
-class LeveldbDataset : public Dataset<K, V, KCoder, VCoder> {
- public:
- typedef Dataset<K, V, KCoder, VCoder> Base;
- typedef typename Base::key_type key_type;
- typedef typename Base::value_type value_type;
- typedef typename Base::DatasetState DatasetState;
- typedef typename Base::Mode Mode;
- typedef typename Base::const_iterator const_iterator;
- typedef typename Base::KV KV;
-
- bool open(const string& filename, Mode mode);
- bool put(const K& key, const V& value);
- bool get(const K& key, V* value);
- bool first_key(K* key);
- bool last_key(K* key);
- bool commit();
- void close();
-
- void keys(vector<K>* keys);
-
- const_iterator begin() const;
- const_iterator cbegin() const;
- const_iterator end() const;
- const_iterator cend() const;
-
- protected:
- class LeveldbState : public DatasetState {
- public:
- explicit LeveldbState(shared_ptr<leveldb::DB> db,
- shared_ptr<leveldb::Iterator> iter)
- : DatasetState(),
- db_(db),
- iter_(iter) { }
-
- ~LeveldbState() {
- // This order is very important.
- // Iterators must be destroyed before their associated DB
- // is destroyed.
- iter_.reset();
- db_.reset();
- }
-
- shared_ptr<DatasetState> clone() {
- shared_ptr<leveldb::Iterator> new_iter;
-
- CHECK(iter_.get());
- new_iter.reset(db_->NewIterator(leveldb::ReadOptions()));
- CHECK(iter_->Valid());
- new_iter->Seek(iter_->key());
- CHECK(new_iter->Valid());
-
- return shared_ptr<DatasetState>(new LeveldbState(db_, new_iter));
- }
-
- shared_ptr<leveldb::DB> db_;
- shared_ptr<leveldb::Iterator> iter_;
- KV kv_pair_;
- };
-
- bool equal(shared_ptr<DatasetState> state1,
- shared_ptr<DatasetState> state2) const;
- void increment(shared_ptr<DatasetState>* state) const;
- KV& dereference(shared_ptr<DatasetState> state) const;
-
- shared_ptr<leveldb::DB> db_;
- shared_ptr<leveldb::WriteBatch> batch_;
- bool read_only_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_LEVELDB_DATASET_H_
+++ /dev/null
-#ifndef CAFFE_LMDB_DATASET_H_
-#define CAFFE_LMDB_DATASET_H_
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "lmdb.h"
-
-#include "caffe/common.hpp"
-#include "caffe/dataset.hpp"
-
-namespace caffe {
-
-template <typename K, typename V,
- typename KCoder = dataset_internal::DefaultCoder<K>,
- typename VCoder = dataset_internal::DefaultCoder<V> >
-class LmdbDataset : public Dataset<K, V, KCoder, VCoder> {
- public:
- typedef Dataset<K, V, KCoder, VCoder> Base;
- typedef typename Base::key_type key_type;
- typedef typename Base::value_type value_type;
- typedef typename Base::DatasetState DatasetState;
- typedef typename Base::Mode Mode;
- typedef typename Base::const_iterator const_iterator;
- typedef typename Base::KV KV;
-
- LmdbDataset()
- : env_(NULL),
- dbi_(0),
- write_txn_(NULL),
- read_txn_(NULL) { }
-
- bool open(const string& filename, Mode mode);
- bool put(const K& key, const V& value);
- bool get(const K& key, V* value);
- bool first_key(K* key);
- bool last_key(K* key);
- bool commit();
- void close();
-
- void keys(vector<K>* keys);
-
- const_iterator begin() const;
- const_iterator cbegin() const;
- const_iterator end() const;
- const_iterator cend() const;
-
- protected:
- class LmdbState : public DatasetState {
- public:
- explicit LmdbState(MDB_cursor* cursor, MDB_txn* txn, const MDB_dbi* dbi)
- : DatasetState(),
- cursor_(cursor),
- txn_(txn),
- dbi_(dbi) { }
-
- shared_ptr<DatasetState> clone() {
- CHECK(cursor_);
-
- MDB_cursor* new_cursor;
- int retval;
-
- retval = mdb_cursor_open(txn_, *dbi_, &new_cursor);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- MDB_val key;
- MDB_val val;
- retval = mdb_cursor_get(cursor_, &key, &val, MDB_GET_CURRENT);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- retval = mdb_cursor_get(new_cursor, &key, &val, MDB_SET);
- CHECK_EQ(MDB_SUCCESS, retval) << mdb_strerror(retval);
-
- return shared_ptr<DatasetState>(new LmdbState(new_cursor, txn_, dbi_));
- }
-
- MDB_cursor* cursor_;
- MDB_txn* txn_;
- const MDB_dbi* dbi_;
- KV kv_pair_;
- };
-
- bool equal(shared_ptr<DatasetState> state1,
- shared_ptr<DatasetState> state2) const;
- void increment(shared_ptr<DatasetState>* state) const;
- KV& dereference(shared_ptr<DatasetState> state) const;
-
- MDB_env* env_;
- MDB_dbi dbi_;
- MDB_txn* write_txn_;
- MDB_txn* read_txn_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_LMDB_DATASET_H_
+++ /dev/null
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/dataset_factory.hpp"
-#include "caffe/leveldb_dataset.hpp"
-#include "caffe/lmdb_dataset.hpp"
-
-namespace caffe {
-
-template <typename K, typename V>
-shared_ptr<Dataset<K, V> > DatasetFactory(const DataParameter_DB& type) {
- switch (type) {
- case DataParameter_DB_LEVELDB:
- return shared_ptr<Dataset<K, V> >(new LeveldbDataset<K, V>());
- case DataParameter_DB_LMDB:
- return shared_ptr<Dataset<K, V> >(new LmdbDataset<K, V>());
- default:
- LOG(FATAL) << "Unknown dataset type " << type;
- return shared_ptr<Dataset<K, V> >();
- }
-}
-
-template <typename K, typename V>
-shared_ptr<Dataset<K, V> > DatasetFactory(const string& type) {
- if ("leveldb" == type) {
- return DatasetFactory<K, V>(DataParameter_DB_LEVELDB);
- } else if ("lmdb" == type) {
- return DatasetFactory<K, V>(DataParameter_DB_LMDB);
- } else {
- LOG(FATAL) << "Unknown dataset type " << type;
- return shared_ptr<Dataset<K, V> >();
- }
-}
-
-#define REGISTER_DATASET(key_type, value_type) \
- template shared_ptr<Dataset<key_type, value_type> > \
- DatasetFactory(const string& type); \
- template shared_ptr<Dataset<key_type, value_type> > \
- DatasetFactory(const DataParameter_DB& type); \
-
-REGISTER_DATASET(string, string);
-REGISTER_DATASET(string, vector<char>);
-REGISTER_DATASET(string, Datum);
-
-#undef REGISTER_DATASET
-
-} // namespace caffe
-
-
+++ /dev/null
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/caffe.hpp"
-#include "caffe/leveldb_dataset.hpp"
-
-namespace caffe {
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::open(const string& filename,
- Mode mode) {
- DLOG(INFO) << "LevelDB: Open " << filename;
-
- leveldb::Options options;
- switch (mode) {
- case Base::New:
- DLOG(INFO) << " mode NEW";
- options.error_if_exists = true;
- options.create_if_missing = true;
- read_only_ = false;
- break;
- case Base::ReadWrite:
- DLOG(INFO) << " mode RW";
- options.error_if_exists = false;
- options.create_if_missing = true;
- read_only_ = false;
- break;
- case Base::ReadOnly:
- DLOG(INFO) << " mode RO";
- options.error_if_exists = false;
- options.create_if_missing = false;
- read_only_ = true;
- break;
- default:
- DLOG(FATAL) << "unknown mode " << mode;
- }
- options.write_buffer_size = 268435456;
- options.max_open_files = 100;
-
- leveldb::DB* db;
-
- LOG(INFO) << "Opening leveldb " << filename;
- leveldb::Status status = leveldb::DB::Open(
- options, filename, &db);
- db_.reset(db);
-
- if (!status.ok()) {
- LOG(ERROR) << "Failed to open leveldb " << filename
- << ". Is it already existing?";
- return false;
- }
-
- batch_.reset(new leveldb::WriteBatch());
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::put(const K& key, const V& value) {
- DLOG(INFO) << "LevelDB: Put";
-
- if (read_only_) {
- LOG(ERROR) << "put can not be used on a dataset in ReadOnly mode";
- return false;
- }
-
- CHECK_NOTNULL(batch_.get());
-
- string serialized_key;
- if (!KCoder::serialize(key, &serialized_key)) {
- return false;
- }
-
- string serialized_value;
- if (!VCoder::serialize(value, &serialized_value)) {
- return false;
- }
-
- batch_->Put(serialized_key, serialized_value);
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::get(const K& key, V* value) {
- DLOG(INFO) << "LevelDB: Get";
-
- string serialized_key;
- if (!KCoder::serialize(key, &serialized_key)) {
- return false;
- }
-
- string serialized_value;
- leveldb::Status status =
- db_->Get(leveldb::ReadOptions(), serialized_key, &serialized_value);
-
- if (!status.ok()) {
- LOG(ERROR) << "leveldb get failed";
- return false;
- }
-
- if (!VCoder::deserialize(serialized_value, value)) {
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::first_key(K* key) {
- DLOG(INFO) << "LevelDB: First key";
-
- CHECK_NOTNULL(db_.get());
- shared_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
- iter->SeekToFirst();
- CHECK(iter->Valid());
- const leveldb::Slice& key_slice = iter->key();
- return KCoder::deserialize(key_slice.data(), key_slice.size(), key);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::last_key(K* key) {
- DLOG(INFO) << "LevelDB: Last key";
-
- CHECK_NOTNULL(db_.get());
- shared_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
- iter->SeekToLast();
- CHECK(iter->Valid());
- const leveldb::Slice& key_slice = iter->key();
- return KCoder::deserialize(key_slice.data(), key_slice.size(), key);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::commit() {
- DLOG(INFO) << "LevelDB: Commit";
-
- if (read_only_) {
- LOG(ERROR) << "commit can not be used on a dataset in ReadOnly mode";
- return false;
- }
-
- CHECK_NOTNULL(db_.get());
- CHECK_NOTNULL(batch_.get());
-
- leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch_.get());
-
- batch_.reset(new leveldb::WriteBatch());
-
- return status.ok();
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LeveldbDataset<K, V, KCoder, VCoder>::close() {
- DLOG(INFO) << "LevelDB: Close";
-
- batch_.reset();
- db_.reset();
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LeveldbDataset<K, V, KCoder, VCoder>::keys(vector<K>* keys) {
- DLOG(INFO) << "LevelDB: Keys";
-
- keys->clear();
- for (const_iterator iter = begin(); iter != end(); ++iter) {
- keys->push_back(iter->key);
- }
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LeveldbDataset<K, V, KCoder, VCoder>::const_iterator
- LeveldbDataset<K, V, KCoder, VCoder>::begin() const {
- CHECK_NOTNULL(db_.get());
- shared_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
- iter->SeekToFirst();
- if (!iter->Valid()) {
- iter.reset();
- }
-
- shared_ptr<DatasetState> state;
- if (iter) {
- state.reset(new LeveldbState(db_, iter));
- }
- return const_iterator(this, state);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LeveldbDataset<K, V, KCoder, VCoder>::const_iterator
- LeveldbDataset<K, V, KCoder, VCoder>::end() const {
- shared_ptr<DatasetState> state;
- return const_iterator(this, state);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LeveldbDataset<K, V, KCoder, VCoder>::const_iterator
- LeveldbDataset<K, V, KCoder, VCoder>::cbegin() const {
- return begin();
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LeveldbDataset<K, V, KCoder, VCoder>::const_iterator
- LeveldbDataset<K, V, KCoder, VCoder>::cend() const { return end(); }
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LeveldbDataset<K, V, KCoder, VCoder>::equal(
- shared_ptr<DatasetState> state1, shared_ptr<DatasetState> state2) const {
- shared_ptr<LeveldbState> leveldb_state1 =
- boost::dynamic_pointer_cast<LeveldbState>(state1);
-
- shared_ptr<LeveldbState> leveldb_state2 =
- boost::dynamic_pointer_cast<LeveldbState>(state2);
-
- // The KV store doesn't really have any sort of ordering,
- // so while we can do a sequential scan over the collection,
- // we can't really use subranges.
- return !leveldb_state1 && !leveldb_state2;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LeveldbDataset<K, V, KCoder, VCoder>::increment(
- shared_ptr<DatasetState>* state) const {
- shared_ptr<LeveldbState> leveldb_state =
- boost::dynamic_pointer_cast<LeveldbState>(*state);
-
- CHECK_NOTNULL(leveldb_state.get());
-
- shared_ptr<leveldb::Iterator>& iter = leveldb_state->iter_;
-
- CHECK_NOTNULL(iter.get());
- CHECK(iter->Valid());
-
- iter->Next();
- if (!iter->Valid()) {
- state->reset();
- }
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename Dataset<K, V, KCoder, VCoder>::KV&
- LeveldbDataset<K, V, KCoder, VCoder>::dereference(
- shared_ptr<DatasetState> state) const {
- shared_ptr<LeveldbState> leveldb_state =
- boost::dynamic_pointer_cast<LeveldbState>(state);
-
- CHECK_NOTNULL(leveldb_state.get());
-
- shared_ptr<leveldb::Iterator>& iter = leveldb_state->iter_;
-
- CHECK_NOTNULL(iter.get());
-
- CHECK(iter->Valid());
-
- const leveldb::Slice& key = iter->key();
- const leveldb::Slice& value = iter->value();
- CHECK(KCoder::deserialize(key.data(), key.size(),
- &leveldb_state->kv_pair_.key));
- CHECK(VCoder::deserialize(value.data(), value.size(),
- &leveldb_state->kv_pair_.value));
-
- return leveldb_state->kv_pair_;
-}
-
-INSTANTIATE_DATASET(LeveldbDataset);
-
-} // namespace caffe
+++ /dev/null
-#include <sys/stat.h>
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/caffe.hpp"
-#include "caffe/lmdb_dataset.hpp"
-
-namespace caffe {
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::open(const string& filename,
- Mode mode) {
- DLOG(INFO) << "LMDB: Open " << filename;
-
- CHECK(NULL == env_);
- CHECK(NULL == write_txn_);
- CHECK(NULL == read_txn_);
- CHECK_EQ(0, dbi_);
-
- int retval;
- if (mode != Base::ReadOnly) {
- retval = mkdir(filename.c_str(), 0744);
- switch (mode) {
- case Base::New:
- if (0 != retval) {
- LOG(ERROR) << "mkdir " << filename << " failed";
- return false;
- }
- break;
- case Base::ReadWrite:
- if (-1 == retval && EEXIST != errno) {
- LOG(ERROR) << "mkdir " << filename << " failed ("
- << strerror(errno) << ")";
- return false;
- }
- break;
- default:
- LOG(FATAL) << "Invalid mode " << mode;
- }
- }
-
- retval = mdb_env_create(&env_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_env_create failed "
- << mdb_strerror(retval);
- return false;
- }
-
- retval = mdb_env_set_mapsize(env_, 1099511627776);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_env_set_mapsize failed " << mdb_strerror(retval);
- return false;
- }
-
- int flag1 = 0;
- int flag2 = 0;
- if (mode == Base::ReadOnly) {
- flag1 = MDB_RDONLY | MDB_NOTLS;
- flag2 = MDB_RDONLY;
- }
-
- retval = mdb_env_open(env_, filename.c_str(), flag1, 0664);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_env_open failed " << mdb_strerror(retval);
- return false;
- }
-
- retval = mdb_txn_begin(env_, NULL, MDB_RDONLY, &read_txn_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_txn_begin failed " << mdb_strerror(retval);
- return false;
- }
-
- retval = mdb_txn_begin(env_, NULL, flag2, &write_txn_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_txn_begin failed " << mdb_strerror(retval);
- return false;
- }
-
- retval = mdb_open(write_txn_, NULL, 0, &dbi_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_open failed" << mdb_strerror(retval);
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::put(const K& key, const V& value) {
- DLOG(INFO) << "LMDB: Put";
-
- vector<char> serialized_key;
- if (!KCoder::serialize(key, &serialized_key)) {
- LOG(ERROR) << "failed to serialize key";
- return false;
- }
-
- vector<char> serialized_value;
- if (!VCoder::serialize(value, &serialized_value)) {
- LOG(ERROR) << "failed to serialized value";
- return false;
- }
-
- MDB_val mdbkey, mdbdata;
- mdbdata.mv_size = serialized_value.size();
- mdbdata.mv_data = serialized_value.data();
- mdbkey.mv_size = serialized_key.size();
- mdbkey.mv_data = serialized_key.data();
-
- CHECK_NOTNULL(write_txn_);
- CHECK_NE(0, dbi_);
-
- int retval = mdb_put(write_txn_, dbi_, &mdbkey, &mdbdata, 0);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_put failed " << mdb_strerror(retval);
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::get(const K& key, V* value) {
- DLOG(INFO) << "LMDB: Get";
-
- vector<char> serialized_key;
- if (!KCoder::serialize(key, &serialized_key)) {
- LOG(ERROR) << "failed to serialized key";
- return false;
- }
-
- MDB_val mdbkey, mdbdata;
- mdbkey.mv_data = serialized_key.data();
- mdbkey.mv_size = serialized_key.size();
-
- int retval;
- retval = mdb_get(read_txn_, dbi_, &mdbkey, &mdbdata);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_get failed " << mdb_strerror(retval);
- return false;
- }
-
- if (!VCoder::deserialize(reinterpret_cast<char*>(mdbdata.mv_data),
- mdbdata.mv_size, value)) {
- LOG(ERROR) << "failed to deserialize value";
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::first_key(K* key) {
- DLOG(INFO) << "LMDB: First key";
-
- int retval;
-
- MDB_cursor* cursor;
- retval = mdb_cursor_open(read_txn_, dbi_, &cursor);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- MDB_val mdbkey;
- MDB_val mdbval;
- retval = mdb_cursor_get(cursor, &mdbkey, &mdbval, MDB_FIRST);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
-
- mdb_cursor_close(cursor);
-
- if (!KCoder::deserialize(reinterpret_cast<char*>(mdbkey.mv_data),
- mdbkey.mv_size, key)) {
- LOG(ERROR) << "failed to deserialize key";
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::last_key(K* key) {
- DLOG(INFO) << "LMDB: Last key";
-
- int retval;
-
- MDB_cursor* cursor;
- retval = mdb_cursor_open(read_txn_, dbi_, &cursor);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- MDB_val mdbkey;
- MDB_val mdbval;
- retval = mdb_cursor_get(cursor, &mdbkey, &mdbval, MDB_LAST);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
-
- mdb_cursor_close(cursor);
-
- if (!KCoder::deserialize(reinterpret_cast<char*>(mdbkey.mv_data),
- mdbkey.mv_size, key)) {
- LOG(ERROR) << "failed to deserialize key";
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::commit() {
- DLOG(INFO) << "LMDB: Commit";
-
- CHECK_NOTNULL(write_txn_);
-
- int retval;
- retval = mdb_txn_commit(write_txn_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_txn_commit failed " << mdb_strerror(retval);
- return false;
- }
-
- mdb_txn_abort(read_txn_);
-
- retval = mdb_txn_begin(env_, NULL, 0, &write_txn_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_txn_begin failed " << mdb_strerror(retval);
- return false;
- }
-
- retval = mdb_txn_begin(env_, NULL, MDB_RDONLY, &read_txn_);
- if (MDB_SUCCESS != retval) {
- LOG(ERROR) << "mdb_txn_begin failed " << mdb_strerror(retval);
- return false;
- }
-
- return true;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LmdbDataset<K, V, KCoder, VCoder>::close() {
- DLOG(INFO) << "LMDB: Close";
-
- if (env_ && dbi_) {
- mdb_txn_abort(write_txn_);
- mdb_txn_abort(read_txn_);
- mdb_close(env_, dbi_);
- mdb_env_close(env_);
- env_ = NULL;
- dbi_ = 0;
- write_txn_ = NULL;
- read_txn_ = NULL;
- }
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LmdbDataset<K, V, KCoder, VCoder>::keys(vector<K>* keys) {
- DLOG(INFO) << "LMDB: Keys";
-
- keys->clear();
- for (const_iterator iter = begin(); iter != end(); ++iter) {
- keys->push_back(iter->key);
- }
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LmdbDataset<K, V, KCoder, VCoder>::const_iterator
- LmdbDataset<K, V, KCoder, VCoder>::begin() const {
- int retval;
-
- MDB_cursor* cursor;
- retval = mdb_cursor_open(read_txn_, dbi_, &cursor);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
- MDB_val key;
- MDB_val val;
- retval = mdb_cursor_get(cursor, &key, &val, MDB_FIRST);
-
- CHECK(MDB_SUCCESS == retval || MDB_NOTFOUND == retval)
- << mdb_strerror(retval);
-
- shared_ptr<DatasetState> state;
- if (MDB_SUCCESS == retval) {
- state.reset(new LmdbState(cursor, read_txn_, &dbi_));
- } else {
- mdb_cursor_close(cursor);
- }
- return const_iterator(this, state);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LmdbDataset<K, V, KCoder, VCoder>::const_iterator
- LmdbDataset<K, V, KCoder, VCoder>::end() const {
- shared_ptr<DatasetState> state;
- return const_iterator(this, state);
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LmdbDataset<K, V, KCoder, VCoder>::const_iterator
- LmdbDataset<K, V, KCoder, VCoder>::cbegin() const { return begin(); }
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename LmdbDataset<K, V, KCoder, VCoder>::const_iterator
- LmdbDataset<K, V, KCoder, VCoder>::cend() const { return end(); }
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-bool LmdbDataset<K, V, KCoder, VCoder>::equal(shared_ptr<DatasetState> state1,
- shared_ptr<DatasetState> state2) const {
- shared_ptr<LmdbState> lmdb_state1 =
- boost::dynamic_pointer_cast<LmdbState>(state1);
-
- shared_ptr<LmdbState> lmdb_state2 =
- boost::dynamic_pointer_cast<LmdbState>(state2);
-
- // The KV store doesn't really have any sort of ordering,
- // so while we can do a sequential scan over the collection,
- // we can't really use subranges.
- return !lmdb_state1 && !lmdb_state2;
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-void LmdbDataset<K, V, KCoder, VCoder>::increment(
- shared_ptr<DatasetState>* state) const {
- shared_ptr<LmdbState> lmdb_state =
- boost::dynamic_pointer_cast<LmdbState>(*state);
-
- CHECK_NOTNULL(lmdb_state.get());
-
- MDB_cursor*& cursor = lmdb_state->cursor_;
-
- CHECK_NOTNULL(cursor);
-
- MDB_val key;
- MDB_val val;
- int retval = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
- if (MDB_NOTFOUND == retval) {
- mdb_cursor_close(cursor);
- state->reset();
- } else {
- CHECK_EQ(MDB_SUCCESS, retval) << mdb_strerror(retval);
- }
-}
-
-template <typename K, typename V, typename KCoder, typename VCoder>
-typename Dataset<K, V, KCoder, VCoder>::KV&
- LmdbDataset<K, V, KCoder, VCoder>::dereference(
- shared_ptr<DatasetState> state) const {
- shared_ptr<LmdbState> lmdb_state =
- boost::dynamic_pointer_cast<LmdbState>(state);
-
- CHECK_NOTNULL(lmdb_state.get());
-
- MDB_cursor*& cursor = lmdb_state->cursor_;
-
- CHECK_NOTNULL(cursor);
-
- MDB_val mdb_key;
- MDB_val mdb_val;
- int retval = mdb_cursor_get(cursor, &mdb_key, &mdb_val, MDB_GET_CURRENT);
- CHECK_EQ(retval, MDB_SUCCESS) << mdb_strerror(retval);
-
- CHECK(KCoder::deserialize(reinterpret_cast<char*>(mdb_key.mv_data),
- mdb_key.mv_size, &lmdb_state->kv_pair_.key));
- CHECK(VCoder::deserialize(reinterpret_cast<char*>(mdb_val.mv_data),
- mdb_val.mv_size, &lmdb_state->kv_pair_.value));
-
- return lmdb_state->kv_pair_;
-}
-
-INSTANTIATE_DATASET(LmdbDataset);
-
-} // namespace caffe
+++ /dev/null
-#include <string>
-#include <vector>
-
-#include "caffe/util/io.hpp"
-
-#include "gtest/gtest.h"
-
-#include "caffe/dataset_factory.hpp"
-
-#include "caffe/test/test_caffe_main.hpp"
-
-namespace caffe {
-
-namespace DatasetTest_internal {
-
-template <typename T>
-struct TestData {
- static T TestValue();
- static T TestAltValue();
- static bool equals(const T& a, const T& b);
-};
-
-template <>
-string TestData<string>::TestValue() {
- return "world";
-}
-
-template <>
-string TestData<string>::TestAltValue() {
- return "bar";
-}
-
-template <>
-bool TestData<string>::equals(const string& a, const string& b) {
- return a == b;
-}
-
-template <>
-vector<char> TestData<vector<char> >::TestValue() {
- string str = "world";
- vector<char> val(str.data(), str.data() + str.size());
- return val;
-}
-
-template <>
-vector<char> TestData<vector<char> >::TestAltValue() {
- string str = "bar";
- vector<char> val(str.data(), str.data() + str.size());
- return val;
-}
-
-template <>
-bool TestData<vector<char> >::equals(const vector<char>& a,
- const vector<char>& b) {
- if (a.size() != b.size()) {
- return false;
- }
- for (size_t i = 0; i < a.size(); ++i) {
- if (a.at(i) != b.at(i)) {
- return false;
- }
- }
-
- return true;
-}
-
-template <>
-Datum TestData<Datum>::TestValue() {
- Datum datum;
- datum.set_channels(3);
- datum.set_height(32);
- datum.set_width(32);
- datum.set_data(string(32 * 32 * 3 * 4, ' '));
- datum.set_label(0);
- return datum;
-}
-
-template <>
-Datum TestData<Datum>::TestAltValue() {
- Datum datum;
- datum.set_channels(1);
- datum.set_height(64);
- datum.set_width(64);
- datum.set_data(string(64 * 64 * 1 * 4, ' '));
- datum.set_label(1);
- return datum;
-}
-
-template <>
-bool TestData<Datum>::equals(const Datum& a, const Datum& b) {
- string serialized_a;
- a.SerializeToString(&serialized_a);
-
- string serialized_b;
- b.SerializeToString(&serialized_b);
-
- return serialized_a == serialized_b;
-}
-
-} // namespace DatasetTest_internal
-
-#define UNPACK_TYPES \
- typedef typename TypeParam::value_type value_type; \
- const DataParameter_DB backend = TypeParam::backend;
-
-template <typename TypeParam>
-class DatasetTest : public ::testing::Test {
- protected:
- typedef typename TypeParam::value_type value_type;
-
- string DBName() {
- string filename;
- MakeTempDir(&filename);
- filename += "/db";
- return filename;
- }
-
- string TestKey() {
- return "hello";
- }
-
- value_type TestValue() {
- return DatasetTest_internal::TestData<value_type>::TestValue();
- }
-
- string TestAltKey() {
- return "foo";
- }
-
- value_type TestAltValue() {
- return DatasetTest_internal::TestData<value_type>::TestAltValue();
- }
-
- template <typename T>
- bool equals(const T& a, const T& b) {
- return DatasetTest_internal::TestData<T>::equals(a, b);
- }
-};
-
-struct StringLeveldb {
- typedef string value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB StringLeveldb::backend = DataParameter_DB_LEVELDB;
-
-struct StringLmdb {
- typedef string value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB StringLmdb::backend = DataParameter_DB_LMDB;
-
-struct VectorLeveldb {
- typedef vector<char> value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB VectorLeveldb::backend = DataParameter_DB_LEVELDB;
-
-struct VectorLmdb {
- typedef vector<char> value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB VectorLmdb::backend = DataParameter_DB_LMDB;
-
-struct DatumLeveldb {
- typedef Datum value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB DatumLeveldb::backend = DataParameter_DB_LEVELDB;
-
-struct DatumLmdb {
- typedef Datum value_type;
- static const DataParameter_DB backend;
-};
-const DataParameter_DB DatumLmdb::backend = DataParameter_DB_LMDB;
-
-typedef ::testing::Types<StringLeveldb, StringLmdb, VectorLeveldb, VectorLmdb,
- DatumLeveldb, DatumLmdb> TestTypes;
-
-TYPED_TEST_CASE(DatasetTest, TestTypes);
-
-TYPED_TEST(DatasetTest, TestNewDoesntExistPasses) {
- UNPACK_TYPES;
-
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(this->DBName(),
- Dataset<string, value_type>::New));
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestNewExistsFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
- dataset->close();
-
- EXPECT_FALSE(dataset->open(name, Dataset<string, value_type>::New));
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyExistsPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyDoesntExistFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_FALSE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
-}
-
-TYPED_TEST(DatasetTest, TestReadWriteExistsPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadWrite));
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestReadWriteDoesntExistPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadWrite));
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestKeys) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key1 = this->TestKey();
- value_type value1 = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key1, value1));
-
- string key2 = this->TestAltKey();
- value_type value2 = this->TestAltValue();
-
- EXPECT_TRUE(dataset->put(key2, value2));
-
- EXPECT_TRUE(dataset->commit());
-
- vector<string> keys;
- dataset->keys(&keys);
-
- EXPECT_EQ(2, keys.size());
-
- EXPECT_TRUE(this->equals(keys.at(0), key1) ||
- this->equals(keys.at(0), key2));
- EXPECT_TRUE(this->equals(keys.at(1), key1) ||
- this->equals(keys.at(2), key2));
- EXPECT_FALSE(this->equals(keys.at(0), keys.at(1)));
-}
-
-TYPED_TEST(DatasetTest, TestFirstKey) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- value_type value = this->TestValue();
-
- string key1 = "01";
- EXPECT_TRUE(dataset->put(key1, value));
-
- string key2 = "02";
- EXPECT_TRUE(dataset->put(key2, value));
-
- string key3 = "03";
- EXPECT_TRUE(dataset->put(key3, value));
-
- EXPECT_TRUE(dataset->commit());
-
- string first_key;
- dataset->first_key(&first_key);
-
- EXPECT_TRUE(this->equals(first_key, key1));
-}
-
-TYPED_TEST(DatasetTest, TestLastKey) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- value_type value = this->TestValue();
-
- string key1 = "01";
- EXPECT_TRUE(dataset->put(key1, value));
-
- string key2 = "02";
- EXPECT_TRUE(dataset->put(key2, value));
-
- string key3 = "03";
- EXPECT_TRUE(dataset->put(key3, value));
-
- EXPECT_TRUE(dataset->commit());
-
- string last_key;
- dataset->last_key(&last_key);
-
- EXPECT_TRUE(this->equals(last_key, key3));
-}
-
-TYPED_TEST(DatasetTest, TestFirstLastKeys) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- value_type value = this->TestValue();
-
- string key1 = "01";
- EXPECT_TRUE(dataset->put(key1, value));
-
- string key2 = "02";
- EXPECT_TRUE(dataset->put(key2, value));
-
- string key3 = "03";
- EXPECT_TRUE(dataset->put(key3, value));
-
- EXPECT_TRUE(dataset->commit());
-
- string first_key;
- dataset->first_key(&first_key);
- string last_key;
- dataset->last_key(&last_key);
-
- EXPECT_TRUE(this->equals(first_key, key1));
- EXPECT_TRUE(this->equals(last_key, key3));
-}
-
-TYPED_TEST(DatasetTest, TestFirstLastKeysUnOrdered) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- value_type value = this->TestValue();
-
- string key3 = "03";
- EXPECT_TRUE(dataset->put(key3, value));
-
- string key1 = "01";
- EXPECT_TRUE(dataset->put(key1, value));
-
- string key2 = "02";
- EXPECT_TRUE(dataset->put(key2, value));
-
- EXPECT_TRUE(dataset->commit());
-
- string first_key;
- dataset->first_key(&first_key);
- string last_key;
- dataset->last_key(&last_key);
-
- EXPECT_TRUE(this->equals(first_key, key1));
- EXPECT_TRUE(this->equals(last_key, key3));
-}
-
-TYPED_TEST(DatasetTest, TestKeysNoCommit) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key1 = this->TestKey();
- value_type value1 = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key1, value1));
-
- string key2 = this->TestAltKey();
- value_type value2 = this->TestAltValue();
-
- EXPECT_TRUE(dataset->put(key2, value2));
-
- vector<string> keys;
- dataset->keys(&keys);
-
- EXPECT_EQ(0, keys.size());
-}
-
-TYPED_TEST(DatasetTest, TestIterators) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- const int kNumExamples = 4;
- for (int i = 0; i < kNumExamples; ++i) {
- stringstream ss;
- ss << i;
- string key = ss.str();
- ss << " here be data";
- value_type value = this->TestValue();
- EXPECT_TRUE(dataset->put(key, value));
- }
- EXPECT_TRUE(dataset->commit());
-
- int count = 0;
- typedef typename Dataset<string, value_type>::const_iterator Iter;
- for (Iter iter = dataset->begin(); iter != dataset->end(); ++iter) {
- (void)iter;
- ++count;
- }
-
- EXPECT_EQ(kNumExamples, count);
-}
-
-TYPED_TEST(DatasetTest, TestIteratorsPreIncrement) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key1 = this->TestAltKey();
- value_type value1 = this->TestAltValue();
-
- string key2 = this->TestKey();
- value_type value2 = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key1, value1));
- EXPECT_TRUE(dataset->put(key2, value2));
- EXPECT_TRUE(dataset->commit());
-
- typename Dataset<string, value_type>::const_iterator iter1 =
- dataset->begin();
-
- EXPECT_FALSE(dataset->end() == iter1);
-
- EXPECT_TRUE(this->equals(iter1->key, key1));
-
- typename Dataset<string, value_type>::const_iterator iter2 = ++iter1;
-
- EXPECT_FALSE(dataset->end() == iter1);
- EXPECT_FALSE(dataset->end() == iter2);
-
- EXPECT_TRUE(this->equals(iter2->key, key2));
-
- typename Dataset<string, value_type>::const_iterator iter3 = ++iter2;
-
- EXPECT_TRUE(dataset->end() == iter3);
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestIteratorsPostIncrement) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key1 = this->TestAltKey();
- value_type value1 = this->TestAltValue();
-
- string key2 = this->TestKey();
- value_type value2 = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key1, value1));
- EXPECT_TRUE(dataset->put(key2, value2));
- EXPECT_TRUE(dataset->commit());
-
- typename Dataset<string, value_type>::const_iterator iter1 =
- dataset->begin();
-
- EXPECT_FALSE(dataset->end() == iter1);
-
- EXPECT_TRUE(this->equals(iter1->key, key1));
-
- typename Dataset<string, value_type>::const_iterator iter2 = iter1++;
-
- EXPECT_FALSE(dataset->end() == iter1);
- EXPECT_FALSE(dataset->end() == iter2);
-
- EXPECT_TRUE(this->equals(iter2->key, key1));
- EXPECT_TRUE(this->equals(iter1->key, key2));
-
- typename Dataset<string, value_type>::const_iterator iter3 = iter1++;
-
- EXPECT_FALSE(dataset->end() == iter3);
- EXPECT_TRUE(this->equals(iter3->key, key2));
- EXPECT_TRUE(dataset->end() == iter1);
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestNewPutPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- EXPECT_TRUE(dataset->commit());
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestNewCommitPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- EXPECT_TRUE(dataset->commit());
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestNewGetPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- EXPECT_TRUE(dataset->commit());
-
- value_type new_value;
-
- EXPECT_TRUE(dataset->get(key, &new_value));
-
- EXPECT_TRUE(this->equals(value, new_value));
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestNewGetNoCommitFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- value_type new_value;
-
- EXPECT_FALSE(dataset->get(key, &new_value));
-}
-
-
-TYPED_TEST(DatasetTest, TestReadWritePutPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadWrite));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- EXPECT_TRUE(dataset->commit());
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestReadWriteCommitPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadWrite));
-
- EXPECT_TRUE(dataset->commit());
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestReadWriteGetPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- EXPECT_TRUE(dataset->commit());
-
- value_type new_value;
-
- EXPECT_TRUE(dataset->get(key, &new_value));
-
- EXPECT_TRUE(this->equals(value, new_value));
-
- dataset->close();
-}
-
-TYPED_TEST(DatasetTest, TestReadWriteGetNoCommitFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- value_type new_value;
-
- EXPECT_FALSE(dataset->get(key, &new_value));
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyPutFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_FALSE(dataset->put(key, value));
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyCommitFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
-
- EXPECT_FALSE(dataset->commit());
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyGetPasses) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- EXPECT_TRUE(dataset->commit());
-
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
-
- value_type new_value;
-
- EXPECT_TRUE(dataset->get(key, &new_value));
-
- EXPECT_TRUE(this->equals(value, new_value));
-}
-
-TYPED_TEST(DatasetTest, TestReadOnlyGetNoCommitFails) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
-
- EXPECT_TRUE(dataset->put(key, value));
-
- dataset->close();
-
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::ReadOnly));
-
- value_type new_value;
-
- EXPECT_FALSE(dataset->get(key, &new_value));
-}
-
-TYPED_TEST(DatasetTest, TestCreateManyItersShortScope) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
- EXPECT_TRUE(dataset->put(key, value));
- EXPECT_TRUE(dataset->commit());
-
- for (int i = 0; i < 1000; ++i) {
- typename Dataset<string, value_type>::const_iterator iter =
- dataset->begin();
- }
-}
-
-TYPED_TEST(DatasetTest, TestCreateManyItersLongScope) {
- UNPACK_TYPES;
-
- string name = this->DBName();
- shared_ptr<Dataset<string, value_type> > dataset =
- DatasetFactory<string, value_type>(backend);
- EXPECT_TRUE(dataset->open(name, Dataset<string, value_type>::New));
-
- string key = this->TestKey();
- value_type value = this->TestValue();
- EXPECT_TRUE(dataset->put(key, value));
- EXPECT_TRUE(dataset->commit());
-
- vector<typename Dataset<string, value_type>::const_iterator> iters;
- for (int i = 0; i < 1000; ++i) {
- iters.push_back(dataset->begin());
- }
-}
-
-#undef UNPACK_TYPES
-
-} // namespace caffe
projects / platform / upstream / caffeonacl.git / commitdiff