1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
9 #include "base/basictypes.h"
10 #include "base/files/file.h"
11 #include "base/logging.h"
12 #include "base/memory/scoped_ptr.h"
13 #include "base/metrics/histogram.h"
14 #include "base/strings/string16.h"
15 #include "base/strings/string_piece.h"
16 #include "base/strings/stringprintf.h"
17 #include "base/strings/utf_string_conversions.h"
18 #include "base/sys_info.h"
19 #include "content/browser/indexed_db/indexed_db_class_factory.h"
20 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
21 #include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h"
22 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
23 #include "third_party/leveldatabase/env_chromium.h"
24 #include "third_party/leveldatabase/env_idb.h"
25 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
28 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
30 using base::StringPiece;
34 // Forcing flushes to disk at the end of a transaction guarantees that the
35 // data hit disk, but drastically impacts throughput when the filesystem is
36 // busy with background compactions. Not syncing trades off reliability for
37 // performance. Note that background compactions which move data from the
38 // log to SSTs are always done with reliable writes.
40 // Sync writes are necessary on Windows for quota calculations; POSIX
41 // calculates file sizes correctly even when not synced to disk.
43 static const bool kSyncWrites = true;
45 // TODO(dgrogan): Either remove the #if block or change this back to false.
46 // See http://crbug.com/338385.
47 static const bool kSyncWrites = true;
50 static leveldb::Slice MakeSlice(const StringPiece& s) {
51 return leveldb::Slice(s.begin(), s.size());
54 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
55 return StringPiece(s.data(), s.size());
58 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
59 const LevelDBComparator* comparator)
60 : comparator_(comparator) {}
62 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
63 const leveldb::Slice& b) const {
64 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
67 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
68 return comparator_->Name();
71 // TODO(jsbell): Support the methods below in the future.
72 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
74 const leveldb::Slice& limit) const {}
76 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
77 std::string* key) const {}
79 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
80 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
82 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
84 LevelDBDatabase::LevelDBDatabase() {}
86 LevelDBDatabase::~LevelDBDatabase() {
87 // db_'s destructor uses comparator_adapter_; order of deletion is important.
89 comparator_adapter_.reset();
93 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
95 const base::FilePath& path,
97 leveldb::Options options;
98 options.comparator = comparator;
99 options.create_if_missing = true;
100 options.paranoid_checks = true;
101 options.compression = leveldb::kSnappyCompression;
103 // For info about the troubles we've run into with this parameter, see:
104 // https://code.google.com/p/chromium/issues/detail?id=227313#c11
105 options.max_open_files = 80;
108 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
109 return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
112 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
113 leveldb::Options options;
114 options.env = leveldb::IDBEnv();
115 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
116 return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
120 class LockImpl : public LevelDBLock {
122 explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
123 : env_(env), lock_(lock) {}
124 virtual ~LockImpl() { env_->UnlockFile(lock_); }
127 leveldb::FileLock* lock_;
129 DISALLOW_COPY_AND_ASSIGN(LockImpl);
133 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
134 const base::FilePath& file_name) {
135 leveldb::Env* env = leveldb::IDBEnv();
136 base::FilePath lock_path = file_name.AppendASCII("LOCK");
137 leveldb::FileLock* lock = NULL;
138 leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
140 return scoped_ptr<LevelDBLock>();
142 return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
145 static int CheckFreeSpace(const char* const type,
146 const base::FilePath& file_name) {
148 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
149 int64 free_disk_space_in_k_bytes =
150 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
151 if (free_disk_space_in_k_bytes < 0) {
152 base::Histogram::FactoryGet(
153 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
157 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
160 int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
162 : free_disk_space_in_k_bytes;
163 const uint64 histogram_max = static_cast<uint64>(1e9);
164 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
165 base::Histogram::FactoryGet(name,
169 base::HistogramBase::kUmaTargetedHistogramFlag)
170 ->Add(clamped_disk_space_k_bytes);
171 return clamped_disk_space_k_bytes;
174 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
175 const leveldb::Status& s) {
176 leveldb_env::MethodID method;
178 leveldb_env::ErrorParsingResult result =
179 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
180 if (result == leveldb_env::NONE)
182 std::string method_histogram_name(histogram_name);
183 method_histogram_name.append(".EnvMethod");
184 base::LinearHistogram::FactoryGet(
185 method_histogram_name,
187 leveldb_env::kNumEntries,
188 leveldb_env::kNumEntries + 1,
189 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
191 std::string error_histogram_name(histogram_name);
193 if (result == leveldb_env::METHOD_AND_PFE) {
195 error_histogram_name.append(std::string(".PFE.") +
196 leveldb_env::MethodIDToString(method));
197 base::LinearHistogram::FactoryGet(
198 error_histogram_name,
200 -base::File::FILE_ERROR_MAX,
201 -base::File::FILE_ERROR_MAX + 1,
202 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
203 } else if (result == leveldb_env::METHOD_AND_ERRNO) {
204 error_histogram_name.append(std::string(".Errno.") +
205 leveldb_env::MethodIDToString(method));
206 base::LinearHistogram::FactoryGet(
207 error_histogram_name,
211 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
215 static void ParseAndHistogramCorruptionDetails(
216 const std::string& histogram_name,
217 const leveldb::Status& status) {
218 int error = leveldb_env::GetCorruptionCode(status);
220 std::string corruption_histogram_name(histogram_name);
221 corruption_histogram_name.append(".Corruption");
222 const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
223 base::LinearHistogram::FactoryGet(
224 corruption_histogram_name,
228 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
231 static void HistogramLevelDBError(const std::string& histogram_name,
232 const leveldb::Status& s) {
244 int leveldb_error = LEVEL_DB_OTHER;
246 leveldb_error = LEVEL_DB_NOT_FOUND;
247 else if (s.IsCorruption())
248 leveldb_error = LEVEL_DB_CORRUPTION;
249 else if (s.IsIOError())
250 leveldb_error = LEVEL_DB_IO_ERROR;
251 base::Histogram::FactoryGet(histogram_name,
254 LEVEL_DB_MAX_ERROR + 1,
255 base::HistogramBase::kUmaTargetedHistogramFlag)
256 ->Add(leveldb_error);
258 ParseAndHistogramIOErrorDetails(histogram_name, s);
260 ParseAndHistogramCorruptionDetails(histogram_name, s);
263 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
264 const LevelDBComparator* comparator,
265 scoped_ptr<LevelDBDatabase>* result,
266 bool* is_disk_full) {
267 base::TimeTicks begin_time = base::TimeTicks::Now();
269 scoped_ptr<ComparatorAdapter> comparator_adapter(
270 new ComparatorAdapter(comparator));
273 const leveldb::Status s =
274 OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
277 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
278 int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
279 // Disks with <100k of free space almost never succeed in opening a
282 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
284 LOG(ERROR) << "Failed to open LevelDB database from "
285 << file_name.AsUTF8Unsafe() << "," << s.ToString();
289 UMA_HISTOGRAM_MEDIUM_TIMES("WebCore.IndexedDB.LevelDB.OpenTime",
290 base::TimeTicks::Now() - begin_time);
292 CheckFreeSpace("Success", file_name);
294 (*result).reset(new LevelDBDatabase);
295 (*result)->db_ = make_scoped_ptr(db);
296 (*result)->comparator_adapter_ = comparator_adapter.Pass();
297 (*result)->comparator_ = comparator;
302 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
303 const LevelDBComparator* comparator) {
304 scoped_ptr<ComparatorAdapter> comparator_adapter(
305 new ComparatorAdapter(comparator));
306 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
309 const leveldb::Status s = OpenDB(
310 comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
313 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
314 return scoped_ptr<LevelDBDatabase>();
317 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
318 result->env_ = in_memory_env.Pass();
319 result->db_ = make_scoped_ptr(db);
320 result->comparator_adapter_ = comparator_adapter.Pass();
321 result->comparator_ = comparator;
323 return result.Pass();
326 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
327 std::string* value) {
328 base::TimeTicks begin_time = base::TimeTicks::Now();
330 leveldb::WriteOptions write_options;
331 write_options.sync = kSyncWrites;
333 const leveldb::Status s =
334 db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
336 LOG(ERROR) << "LevelDB put failed: " << s.ToString();
338 UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.PutTime",
339 base::TimeTicks::Now() - begin_time);
343 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
344 leveldb::WriteOptions write_options;
345 write_options.sync = kSyncWrites;
347 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
349 LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
353 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
356 const LevelDBSnapshot* snapshot) {
358 leveldb::ReadOptions read_options;
359 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
360 // performance impact is too great.
361 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
363 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
369 return leveldb::Status::OK();
370 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
371 LOG(ERROR) << "LevelDB get failed: " << s.ToString();
375 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
376 base::TimeTicks begin_time = base::TimeTicks::Now();
377 leveldb::WriteOptions write_options;
378 write_options.sync = kSyncWrites;
380 const leveldb::Status s =
381 db_->Write(write_options, write_batch.write_batch_.get());
383 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
384 LOG(ERROR) << "LevelDB write failed: " << s.ToString();
386 UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.WriteTime",
387 base::TimeTicks::Now() - begin_time);
392 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
393 const LevelDBSnapshot* snapshot) {
394 leveldb::ReadOptions read_options;
395 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
396 // performance impact is too great.
397 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
399 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
400 return scoped_ptr<LevelDBIterator>(
401 IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass()));
404 const LevelDBComparator* LevelDBDatabase::Comparator() const {
408 void LevelDBDatabase::Compact(const base::StringPiece& start,
409 const base::StringPiece& stop) {
410 const leveldb::Slice start_slice = MakeSlice(start);
411 const leveldb::Slice stop_slice = MakeSlice(stop);
412 // NULL batch means just wait for earlier writes to be done
413 db_->Write(leveldb::WriteOptions(), NULL);
414 db_->CompactRange(&start_slice, &stop_slice);
417 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
419 } // namespace content