src/content/browser/indexed_db/leveldb/leveldb_database.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
   6
   7 #include <cerrno>
   8
   9 #include "base/basictypes.h"
  10 #include "base/files/file.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/strings/string16.h"
  15 #include "base/strings/string_piece.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "base/sys_info.h"
  19 #include "content/browser/indexed_db/indexed_db_class_factory.h"
  20 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
  21 #include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h"
  22 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
  23 #include "third_party/leveldatabase/env_chromium.h"
  24 #include "third_party/leveldatabase/env_idb.h"
  25 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
  26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
  28 #include "third_party/leveldatabase/src/include/leveldb/filter_policy.h"
  29 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  30
  31 using base::StringPiece;
  32
  33 namespace content {
  34
  35 // Forcing flushes to disk at the end of a transaction guarantees that the
  36 // data hit disk, but drastically impacts throughput when the filesystem is
  37 // busy with background compactions. Not syncing trades off reliability for
  38 // performance. Note that background compactions which move data from the
  39 // log to SSTs are always done with reliable writes.
  40 //
  41 // Sync writes are necessary on Windows for quota calculations; POSIX
  42 // calculates file sizes correctly even when not synced to disk.
  43 #if defined(OS_WIN)
  44 static const bool kSyncWrites = true;
  45 #else
  46 // TODO(dgrogan): Either remove the #if block or change this back to false.
  47 // See http://crbug.com/338385.
  48 static const bool kSyncWrites = true;
  49 #endif
  50
  51 static leveldb::Slice MakeSlice(const StringPiece& s) {
  52   return leveldb::Slice(s.begin(), s.size());
  53 }
  54
  55 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
  56   return StringPiece(s.data(), s.size());
  57 }
  58
  59 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
  60     const LevelDBComparator* comparator)
  61     : comparator_(comparator) {}
  62
  63 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
  64                                                 const leveldb::Slice& b) const {
  65   return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
  66 }
  67
  68 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
  69   return comparator_->Name();
  70 }
  71
  72 // TODO(jsbell): Support the methods below in the future.
  73 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
  74     std::string* start,
  75     const leveldb::Slice& limit) const {}
  76
  77 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
  78     std::string* key) const {}
  79
  80 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
  81     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
  82
  83 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
  84
  85 LevelDBDatabase::LevelDBDatabase() {}
  86
  87 LevelDBDatabase::~LevelDBDatabase() {
  88   // db_'s destructor uses comparator_adapter_; order of deletion is important.
  89   db_.reset();
  90   comparator_adapter_.reset();
  91   env_.reset();
  92 }
  93
  94 static leveldb::Status OpenDB(
  95     leveldb::Comparator* comparator,
  96     leveldb::Env* env,
  97     const base::FilePath& path,
  98     leveldb::DB** db,
  99     scoped_ptr<const leveldb::FilterPolicy>* filter_policy) {
 100   filter_policy->reset(leveldb::NewBloomFilterPolicy(10));
 101   leveldb::Options options;
 102   options.comparator = comparator;
 103   options.create_if_missing = true;
 104   options.paranoid_checks = true;
 105   options.filter_policy = filter_policy->get();
 106   options.compression = leveldb::kSnappyCompression;
 107
 108   // For info about the troubles we've run into with this parameter, see:
 109   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
 110   options.max_open_files = 80;
 111   options.env = env;
 112
 113   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 114   leveldb::Status s = leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
 115
 116   return s;
 117 }
 118
 119 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
 120   leveldb::Options options;
 121   options.env = leveldb::IDBEnv();
 122   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 123   return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
 124 }
 125
 126 namespace {
 127 class LockImpl : public LevelDBLock {
 128  public:
 129   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
 130       : env_(env), lock_(lock) {}
 131   virtual ~LockImpl() { env_->UnlockFile(lock_); }
 132  private:
 133   leveldb::Env* env_;
 134   leveldb::FileLock* lock_;
 135
 136   DISALLOW_COPY_AND_ASSIGN(LockImpl);
 137 };
 138 }  // namespace
 139
 140 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
 141     const base::FilePath& file_name) {
 142   leveldb::Env* env = leveldb::IDBEnv();
 143   base::FilePath lock_path = file_name.AppendASCII("LOCK");
 144   leveldb::FileLock* lock = NULL;
 145   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
 146   if (!status.ok())
 147     return scoped_ptr<LevelDBLock>();
 148   DCHECK(lock);
 149   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
 150 }
 151
 152 static int CheckFreeSpace(const char* const type,
 153                           const base::FilePath& file_name) {
 154   std::string name =
 155       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
 156   int64 free_disk_space_in_k_bytes =
 157       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
 158   if (free_disk_space_in_k_bytes < 0) {
 159     base::Histogram::FactoryGet(
 160         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
 161         1,
 162         2 /*boundary*/,
 163         2 /*boundary*/ + 1,
 164         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
 165     return -1;
 166   }
 167   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
 168                                        ? INT_MAX
 169                                        : free_disk_space_in_k_bytes;
 170   const uint64 histogram_max = static_cast<uint64>(1e9);
 171   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
 172   base::Histogram::FactoryGet(name,
 173                               1,
 174                               histogram_max,
 175                               11 /*buckets*/,
 176                               base::HistogramBase::kUmaTargetedHistogramFlag)
 177       ->Add(clamped_disk_space_k_bytes);
 178   return clamped_disk_space_k_bytes;
 179 }
 180
 181 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
 182                                             const leveldb::Status& s) {
 183   leveldb_env::MethodID method;
 184   int error = -1;
 185   leveldb_env::ErrorParsingResult result =
 186       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
 187   if (result == leveldb_env::NONE)
 188     return;
 189   std::string method_histogram_name(histogram_name);
 190   method_histogram_name.append(".EnvMethod");
 191   base::LinearHistogram::FactoryGet(
 192       method_histogram_name,
 193       1,
 194       leveldb_env::kNumEntries,
 195       leveldb_env::kNumEntries + 1,
 196       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
 197
 198   std::string error_histogram_name(histogram_name);
 199
 200   if (result == leveldb_env::METHOD_AND_PFE) {
 201     DCHECK_LT(error, 0);
 202     error_histogram_name.append(std::string(".PFE.") +
 203                                 leveldb_env::MethodIDToString(method));
 204     base::LinearHistogram::FactoryGet(
 205         error_histogram_name,
 206         1,
 207         -base::File::FILE_ERROR_MAX,
 208         -base::File::FILE_ERROR_MAX + 1,
 209         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
 210   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
 211     error_histogram_name.append(std::string(".Errno.") +
 212                                 leveldb_env::MethodIDToString(method));
 213     base::LinearHistogram::FactoryGet(
 214         error_histogram_name,
 215         1,
 216         ERANGE + 1,
 217         ERANGE + 2,
 218         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 219   }
 220 }
 221
 222 static void ParseAndHistogramCorruptionDetails(
 223     const std::string& histogram_name,
 224     const leveldb::Status& status) {
 225   int error = leveldb_env::GetCorruptionCode(status);
 226   DCHECK_GE(error, 0);
 227   std::string corruption_histogram_name(histogram_name);
 228   corruption_histogram_name.append(".Corruption");
 229   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
 230   base::LinearHistogram::FactoryGet(
 231       corruption_histogram_name,
 232       1,
 233       kNumPatterns,
 234       kNumPatterns + 1,
 235       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 236 }
 237
 238 static void HistogramLevelDBError(const std::string& histogram_name,
 239                                   const leveldb::Status& s) {
 240   if (s.ok()) {
 241     NOTREACHED();
 242     return;
 243   }
 244   enum {
 245     LEVEL_DB_NOT_FOUND,
 246     LEVEL_DB_CORRUPTION,
 247     LEVEL_DB_IO_ERROR,
 248     LEVEL_DB_OTHER,
 249     LEVEL_DB_MAX_ERROR
 250   };
 251   int leveldb_error = LEVEL_DB_OTHER;
 252   if (s.IsNotFound())
 253     leveldb_error = LEVEL_DB_NOT_FOUND;
 254   else if (s.IsCorruption())
 255     leveldb_error = LEVEL_DB_CORRUPTION;
 256   else if (s.IsIOError())
 257     leveldb_error = LEVEL_DB_IO_ERROR;
 258   base::Histogram::FactoryGet(histogram_name,
 259                               1,
 260                               LEVEL_DB_MAX_ERROR,
 261                               LEVEL_DB_MAX_ERROR + 1,
 262                               base::HistogramBase::kUmaTargetedHistogramFlag)
 263       ->Add(leveldb_error);
 264   if (s.IsIOError())
 265     ParseAndHistogramIOErrorDetails(histogram_name, s);
 266   else
 267     ParseAndHistogramCorruptionDetails(histogram_name, s);
 268 }
 269
 270 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
 271                                       const LevelDBComparator* comparator,
 272                                       scoped_ptr<LevelDBDatabase>* result,
 273                                       bool* is_disk_full) {
 274   base::TimeTicks begin_time = base::TimeTicks::Now();
 275
 276   scoped_ptr<ComparatorAdapter> comparator_adapter(
 277       new ComparatorAdapter(comparator));
 278
 279   leveldb::DB* db;
 280   scoped_ptr<const leveldb::FilterPolicy> filter_policy;
 281   const leveldb::Status s = OpenDB(comparator_adapter.get(),
 282                                    leveldb::IDBEnv(),
 283                                    file_name,
 284                                    &db,
 285                                    &filter_policy);
 286
 287   if (!s.ok()) {
 288     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
 289     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
 290     // Disks with <100k of free space almost never succeed in opening a
 291     // leveldb database.
 292     if (is_disk_full)
 293       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
 294
 295     LOG(ERROR) << "Failed to open LevelDB database from "
 296                << file_name.AsUTF8Unsafe() << "," << s.ToString();
 297     return s;
 298   }
 299
 300   UMA_HISTOGRAM_MEDIUM_TIMES("WebCore.IndexedDB.LevelDB.OpenTime",
 301                              base::TimeTicks::Now() - begin_time);
 302
 303   CheckFreeSpace("Success", file_name);
 304
 305   (*result).reset(new LevelDBDatabase);
 306   (*result)->db_ = make_scoped_ptr(db);
 307   (*result)->comparator_adapter_ = comparator_adapter.Pass();
 308   (*result)->comparator_ = comparator;
 309   (*result)->filter_policy_ = filter_policy.Pass();
 310
 311   return s;
 312 }
 313
 314 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
 315     const LevelDBComparator* comparator) {
 316   scoped_ptr<ComparatorAdapter> comparator_adapter(
 317       new ComparatorAdapter(comparator));
 318   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
 319
 320   leveldb::DB* db;
 321   scoped_ptr<const leveldb::FilterPolicy> filter_policy;
 322   const leveldb::Status s = OpenDB(comparator_adapter.get(),
 323                                    in_memory_env.get(),
 324                                    base::FilePath(),
 325                                    &db,
 326                                    &filter_policy);
 327
 328   if (!s.ok()) {
 329     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
 330     return scoped_ptr<LevelDBDatabase>();
 331   }
 332
 333   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 334   result->env_ = in_memory_env.Pass();
 335   result->db_ = make_scoped_ptr(db);
 336   result->comparator_adapter_ = comparator_adapter.Pass();
 337   result->comparator_ = comparator;
 338   result->filter_policy_ = filter_policy.Pass();
 339
 340   return result.Pass();
 341 }
 342
 343 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
 344                                      std::string* value) {
 345   base::TimeTicks begin_time = base::TimeTicks::Now();
 346
 347   leveldb::WriteOptions write_options;
 348   write_options.sync = kSyncWrites;
 349
 350   const leveldb::Status s =
 351       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
 352   if (!s.ok())
 353     LOG(ERROR) << "LevelDB put failed: " << s.ToString();
 354   else
 355     UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.PutTime",
 356                         base::TimeTicks::Now() - begin_time);
 357   return s;
 358 }
 359
 360 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
 361   leveldb::WriteOptions write_options;
 362   write_options.sync = kSyncWrites;
 363
 364   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
 365   if (!s.IsNotFound())
 366     LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
 367   return s;
 368 }
 369
 370 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
 371                                      std::string* value,
 372                                      bool* found,
 373                                      const LevelDBSnapshot* snapshot) {
 374   *found = false;
 375   leveldb::ReadOptions read_options;
 376   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 377                                          // performance impact is too great.
 378   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 379
 380   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
 381   if (s.ok()) {
 382     *found = true;
 383     return s;
 384   }
 385   if (s.IsNotFound())
 386     return leveldb::Status::OK();
 387   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
 388   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
 389   return s;
 390 }
 391
 392 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
 393   base::TimeTicks begin_time = base::TimeTicks::Now();
 394   leveldb::WriteOptions write_options;
 395   write_options.sync = kSyncWrites;
 396
 397   const leveldb::Status s =
 398       db_->Write(write_options, write_batch.write_batch_.get());
 399   if (!s.ok()) {
 400     HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
 401     LOG(ERROR) << "LevelDB write failed: " << s.ToString();
 402   } else {
 403     UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.WriteTime",
 404                         base::TimeTicks::Now() - begin_time);
 405   }
 406   return s;
 407 }
 408
 409 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
 410     const LevelDBSnapshot* snapshot) {
 411   leveldb::ReadOptions read_options;
 412   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 413                                          // performance impact is too great.
 414   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 415
 416   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
 417   return scoped_ptr<LevelDBIterator>(
 418       IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass()));
 419 }
 420
 421 const LevelDBComparator* LevelDBDatabase::Comparator() const {
 422   return comparator_;
 423 }
 424
 425 void LevelDBDatabase::Compact(const base::StringPiece& start,
 426                               const base::StringPiece& stop) {
 427   const leveldb::Slice start_slice = MakeSlice(start);
 428   const leveldb::Slice stop_slice = MakeSlice(stop);
 429   // NULL batch means just wait for earlier writes to be done
 430   db_->Write(leveldb::WriteOptions(), NULL);
 431   db_->CompactRange(&start_slice, &stop_slice);
 432 }
 433
 434 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
 435
 436 }  // namespace content