1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/dom_storage/session_storage_database.h"
7 #include "base/files/file_util.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_number_conversions.h"
11 #include "base/strings/stringprintf.h"
12 #include "base/strings/utf_string_conversions.h"
13 #include "third_party/leveldatabase/src/include/leveldb/db.h"
14 #include "third_party/leveldatabase/src/include/leveldb/iterator.h"
15 #include "third_party/leveldatabase/src/include/leveldb/options.h"
16 #include "third_party/leveldatabase/src/include/leveldb/status.h"
17 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
23 const char session_storage_uma_name[] = "SessionStorageDatabase.Open";
25 enum SessionStorageUMA {
26 SESSION_STORAGE_UMA_SUCCESS,
27 SESSION_STORAGE_UMA_RECREATED,
28 SESSION_STORAGE_UMA_FAIL,
29 SESSION_STORAGE_UMA_MAX
34 // Layout of the database:
36 // -----------------------------------------------------------------------
37 // | map-1- | 2 (refcount, start of map-1-* keys)|
38 // | map-1-a | b (a = b in map 1) |
40 // | namespace- | dummy (start of namespace-* keys) |
41 // | namespace-1- (1 = namespace id)| dummy (start of namespace-1-* keys)|
42 // | namespace-1-origin1 | 1 (mapid) |
43 // | namespace-1-origin2 | 2 |
44 // | namespace-2- | dummy |
45 // | namespace-2-origin1 | 1 (shallow copy) |
46 // | namespace-2-origin2 | 2 (shallow copy) |
47 // | namespace-3- | dummy |
48 // | namespace-3-origin1 | 3 (deep copy) |
49 // | namespace-3-origin2 | 2 (shallow copy) |
50 // | next-map-id | 4 |
54 // This class keeps track of ongoing operations across different threads. When
55 // DB inconsistency is detected, we need to 1) make sure no new operations start
56 // 2) wait until all current operations finish, and let the last one of them
57 // close the DB and delete the data. The DB will remain empty for the rest of
58 // the run, and will be recreated during the next run. We cannot hope to recover
59 // during this run, since the upper layer will have a different idea about what
60 // should be in the database.
61 class SessionStorageDatabase::DBOperation {
63 DBOperation(SessionStorageDatabase* session_storage_database)
64 : session_storage_database_(session_storage_database) {
65 base::AutoLock auto_lock(session_storage_database_->db_lock_);
66 ++session_storage_database_->operation_count_;
70 base::AutoLock auto_lock(session_storage_database_->db_lock_);
71 --session_storage_database_->operation_count_;
72 if ((session_storage_database_->is_inconsistent_ ||
73 session_storage_database_->db_error_) &&
74 session_storage_database_->operation_count_ == 0 &&
75 !session_storage_database_->invalid_db_deleted_) {
76 // No other operations are ongoing and the data is bad -> delete it now.
77 session_storage_database_->db_.reset();
80 base::WideToUTF8(session_storage_database_->file_path_.value()),
83 leveldb::DestroyDB(session_storage_database_->file_path_.value(),
86 session_storage_database_->invalid_db_deleted_ = true;
91 SessionStorageDatabase* session_storage_database_;
95 SessionStorageDatabase::SessionStorageDatabase(const base::FilePath& file_path)
96 : file_path_(file_path),
98 is_inconsistent_(false),
99 invalid_db_deleted_(false),
100 operation_count_(0) {
103 SessionStorageDatabase::~SessionStorageDatabase() {
106 void SessionStorageDatabase::ReadAreaValues(const std::string& namespace_id,
108 DOMStorageValuesMap* result) {
109 // We don't create a database if it doesn't exist. In that case, there is
110 // nothing to be added to the result.
111 if (!LazyOpen(false))
113 DBOperation operation(this);
115 // While ReadAreaValues is in progress, another thread can call
116 // CommitAreaChanges. CommitAreaChanges might update map ref count key while
117 // this thread is iterating over the map ref count key. To protect the reading
118 // operation, create a snapshot and read from it.
119 leveldb::ReadOptions options;
120 options.snapshot = db_->GetSnapshot();
124 if (GetMapForArea(namespace_id, origin.spec(), options, &exists, &map_id) &&
126 ReadMap(map_id, options, result, false);
127 db_->ReleaseSnapshot(options.snapshot);
130 bool SessionStorageDatabase::CommitAreaChanges(
131 const std::string& namespace_id,
133 bool clear_all_first,
134 const DOMStorageValuesMap& changes) {
135 // Even if |changes| is empty, we need to write the appropriate placeholders
136 // in the database, so that it can be later shallow-copied succssfully.
139 DBOperation operation(this);
141 leveldb::WriteBatch batch;
142 // Ensure that the keys "namespace-" "namespace-N" (see the schema above)
144 const bool kOkIfExists = true;
145 if (!CreateNamespace(namespace_id, kOkIfExists, &batch))
150 if (!GetMapForArea(namespace_id, origin.spec(), leveldb::ReadOptions(),
155 if (!GetMapRefCount(map_id, &ref_count))
158 if (!DeepCopyArea(namespace_id, origin, !clear_all_first,
162 else if (clear_all_first) {
163 if (!ClearMap(map_id, &batch))
167 // Map doesn't exist, create it now if needed.
168 if (!changes.empty()) {
169 if (!CreateMapForArea(namespace_id, origin, &map_id, &batch))
174 WriteValuesToMap(map_id, changes, &batch);
176 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
177 return DatabaseErrorCheck(s.ok());
180 bool SessionStorageDatabase::CloneNamespace(
181 const std::string& namespace_id, const std::string& new_namespace_id) {
182 // Go through all origins in the namespace |namespace_id|, create placeholders
183 // for them in |new_namespace_id|, and associate them with the existing maps.
185 // Example, data before shallow copy:
186 // | map-1- | 1 (refcount) |
188 // | namespace-1- (1 = namespace id)| dummy |
189 // | namespace-1-origin1 | 1 (mapid) |
191 // Example, data after shallow copy:
192 // | map-1- | 2 (inc. refcount) |
194 // | namespace-1-(1 = namespace id) | dummy |
195 // | namespace-1-origin1 | 1 (mapid) |
196 // | namespace-2- | dummy |
197 // | namespace-2-origin1 | 1 (mapid) << references the same map
201 DBOperation operation(this);
203 leveldb::WriteBatch batch;
204 const bool kOkIfExists = false;
205 if (!CreateNamespace(new_namespace_id, kOkIfExists, &batch))
208 std::map<std::string, std::string> areas;
209 if (!GetAreasInNamespace(namespace_id, &areas))
212 for (std::map<std::string, std::string>::const_iterator it = areas.begin();
213 it != areas.end(); ++it) {
214 const std::string& origin = it->first;
215 const std::string& map_id = it->second;
216 if (!IncreaseMapRefCount(map_id, &batch))
218 AddAreaToNamespace(new_namespace_id, origin, map_id, &batch);
220 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
221 return DatabaseErrorCheck(s.ok());
224 bool SessionStorageDatabase::DeleteArea(const std::string& namespace_id,
225 const GURL& origin) {
226 if (!LazyOpen(false)) {
227 // No need to create the database if it doesn't exist.
230 DBOperation operation(this);
231 leveldb::WriteBatch batch;
232 if (!DeleteAreaHelper(namespace_id, origin.spec(), &batch))
234 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
235 return DatabaseErrorCheck(s.ok());
238 bool SessionStorageDatabase::DeleteNamespace(const std::string& namespace_id) {
239 if (!LazyOpen(false)) {
240 // No need to create the database if it doesn't exist.
243 DBOperation operation(this);
244 // Itereate through the areas in the namespace.
245 leveldb::WriteBatch batch;
246 std::map<std::string, std::string> areas;
247 if (!GetAreasInNamespace(namespace_id, &areas))
249 for (std::map<std::string, std::string>::const_iterator it = areas.begin();
250 it != areas.end(); ++it) {
251 const std::string& origin = it->first;
252 if (!DeleteAreaHelper(namespace_id, origin, &batch))
255 batch.Delete(NamespaceStartKey(namespace_id));
256 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
257 return DatabaseErrorCheck(s.ok());
260 bool SessionStorageDatabase::ReadNamespacesAndOrigins(
261 std::map<std::string, std::vector<GURL> >* namespaces_and_origins) {
264 DBOperation operation(this);
266 // While ReadNamespacesAndOrigins is in progress, another thread can call
267 // CommitAreaChanges. To protect the reading operation, create a snapshot and
269 leveldb::ReadOptions options;
270 options.snapshot = db_->GetSnapshot();
272 std::string namespace_prefix = NamespacePrefix();
273 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(options));
274 it->Seek(namespace_prefix);
275 // If the key is not found, the status of the iterator won't be IsNotFound(),
276 // but the iterator will be invalid.
278 db_->ReleaseSnapshot(options.snapshot);
282 if (!DatabaseErrorCheck(it->status().ok())) {
283 db_->ReleaseSnapshot(options.snapshot);
287 // Skip the dummy entry "namespace-" and iterate the namespaces.
288 std::string current_namespace_start_key;
289 std::string current_namespace_id;
290 for (it->Next(); it->Valid(); it->Next()) {
291 std::string key = it->key().ToString();
292 if (key.find(namespace_prefix) != 0) {
293 // Iterated past the "namespace-" keys.
296 // For each namespace, the first key is "namespace-<namespaceid>-", and the
297 // subsequent keys are "namespace-<namespaceid>-<origin>". Read the unique
298 // "<namespaceid>" parts from the keys.
299 if (current_namespace_start_key.empty() ||
300 key.substr(0, current_namespace_start_key.length()) !=
301 current_namespace_start_key) {
302 // The key is of the form "namespace-<namespaceid>-" for a new
304 current_namespace_start_key = key;
305 current_namespace_id =
306 key.substr(namespace_prefix.length(),
307 key.length() - namespace_prefix.length() - 1);
308 // Ensure that we keep track of the namespace even if it doesn't contain
310 namespaces_and_origins->insert(
311 std::make_pair(current_namespace_id, std::vector<GURL>()));
313 // The key is of the form "namespace-<namespaceid>-<origin>".
314 std::string origin = key.substr(current_namespace_start_key.length());
315 (*namespaces_and_origins)[current_namespace_id].push_back(GURL(origin));
318 db_->ReleaseSnapshot(options.snapshot);
322 bool SessionStorageDatabase::LazyOpen(bool create_if_needed) {
323 base::AutoLock auto_lock(db_lock_);
324 if (db_error_ || is_inconsistent_) {
325 // Don't try to open a database that we know has failed already.
331 if (!create_if_needed &&
332 (!base::PathExists(file_path_) || base::IsDirectoryEmpty(file_path_))) {
333 // If the directory doesn't exist already and we haven't been asked to
334 // create a file on disk, then we don't bother opening the database. This
335 // means we wait until we absolutely need to put something onto disk before
341 leveldb::Status s = TryToOpen(&db);
343 LOG(WARNING) << "Failed to open leveldb in " << file_path_.value()
344 << ", error: " << s.ToString();
347 // Clear the directory and try again.
348 base::DeleteFile(file_path_, true);
351 LOG(WARNING) << "Failed to open leveldb in " << file_path_.value()
352 << ", error: " << s.ToString();
353 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
354 SESSION_STORAGE_UMA_FAIL,
355 SESSION_STORAGE_UMA_MAX);
360 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
361 SESSION_STORAGE_UMA_RECREATED,
362 SESSION_STORAGE_UMA_MAX);
364 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
365 SESSION_STORAGE_UMA_SUCCESS,
366 SESSION_STORAGE_UMA_MAX);
372 leveldb::Status SessionStorageDatabase::TryToOpen(leveldb::DB** db) {
373 leveldb::Options options;
374 // The directory exists but a valid leveldb database might not exist inside it
375 // (e.g., a subset of the needed files might be missing). Handle this
376 // situation gracefully by creating the database now.
377 options.max_open_files = 0; // Use minimum.
378 options.create_if_missing = true;
380 return leveldb::DB::Open(options, base::WideToUTF8(file_path_.value()), db);
381 #elif defined(OS_POSIX)
382 return leveldb::DB::Open(options, file_path_.value(), db);
386 bool SessionStorageDatabase::IsOpen() const {
387 return db_.get() != NULL;
390 bool SessionStorageDatabase::CallerErrorCheck(bool ok) const {
395 bool SessionStorageDatabase::ConsistencyCheck(bool ok) {
398 base::AutoLock auto_lock(db_lock_);
399 // We cannot recover the database during this run, e.g., the upper layer can
400 // have a different understanding of the database state (shallow and deep
401 // copies). Make further operations fail. The next operation that finishes
402 // will delete the data, and next run will recerate the database.
403 is_inconsistent_ = true;
407 bool SessionStorageDatabase::DatabaseErrorCheck(bool ok) {
410 base::AutoLock auto_lock(db_lock_);
411 // Make further operations fail. The next operation that finishes
412 // will delete the data, and next run will recerate the database.
417 bool SessionStorageDatabase::CreateNamespace(const std::string& namespace_id,
419 leveldb::WriteBatch* batch) {
420 leveldb::Slice namespace_prefix = NamespacePrefix();
422 leveldb::Status s = db_->Get(leveldb::ReadOptions(), namespace_prefix,
424 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
427 batch->Put(namespace_prefix, "");
429 std::string namespace_start_key = NamespaceStartKey(namespace_id);
430 s = db_->Get(leveldb::ReadOptions(), namespace_start_key, &dummy);
431 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
433 if (s.IsNotFound()) {
434 batch->Put(namespace_start_key, "");
437 return CallerErrorCheck(ok_if_exists);
440 bool SessionStorageDatabase::GetAreasInNamespace(
441 const std::string& namespace_id,
442 std::map<std::string, std::string>* areas) {
443 std::string namespace_start_key = NamespaceStartKey(namespace_id);
444 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(leveldb::ReadOptions()));
445 it->Seek(namespace_start_key);
446 // If the key is not found, the status of the iterator won't be IsNotFound(),
447 // but the iterator will be invalid.
449 // The namespace_start_key is not found when the namespace doesn't contain
450 // any areas. We don't need to do anything.
453 if (!DatabaseErrorCheck(it->status().ok()))
456 // Skip the dummy entry "namespace-<namespaceid>-" and iterate the origins.
457 for (it->Next(); it->Valid(); it->Next()) {
458 std::string key = it->key().ToString();
459 if (key.find(namespace_start_key) != 0) {
460 // Iterated past the origins for this namespace.
463 std::string origin = key.substr(namespace_start_key.length());
464 std::string map_id = it->value().ToString();
465 (*areas)[origin] = map_id;
470 void SessionStorageDatabase::AddAreaToNamespace(const std::string& namespace_id,
471 const std::string& origin,
472 const std::string& map_id,
473 leveldb::WriteBatch* batch) {
474 std::string namespace_key = NamespaceKey(namespace_id, origin);
475 batch->Put(namespace_key, map_id);
478 bool SessionStorageDatabase::DeleteAreaHelper(
479 const std::string& namespace_id,
480 const std::string& origin,
481 leveldb::WriteBatch* batch) {
484 if (!GetMapForArea(namespace_id, origin, leveldb::ReadOptions(), &exists,
488 return true; // Nothing to delete.
489 if (!DecreaseMapRefCount(map_id, 1, batch))
491 std::string namespace_key = NamespaceKey(namespace_id, origin);
492 batch->Delete(namespace_key);
494 // If this was the only area in the namespace, delete the namespace start key,
496 std::string namespace_start_key = NamespaceStartKey(namespace_id);
497 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(leveldb::ReadOptions()));
498 it->Seek(namespace_start_key);
499 if (!ConsistencyCheck(it->Valid()))
501 // Advance the iterator 2 times (we still haven't really deleted
504 if (!ConsistencyCheck(it->Valid()))
509 std::string key = it->key().ToString();
510 if (key.find(namespace_start_key) != 0)
511 batch->Delete(namespace_start_key);
515 bool SessionStorageDatabase::GetMapForArea(const std::string& namespace_id,
516 const std::string& origin,
517 const leveldb::ReadOptions& options,
518 bool* exists, std::string* map_id) {
519 std::string namespace_key = NamespaceKey(namespace_id, origin);
520 leveldb::Status s = db_->Get(options, namespace_key, map_id);
521 if (s.IsNotFound()) {
526 return DatabaseErrorCheck(s.ok());
529 bool SessionStorageDatabase::CreateMapForArea(const std::string& namespace_id,
532 leveldb::WriteBatch* batch) {
533 leveldb::Slice next_map_id_key = NextMapIdKey();
534 leveldb::Status s = db_->Get(leveldb::ReadOptions(), next_map_id_key, map_id);
535 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
537 int64 next_map_id = 0;
538 if (s.IsNotFound()) {
541 bool conversion_ok = base::StringToInt64(*map_id, &next_map_id);
542 if (!ConsistencyCheck(conversion_ok))
545 batch->Put(next_map_id_key, base::Int64ToString(++next_map_id));
546 std::string namespace_key = NamespaceKey(namespace_id, origin.spec());
547 batch->Put(namespace_key, *map_id);
548 batch->Put(MapRefCountKey(*map_id), "1");
552 bool SessionStorageDatabase::ReadMap(const std::string& map_id,
553 const leveldb::ReadOptions& options,
554 DOMStorageValuesMap* result,
556 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(options));
557 std::string map_start_key = MapRefCountKey(map_id);
558 it->Seek(map_start_key);
559 // If the key is not found, the status of the iterator won't be IsNotFound(),
560 // but the iterator will be invalid. The map needs to exist, otherwise we have
561 // a stale map_id in the database.
562 if (!ConsistencyCheck(it->Valid()))
564 if (!DatabaseErrorCheck(it->status().ok()))
566 // Skip the dummy entry "map-<mapid>-".
567 for (it->Next(); it->Valid(); it->Next()) {
568 std::string key = it->key().ToString();
569 if (key.find(map_start_key) != 0) {
570 // Iterated past the keys in this map.
573 // Key is of the form "map-<mapid>-<key>".
574 base::string16 key16 =
575 base::UTF8ToUTF16(key.substr(map_start_key.length()));
577 (*result)[key16] = base::NullableString16();
579 // Convert the raw data stored in std::string (it->value()) to raw data
580 // stored in base::string16.
581 size_t len = it->value().size() / sizeof(base::char16);
582 const base::char16* data_ptr =
583 reinterpret_cast<const base::char16*>(it->value().data());
585 base::NullableString16(base::string16(data_ptr, len), false);
591 void SessionStorageDatabase::WriteValuesToMap(const std::string& map_id,
592 const DOMStorageValuesMap& values,
593 leveldb::WriteBatch* batch) {
594 for (DOMStorageValuesMap::const_iterator it = values.begin();
597 base::NullableString16 value = it->second;
598 std::string key = MapKey(map_id, base::UTF16ToUTF8(it->first));
599 if (value.is_null()) {
602 // Convert the raw data stored in base::string16 to raw data stored in
604 const char* data = reinterpret_cast<const char*>(value.string().data());
605 size_t size = value.string().size() * 2;
606 batch->Put(key, leveldb::Slice(data, size));
611 bool SessionStorageDatabase::GetMapRefCount(const std::string& map_id,
613 std::string ref_count_string;
614 leveldb::Status s = db_->Get(leveldb::ReadOptions(),
615 MapRefCountKey(map_id), &ref_count_string);
616 if (!ConsistencyCheck(s.ok()))
618 bool conversion_ok = base::StringToInt64(ref_count_string, ref_count);
619 return ConsistencyCheck(conversion_ok);
622 bool SessionStorageDatabase::IncreaseMapRefCount(const std::string& map_id,
623 leveldb::WriteBatch* batch) {
624 // Increase the ref count for the map.
626 if (!GetMapRefCount(map_id, &old_ref_count))
628 batch->Put(MapRefCountKey(map_id), base::Int64ToString(++old_ref_count));
632 bool SessionStorageDatabase::DecreaseMapRefCount(const std::string& map_id,
634 leveldb::WriteBatch* batch) {
635 // Decrease the ref count for the map.
637 if (!GetMapRefCount(map_id, &ref_count))
639 if (!ConsistencyCheck(decrease <= ref_count))
641 ref_count -= decrease;
643 batch->Put(MapRefCountKey(map_id), base::Int64ToString(ref_count));
645 // Clear all keys in the map.
646 if (!ClearMap(map_id, batch))
648 batch->Delete(MapRefCountKey(map_id));
653 bool SessionStorageDatabase::ClearMap(const std::string& map_id,
654 leveldb::WriteBatch* batch) {
655 DOMStorageValuesMap values;
656 if (!ReadMap(map_id, leveldb::ReadOptions(), &values, true))
658 for (DOMStorageValuesMap::const_iterator it = values.begin();
659 it != values.end(); ++it)
660 batch->Delete(MapKey(map_id, base::UTF16ToUTF8(it->first)));
664 bool SessionStorageDatabase::DeepCopyArea(
665 const std::string& namespace_id, const GURL& origin, bool copy_data,
666 std::string* map_id, leveldb::WriteBatch* batch) {
667 // Example, data before deep copy:
668 // | namespace-1- (1 = namespace id)| dummy |
669 // | namespace-1-origin1 | 1 (mapid) |
670 // | namespace-2- | dummy |
671 // | namespace-2-origin1 | 1 (mapid) << references the same map
672 // | map-1- | 2 (refcount) |
675 // Example, data after deep copy copy:
676 // | namespace-1-(1 = namespace id) | dummy |
677 // | namespace-1-origin1 | 1 (mapid) |
678 // | namespace-2- | dummy |
679 // | namespace-2-origin1 | 2 (mapid) << references the new map
680 // | map-1- | 1 (dec. refcount) |
682 // | map-2- | 1 (refcount) |
685 // Read the values from the old map here. If we don't need to copy the data,
686 // this can stay empty.
687 DOMStorageValuesMap values;
688 if (copy_data && !ReadMap(*map_id, leveldb::ReadOptions(), &values, false))
690 if (!DecreaseMapRefCount(*map_id, 1, batch))
692 // Create a new map (this will also break the association to the old map) and
693 // write the old data into it. This will write the id of the created map into
695 if (!CreateMapForArea(namespace_id, origin, map_id, batch))
697 WriteValuesToMap(*map_id, values, batch);
701 std::string SessionStorageDatabase::NamespaceStartKey(
702 const std::string& namespace_id) {
703 return base::StringPrintf("namespace-%s-", namespace_id.c_str());
706 std::string SessionStorageDatabase::NamespaceKey(
707 const std::string& namespace_id, const std::string& origin) {
708 return base::StringPrintf("namespace-%s-%s", namespace_id.c_str(),
712 const char* SessionStorageDatabase::NamespacePrefix() {
716 std::string SessionStorageDatabase::MapRefCountKey(const std::string& map_id) {
717 return base::StringPrintf("map-%s-", map_id.c_str());
720 std::string SessionStorageDatabase::MapKey(const std::string& map_id,
721 const std::string& key) {
722 return base::StringPrintf("map-%s-%s", map_id.c_str(), key.c_str());
725 const char* SessionStorageDatabase::NextMapIdKey() {
726 return "next-map-id";
729 } // namespace content