1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/dom_storage/session_storage_database.h"
7 #include "base/file_util.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_number_conversions.h"
11 #include "base/strings/stringprintf.h"
12 #include "base/strings/utf_string_conversions.h"
13 #include "third_party/leveldatabase/src/include/leveldb/db.h"
14 #include "third_party/leveldatabase/src/include/leveldb/iterator.h"
15 #include "third_party/leveldatabase/src/include/leveldb/options.h"
16 #include "third_party/leveldatabase/src/include/leveldb/status.h"
17 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
23 const char session_storage_uma_name[] = "SessionStorageDatabase.Open";
25 enum SessionStorageUMA {
26 SESSION_STORAGE_UMA_SUCCESS,
27 SESSION_STORAGE_UMA_RECREATED,
28 SESSION_STORAGE_UMA_FAIL,
29 SESSION_STORAGE_UMA_MAX
34 // Layout of the database:
36 // -----------------------------------------------------------------------
37 // | map-1- | 2 (refcount, start of map-1-* keys)|
38 // | map-1-a | b (a = b in map 1) |
40 // | namespace- | dummy (start of namespace-* keys) |
41 // | namespace-1- (1 = namespace id)| dummy (start of namespace-1-* keys)|
42 // | namespace-1-origin1 | 1 (mapid) |
43 // | namespace-1-origin2 | 2 |
44 // | namespace-2- | dummy |
45 // | namespace-2-origin1 | 1 (shallow copy) |
46 // | namespace-2-origin2 | 2 (shallow copy) |
47 // | namespace-3- | dummy |
48 // | namespace-3-origin1 | 3 (deep copy) |
49 // | namespace-3-origin2 | 2 (shallow copy) |
50 // | next-map-id | 4 |
54 SessionStorageDatabase::SessionStorageDatabase(const base::FilePath& file_path)
55 : file_path_(file_path),
57 is_inconsistent_(false) {
60 SessionStorageDatabase::~SessionStorageDatabase() {
63 void SessionStorageDatabase::ReadAreaValues(const std::string& namespace_id,
65 DOMStorageValuesMap* result) {
66 // We don't create a database if it doesn't exist. In that case, there is
67 // nothing to be added to the result.
71 // While ReadAreaValues is in progress, another thread can call
72 // CommitAreaChanges. CommitAreaChanges might update map ref count key while
73 // this thread is iterating over the map ref count key. To protect the reading
74 // operation, create a snapshot and read from it.
75 leveldb::ReadOptions options;
76 options.snapshot = db_->GetSnapshot();
80 if (GetMapForArea(namespace_id, origin.spec(), options, &exists, &map_id) &&
82 ReadMap(map_id, options, result, false);
83 db_->ReleaseSnapshot(options.snapshot);
86 bool SessionStorageDatabase::CommitAreaChanges(
87 const std::string& namespace_id,
90 const DOMStorageValuesMap& changes) {
91 // Even if |changes| is empty, we need to write the appropriate placeholders
92 // in the database, so that it can be later shallow-copied succssfully.
96 leveldb::WriteBatch batch;
97 // Ensure that the keys "namespace-" "namespace-N" (see the schema above)
99 const bool kOkIfExists = true;
100 if (!CreateNamespace(namespace_id, kOkIfExists, &batch))
105 if (!GetMapForArea(namespace_id, origin.spec(), leveldb::ReadOptions(),
110 if (!GetMapRefCount(map_id, &ref_count))
113 if (!DeepCopyArea(namespace_id, origin, !clear_all_first,
117 else if (clear_all_first) {
118 if (!ClearMap(map_id, &batch))
122 // Map doesn't exist, create it now if needed.
123 if (!changes.empty()) {
124 if (!CreateMapForArea(namespace_id, origin, &map_id, &batch))
129 WriteValuesToMap(map_id, changes, &batch);
131 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
132 return DatabaseErrorCheck(s.ok());
135 bool SessionStorageDatabase::CloneNamespace(
136 const std::string& namespace_id, const std::string& new_namespace_id) {
137 // Go through all origins in the namespace |namespace_id|, create placeholders
138 // for them in |new_namespace_id|, and associate them with the existing maps.
140 // Example, data before shallow copy:
141 // | map-1- | 1 (refcount) |
143 // | namespace-1- (1 = namespace id)| dummy |
144 // | namespace-1-origin1 | 1 (mapid) |
146 // Example, data after shallow copy:
147 // | map-1- | 2 (inc. refcount) |
149 // | namespace-1-(1 = namespace id) | dummy |
150 // | namespace-1-origin1 | 1 (mapid) |
151 // | namespace-2- | dummy |
152 // | namespace-2-origin1 | 1 (mapid) << references the same map
157 leveldb::WriteBatch batch;
158 const bool kOkIfExists = false;
159 if (!CreateNamespace(new_namespace_id, kOkIfExists, &batch))
162 std::map<std::string, std::string> areas;
163 if (!GetAreasInNamespace(namespace_id, &areas))
166 for (std::map<std::string, std::string>::const_iterator it = areas.begin();
167 it != areas.end(); ++it) {
168 const std::string& origin = it->first;
169 const std::string& map_id = it->second;
170 if (!IncreaseMapRefCount(map_id, &batch))
172 AddAreaToNamespace(new_namespace_id, origin, map_id, &batch);
174 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
175 return DatabaseErrorCheck(s.ok());
178 bool SessionStorageDatabase::DeleteArea(const std::string& namespace_id,
179 const GURL& origin) {
180 if (!LazyOpen(false)) {
181 // No need to create the database if it doesn't exist.
184 leveldb::WriteBatch batch;
185 if (!DeleteAreaHelper(namespace_id, origin.spec(), &batch))
187 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
188 return DatabaseErrorCheck(s.ok());
191 bool SessionStorageDatabase::DeleteNamespace(const std::string& namespace_id) {
192 if (!LazyOpen(false)) {
193 // No need to create the database if it doesn't exist.
196 // Itereate through the areas in the namespace.
197 leveldb::WriteBatch batch;
198 std::map<std::string, std::string> areas;
199 if (!GetAreasInNamespace(namespace_id, &areas))
201 for (std::map<std::string, std::string>::const_iterator it = areas.begin();
202 it != areas.end(); ++it) {
203 const std::string& origin = it->first;
204 if (!DeleteAreaHelper(namespace_id, origin, &batch))
207 batch.Delete(NamespaceStartKey(namespace_id));
208 leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch);
209 return DatabaseErrorCheck(s.ok());
212 bool SessionStorageDatabase::ReadNamespacesAndOrigins(
213 std::map<std::string, std::vector<GURL> >* namespaces_and_origins) {
217 // While ReadNamespacesAndOrigins is in progress, another thread can call
218 // CommitAreaChanges. To protect the reading operation, create a snapshot and
220 leveldb::ReadOptions options;
221 options.snapshot = db_->GetSnapshot();
223 std::string namespace_prefix = NamespacePrefix();
224 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(options));
225 it->Seek(namespace_prefix);
226 // If the key is not found, the status of the iterator won't be IsNotFound(),
227 // but the iterator will be invalid.
229 db_->ReleaseSnapshot(options.snapshot);
233 if (!DatabaseErrorCheck(it->status().ok())) {
234 db_->ReleaseSnapshot(options.snapshot);
238 // Skip the dummy entry "namespace-" and iterate the namespaces.
239 std::string current_namespace_start_key;
240 std::string current_namespace_id;
241 for (it->Next(); it->Valid(); it->Next()) {
242 std::string key = it->key().ToString();
243 if (key.find(namespace_prefix) != 0) {
244 // Iterated past the "namespace-" keys.
247 // For each namespace, the first key is "namespace-<namespaceid>-", and the
248 // subsequent keys are "namespace-<namespaceid>-<origin>". Read the unique
249 // "<namespaceid>" parts from the keys.
250 if (current_namespace_start_key.empty() ||
251 key.substr(0, current_namespace_start_key.length()) !=
252 current_namespace_start_key) {
253 // The key is of the form "namespace-<namespaceid>-" for a new
255 current_namespace_start_key = key;
256 current_namespace_id =
257 key.substr(namespace_prefix.length(),
258 key.length() - namespace_prefix.length() - 1);
259 // Ensure that we keep track of the namespace even if it doesn't contain
261 namespaces_and_origins->insert(
262 std::make_pair(current_namespace_id, std::vector<GURL>()));
264 // The key is of the form "namespace-<namespaceid>-<origin>".
265 std::string origin = key.substr(current_namespace_start_key.length());
266 (*namespaces_and_origins)[current_namespace_id].push_back(GURL(origin));
269 db_->ReleaseSnapshot(options.snapshot);
273 bool SessionStorageDatabase::LazyOpen(bool create_if_needed) {
274 base::AutoLock auto_lock(db_lock_);
275 if (db_error_ || is_inconsistent_) {
276 // Don't try to open a database that we know has failed already.
282 if (!create_if_needed &&
283 (!base::PathExists(file_path_) ||
284 file_util::IsDirectoryEmpty(file_path_))) {
285 // If the directory doesn't exist already and we haven't been asked to
286 // create a file on disk, then we don't bother opening the database. This
287 // means we wait until we absolutely need to put something onto disk before
293 leveldb::Status s = TryToOpen(&db);
295 LOG(WARNING) << "Failed to open leveldb in " << file_path_.value()
296 << ", error: " << s.ToString();
299 // Clear the directory and try again.
300 base::DeleteFile(file_path_, true);
303 LOG(WARNING) << "Failed to open leveldb in " << file_path_.value()
304 << ", error: " << s.ToString();
305 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
306 SESSION_STORAGE_UMA_FAIL,
307 SESSION_STORAGE_UMA_MAX);
312 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
313 SESSION_STORAGE_UMA_RECREATED,
314 SESSION_STORAGE_UMA_MAX);
316 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name,
317 SESSION_STORAGE_UMA_SUCCESS,
318 SESSION_STORAGE_UMA_MAX);
324 leveldb::Status SessionStorageDatabase::TryToOpen(leveldb::DB** db) {
325 leveldb::Options options;
326 // The directory exists but a valid leveldb database might not exist inside it
327 // (e.g., a subset of the needed files might be missing). Handle this
328 // situation gracefully by creating the database now.
329 options.max_open_files = 0; // Use minimum.
330 options.create_if_missing = true;
332 return leveldb::DB::Open(options, WideToUTF8(file_path_.value()), db);
333 #elif defined(OS_POSIX)
334 return leveldb::DB::Open(options, file_path_.value(), db);
338 bool SessionStorageDatabase::IsOpen() const {
339 return db_.get() != NULL;
342 bool SessionStorageDatabase::CallerErrorCheck(bool ok) const {
347 bool SessionStorageDatabase::ConsistencyCheck(bool ok) {
350 base::AutoLock auto_lock(db_lock_);
352 is_inconsistent_ = true;
353 // We cannot recover the database during this run, e.g., the upper layer can
354 // have a different understanding of the database state (shallow and deep
356 // TODO(marja): Error handling.
360 bool SessionStorageDatabase::DatabaseErrorCheck(bool ok) {
363 base::AutoLock auto_lock(db_lock_);
365 // TODO(marja): Error handling.
369 bool SessionStorageDatabase::CreateNamespace(const std::string& namespace_id,
371 leveldb::WriteBatch* batch) {
372 leveldb::Slice namespace_prefix = NamespacePrefix();
374 leveldb::Status s = db_->Get(leveldb::ReadOptions(), namespace_prefix,
376 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
379 batch->Put(namespace_prefix, "");
381 std::string namespace_start_key = NamespaceStartKey(namespace_id);
382 s = db_->Get(leveldb::ReadOptions(), namespace_start_key, &dummy);
383 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
385 if (s.IsNotFound()) {
386 batch->Put(namespace_start_key, "");
389 return CallerErrorCheck(ok_if_exists);
392 bool SessionStorageDatabase::GetAreasInNamespace(
393 const std::string& namespace_id,
394 std::map<std::string, std::string>* areas) {
395 std::string namespace_start_key = NamespaceStartKey(namespace_id);
396 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(leveldb::ReadOptions()));
397 it->Seek(namespace_start_key);
398 // If the key is not found, the status of the iterator won't be IsNotFound(),
399 // but the iterator will be invalid.
401 // The namespace_start_key is not found when the namespace doesn't contain
402 // any areas. We don't need to do anything.
405 if (!DatabaseErrorCheck(it->status().ok()))
408 // Skip the dummy entry "namespace-<namespaceid>-" and iterate the origins.
409 for (it->Next(); it->Valid(); it->Next()) {
410 std::string key = it->key().ToString();
411 if (key.find(namespace_start_key) != 0) {
412 // Iterated past the origins for this namespace.
415 std::string origin = key.substr(namespace_start_key.length());
416 std::string map_id = it->value().ToString();
417 (*areas)[origin] = map_id;
422 void SessionStorageDatabase::AddAreaToNamespace(const std::string& namespace_id,
423 const std::string& origin,
424 const std::string& map_id,
425 leveldb::WriteBatch* batch) {
426 std::string namespace_key = NamespaceKey(namespace_id, origin);
427 batch->Put(namespace_key, map_id);
430 bool SessionStorageDatabase::DeleteAreaHelper(
431 const std::string& namespace_id,
432 const std::string& origin,
433 leveldb::WriteBatch* batch) {
436 if (!GetMapForArea(namespace_id, origin, leveldb::ReadOptions(), &exists,
440 return true; // Nothing to delete.
441 if (!DecreaseMapRefCount(map_id, 1, batch))
443 std::string namespace_key = NamespaceKey(namespace_id, origin);
444 batch->Delete(namespace_key);
446 // If this was the only area in the namespace, delete the namespace start key,
448 std::string namespace_start_key = NamespaceStartKey(namespace_id);
449 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(leveldb::ReadOptions()));
450 it->Seek(namespace_start_key);
451 if (!ConsistencyCheck(it->Valid()))
453 // Advance the iterator 2 times (we still haven't really deleted
456 if (!ConsistencyCheck(it->Valid()))
461 std::string key = it->key().ToString();
462 if (key.find(namespace_start_key) != 0)
463 batch->Delete(namespace_start_key);
467 bool SessionStorageDatabase::GetMapForArea(const std::string& namespace_id,
468 const std::string& origin,
469 const leveldb::ReadOptions& options,
470 bool* exists, std::string* map_id) {
471 std::string namespace_key = NamespaceKey(namespace_id, origin);
472 leveldb::Status s = db_->Get(options, namespace_key, map_id);
473 if (s.IsNotFound()) {
478 return DatabaseErrorCheck(s.ok());
481 bool SessionStorageDatabase::CreateMapForArea(const std::string& namespace_id,
484 leveldb::WriteBatch* batch) {
485 leveldb::Slice next_map_id_key = NextMapIdKey();
486 leveldb::Status s = db_->Get(leveldb::ReadOptions(), next_map_id_key, map_id);
487 if (!DatabaseErrorCheck(s.ok() || s.IsNotFound()))
489 int64 next_map_id = 0;
490 if (s.IsNotFound()) {
493 bool conversion_ok = base::StringToInt64(*map_id, &next_map_id);
494 if (!ConsistencyCheck(conversion_ok))
497 batch->Put(next_map_id_key, base::Int64ToString(++next_map_id));
498 std::string namespace_key = NamespaceKey(namespace_id, origin.spec());
499 batch->Put(namespace_key, *map_id);
500 batch->Put(MapRefCountKey(*map_id), "1");
504 bool SessionStorageDatabase::ReadMap(const std::string& map_id,
505 const leveldb::ReadOptions& options,
506 DOMStorageValuesMap* result,
508 scoped_ptr<leveldb::Iterator> it(db_->NewIterator(options));
509 std::string map_start_key = MapRefCountKey(map_id);
510 it->Seek(map_start_key);
511 // If the key is not found, the status of the iterator won't be IsNotFound(),
512 // but the iterator will be invalid. The map needs to exist, otherwise we have
513 // a stale map_id in the database.
514 if (!ConsistencyCheck(it->Valid()))
516 if (!DatabaseErrorCheck(it->status().ok()))
518 // Skip the dummy entry "map-<mapid>-".
519 for (it->Next(); it->Valid(); it->Next()) {
520 std::string key = it->key().ToString();
521 if (key.find(map_start_key) != 0) {
522 // Iterated past the keys in this map.
525 // Key is of the form "map-<mapid>-<key>".
526 base::string16 key16 = UTF8ToUTF16(key.substr(map_start_key.length()));
528 (*result)[key16] = base::NullableString16();
530 // Convert the raw data stored in std::string (it->value()) to raw data
531 // stored in base::string16.
532 size_t len = it->value().size() / sizeof(char16);
533 const char16* data_ptr =
534 reinterpret_cast<const char16*>(it->value().data());
536 base::NullableString16(base::string16(data_ptr, len), false);
542 void SessionStorageDatabase::WriteValuesToMap(const std::string& map_id,
543 const DOMStorageValuesMap& values,
544 leveldb::WriteBatch* batch) {
545 for (DOMStorageValuesMap::const_iterator it = values.begin();
548 base::NullableString16 value = it->second;
549 std::string key = MapKey(map_id, UTF16ToUTF8(it->first));
550 if (value.is_null()) {
553 // Convert the raw data stored in base::string16 to raw data stored in
555 const char* data = reinterpret_cast<const char*>(value.string().data());
556 size_t size = value.string().size() * 2;
557 batch->Put(key, leveldb::Slice(data, size));
562 bool SessionStorageDatabase::GetMapRefCount(const std::string& map_id,
564 std::string ref_count_string;
565 leveldb::Status s = db_->Get(leveldb::ReadOptions(),
566 MapRefCountKey(map_id), &ref_count_string);
567 if (!ConsistencyCheck(s.ok()))
569 bool conversion_ok = base::StringToInt64(ref_count_string, ref_count);
570 return ConsistencyCheck(conversion_ok);
573 bool SessionStorageDatabase::IncreaseMapRefCount(const std::string& map_id,
574 leveldb::WriteBatch* batch) {
575 // Increase the ref count for the map.
577 if (!GetMapRefCount(map_id, &old_ref_count))
579 batch->Put(MapRefCountKey(map_id), base::Int64ToString(++old_ref_count));
583 bool SessionStorageDatabase::DecreaseMapRefCount(const std::string& map_id,
585 leveldb::WriteBatch* batch) {
586 // Decrease the ref count for the map.
588 if (!GetMapRefCount(map_id, &ref_count))
590 if (!ConsistencyCheck(decrease <= ref_count))
592 ref_count -= decrease;
594 batch->Put(MapRefCountKey(map_id), base::Int64ToString(ref_count));
596 // Clear all keys in the map.
597 if (!ClearMap(map_id, batch))
599 batch->Delete(MapRefCountKey(map_id));
604 bool SessionStorageDatabase::ClearMap(const std::string& map_id,
605 leveldb::WriteBatch* batch) {
606 DOMStorageValuesMap values;
607 if (!ReadMap(map_id, leveldb::ReadOptions(), &values, true))
609 for (DOMStorageValuesMap::const_iterator it = values.begin();
610 it != values.end(); ++it)
611 batch->Delete(MapKey(map_id, UTF16ToUTF8(it->first)));
615 bool SessionStorageDatabase::DeepCopyArea(
616 const std::string& namespace_id, const GURL& origin, bool copy_data,
617 std::string* map_id, leveldb::WriteBatch* batch) {
618 // Example, data before deep copy:
619 // | namespace-1- (1 = namespace id)| dummy |
620 // | namespace-1-origin1 | 1 (mapid) |
621 // | namespace-2- | dummy |
622 // | namespace-2-origin1 | 1 (mapid) << references the same map
623 // | map-1- | 2 (refcount) |
626 // Example, data after deep copy copy:
627 // | namespace-1-(1 = namespace id) | dummy |
628 // | namespace-1-origin1 | 1 (mapid) |
629 // | namespace-2- | dummy |
630 // | namespace-2-origin1 | 2 (mapid) << references the new map
631 // | map-1- | 1 (dec. refcount) |
633 // | map-2- | 1 (refcount) |
636 // Read the values from the old map here. If we don't need to copy the data,
637 // this can stay empty.
638 DOMStorageValuesMap values;
639 if (copy_data && !ReadMap(*map_id, leveldb::ReadOptions(), &values, false))
641 if (!DecreaseMapRefCount(*map_id, 1, batch))
643 // Create a new map (this will also break the association to the old map) and
644 // write the old data into it. This will write the id of the created map into
646 if (!CreateMapForArea(namespace_id, origin, map_id, batch))
648 WriteValuesToMap(*map_id, values, batch);
652 std::string SessionStorageDatabase::NamespaceStartKey(
653 const std::string& namespace_id) {
654 return base::StringPrintf("namespace-%s-", namespace_id.c_str());
657 std::string SessionStorageDatabase::NamespaceKey(
658 const std::string& namespace_id, const std::string& origin) {
659 return base::StringPrintf("namespace-%s-%s", namespace_id.c_str(),
663 const char* SessionStorageDatabase::NamespacePrefix() {
667 std::string SessionStorageDatabase::MapRefCountKey(const std::string& map_id) {
668 return base::StringPrintf("map-%s-", map_id.c_str());
671 std::string SessionStorageDatabase::MapKey(const std::string& map_id,
672 const std::string& key) {
673 return base::StringPrintf("map-%s-%s", map_id.c_str(), key.c_str());
676 const char* SessionStorageDatabase::NextMapIdKey() {
677 return "next-map-id";
680 } // namespace content