1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/performance_monitor/database.h"
7 #include "base/file_util.h"
8 #include "base/files/file_path.h"
9 #include "base/json/json_reader.h"
10 #include "base/json/json_writer.h"
11 #include "base/logging.h"
12 #include "base/path_service.h"
13 #include "base/stl_util.h"
14 #include "base/strings/string_number_conversions.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "base/time/time.h"
17 #include "chrome/browser/performance_monitor/key_builder.h"
18 #include "chrome/common/chrome_paths.h"
19 #include "content/public/browser/browser_thread.h"
20 #include "third_party/leveldatabase/src/include/leveldb/db.h"
21 #include "third_party/leveldatabase/src/include/leveldb/iterator.h"
22 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
24 namespace performance_monitor {
26 const char kDbDir[] = "Performance Monitor Databases";
27 const char kRecentDb[] = "Recent Metrics";
28 const char kMaxValueDb[] = "Max Value Metrics";
29 const char kEventDb[] = "Events";
30 const char kStateDb[] = "Configuration";
31 const char kActiveIntervalDb[] = "Active Interval";
32 const char kMetricDb[] = "Metrics";
33 const double kDefaultMaxValue = 0.0;
35 // If the db is quiet for this number of minutes, then it is considered down.
36 const base::TimeDelta kActiveIntervalTimeout = base::TimeDelta::FromMinutes(5);
38 TimeRange ActiveIntervalToTimeRange(const std::string& start_time,
39 const std::string& end_time) {
40 int64 start_time_int = 0;
41 int64 end_time_int = 0;
42 base::StringToInt64(start_time, &start_time_int);
43 base::StringToInt64(end_time, &end_time_int);
44 return TimeRange(base::Time::FromInternalValue(start_time_int),
45 base::Time::FromInternalValue(end_time_int));
48 double StringToDouble(const std::string& s) {
50 if (!base::StringToDouble(s, &value))
51 LOG(ERROR) << "Failed to convert " << s << " to double.";
55 // Returns an event from the given JSON string; the scoped_ptr will be NULL if
56 // we are unable to properly parse the JSON.
57 scoped_ptr<Event> EventFromJSON(const std::string& data) {
58 Value* value = base::JSONReader::Read(data);
59 DictionaryValue* dict = NULL;
60 if (!value || !value->GetAsDictionary(&dict))
61 return scoped_ptr<Event>();
63 return Event::FromValue(scoped_ptr<DictionaryValue>(dict));
68 const char Database::kDatabaseSequenceToken[] =
69 "_performance_monitor_db_sequence_token_";
71 TimeRange::TimeRange() {
74 TimeRange::TimeRange(base::Time start_time, base::Time end_time)
79 TimeRange::~TimeRange() {
82 base::Time Database::SystemClock::GetTime() {
83 return base::Time::Now();
87 scoped_ptr<Database> Database::Create(base::FilePath path) {
88 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
90 CHECK(PathService::Get(chrome::DIR_USER_DATA, &path));
91 path = path.AppendASCII(kDbDir);
93 scoped_ptr<Database> database;
94 if (!base::DirectoryExists(path) && !file_util::CreateDirectory(path))
95 return database.Pass();
96 database.reset(new Database(path));
98 // If the database did not initialize correctly, return a NULL scoped_ptr.
99 if (!database->valid_)
101 return database.Pass();
104 bool Database::AddStateValue(const std::string& key, const std::string& value) {
105 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
106 UpdateActiveInterval();
107 leveldb::Status insert_status = state_db_->Put(write_options_, key, value);
108 return insert_status.ok();
111 std::string Database::GetStateValue(const std::string& key) {
112 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
114 state_db_->Get(read_options_, key, &result);
118 bool Database::AddEvent(const Event& event) {
119 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
120 UpdateActiveInterval();
122 base::JSONWriter::Write(event.data(), &value);
123 std::string key = key_builder_->CreateEventKey(event.time(), event.type());
124 leveldb::Status status = event_db_->Put(write_options_, key, value);
128 std::vector<TimeRange> Database::GetActiveIntervals(const base::Time& start,
129 const base::Time& end) {
130 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
131 std::vector<TimeRange> results;
132 std::string start_key = key_builder_->CreateActiveIntervalKey(start);
133 std::string end_key = key_builder_->CreateActiveIntervalKey(end);
134 scoped_ptr<leveldb::Iterator> it(active_interval_db_->NewIterator(
137 // If the interator is valid, we check the previous value in case we jumped
138 // into the middle of an active interval. If the iterator is not valid, then
139 // the key may be in the current active interval.
144 if (it->Valid() && it->value().ToString() > start_key) {
145 results.push_back(ActiveIntervalToTimeRange(it->key().ToString(),
146 it->value().ToString()));
149 for (it->Seek(start_key);
150 it->Valid() && it->key().ToString() < end_key;
152 results.push_back(ActiveIntervalToTimeRange(it->key().ToString(),
153 it->value().ToString()));
158 Database::EventVector Database::GetEvents(EventType type,
159 const base::Time& start,
160 const base::Time& end) {
161 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
163 std::string start_key =
164 key_builder_->CreateEventKey(start, EVENT_UNDEFINED);
165 std::string end_key =
166 key_builder_->CreateEventKey(end, EVENT_NUMBER_OF_EVENTS);
167 leveldb::WriteBatch invalid_entries;
168 scoped_ptr<leveldb::Iterator> it(event_db_->NewIterator(read_options_));
169 for (it->Seek(start_key);
170 it->Valid() && it->key().ToString() <= end_key;
172 if (type != EVENT_UNDEFINED) {
174 key_builder_->EventKeyToEventType(it->key().ToString());
175 if (key_type != type)
178 scoped_ptr<Event> event = EventFromJSON(it->value().ToString());
180 invalid_entries.Delete(it->key());
181 LOG(ERROR) << "Found invalid event in the database. JSON: '"
182 << it->value().ToString()
183 << "'. Erasing event from the database.";
186 events.push_back(linked_ptr<Event>(event.release()));
188 event_db_->Write(write_options_, &invalid_entries);
192 Database::EventTypeSet Database::GetEventTypes(const base::Time& start,
193 const base::Time& end) {
194 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
195 EventTypeSet results;
196 std::string start_key =
197 key_builder_->CreateEventKey(start, EVENT_UNDEFINED);
198 std::string end_key =
199 key_builder_->CreateEventKey(end, EVENT_NUMBER_OF_EVENTS);
200 scoped_ptr<leveldb::Iterator> it(event_db_->NewIterator(read_options_));
201 for (it->Seek(start_key);
202 it->Valid() && it->key().ToString() <= end_key;
205 key_builder_->EventKeyToEventType(it->key().ToString());
206 results.insert(key_type);
211 bool Database::AddMetric(const std::string& activity,
212 const Metric& metric) {
213 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
214 if (!metric.IsValid()) {
215 DLOG(ERROR) << "Metric to be added is invalid. Type: " << metric.type
216 << ", Time: " << metric.time.ToInternalValue()
217 << ", Value: " << metric.value << ". Ignoring.";
221 UpdateActiveInterval();
222 std::string recent_key =
223 key_builder_->CreateRecentKey(metric.time, metric.type, activity);
224 std::string metric_key =
225 key_builder_->CreateMetricKey(metric.time, metric.type, activity);
226 std::string recent_map_key =
227 key_builder_->CreateRecentMapKey(metric.type, activity);
228 // Use recent_map_ to quickly find the key that must be removed.
229 RecentMap::iterator old_it = recent_map_.find(recent_map_key);
230 if (old_it != recent_map_.end())
231 recent_db_->Delete(write_options_, old_it->second);
232 recent_map_[recent_map_key] = recent_key;
233 leveldb::Status recent_status =
234 recent_db_->Put(write_options_, recent_key, metric.ValueAsString());
235 leveldb::Status metric_status =
236 metric_db_->Put(write_options_, metric_key, metric.ValueAsString());
238 bool max_value_success =
239 UpdateMaxValue(activity, metric.type, metric.ValueAsString());
240 return recent_status.ok() && metric_status.ok() && max_value_success;
243 bool Database::UpdateMaxValue(const std::string& activity,
245 const std::string& value) {
246 std::string max_value_key(
247 key_builder_->CreateMaxValueKey(metric, activity));
248 bool has_key = ContainsKey(max_value_map_, max_value_key);
249 if ((has_key && StringToDouble(value) > max_value_map_[max_value_key]) ||
251 max_value_map_[max_value_key] = StringToDouble(value);
252 return max_value_db_->Put(write_options_, max_value_key, value).ok();
258 Database::MetricTypeSet Database::GetActiveMetrics(const base::Time& start,
259 const base::Time& end) {
260 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
261 std::string recent_start_key = key_builder_->CreateRecentKey(
262 start, static_cast<MetricType>(0), std::string());
263 std::string recent_end_key = key_builder_->CreateRecentKey(
264 end, METRIC_NUMBER_OF_METRICS, std::string());
265 std::string recent_end_of_time_key = key_builder_->CreateRecentKey(
266 clock_->GetTime(), METRIC_NUMBER_OF_METRICS, std::string());
268 MetricTypeSet active_metrics;
269 // Get all the guaranteed metrics.
270 scoped_ptr<leveldb::Iterator> recent_it(
271 recent_db_->NewIterator(read_options_));
272 for (recent_it->Seek(recent_start_key);
273 recent_it->Valid() && recent_it->key().ToString() <= recent_end_key;
275 RecentKey split_key =
276 key_builder_->SplitRecentKey(recent_it->key().ToString());
277 active_metrics.insert(split_key.type);
279 // Get all the possible metrics (metrics that may have been updated after
281 MetricTypeSet possible_metrics;
282 for (recent_it->Seek(recent_end_key);
283 recent_it->Valid() &&
284 recent_it->key().ToString() <= recent_end_of_time_key;
286 RecentKey split_key =
287 key_builder_->SplitRecentKey(recent_it->key().ToString());
288 possible_metrics.insert(split_key.type);
290 MetricTypeSet::iterator possible_it;
291 scoped_ptr<leveldb::Iterator> metric_it(
292 metric_db_->NewIterator(read_options_));
293 for (possible_it = possible_metrics.begin();
294 possible_it != possible_metrics.end();
296 std::string metric_start_key =
297 key_builder_->CreateMetricKey(start, *possible_it,std::string());
298 std::string metric_end_key =
299 key_builder_->CreateMetricKey(end, *possible_it, std::string());
300 metric_it->Seek(metric_start_key);
301 // Stats in the timerange from any activity makes the metric active.
302 if (metric_it->Valid() && metric_it->key().ToString() <= metric_end_key) {
303 active_metrics.insert(*possible_it);
307 return active_metrics;
310 std::set<std::string> Database::GetActiveActivities(MetricType metric_type,
311 const base::Time& start) {
312 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
313 std::set<std::string> results;
314 std::string start_key = key_builder_->CreateRecentKey(
315 start, static_cast<MetricType>(0), std::string());
316 scoped_ptr<leveldb::Iterator> it(recent_db_->NewIterator(read_options_));
317 for (it->Seek(start_key); it->Valid(); it->Next()) {
318 RecentKey split_key =
319 key_builder_->SplitRecentKey(it->key().ToString());
320 if (split_key.type == metric_type)
321 results.insert(split_key.activity);
326 double Database::GetMaxStatsForActivityAndMetric(const std::string& activity,
328 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
329 std::string max_value_key(
330 key_builder_->CreateMaxValueKey(metric, activity));
331 if (ContainsKey(max_value_map_, max_value_key))
332 return max_value_map_[max_value_key];
333 return kDefaultMaxValue;
336 bool Database::GetRecentStatsForActivityAndMetric(const std::string& activity,
337 MetricType metric_type,
339 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
340 std::string recent_map_key =
341 key_builder_->CreateRecentMapKey(metric_type, activity);
342 if (!ContainsKey(recent_map_, recent_map_key))
344 std::string recent_key = recent_map_[recent_map_key];
347 leveldb::Status status = recent_db_->Get(read_options_, recent_key, &result);
349 *metric = Metric(metric_type,
350 key_builder_->SplitRecentKey(recent_key).time,
355 scoped_ptr<Database::MetricVector> Database::GetStatsForActivityAndMetric(
356 const std::string& activity,
357 MetricType metric_type,
358 const base::Time& start,
359 const base::Time& end) {
360 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
361 scoped_ptr<MetricVector> results(new MetricVector());
362 std::string start_key =
363 key_builder_->CreateMetricKey(start, metric_type, activity);
364 std::string end_key =
365 key_builder_->CreateMetricKey(end, metric_type, activity);
366 leveldb::WriteBatch invalid_entries;
367 scoped_ptr<leveldb::Iterator> it(metric_db_->NewIterator(read_options_));
368 for (it->Seek(start_key);
369 it->Valid() && it->key().ToString() <= end_key;
371 MetricKey split_key =
372 key_builder_->SplitMetricKey(it->key().ToString());
373 if (split_key.activity == activity) {
374 Metric metric(metric_type, split_key.time, it->value().ToString());
375 if (!metric.IsValid()) {
376 invalid_entries.Delete(it->key());
377 LOG(ERROR) << "Found bad metric in the database. Type: "
378 << metric.type << ", Time: " << metric.time.ToInternalValue()
379 << ", Value: " << metric.value
380 << ". Erasing metric from database.";
383 results->push_back(metric);
386 metric_db_->Write(write_options_, &invalid_entries);
387 return results.Pass();
390 Database::MetricVectorMap Database::GetStatsForMetricByActivity(
391 MetricType metric_type,
392 const base::Time& start,
393 const base::Time& end) {
394 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
395 MetricVectorMap results;
396 std::string start_key =
397 key_builder_->CreateMetricKey(start, metric_type, std::string());
398 std::string end_key =
399 key_builder_->CreateMetricKey(end, metric_type, std::string());
400 leveldb::WriteBatch invalid_entries;
401 scoped_ptr<leveldb::Iterator> it(metric_db_->NewIterator(read_options_));
402 for (it->Seek(start_key);
403 it->Valid() && it->key().ToString() <= end_key;
405 MetricKey split_key = key_builder_->SplitMetricKey(it->key().ToString());
406 if (!results[split_key.activity].get()) {
407 results[split_key.activity] =
408 linked_ptr<MetricVector >(new MetricVector());
410 Metric metric(metric_type, split_key.time, it->value().ToString());
411 if (!metric.IsValid()) {
412 invalid_entries.Delete(it->key());
413 LOG(ERROR) << "Found bad metric in the database. Type: "
414 << metric.type << ", Time: " << metric.time.ToInternalValue()
415 << ", Value: " << metric.value
416 << ". Erasing metric from database.";
419 results[split_key.activity]->push_back(metric);
421 metric_db_->Write(write_options_, &invalid_entries);
425 Database::Database(const base::FilePath& path)
426 : key_builder_(new KeyBuilder()),
428 read_options_(leveldb::ReadOptions()),
429 write_options_(leveldb::WriteOptions()),
435 clock_ = scoped_ptr<Clock>(new SystemClock());
439 Database::~Database() {
442 bool Database::InitDBs() {
443 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
444 leveldb::Options open_options;
445 open_options.max_open_files = 0; // Use minimum.
446 open_options.create_if_missing = true;
448 // TODO (rdevlin.cronin): This code is ugly. Fix it.
449 recent_db_ = SafelyOpenDatabase(open_options,
451 true); // fix if damaged
452 max_value_db_ = SafelyOpenDatabase(open_options,
454 true); // fix if damaged
455 state_db_ = SafelyOpenDatabase(open_options,
457 true); // fix if damaged
458 active_interval_db_ = SafelyOpenDatabase(open_options,
460 true); // fix if damaged
461 metric_db_ = SafelyOpenDatabase(open_options,
463 true); // fix if damaged
464 event_db_ = SafelyOpenDatabase(open_options,
466 true); // fix if damaged
467 return recent_db_ && max_value_db_ && state_db_ &&
468 active_interval_db_ && metric_db_ && event_db_;
471 scoped_ptr<leveldb::DB> Database::SafelyOpenDatabase(
472 const leveldb::Options& options,
473 const std::string& path,
474 bool fix_if_damaged) {
475 #if defined(OS_POSIX)
476 std::string name = path_.AppendASCII(path).value();
477 #elif defined(OS_WIN)
478 std::string name = WideToUTF8(path_.AppendASCII(path).value());
481 leveldb::DB* database;
482 leveldb::Status status = leveldb::DB::Open(options, name, &database);
483 // If all goes well, return the database.
485 return scoped_ptr<leveldb::DB>(database);
487 // Return NULL and print the error if we either didn't find the database and
488 // don't want to create it, or if we don't want to try to fix it.
489 if ((status.IsNotFound() && !options.create_if_missing) || !fix_if_damaged) {
490 LOG(ERROR) << status.ToString();
491 return scoped_ptr<leveldb::DB>();
493 // Otherwise, we have an error (corruption, io error, or a not found error
494 // even if we tried to create it).
496 // First, we try again.
497 LOG(ERROR) << "Database error: " << status.ToString() << ". Trying again.";
498 status = leveldb::DB::Open(options, name, &database);
499 // If we fail on corruption, we can try to repair it.
500 if (status.IsCorruption()) {
501 LOG(ERROR) << "Database corrupt (second attempt). Trying to repair.";
502 status = leveldb::RepairDB(name, options);
503 // If the repair succeeds and we can open the database, return the
504 // database. Otherwise, continue on.
506 status = leveldb::DB::Open(options, name, &database);
508 return scoped_ptr<leveldb::DB>(database);
510 LOG(ERROR) << "Repair failed. Deleting database.";
512 // Next, try to delete and recreate the database. Return NULL if we fail
513 // on either of these steps.
514 status = leveldb::DestroyDB(name, options);
516 LOG(ERROR) << "Failed to delete database. " << status.ToString();
517 return scoped_ptr<leveldb::DB>();
519 // If we don't have the create_if_missing option, add it (it's safe to
520 // assume this is okay, since we have permission to |fix_if_damaged|).
521 if (!options.create_if_missing) {
522 leveldb::Options create_options(options);
523 create_options.create_if_missing = true;
524 status = leveldb::DB::Open(create_options, name, &database);
526 status = leveldb::DB::Open(options, name, &database);
528 // There's nothing else we can try at this point.
530 return scoped_ptr<leveldb::DB>(database);
531 // Return the database if we succeeded, or NULL on failure.
532 LOG(ERROR) << "Failed to recreate database. " << status.ToString();
533 return scoped_ptr<leveldb::DB>();
536 bool Database::Close() {
537 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
541 max_value_db_.reset();
543 active_interval_db_.reset();
544 start_time_key_.clear();
548 void Database::LoadRecents() {
549 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
551 scoped_ptr<leveldb::Iterator> it(recent_db_->NewIterator(read_options_));
552 for (it->SeekToFirst(); it->Valid(); it->Next()) {
553 RecentKey split_key = key_builder_->SplitRecentKey(it->key().ToString());
554 recent_map_[key_builder_->
555 CreateRecentMapKey(split_key.type, split_key.activity)] =
556 it->key().ToString();
560 void Database::LoadMaxValues() {
561 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
562 max_value_map_.clear();
563 scoped_ptr<leveldb::Iterator> it(max_value_db_->NewIterator(read_options_));
564 for (it->SeekToFirst(); it->Valid(); it->Next()) {
565 max_value_map_[it->key().ToString()] =
566 StringToDouble(it->value().ToString());
570 // TODO(chebert): Only update the active interval under certian circumstances
571 // eg. every 10 times or when forced.
572 void Database::UpdateActiveInterval() {
573 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
574 base::Time current_time = clock_->GetTime();
575 std::string end_time;
576 // If the last update was too long ago.
577 if (start_time_key_.empty() ||
578 current_time - last_update_time_ > kActiveIntervalTimeout) {
579 start_time_key_ = key_builder_->CreateActiveIntervalKey(current_time);
580 end_time = start_time_key_;
582 end_time = key_builder_->CreateActiveIntervalKey(clock_->GetTime());
584 last_update_time_ = current_time;
585 active_interval_db_->Put(write_options_, start_time_key_, end_time);
588 } // namespace performance_monitor