1 // Copyright (C) 2013 Google Inc.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 #include "retriever.h"
17 #include <libaddressinput/callback.h>
18 #include <libaddressinput/downloader.h>
19 #include <libaddressinput/storage.h>
20 #include <libaddressinput/util/basictypes.h>
21 #include <libaddressinput/util/scoped_ptr.h>
30 #include "fallback_data_store.h"
31 #include "time_to_string.h"
33 #include "util/stl_util.h"
36 namespace addressinput {
40 // The number of seconds after which data is considered stale. The staleness
41 // threshold is 30 days:
44 // 60 minutes per hour *
45 // 60 seconds per minute.
46 static const double kStaleDataAgeInSeconds = 30.0 * 24.0 * 60.0 * 60.0;
48 // The prefix for the timestamp line in the footer.
49 const char kTimestampPrefix[] = "timestamp=";
51 // The prefix for the checksum line in the footer.
52 const char kChecksumPrefix[] = "checksum=";
54 // The separator between lines of footer and data.
55 const char kSeparator = '\n';
57 // Returns |data| with attached checksum and current timestamp. Format:
60 // checksum=<checksum>
61 // timestamp=<timestamp>
63 // The timestamp is the time_t that was returned from time(NULL) function. The
64 // timestamp does not need to be portable because it is written and read only by
65 // Retriever. The value is somewhat human-readable: it is the number of seconds
68 // The checksum is the 32-character hexadecimal MD5 checksum of <data>. It is
69 // meant to protect from random file changes on disk.
70 void AppendTimestamp(std::string* data) {
71 std::string md5 = MD5String(*data);
73 data->push_back(kSeparator);
74 data->append(kChecksumPrefix);
77 data->push_back(kSeparator);
78 data->append(kTimestampPrefix);
79 data->append(TimeToString(time(NULL)));
82 // Places the footer value into |footer_value| parameter and the rest of the
83 // data into |data| parameter. Returns |true| if the footer format is valid.
84 bool ExtractFooter(scoped_ptr<std::string> data_and_footer,
85 const std::string& footer_prefix,
86 std::string* footer_value,
87 scoped_ptr<std::string>* data) {
88 assert(footer_value != NULL);
91 std::string::size_type separator_position =
92 data_and_footer->rfind(kSeparator);
93 if (separator_position == std::string::npos) {
97 std::string::size_type footer_start = separator_position + 1;
98 if (data_and_footer->compare(footer_start,
99 footer_prefix.length(),
100 footer_prefix) != 0) {
105 data_and_footer->substr(footer_start + footer_prefix.length());
106 *data = data_and_footer.Pass();
107 (*data)->resize(separator_position);
111 // Strips out the timestamp and checksum from |data_and_footer|. Validates the
112 // checksum. Saves the footer-less data into |data|. Compares the parsed
113 // timestamp with current time and saves the difference into |age_in_seconds|.
115 // The parameters should not be NULL.
117 // Returns |true| if |data_and_footer| is correctly formatted and has the
119 bool VerifyAndExtractTimestamp(const std::string& data_and_footer,
120 scoped_ptr<std::string>* data,
121 double* age_in_seconds) {
122 assert(data != NULL);
123 assert(age_in_seconds != NULL);
125 std::string timestamp_string;
126 scoped_ptr<std::string> checksum_and_data;
127 if (!ExtractFooter(make_scoped_ptr(new std::string(data_and_footer)),
128 kTimestampPrefix, ×tamp_string, &checksum_and_data)) {
132 time_t timestamp = atol(timestamp_string.c_str());
137 *age_in_seconds = difftime(time(NULL), timestamp);
138 if (*age_in_seconds < 0.0) {
142 std::string checksum;
143 if (!ExtractFooter(checksum_and_data.Pass(),
144 kChecksumPrefix, &checksum, data)) {
148 return checksum == MD5String(**data);
153 Retriever::Retriever(const std::string& validation_data_url,
154 scoped_ptr<Downloader> downloader,
155 scoped_ptr<Storage> storage)
156 : validation_data_url_(validation_data_url),
157 downloader_(downloader.Pass()),
158 storage_(storage.Pass()),
160 assert(validation_data_url_.length() > 0);
161 assert(validation_data_url_[validation_data_url_.length() - 1] == '/');
162 assert(storage_ != NULL);
163 assert(downloader_ != NULL);
166 Retriever::~Retriever() {
167 STLDeleteValues(&requests_);
170 void Retriever::Retrieve(const std::string& key,
171 scoped_ptr<Callback> retrieved) {
172 std::map<std::string, Callback*>::iterator request_it =
174 if (request_it != requests_.end()) {
175 // Abandon a previous request.
176 delete request_it->second;
177 requests_.erase(request_it);
180 requests_[key] = retrieved.release();
182 BuildCallback(this, &Retriever::OnDataRetrievedFromStorage));
185 void Retriever::OnDataRetrievedFromStorage(bool success,
186 const std::string& key,
187 const std::string& stored_data) {
189 scoped_ptr<std::string> unwrapped;
190 double age_in_seconds = 0.0;
191 if (VerifyAndExtractTimestamp(stored_data, &unwrapped, &age_in_seconds)) {
192 if (age_in_seconds < kStaleDataAgeInSeconds) {
193 InvokeCallbackForKey(key, success, *unwrapped);
196 stale_data_[key].swap(*unwrapped);
200 downloader_->Download(GetUrlForKey(key),
201 BuildScopedPtrCallback(this, &Retriever::OnDownloaded));
204 void Retriever::OnDownloaded(bool success,
205 const std::string& url,
206 scoped_ptr<std::string> downloaded_data) {
207 const std::string& key = GetKeyForUrl(url);
208 std::map<std::string, std::string>::iterator stale_data_it =
209 stale_data_.find(key);
212 InvokeCallbackForKey(key, success, *downloaded_data);
213 AppendTimestamp(downloaded_data.get());
214 storage_->Put(key, downloaded_data.Pass());
215 } else if (stale_data_it != stale_data_.end()) {
216 InvokeCallbackForKey(key, true, stale_data_it->second);
218 std::string fallback;
219 success = FallbackDataStore::Get(key, &fallback);
220 InvokeCallbackForKey(key, success, fallback);
223 if (stale_data_it != stale_data_.end()) {
224 stale_data_.erase(stale_data_it);
228 std::string Retriever::GetUrlForKey(const std::string& key) const {
229 return validation_data_url_ + key;
232 std::string Retriever::GetKeyForUrl(const std::string& url) const {
234 url.compare(0, validation_data_url_.length(), validation_data_url_) == 0
235 ? url.substr(validation_data_url_.length())
239 bool Retriever::IsValidationDataUrl(const std::string& url) const {
241 url.compare(0, validation_data_url_.length(), validation_data_url_) == 0;
244 void Retriever::InvokeCallbackForKey(const std::string& key,
246 const std::string& data) {
247 std::map<std::string, Callback*>::iterator iter =
249 if (iter == requests_.end()) {
250 // An abandoned request.
253 scoped_ptr<Callback> callback(iter->second);
254 requests_.erase(iter);
255 if (callback == NULL) {
258 (*callback)(success, key, data);
261 } // namespace addressinput