// The environment variable that overrides the maximum age of entries in the
// Stat cache. A value of 0 (the default) means nothing is cached.
constexpr char kStatCacheMaxAge[] = "GCS_STAT_CACHE_MAX_AGE";
-constexpr uint64 kStatCacheDefaultMaxAge = 0;
+constexpr uint64 kStatCacheDefaultMaxAge = 5;
// The environment variable that overrides the maximum number of entries in the
// Stat cache.
constexpr char kStatCacheMaxEntries[] = "GCS_STAT_CACHE_MAX_ENTRIES";
/// A GCS-based implementation of a random access file with an LRU block cache.
class GcsRandomAccessFile : public RandomAccessFile {
public:
- GcsRandomAccessFile(const string& filename, FileBlockCache* file_block_cache)
- : filename_(filename), file_block_cache_(file_block_cache) {}
+ using SignatureGenFun =
+ std::function<Status(const string& filename, int64* file_signature)>;
+
+ GcsRandomAccessFile(const string& filename, FileBlockCache* file_block_cache,
+ const SignatureGenFun& signature_gen_fun)
+ : filename_(filename),
+ file_block_cache_(file_block_cache),
+ signature_gen_fun_(signature_gen_fun) {}
/// The implementation of reads with an LRU block cache. Thread safe.
Status Read(uint64 offset, size_t n, StringPiece* result,
char* scratch) const override {
+ if (file_block_cache_->IsCacheEnabled()) {
+ int64 signature;
+ TF_RETURN_IF_ERROR(signature_gen_fun_(filename_, &signature));
+ if (!file_block_cache_->ValidateAndUpdateFileSignature(filename_,
+ signature)) {
+ VLOG(1) << "File " << filename_
+ << " signature has been changed. Refreshing the cache.";
+ }
+ }
+
*result = StringPiece();
size_t bytes_transferred;
TF_RETURN_IF_ERROR(file_block_cache_->Read(filename_, offset, n, scratch,
&bytes_transferred));
*result = StringPiece(scratch, bytes_transferred);
- string checkpoint_ending = "/checkpoint";
- // Check if the file is the checkpoint file as we should not be caching
- // that. As it's contents are updated and used for iterating checkpoints.
- if (std::equal(checkpoint_ending.rbegin(), checkpoint_ending.rend(),
- filename_.rbegin())) {
- // Remove the checkpoint file from the cache
- file_block_cache_->RemoveFile(filename_);
- }
+
if (bytes_transferred < n) {
// This is not an error per se. The RandomAccessFile interface expects
// that Read returns OutOfRange if fewer bytes were read than requested.
const string filename_;
/// The LRU block cache for this file.
mutable FileBlockCache* file_block_cache_; // not owned
+
+ const SignatureGenFun signature_gen_fun_;
};
/// \brief GCS-based implementation of a writeable file.
if (GetEnvVar(kStatCacheMaxEntries, strings::safe_strtou64, &value)) {
stat_cache_max_entries = value;
}
- stat_cache_.reset(new ExpiringLRUCache<FileStatistics>(
- stat_cache_max_age, stat_cache_max_entries));
+ stat_cache_.reset(new ExpiringLRUCache<GcsFileStat>(stat_cache_max_age,
+ stat_cache_max_entries));
// Apply overrides for the matching paths cache max age and max entries, if
// provided.
uint64 matching_paths_cache_max_age = kMatchingPathsCacheDefaultMaxAge;
const string& fname, std::unique_ptr<RandomAccessFile>* result) {
string bucket, object;
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
- result->reset(new GcsRandomAccessFile(fname, file_block_cache_.get()));
+ result->reset(new GcsRandomAccessFile(
+ fname, file_block_cache_.get(),
+ [this, bucket, object](const string& fname, int64* signature) {
+ GcsFileStat stat;
+ TF_RETURN_IF_ERROR(stat_cache_->LookupOrCompute(
+ fname, &stat,
+ [this, bucket, object](const string& fname, GcsFileStat* stat) {
+ return UncachedStatForObject(fname, bucket, object, stat);
+ }));
+ *signature = stat.generation_number;
+ return Status::OK();
+ }));
return Status::OK();
}
if (bytes_read < n) {
// Check stat cache to see if we encountered an interrupted read.
- FileStatistics stat;
+ GcsFileStat stat;
if (stat_cache_->Lookup(filename, &stat)) {
- if (offset + bytes_read < stat.length) {
+ if (offset + bytes_read < stat.base.length) {
return errors::Internal(strings::Printf(
"File contents are inconsistent for file: %s @ %lu.",
filename.c_str(), offset));
if (!result) {
return errors::Internal("'result' cannot be nullptr.");
}
- FileStatistics not_used_stat;
+ GcsFileStat not_used_stat;
const Status status = StatForObject(fname, bucket, object, ¬_used_stat);
switch (status.code()) {
case errors::Code::OK:
}
}
-Status GcsFileSystem::StatForObject(const string& fname, const string& bucket,
- const string& object,
- FileStatistics* stat) {
- if (!stat) {
- return errors::Internal("'stat' cannot be nullptr.");
- }
- if (object.empty()) {
- return errors::InvalidArgument(strings::Printf(
- "'object' must be a non-empty string. (File: %s)", fname.c_str()));
- }
+Status GcsFileSystem::UncachedStatForObject(const string& fname,
+ const string& bucket,
+ const string& object,
+ GcsFileStat* stat) {
+ std::vector<char> output_buffer;
+ std::unique_ptr<HttpRequest> request;
+ TF_RETURN_WITH_CONTEXT_IF_ERROR(CreateHttpRequest(&request),
+ " when reading metadata of gs://", bucket,
+ "/", object);
- StatCache::ComputeFunc compute_func = [this, &bucket, &object](
- const string& fname,
- FileStatistics* stat) {
- std::vector<char> output_buffer;
- std::unique_ptr<HttpRequest> request;
- TF_RETURN_WITH_CONTEXT_IF_ERROR(CreateHttpRequest(&request),
- " when reading metadata of gs://", bucket,
- "/", object);
+ request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket, "/o/",
+ request->EscapeString(object),
+ "?fields=size%2Cgeneration%2Cupdated"));
+ request->SetResultBuffer(&output_buffer);
+ request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata);
- request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket, "/o/",
- request->EscapeString(object),
- "?fields=size%2Cupdated"));
- request->SetResultBuffer(&output_buffer);
- request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata);
+ if (stats_ != nullptr) {
+ stats_->RecordStatObjectRequest();
+ }
- if (stats_ != nullptr) {
- stats_->RecordStatObjectRequest();
- }
+ TF_RETURN_WITH_CONTEXT_IF_ERROR(
+ request->Send(), " when reading metadata of gs://", bucket, "/", object);
- TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(),
- " when reading metadata of gs://", bucket,
- "/", object);
+ Json::Value root;
+ TF_RETURN_IF_ERROR(ParseJson(output_buffer, &root));
- Json::Value root;
- TF_RETURN_IF_ERROR(ParseJson(output_buffer, &root));
+ // Parse file size.
+ TF_RETURN_IF_ERROR(GetInt64Value(root, "size", &stat->base.length));
- // Parse file size.
- TF_RETURN_IF_ERROR(GetInt64Value(root, "size", &stat->length));
+ // Parse generation number.
+ TF_RETURN_IF_ERROR(
+ GetInt64Value(root, "generation", &stat->generation_number));
- // Parse file modification time.
- string updated;
- TF_RETURN_IF_ERROR(GetStringValue(root, "updated", &updated));
- TF_RETURN_IF_ERROR(ParseRfc3339Time(updated, &(stat->mtime_nsec)));
+ // Parse file modification time.
+ string updated;
+ TF_RETURN_IF_ERROR(GetStringValue(root, "updated", &updated));
+ TF_RETURN_IF_ERROR(ParseRfc3339Time(updated, &(stat->base.mtime_nsec)));
- VLOG(1) << "Stat of: gs://" << bucket << "/" << object << " -- "
- << " length: " << stat->length
- << "; mtime_nsec: " << stat->mtime_nsec << "; updated: " << updated;
+ VLOG(1) << "Stat of: gs://" << bucket << "/" << object << " -- "
+ << " length: " << stat->base.length
+ << " generation: " << stat->generation_number
+ << "; mtime_nsec: " << stat->base.mtime_nsec
+ << "; updated: " << updated;
- stat->is_directory = false;
- return Status::OK();
- };
+ stat->base.is_directory = false;
+ return Status::OK();
+}
- TF_RETURN_IF_ERROR(stat_cache_->LookupOrCompute(fname, stat, compute_func));
- if (stat->is_directory) {
+Status GcsFileSystem::StatForObject(const string& fname, const string& bucket,
+ const string& object, GcsFileStat* stat) {
+ if (!stat) {
+ return errors::Internal("'stat' cannot be nullptr.");
+ }
+ if (object.empty()) {
+ return errors::InvalidArgument(strings::Printf(
+ "'object' must be a non-empty string. (File: %s)", fname.c_str()));
+ }
+
+ TF_RETURN_IF_ERROR(stat_cache_->LookupOrCompute(
+ fname, stat,
+ [this, &bucket, &object](const string& fname, GcsFileStat* stat) {
+ return UncachedStatForObject(fname, bucket, object, stat);
+ }));
+ if (stat->base.is_directory) {
return errors::NotFound(fname, " is a directory.");
} else {
return Status::OK();
return errors::Internal("'result' cannot be nullptr.");
}
StatCache::ComputeFunc compute_func = [this](const string& dirname,
- FileStatistics* stat) {
+ GcsFileStat* stat) {
std::vector<string> children;
TF_RETURN_IF_ERROR(
GetChildrenBounded(dirname, 1, &children, true /* recursively */,
true /* include_self_directory_marker */));
if (!children.empty()) {
- *stat = DIRECTORY_STAT;
+ stat->base = DIRECTORY_STAT;
return Status::OK();
} else {
return errors::InvalidArgument("Not a directory!");
}
};
- FileStatistics stat;
+ GcsFileStat stat;
Status s = stat_cache_->LookupOrCompute(dirname, &stat, compute_func);
if (s.ok()) {
- *result = stat.is_directory;
+ *result = stat.base.is_directory;
return Status::OK();
}
if (errors::IsInvalidArgument(s)) {
return errors::NotFound("The specified bucket ", fname, " was not found.");
}
- const Status status = StatForObject(fname, bucket, object, stat);
+ GcsFileStat gcs_stat;
+ const Status status = StatForObject(fname, bucket, object, &gcs_stat);
if (status.ok()) {
+ *stat = gcs_stat.base;
return Status::OK();
}
if (status.code() != errors::Code::NOT_FOUND) {
EXPECT_EQ("6789", result);
}
-TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) {
+TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
// "0123456789abcde".
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
"Range: 0-8\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
kTestTimeoutConfig, nullptr /* gcs additional header */);
EXPECT_EQ("0123", result);
}
-TEST(GcsFileSystemTest, NewRandomAccessFile_CheckpointFile_WithBlockCache) {
- // Our underlying file in this test changes as new data comes in
- std::vector<HttpRequest*> requests(
- {new FakeHttpRequest(
- "Uri: https://storage.googleapis.com/bucket/checkpoint\n"
- "Auth Token: fake_token\n"
- "Range: 0-8\n"
- "Timeouts: 5 1 20\n",
- "012345678"),
- new FakeHttpRequest(
- "Uri: https://storage.googleapis.com/bucket/checkpoint\n"
- "Auth Token: fake_token\n"
- "Range: 0-8\n"
- "Timeouts: 5 1 20\n",
- "abcdefghi")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
-
- char scratch[100];
- StringPiece result;
- {
- // We are instantiating this in an enclosed scope to make sure after the
- // unique ptr goes out of scope, we can still access result.
- std::unique_ptr<RandomAccessFile> file;
- TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/checkpoint", &file));
-
- // Read the first chunk. The cache will be populated with the first block of
- // 9 bytes.
- scratch[5] = 'x';
- TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
- EXPECT_EQ("0123", result);
- EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
-
- // The second chunk should not be in cache so we make a new request
- // As the checkpoint file should not be cached
- TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
- EXPECT_EQ("abcd", result);
- EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
- }
-}
-
TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) {
// Our underlying file in this test is a 15 byte file with contents
// "0123456789abcde".
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
"Range: 0-8\n"
"Timeouts: 5 1 20\n",
"012345678"),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
"Range: 0-8\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
kTestTimeoutConfig, nullptr /* gcs additional header */);
// Our underlying file in this test is a 16 byte file with contents
// "0123456789abcdef".
std::vector<HttpRequest*> requests(
- {new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
+ {new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "object?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
"Auth Token: fake_token\n"
"Range: 0-7\n"
"Timeouts: 5 1 20\n",
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
kTestTimeoutConfig, nullptr /* gcs additional header */);
}
}
+TEST(GcsFileSystemTest,
+ NewRandomAccessFile_WithBlockCache_FileSignatureChanges) {
+ std::vector<HttpRequest*> requests(
+ {new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-8\n"
+ "Timeouts: 5 1 20\n",
+ "01234"),
+ new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"5\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-8\n"
+ "Timeouts: 5 1 20\n",
+ "43210")});
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
+ 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, nullptr /* gcs additional header */);
+
+ std::unique_ptr<RandomAccessFile> file;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
+
+ char scratch[5];
+ StringPiece result;
+
+ // First read.
+ TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
+ EXPECT_EQ("01234", result);
+
+ // Second read. File signatures are different.
+ TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
+ EXPECT_EQ("43210", result);
+}
+
TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
std::vector<HttpRequest*> requests;
GcsFileSystem fs(
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "random_access.txt?fields=size%2Cupdated\n"
+ "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"6\","
+ strings::StrCat("{\"size\": \"6\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
TEST(GcsFileSystemTest, NewWritableFile) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Range: 0-7\n"
"Put body: content1,content2\n",
""),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:15:34.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Range: 0-7\n"
"Timeouts: 5 1 20\n",
"01234567")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, nullptr /* gcs additional header */);
// Read from the file first, to fill the block cache.
std::unique_ptr<RandomAccessFile> rfile;
// path.
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Range: 0-7\n"
"Put: yes\n",
"", Status::OK(), nullptr, {}, 201),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Range: 0-7\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
kTestTimeoutConfig, nullptr /* gcs additional header */);
TEST(GcsFileSystemTest, NewAppendableFile) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
"Auth Token: fake_token\n"
"Range: 0-31\n"
"Put body: content1,content2\n",
""),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:25:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
"Auth Token: fake_token\n"
"Range: 0-31\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
32 /* block size */, 32 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
kTestTimeoutConfig, nullptr /* gcs additional header */);
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Frandom_access.txt?fields=size%2Cupdated\n"
+ "path%2Frandom_access.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"", content.size(),
- "\", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ strings::StrCat("{\"size\": \"", content.size(), "\"",
+ ", \"generation\": \"1\"",
+ ", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
strings::StrCat("Uri: https://storage.googleapis.com/bucket/"
"path%2Frandom_access.txt\n"
TEST(GcsFileSystemTest, FileExists_YesAsObject) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Ffile1.txt?fields=size%2Cupdated\n"
+ "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsubfolder?fields=size%2Cupdated\n"
+ "path%2Fsubfolder?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Ffile1.txt?fields=size%2Cupdated\n"
+ "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Ffile1.txt?fields=size%2Cupdated\n"
+ "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsubfolder?fields=size%2Cupdated\n"
+ "path%2Fsubfolder?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
TEST(GcsFileSystemTest, DeleteFile) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
"Delete: yes\n",
""),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
16 /* block size */, 16 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
kTestTimeoutConfig, nullptr /* gcs additional header */);
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/file.txt\n"
""),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
TEST(GcsFileSystemTest, GetFileSize) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
TEST(GcsFileSystemTest, RenameFile_Object) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
"Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
"Timeouts: 5 1 20\n",
"76543210"),
- // IsDirectory is checking whether there are children objects.
- new FakeHttpRequest(
- "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
- "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
- "&maxResults=1\n"
- "Auth Token: fake_token\n"
- "Timeouts: 5 1 10\n",
- "{}"),
- // IsDirectory is checking if the path exists as an object.
- new FakeHttpRequest(
- "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsrc.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n"
- "Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
- "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"Delete: yes\n",
""),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
"Timeouts: 5 1 20\n",
"89abcdef"),
new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+ "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
+ "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+ new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
"Range: 0-15\n"
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
16 /* block size */, 64 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
kTestTimeoutConfig, nullptr /* gcs additional header */);
{// Stat the target file.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fdst.txt?fields=size%2Cupdated\n"
+ "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1000\","
+ strings::StrCat("{\"size\": \"1000\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// IsDirectory is checking whether there are children objects.
new FakeHttpRequest(
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsrc.txt?fields=size%2Cupdated\n"
+ "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
new FakeHttpRequest(
""),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fdst.txt?fields=size%2Cupdated\n"
+ "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsrc.txt?fields=size%2Cupdated\n"
+ "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
new FakeHttpRequest(
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Fsrc.txt?fields=size%2Cupdated\n"
+ "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
new FakeHttpRequest(
TEST(GcsFileSystemTest, Stat_Object) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "subfolder?fields=size%2Cupdated\n"
+ "subfolder?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path?fields=size%2Cupdated\n"
+ "path?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "subfolder?fields=size%2Cupdated\n"
+ "subfolder?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
"{}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
"{}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
// Checking if gs://bucket/path/file3.txt is an object - fails with 404.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path%2Ffile3.txt?fields=size%2Cupdated\n"
+ "path%2Ffile3.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "path?fields=size%2Cupdated\n"
+ "path?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
TEST(GcsFileSystemTest, Stat_StatsRecording) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
- "file.txt?fields=size%2Cupdated\n"
+ "file.txt?fields=size%2Cgeneration%2Cupdated\n"
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
- strings::StrCat("{\"size\": \"1010\","
+ strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(