cc_library(
name = "retrying_file_system",
- srcs = [
- "retrying_file_system.cc",
- ],
hdrs = [
"retrying_file_system.h",
],
};
/// Google Cloud Storage implementation of a file system with retry on failures.
-class RetryingGcsFileSystem : public RetryingFileSystem {
+class RetryingGcsFileSystem : public RetryingFileSystem<GcsFileSystem> {
public:
- RetryingGcsFileSystem() : RetryingGcsFileSystem(new GcsFileSystem) {}
-
- void SetStats(GcsStatsInterface* stats) { underlying_->SetStats(stats); }
-
- private:
- explicit RetryingGcsFileSystem(GcsFileSystem* fs)
- : RetryingFileSystem(std::unique_ptr<FileSystem>(fs)), underlying_(fs) {}
-
- // TODO(b/74259157): Refactor RetryingFileSystem to avoid holding this ptr.
- GcsFileSystem* underlying_;
+ RetryingGcsFileSystem()
+ : RetryingFileSystem(std::unique_ptr<GcsFileSystem>(new GcsFileSystem)) {}
};
} // namespace tensorflow
+++ /dev/null
-/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/platform/cloud/retrying_file_system.h"
-#include <functional>
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/platform/cloud/retrying_utils.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/file_system.h"
-
-namespace tensorflow {
-
-namespace {
-
-class RetryingRandomAccessFile : public RandomAccessFile {
- public:
- RetryingRandomAccessFile(std::unique_ptr<RandomAccessFile> base_file,
- int64 delay_microseconds)
- : base_file_(std::move(base_file)),
- initial_delay_microseconds_(delay_microseconds) {}
-
- Status Read(uint64 offset, size_t n, StringPiece* result,
- char* scratch) const override {
- return RetryingUtils::CallWithRetries(
- std::bind(&RandomAccessFile::Read, base_file_.get(), offset, n, result,
- scratch),
- initial_delay_microseconds_);
- }
-
- private:
- std::unique_ptr<RandomAccessFile> base_file_;
- const int64 initial_delay_microseconds_;
-};
-
-class RetryingWritableFile : public WritableFile {
- public:
- RetryingWritableFile(std::unique_ptr<WritableFile> base_file,
- int64 delay_microseconds)
- : base_file_(std::move(base_file)),
- initial_delay_microseconds_(delay_microseconds) {}
-
- ~RetryingWritableFile() override {
- // Makes sure the retrying version of Close() is called in the destructor.
- Close().IgnoreError();
- }
-
- Status Append(const StringPiece& data) override {
- return RetryingUtils::CallWithRetries(
- std::bind(&WritableFile::Append, base_file_.get(), data),
- initial_delay_microseconds_);
- }
- Status Close() override {
- return RetryingUtils::CallWithRetries(
- std::bind(&WritableFile::Close, base_file_.get()),
- initial_delay_microseconds_);
- }
- Status Flush() override {
- return RetryingUtils::CallWithRetries(
- std::bind(&WritableFile::Flush, base_file_.get()),
- initial_delay_microseconds_);
- }
- Status Sync() override {
- return RetryingUtils::CallWithRetries(
- std::bind(&WritableFile::Sync, base_file_.get()),
- initial_delay_microseconds_);
- }
-
- private:
- std::unique_ptr<WritableFile> base_file_;
- const int64 initial_delay_microseconds_;
-};
-
-} // namespace
-
-Status RetryingFileSystem::NewRandomAccessFile(
- const string& filename, std::unique_ptr<RandomAccessFile>* result) {
- std::unique_ptr<RandomAccessFile> base_file;
- TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::NewRandomAccessFile, base_file_system_.get(),
- filename, &base_file),
- initial_delay_microseconds_));
- result->reset(new RetryingRandomAccessFile(std::move(base_file),
- initial_delay_microseconds_));
- return Status::OK();
-}
-
-Status RetryingFileSystem::NewWritableFile(
- const string& filename, std::unique_ptr<WritableFile>* result) {
- std::unique_ptr<WritableFile> base_file;
- TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::NewWritableFile, base_file_system_.get(), filename,
- &base_file),
- initial_delay_microseconds_));
- result->reset(new RetryingWritableFile(std::move(base_file),
- initial_delay_microseconds_));
- return Status::OK();
-}
-
-Status RetryingFileSystem::NewAppendableFile(
- const string& filename, std::unique_ptr<WritableFile>* result) {
- std::unique_ptr<WritableFile> base_file;
- TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::NewAppendableFile, base_file_system_.get(),
- filename, &base_file),
- initial_delay_microseconds_));
- result->reset(new RetryingWritableFile(std::move(base_file),
- initial_delay_microseconds_));
- return Status::OK();
-}
-
-Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile(
- const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>* result) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::NewReadOnlyMemoryRegionFromFile,
- base_file_system_.get(), filename, result),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::FileExists(const string& fname) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::FileExists, base_file_system_.get(), fname),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::Stat(const string& fname, FileStatistics* stat) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::Stat, base_file_system_.get(), fname, stat),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::GetChildren(const string& dir,
- std::vector<string>* result) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::GetChildren, base_file_system_.get(), dir, result),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::GetMatchingPaths(const string& pattern,
- std::vector<string>* result) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::GetMatchingPaths, base_file_system_.get(), pattern,
- result),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::DeleteFile(const string& fname) {
- return RetryingUtils::DeleteWithRetries(
- std::bind(&FileSystem::DeleteFile, base_file_system_.get(), fname),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::CreateDir(const string& dirname) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::CreateDir, base_file_system_.get(), dirname),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::DeleteDir(const string& dirname) {
- return RetryingUtils::DeleteWithRetries(
- std::bind(&FileSystem::DeleteDir, base_file_system_.get(), dirname),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::GetFileSize(const string& fname, uint64* file_size) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::GetFileSize, base_file_system_.get(), fname,
- file_size),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::RenameFile(const string& src, const string& target) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::RenameFile, base_file_system_.get(), src, target),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::IsDirectory(const string& dirname) {
- return RetryingUtils::CallWithRetries(
- std::bind(&FileSystem::IsDirectory, base_file_system_.get(), dirname),
- initial_delay_microseconds_);
-}
-
-Status RetryingFileSystem::DeleteRecursively(const string& dirname,
- int64* undeleted_files,
- int64* undeleted_dirs) {
- return RetryingUtils::DeleteWithRetries(
- std::bind(&FileSystem::DeleteRecursively, base_file_system_.get(),
- dirname, undeleted_files, undeleted_dirs),
- initial_delay_microseconds_);
-}
-
-void RetryingFileSystem::FlushCaches() { base_file_system_->FlushCaches(); }
-
-} // namespace tensorflow
#ifndef TENSORFLOW_CORE_PLATFORM_RETRYING_FILE_SYSTEM_H_
#define TENSORFLOW_CORE_PLATFORM_RETRYING_FILE_SYSTEM_H_
+#include <functional>
#include <string>
#include <vector>
+
+#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/platform/cloud/retrying_utils.h"
+#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
namespace tensorflow {
/// A wrapper to add retry logic to another file system.
+template <typename Underlying>
class RetryingFileSystem : public FileSystem {
public:
- RetryingFileSystem(std::unique_ptr<FileSystem> base_file_system,
+ RetryingFileSystem(std::unique_ptr<Underlying> base_file_system,
int64 delay_microseconds = 1000000)
: base_file_system_(std::move(base_file_system)),
initial_delay_microseconds_(delay_microseconds) {}
const string& filename,
std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
- Status FileExists(const string& fname) override;
+ Status FileExists(const string& fname) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::FileExists, base_file_system_.get(), fname),
+ initial_delay_microseconds_);
+ }
+
+ Status GetChildren(const string& dir, std::vector<string>* result) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::GetChildren, base_file_system_.get(), dir,
+ result),
+ initial_delay_microseconds_);
+ }
+
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* result) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::GetMatchingPaths, base_file_system_.get(),
+ pattern, result),
+ initial_delay_microseconds_);
+ }
+
+ Status Stat(const string& fname, FileStatistics* stat) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::Stat, base_file_system_.get(), fname, stat),
+ initial_delay_microseconds_);
+ }
+
+ Status DeleteFile(const string& fname) override {
+ return RetryingUtils::DeleteWithRetries(
+ std::bind(&FileSystem::DeleteFile, base_file_system_.get(), fname),
+ initial_delay_microseconds_);
+ }
+
+ Status CreateDir(const string& dirname) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::CreateDir, base_file_system_.get(), dirname),
+ initial_delay_microseconds_);
+ }
+
+ Status DeleteDir(const string& dirname) override {
+ return RetryingUtils::DeleteWithRetries(
+ std::bind(&FileSystem::DeleteDir, base_file_system_.get(), dirname),
+ initial_delay_microseconds_);
+ }
+
+ Status GetFileSize(const string& fname, uint64* file_size) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::GetFileSize, base_file_system_.get(), fname,
+ file_size),
+ initial_delay_microseconds_);
+ }
+
+ Status RenameFile(const string& src, const string& target) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::RenameFile, base_file_system_.get(), src,
+ target),
+ initial_delay_microseconds_);
+ }
+
+ Status IsDirectory(const string& dirname) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::IsDirectory, base_file_system_.get(), dirname),
+ initial_delay_microseconds_);
+ }
- Status GetChildren(const string& dir, std::vector<string>* result) override;
+ Status DeleteRecursively(const string& dirname, int64* undeleted_files,
+ int64* undeleted_dirs) override {
+ return RetryingUtils::DeleteWithRetries(
+ std::bind(&FileSystem::DeleteRecursively, base_file_system_.get(),
+ dirname, undeleted_files, undeleted_dirs),
+ initial_delay_microseconds_);
+ }
- Status GetMatchingPaths(const string& dir,
- std::vector<string>* result) override;
+ void FlushCaches() override { base_file_system_->FlushCaches(); }
- Status Stat(const string& fname, FileStatistics* stat) override;
+ Underlying* underlying() const { return base_file_system_.get(); }
- Status DeleteFile(const string& fname) override;
+ private:
+ std::unique_ptr<Underlying> base_file_system_;
+ const int64 initial_delay_microseconds_;
- Status CreateDir(const string& dirname) override;
+ TF_DISALLOW_COPY_AND_ASSIGN(RetryingFileSystem);
+};
- Status DeleteDir(const string& dirname) override;
+namespace retrying_internals {
- Status GetFileSize(const string& fname, uint64* file_size) override;
+class RetryingRandomAccessFile : public RandomAccessFile {
+ public:
+ RetryingRandomAccessFile(std::unique_ptr<RandomAccessFile> base_file,
+ int64 delay_microseconds)
+ : base_file_(std::move(base_file)),
+ initial_delay_microseconds_(delay_microseconds) {}
- Status RenameFile(const string& src, const string& target) override;
+ Status Read(uint64 offset, size_t n, StringPiece* result,
+ char* scratch) const override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&RandomAccessFile::Read, base_file_.get(), offset, n, result,
+ scratch),
+ initial_delay_microseconds_);
+ }
- Status IsDirectory(const string& dir) override;
+ private:
+ std::unique_ptr<RandomAccessFile> base_file_;
+ const int64 initial_delay_microseconds_;
+};
- Status DeleteRecursively(const string& dirname, int64* undeleted_files,
- int64* undeleted_dirs) override;
+class RetryingWritableFile : public WritableFile {
+ public:
+ RetryingWritableFile(std::unique_ptr<WritableFile> base_file,
+ int64 delay_microseconds)
+ : base_file_(std::move(base_file)),
+ initial_delay_microseconds_(delay_microseconds) {}
- void FlushCaches() override;
+ ~RetryingWritableFile() override {
+ // Makes sure the retrying version of Close() is called in the destructor.
+ Close().IgnoreError();
+ }
+
+ Status Append(const StringPiece& data) override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&WritableFile::Append, base_file_.get(), data),
+ initial_delay_microseconds_);
+ }
+ Status Close() override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&WritableFile::Close, base_file_.get()),
+ initial_delay_microseconds_);
+ }
+ Status Flush() override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&WritableFile::Flush, base_file_.get()),
+ initial_delay_microseconds_);
+ }
+ Status Sync() override {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&WritableFile::Sync, base_file_.get()),
+ initial_delay_microseconds_);
+ }
private:
- std::unique_ptr<FileSystem> base_file_system_;
+ std::unique_ptr<WritableFile> base_file_;
const int64 initial_delay_microseconds_;
-
- TF_DISALLOW_COPY_AND_ASSIGN(RetryingFileSystem);
};
+} // namespace retrying_internals
+
+template <typename Underlying>
+Status RetryingFileSystem<Underlying>::NewRandomAccessFile(
+ const string& filename, std::unique_ptr<RandomAccessFile>* result) {
+ std::unique_ptr<RandomAccessFile> base_file;
+ TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::NewRandomAccessFile, base_file_system_.get(),
+ filename, &base_file),
+ initial_delay_microseconds_));
+ result->reset(new retrying_internals::RetryingRandomAccessFile(
+ std::move(base_file), initial_delay_microseconds_));
+ return Status::OK();
+}
+
+template <typename Underlying>
+Status RetryingFileSystem<Underlying>::NewWritableFile(
+ const string& filename, std::unique_ptr<WritableFile>* result) {
+ std::unique_ptr<WritableFile> base_file;
+ TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::NewWritableFile, base_file_system_.get(), filename,
+ &base_file),
+ initial_delay_microseconds_));
+ result->reset(new retrying_internals::RetryingWritableFile(
+ std::move(base_file), initial_delay_microseconds_));
+ return Status::OK();
+}
+
+template <typename Underlying>
+Status RetryingFileSystem<Underlying>::NewAppendableFile(
+ const string& filename, std::unique_ptr<WritableFile>* result) {
+ std::unique_ptr<WritableFile> base_file;
+ TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::NewAppendableFile, base_file_system_.get(),
+ filename, &base_file),
+ initial_delay_microseconds_));
+ result->reset(new retrying_internals::RetryingWritableFile(
+ std::move(base_file), initial_delay_microseconds_));
+ return Status::OK();
+}
+
+template <typename Underlying>
+Status RetryingFileSystem<Underlying>::NewReadOnlyMemoryRegionFromFile(
+ const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>* result) {
+ return RetryingUtils::CallWithRetries(
+ std::bind(&FileSystem::NewReadOnlyMemoryRegionFromFile,
+ base_file_system_.get(), filename, result),
+ initial_delay_microseconds_);
+}
+
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PLATFORM_RETRYING_FILE_SYSTEM_H_
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->random_access_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped random access file.
std::unique_ptr<RandomAccessFile> random_access_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped appendable file.
std::unique_ptr<WritableFile> writable_file;
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
base_fs->writable_file_to_return = std::move(base_file);
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
// Retrieve the wrapped writable file.
std::unique_ptr<WritableFile> writable_file;
std::make_tuple("NewReadOnlyMemoryRegionFromFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::unique_ptr<ReadOnlyMemoryRegion> result;
TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result));
CreateRetriableErrors("NewReadOnlyMemoryRegionFromFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::unique_ptr<ReadOnlyMemoryRegion> result;
const auto& status =
std::make_tuple("GetChildren", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
TF_EXPECT_OK(fs.GetChildren("gs://path", &result));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetChildren", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
const auto& status = fs.GetChildren("gs://path", &result);
std::make_tuple("GetMatchingPaths", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://path/dir", &result));
CreateRetriableErrors("GetMatchingPaths", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
const auto& status = fs.GetMatchingPaths("gs://path/dir", &result);
std::make_tuple("DeleteFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
TF_EXPECT_OK(fs.DeleteFile("gs://path/file.txt"));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
const auto& status = fs.DeleteFile("gs://path/file.txt");
std::make_tuple("CreateDir", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
TF_EXPECT_OK(fs.CreateDir("gs://path/newdir"));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("CreateDir", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
const auto& status = fs.CreateDir("gs://path/newdir");
std::make_tuple("DeleteDir", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
TF_EXPECT_OK(fs.DeleteDir("gs://path/dir"));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("DeleteDir", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
std::vector<string> result;
const auto& status = fs.DeleteDir("gs://path/dir");
std::make_tuple("GetFileSize", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
uint64 size;
TF_EXPECT_OK(fs.GetFileSize("gs://path/file.txt", &size));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("GetFileSize", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
uint64 size;
const auto& status = fs.GetFileSize("gs://path/file.txt", &size);
std::make_tuple("RenameFile", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
TF_EXPECT_OK(fs.RenameFile("old_name", "new_name"));
}
ExpectedCalls expected_fs_calls = CreateRetriableErrors("RenameFile", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
const auto& status = fs.RenameFile("old_name", "new_name");
EXPECT_TRUE(
std::make_tuple("Stat", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("file_name", &stat));
ExpectedCalls expected_fs_calls = CreateRetriableErrors("Stat", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
FileStatistics stat;
const auto& status = fs.Stat("file_name", &stat);
ExpectedCalls expected_fs_calls = CreateRetriableErrors("FileExists", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
const auto& status = fs.FileExists("file_name");
EXPECT_TRUE(
std::make_tuple("FileExists", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
TF_EXPECT_OK(fs.FileExists("gs://path/dir"));
}
std::make_tuple("IsDirectory", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
TF_EXPECT_OK(fs.IsDirectory("gs://path/dir"));
}
ExpectedCalls expected_fs_calls = CreateRetriableErrors("IsDirectory", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
const auto& status = fs.IsDirectory("gs://path/dir");
EXPECT_TRUE(
std::make_tuple("DeleteRecursively", Status::OK())});
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(
CreateRetriableErrors("DeleteRecursively", 11);
std::unique_ptr<MockFileSystem> base_fs(
new MockFileSystem(expected_fs_calls));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
int64 undeleted_files, undeleted_dirs;
const auto& status =
ExpectedCalls none;
bool flushed = false;
std::unique_ptr<MockFileSystem> base_fs(new MockFileSystem(none, &flushed));
- RetryingFileSystem fs(std::move(base_fs), 0);
+ RetryingFileSystem<MockFileSystem> fs(std::move(base_fs), 0);
fs.FlushCaches();
EXPECT_TRUE(flushed);
}
projects / platform / upstream / tensorflow.git / commitdiff