#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
namespace tensorflow {
namespace {
return AAssetDir_getNextFileName(dir.get()) != NULL;
}
+Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) {
+ return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
Status AssetManagerFileSystem::NewWritableFile(
const string& fname, std::unique_ptr<WritableFile>* result) {
return errors::Unimplemented("Asset storage is read only.");
Status DeleteDir(const string& d) override;
Status RenameFile(const string& s, const string& t) override;
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override;
+
private:
string RemoveAssetPrefix(const string& name);
tensorflow/core/platform/posix/load_library.cc
tensorflow/core/platform/posix/env_time.cc
tensorflow/core/platform/file_system.cc
+tensorflow/core/platform/file_system_helper.cc
tensorflow/core/platform/env.cc
tensorflow/core/platform/env_time.cc
tensorflow/core/platform/setround.cc
"platform/env.h",
"platform/env_time.h",
"platform/file_system.h",
+ "platform/file_system_helper.h",
"platform/fingerprint.h",
"platform/init_main.h",
"platform/logging.h",
#include <deque>
#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
namespace tensorflow {
-namespace {
-
-constexpr int kNumThreads = 8;
-
-// Run a function in parallel using a ThreadPool, but skip the ThreadPool
-// on the iOS platform due to its problems with more than a few threads.
-void ForEach(int first, int last, const std::function<void(int)>& f) {
-#if TARGET_OS_IPHONE
- for (int i = first; i < last; i++) {
- f(i);
- }
-#else
- int num_threads = std::min(kNumThreads, last - first);
- thread::ThreadPool threads(Env::Default(), "ForEach", num_threads);
- for (int i = first; i < last; i++) {
- threads.Schedule([f, i] { f(i); });
- }
-#endif
-}
-
-} // anonymous namespace
-
FileSystem::~FileSystem() {}
string FileSystem::TranslateName(const string& name) const {
return result;
}
-Status FileSystem::GetMatchingPaths(const string& pattern,
- std::vector<string>* results) {
- results->clear();
- // Find the fixed prefix by looking for the first wildcard.
- string fixed_prefix = pattern.substr(0, pattern.find_first_of("*?[\\"));
- string eval_pattern = pattern;
- std::vector<string> all_files;
- string dir = io::Dirname(fixed_prefix).ToString();
- // If dir is empty then we need to fix up fixed_prefix and eval_pattern to
- // include . as the top level directory.
- if (dir.empty()) {
- dir = ".";
- fixed_prefix = io::JoinPath(dir, fixed_prefix);
- eval_pattern = io::JoinPath(dir, pattern);
- }
-
- // Setup a BFS to explore everything under dir.
- std::deque<string> dir_q;
- dir_q.push_back(dir);
- Status ret; // Status to return.
- // children_dir_status holds is_dir status for children. It can have three
- // possible values: OK for true; FAILED_PRECONDITION for false; CANCELLED
- // if we don't calculate IsDirectory (we might do that because there isn't
- // any point in exploring that child path).
- std::vector<Status> children_dir_status;
- while (!dir_q.empty()) {
- string current_dir = dir_q.front();
- dir_q.pop_front();
- std::vector<string> children;
- Status s = GetChildren(current_dir, &children);
- ret.Update(s);
- if (children.empty()) continue;
- // This IsDirectory call can be expensive for some FS. Parallelizing it.
- children_dir_status.resize(children.size());
- ForEach(0, children.size(),
- [this, ¤t_dir, &children, &fixed_prefix,
- &children_dir_status](int i) {
- const string child_path = io::JoinPath(current_dir, children[i]);
- // In case the child_path doesn't start with the fixed_prefix then
- // we don't need to explore this path.
- if (!str_util::StartsWith(child_path, fixed_prefix)) {
- children_dir_status[i] = Status(tensorflow::error::CANCELLED,
- "Operation not needed");
- } else {
- children_dir_status[i] = IsDirectory(child_path);
- }
- });
- for (int i = 0; i < children.size(); ++i) {
- const string child_path = io::JoinPath(current_dir, children[i]);
- // If the IsDirectory call was cancelled we bail.
- if (children_dir_status[i].code() == tensorflow::error::CANCELLED) {
- continue;
- }
- // If the child is a directory add it to the queue.
- if (children_dir_status[i].ok()) {
- dir_q.push_back(child_path);
- }
- all_files.push_back(child_path);
- }
- }
-
- // Match all obtained files to the input pattern.
- for (const auto& f : all_files) {
- if (Env::Default()->MatchPath(f, eval_pattern)) {
- results->push_back(f);
- }
- }
- return ret;
-}
-
Status FileSystem::DeleteRecursively(const string& dirname,
int64* undeleted_files,
int64* undeleted_dirs) {
/// * OK - no errors
/// * UNIMPLEMENTED - Some underlying functions (like GetChildren) are not
/// implemented
- /// The default implementation uses a combination of GetChildren, MatchPath
- /// and IsDirectory.
virtual Status GetMatchingPaths(const string& pattern,
- std::vector<string>* results);
+ std::vector<string>* results) = 0;
/// \brief Obtains statistics for the given path.
virtual Status Stat(const string& fname, FileStatistics* stat) = 0;
--- /dev/null
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/file_system_helper.h"
+
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/platform.h"
+
+namespace tensorflow {
+namespace internal {
+
+namespace {
+
+constexpr int kNumThreads = 8;
+
+// Run a function in parallel using a ThreadPool, but skip the ThreadPool
+// on the iOS platform due to its problems with more than a few threads.
+void ForEach(int first, int last, const std::function<void(int)>& f) {
+#if TARGET_OS_IPHONE
+ for (int i = first; i < last; i++) {
+ f(i);
+ }
+#else
+ int num_threads = std::min(kNumThreads, last - first);
+ thread::ThreadPool threads(Env::Default(), "ForEach", num_threads);
+ for (int i = first; i < last; i++) {
+ threads.Schedule([f, i] { f(i); });
+ }
+#endif
+}
+
+} // namespace
+
+Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern,
+ std::vector<string>* results) {
+ results->clear();
+ // Find the fixed prefix by looking for the first wildcard.
+ string fixed_prefix = pattern.substr(0, pattern.find_first_of("*?[\\"));
+ string eval_pattern = pattern;
+ std::vector<string> all_files;
+ string dir = io::Dirname(fixed_prefix).ToString();
+ // If dir is empty then we need to fix up fixed_prefix and eval_pattern to
+ // include . as the top level directory.
+ if (dir.empty()) {
+ dir = ".";
+ fixed_prefix = io::JoinPath(dir, fixed_prefix);
+ eval_pattern = io::JoinPath(dir, pattern);
+ }
+
+ // Setup a BFS to explore everything under dir.
+ std::deque<string> dir_q;
+ dir_q.push_back(dir);
+ Status ret; // Status to return.
+ // children_dir_status holds is_dir status for children. It can have three
+ // possible values: OK for true; FAILED_PRECONDITION for false; CANCELLED
+ // if we don't calculate IsDirectory (we might do that because there isn't
+ // any point in exploring that child path).
+ std::vector<Status> children_dir_status;
+ while (!dir_q.empty()) {
+ string current_dir = dir_q.front();
+ dir_q.pop_front();
+ std::vector<string> children;
+ Status s = fs->GetChildren(current_dir, &children);
+ ret.Update(s);
+ if (children.empty()) continue;
+ // This IsDirectory call can be expensive for some FS. Parallelizing it.
+ children_dir_status.resize(children.size());
+ ForEach(0, children.size(),
+ [fs, ¤t_dir, &children, &fixed_prefix,
+ &children_dir_status](int i) {
+ const string child_path = io::JoinPath(current_dir, children[i]);
+ // In case the child_path doesn't start with the fixed_prefix then
+ // we don't need to explore this path.
+ if (!str_util::StartsWith(child_path, fixed_prefix)) {
+ children_dir_status[i] = Status(tensorflow::error::CANCELLED,
+ "Operation not needed");
+ } else {
+ children_dir_status[i] = fs->IsDirectory(child_path);
+ }
+ });
+ for (int i = 0; i < children.size(); ++i) {
+ const string child_path = io::JoinPath(current_dir, children[i]);
+ // If the IsDirectory call was cancelled we bail.
+ if (children_dir_status[i].code() == tensorflow::error::CANCELLED) {
+ continue;
+ }
+ // If the child is a directory add it to the queue.
+ if (children_dir_status[i].ok()) {
+ dir_q.push_back(child_path);
+ }
+ all_files.push_back(child_path);
+ }
+ }
+
+ // Match all obtained files to the input pattern.
+ for (const auto& f : all_files) {
+ if (env->MatchPath(f, eval_pattern)) {
+ results->push_back(f);
+ }
+ }
+ return ret;
+}
+
+} // namespace internal
+} // namespace tensorflow
--- /dev/null
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
+#define TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+class FileSystem;
+class Env;
+
+namespace internal {
+
+// Given a pattern, stores in 'results' the set of paths (in the given file
+// system) that match that pattern.
+//
+// This helper may be used by implementations of FileSystem::GetMatchingPaths()
+// in order to provide parallel scanning of subdirectories (except on iOS).
+//
+// Arguments:
+// fs: may not be null and will be used to identify directories and list
+// their contents.
+// env: may not be null and will be used to check if a match has been found.
+// pattern: see FileSystem::GetMatchingPaths() for details.
+// results: will be cleared and may not be null.
+//
+// Returns an error status if any call to 'fs' failed.
+Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern,
+ std::vector<string>* results);
+
+} // namespace internal
+} // namespace tensorflow
+
+#endif // TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/file_system_helper.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/posix/error.h"
return Status::OK();
}
+Status HadoopFileSystem::GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) {
+ return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
Status HadoopFileSystem::DeleteFile(const string& fname) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(fname, &fs));
Status GetChildren(const string& dir, std::vector<string>* result) override;
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override;
+
Status DeleteFile(const string& fname) override;
Status CreateDir(const string& name) override;
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/file_system_helper.h"
namespace tensorflow {
return errors::Unimplemented("GetChildren unimplemented");
}
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override {
+ return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+ }
+
Status DeleteFile(const string& fname) override {
return errors::Unimplemented("DeleteFile unimplemented");
}
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/posix/error.h"
#include "tensorflow/core/platform/posix/posix_file_system.h"
return Status::OK();
}
+Status PosixFileSystem::GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) {
+ return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
Status PosixFileSystem::DeleteFile(const string& fname) {
Status result;
if (unlink(TranslateName(fname).c_str()) != 0) {
Status Stat(const string& fname, FileStatistics* stats) override;
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override;
+
Status DeleteFile(const string& fname) override;
Status CreateDir(const string& name) override;
#include "tensorflow/core/platform/s3/s3_file_system.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/file_system_helper.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/s3/aws_logging.h"
#include "tensorflow/core/platform/s3/s3_crypto.h"
return Status::OK();
}
+Status S3FileSystem::GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) {
+ return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
Status S3FileSystem::DeleteFile(const string& fname) {
string bucket, object;
TF_RETURN_IF_ERROR(ParseS3Path(fname, false, &bucket, &object));
Status Stat(const string& fname, FileStatistics* stat) override;
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override;
+
Status DeleteFile(const string& fname) override;
Status CreateDir(const string& name) override;
#include "tensorflow/core/lib/core/error_codes.pb.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/posix/error.h"
#include "tensorflow/core/platform/windows/error.h"
// but no code appears to rely on this behavior.
string converted_pattern(pattern);
std::replace(converted_pattern.begin(), converted_pattern.end(), '\\', '/');
- TF_RETURN_IF_ERROR(FileSystem::GetMatchingPaths(converted_pattern, results));
+ TF_RETURN_IF_ERROR(internal::GetMatchingPaths(this, Env::Default(),
+ converted_pattern, results));
for (string& result : *results) {
std::replace(result.begin(), result.end(), '/', '\\');
}
return errors::Unimplemented("memmapped format doesn't support GetChildren");
}
+Status MemmappedFileSystem::GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) {
+ return errors::Unimplemented(
+ "memmapped format doesn't support GetMatchingPaths");
+}
+
Status MemmappedFileSystem::DeleteFile(const string& filename) {
return errors::Unimplemented("memmapped format doesn't support DeleteFile");
}
Status NewAppendableFile(const string& fname,
std::unique_ptr<WritableFile>* result) override;
Status GetChildren(const string& dir, std::vector<string>* r) override;
+ Status GetMatchingPaths(const string& pattern,
+ std::vector<string>* results) override;
Status DeleteFile(const string& f) override;
Status CreateDir(const string& d) override;
Status DeleteDir(const string& d) override;
projects / platform / upstream / tensorflow.git / commitdiff