Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix GCS filesystem #48222

Merged
merged 2 commits into from Apr 1, 2021
Merged

Fix GCS filesystem #48222

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
158505e
Switch TF filesystems to keep backwards compatibility.
mihaimaruseac Mar 31, 2021
b9e31e6
Fix typo/logic bug in modular plugins.
mihaimaruseac Mar 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
35 changes: 18 additions & 17 deletions tensorflow/core/platform/env.h
View file
Open in desktop
Expand Up @@ -636,21 +636,22 @@ namespace register_file_system {

template <typename Factory>
struct Register {
Register(Env* env, const std::string& scheme, bool legacy) {
Register(Env* env, const std::string& scheme, bool try_modular_filesystems) {
// TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
// after TF 2.6+.
if (legacy) {
const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
string enable_legacy =
enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
if (!(enable_legacy == "true" || enable_legacy == "1")) {
if (try_modular_filesystems) {
const char* env_value = getenv("TF_USE_MODULAR_FILESYSTEM");
string load_plugin = env_value ? absl::AsciiStrToLower(env_value) : "";
if (load_plugin == "true" || load_plugin == "1") {
// We don't register the static filesystem and wait for SIG IO one
LOG(WARNING) << "Using modular file system for '" << scheme << "."
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
return;
}
LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated"
<< " and will be removed in tensorflow 2.6 or higher."
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
// If the envvar is missing or not "true"/"1", then fall back to legacy
// implementation to be backwards compatible.
}
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
Expand All @@ -666,15 +667,15 @@ struct Register {

// Register a FileSystem implementation for a scheme. Files with names that have
// "scheme://" prefixes are routed to use this implementation.
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, modular) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, modular)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, modular) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular) \
static ::tensorflow::register_file_system::Register<factory> \
register_ff##ctr TF_ATTRIBUTE_UNUSED = \
::tensorflow::register_file_system::Register<factory>(env, scheme, \
legacy)
modular)

#define REGISTER_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \
Expand Down