From ace1e9c30fb67de0605fc57c66a5f427f18c4b87 Mon Sep 17 00:00:00 2001 From: Paarth Madan Date: Fri, 14 Jan 2022 15:05:50 -0500 Subject: [PATCH 1/4] Sort imports --- lib/i18n/backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/i18n/backend.rb b/lib/i18n/backend.rb index 453ed716..52c5498f 100644 --- a/lib/i18n/backend.rb +++ b/lib/i18n/backend.rb @@ -3,7 +3,6 @@ module I18n module Backend autoload :Base, 'i18n/backend/base' - autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler' autoload :Cache, 'i18n/backend/cache' autoload :CacheFile, 'i18n/backend/cache_file' autoload :Cascade, 'i18n/backend/cascade' @@ -11,6 +10,7 @@ module Backend autoload :Fallbacks, 'i18n/backend/fallbacks' autoload :Flatten, 'i18n/backend/flatten' autoload :Gettext, 'i18n/backend/gettext' + autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler' autoload :KeyValue, 'i18n/backend/key_value' autoload :Memoize, 'i18n/backend/memoize' autoload :Metadata, 'i18n/backend/metadata' From 9ec31b05a64e2344967d3089cf9cc786f1323a49 Mon Sep 17 00:00:00 2001 From: Paarth Madan Date: Wed, 2 Feb 2022 19:45:47 -0500 Subject: [PATCH 2/4] Test #available_locales delegates to backend more precisely --- lib/i18n/tests/basics.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/i18n/tests/basics.rb b/lib/i18n/tests/basics.rb index 951d7ab5..be824302 100644 --- a/lib/i18n/tests/basics.rb +++ b/lib/i18n/tests/basics.rb @@ -5,12 +5,11 @@ def teardown I18n.available_locales = nil end - test "available_locales returns the locales stored to the backend by default" do + test "available_locales returns the available_locales produced by the backend, by default" do I18n.backend.store_translations('de', :foo => 'bar') I18n.backend.store_translations('en', :foo => 'foo') - assert I18n.available_locales.include?(:de) - assert I18n.available_locales.include?(:en) + assert_equal I18n.available_locales, I18n.backend.available_locales end test "available_locales can be set to something else independently from the actual locale data" do @@ -24,8 +23,7 @@ def teardown assert_equal [:foo, :bar], I18n.available_locales I18n.available_locales = nil - assert I18n.available_locales.include?(:de) - assert I18n.available_locales.include?(:en) + assert_equal I18n.available_locales, I18n.backend.available_locales end test "available_locales memoizes when set explicitely" do From eb9dbf485f65ac68e730f52977be6af04f1e708b Mon Sep 17 00:00:00 2001 From: Paarth Madan Date: Wed, 2 Feb 2022 19:47:05 -0500 Subject: [PATCH 3/4] Yield loaded translations to block in #load_translations This makes introspecting the translations loaded by a file easier. Maintains backwards compatability as the block is optional. --- lib/i18n/backend/base.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/i18n/backend/base.rb b/lib/i18n/backend/base.rb index b78a6af1..77e832c0 100644 --- a/lib/i18n/backend/base.rb +++ b/lib/i18n/backend/base.rb @@ -13,7 +13,10 @@ module Base # for details. def load_translations(*filenames) filenames = I18n.load_path if filenames.empty? - filenames.flatten.each { |filename| load_file(filename) } + filenames.flatten.each do |filename| + loaded_translations = load_file(filename) + yield filename, loaded_translations if block_given? + end end # This method receives a locale, a data hash and options for storing translations. @@ -226,6 +229,8 @@ def load_file(filename) raise InvalidLocaleData.new(filename, 'expects it to return a hash, but does not') end data.each { |locale, d| store_translations(locale, d || {}, skip_symbolize_keys: keys_symbolized) } + + data end # Loads a plain Ruby translations file. eval'ing the file must yield From 422959a5732b5da25033e551de3aaffb1d3cfd57 Mon Sep 17 00:00:00 2001 From: Paarth Madan Date: Wed, 2 Feb 2022 19:47:41 -0500 Subject: [PATCH 4/4] Introduce LazyLoadable backend --- lib/i18n/backend.rb | 1 + lib/i18n/backend/lazy_loadable.rb | 184 ++++++++++++++++++++++++ lib/i18n/exceptions.rb | 34 +++++ test/api/lazy_loadable_test.rb | 24 ++++ test/backend/lazy_loadable_test.rb | 223 +++++++++++++++++++++++++++++ test/test_data/locales/fr.yml | 3 + 6 files changed, 469 insertions(+) create mode 100644 lib/i18n/backend/lazy_loadable.rb create mode 100644 test/api/lazy_loadable_test.rb create mode 100644 test/backend/lazy_loadable_test.rb create mode 100644 test/test_data/locales/fr.yml diff --git a/lib/i18n/backend.rb b/lib/i18n/backend.rb index 52c5498f..863d6187 100644 --- a/lib/i18n/backend.rb +++ b/lib/i18n/backend.rb @@ -12,6 +12,7 @@ module Backend autoload :Gettext, 'i18n/backend/gettext' autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler' autoload :KeyValue, 'i18n/backend/key_value' + autoload :LazyLoadable, 'i18n/backend/lazy_loadable' autoload :Memoize, 'i18n/backend/memoize' autoload :Metadata, 'i18n/backend/metadata' autoload :Pluralization, 'i18n/backend/pluralization' diff --git a/lib/i18n/backend/lazy_loadable.rb b/lib/i18n/backend/lazy_loadable.rb new file mode 100644 index 00000000..60f21fa1 --- /dev/null +++ b/lib/i18n/backend/lazy_loadable.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +module I18n + module Backend + # Backend that lazy loads translations based on the current locale. This + # implementation avoids loading all translations up front. Instead, it only + # loads the translations that belong to the current locale. This offers a + # performance incentive in local development and test environments for + # applications with many translations for many different locales. It's + # particularly useful when the application only refers to a single locales' + # translations at a time (ex. A Rails workload). The implementation + # identifies which translation files from the load path belong to the + # current locale by pattern matching against their path name. + # + # Specifically, a translation file is considered to belong to a locale if: + # a) the filename is in the I18n load path + # b) the filename ends in a supported extension (ie. .yml, .json, .po, .rb) + # c) the filename starts with the locale identifier + # d) the locale identifier and optional proceeding text is separated by an underscore, ie. "_". + # + # Examples: + # Valid files that will be selected by this backend: + # + # "files/locales/en_translation.yml" (Selected for locale "en") + # "files/locales/fr.po" (Selected for locale "fr") + # + # Invalid files that won't be selected by this backend: + # + # "files/locales/translation-file" + # "files/locales/en-translation.unsupported" + # "files/locales/french/translation.yml" + # "files/locales/fr/translation.yml" + # + # The implementation uses this assumption to defer the loading of + # translation files until the current locale actually requires them. + # + # The backend has two working modes: lazy_load and eager_load. + # + # Note: This backend should only be enabled in test environments! + # When the mode is set to false, the backend behaves exactly like the + # Simple backend, with an additional check that the paths being loaded + # abide by the format. If paths can't be matched to the format, an error is raised. + # + # You can configure lazy loaded backends through the initializer or backends + # accessor: + # + # # In test environments + # + # I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: true) + # + # # In other environments, such as production and CI + # + # I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: false) # default + # + class LocaleExtractor + class << self + def locale_from_path(path) + name = File.basename(path, ".*") + locale = name.split("_").first + locale.to_sym unless locale.nil? + end + end + end + + class LazyLoadable < Simple + def initialize(lazy_load: false) + @lazy_load = lazy_load + end + + # Returns whether the current locale is initialized. + def initialized? + if lazy_load? + initialized_locales[I18n.locale] + else + super + end + end + + # Clean up translations and uninitialize all locales. + def reload! + if lazy_load? + @initialized_locales = nil + @translations = nil + else + super + end + end + + # Eager loading is not supported in the lazy context. + def eager_load! + if lazy_load? + raise UnsupportedMethod.new(__method__, self.class, "Cannot eager load translations because backend was configured with lazy_load: true.") + else + super + end + end + + # Parse the load path and extract all locales. + def available_locales + if lazy_load? + I18n.load_path.map { |path| LocaleExtractor.locale_from_path(path) } + else + super + end + end + + def lookup(locale, key, scope = [], options = EMPTY_HASH) + if lazy_load? + I18n.with_locale(locale) do + super + end + else + super + end + end + + protected + + + # Load translations from files that belong to the current locale. + def init_translations + file_errors = if lazy_load? + initialized_locales[I18n.locale] = true + load_translations_and_collect_file_errors(filenames_for_current_locale) + else + @initialized = true + load_translations_and_collect_file_errors(I18n.load_path) + end + + raise InvalidFilenames.new(file_errors) unless file_errors.empty? + end + + def initialized_locales + @initialized_locales ||= Hash.new(false) + end + + private + + def lazy_load? + @lazy_load + end + + class FilenameIncorrect < StandardError + def initialize(file, expected_locale, unexpected_locales) + super "#{file} can only load translations for \"#{expected_locale}\". Found translations for: #{unexpected_locales}." + end + end + + # Loads each file supplied and asserts that the file only loads + # translations as expected by the name. The method returns a list of + # errors corresponding to offending files. + def load_translations_and_collect_file_errors(files) + errors = [] + + load_translations(files) do |file, loaded_translations| + assert_file_named_correctly!(file, loaded_translations) + rescue FilenameIncorrect => e + errors << e + end + + errors + end + + # Select all files from I18n load path that belong to current locale. + # These files must start with the locale identifier (ie. "en", "pt-BR"), + # followed by an "_" demarcation to separate proceeding text. + def filenames_for_current_locale + I18n.load_path.flatten.select do |path| + LocaleExtractor.locale_from_path(path) == I18n.locale + end + end + + # Checks if a filename is named in correspondence to the translations it loaded. + # The locale extracted from the path must be the single locale loaded in the translations. + def assert_file_named_correctly!(file, translations) + loaded_locales = translations.keys.map(&:to_sym) + expected_locale = LocaleExtractor.locale_from_path(file) + unexpected_locales = loaded_locales.reject { |locale| locale == expected_locale } + + raise FilenameIncorrect.new(file, expected_locale, unexpected_locales) unless unexpected_locales.empty? + end + end + end +end diff --git a/lib/i18n/exceptions.rb b/lib/i18n/exceptions.rb index 212a9039..f66e2076 100644 --- a/lib/i18n/exceptions.rb +++ b/lib/i18n/exceptions.rb @@ -110,4 +110,38 @@ def initialize(type, filename) super "can not load translations from #{filename}, the file type #{type} is not known" end end + + class UnsupportedMethod < ArgumentError + attr_reader :method, :backend_klass, :msg + def initialize(method, backend_klass, msg) + @method = method + @backend_klass = backend_klass + @msg = msg + super "#{backend_klass} does not support the ##{method} method. #{msg}" + end + end + + class InvalidFilenames < ArgumentError + NUMBER_OF_ERRORS_SHOWN = 20 + def initialize(file_errors) + super <<~MSG + Found #{file_errors.count} error(s). + The first #{[file_errors.count, NUMBER_OF_ERRORS_SHOWN].min} error(s): + #{file_errors.map(&:message).first(NUMBER_OF_ERRORS_SHOWN).join("\n")} + + To use the LazyLoadable backend: + 1. Filenames must start with the locale. + 2. An underscore must separate the locale with any optional text that follows. + 3. The file must only contain translation data for the single locale. + + Example: + "/config/locales/fr.yml" which contains: + ```yml + fr: + dog: + chien + ``` + MSG + end + end end diff --git a/test/api/lazy_loadable_test.rb b/test/api/lazy_loadable_test.rb new file mode 100644 index 00000000..a47dac47 --- /dev/null +++ b/test/api/lazy_loadable_test.rb @@ -0,0 +1,24 @@ +require 'test_helper' + +class I18nLazyLoadableBackendApiTest < I18n::TestCase + def setup + I18n.backend = I18n::Backend::LazyLoadable.new + super + end + + include I18n::Tests::Basics + include I18n::Tests::Defaults + include I18n::Tests::Interpolation + include I18n::Tests::Link + include I18n::Tests::Lookup + include I18n::Tests::Pluralization + include I18n::Tests::Procs + include I18n::Tests::Localization::Date + include I18n::Tests::Localization::DateTime + include I18n::Tests::Localization::Time + include I18n::Tests::Localization::Procs + + test "make sure we use the LazyLoadable backend" do + assert_equal I18n::Backend::LazyLoadable, I18n.backend.class + end +end diff --git a/test/backend/lazy_loadable_test.rb b/test/backend/lazy_loadable_test.rb new file mode 100644 index 00000000..a15ab009 --- /dev/null +++ b/test/backend/lazy_loadable_test.rb @@ -0,0 +1,223 @@ +require 'test_helper' + +class I18nBackendLazyLoadableTest < I18n::TestCase + def setup + super + + @lazy_mode_backend = I18n::Backend::LazyLoadable.new(lazy_load: true) + @eager_mode_backend = I18n::Backend::LazyLoadable.new(lazy_load: false) + + I18n.load_path = [File.join(locales_dir, '/en.yml'), File.join(locales_dir, '/fr.yml')] + end + + test "lazy mode: only loads translations for current locale" do + with_lazy_mode do + @backend.reload! + + assert_nil translations + + I18n.with_locale(:en) { I18n.t("foo.bar") } + assert_equal({ en: { foo: { bar: "baz" }}}, translations) + end + end + + test "lazy mode: merges translations for current locale with translations already existing in memory" do + with_lazy_mode do + @backend.reload! + + I18n.with_locale(:en) { I18n.t("foo.bar") } + assert_equal({ en: { foo: { bar: "baz" }}}, translations) + + I18n.with_locale(:fr) { I18n.t("animal.dog") } + assert_equal({ en: { foo: { bar: "baz" } }, fr: { animal: { dog: "chien" } } }, translations) + end + end + + test "lazy mode: #initialized? responds based on whether current locale is initialized" do + with_lazy_mode do + @backend.reload! + + I18n.with_locale(:en) do + refute_predicate @backend, :initialized? + I18n.t("foo.bar") + assert_predicate @backend, :initialized? + end + + I18n.with_locale(:fr) do + refute_predicate @backend, :initialized? + end + end + end + + test "lazy mode: reload! uninitializes all locales" do + with_lazy_mode do + I18n.with_locale(:en) { I18n.t("foo.bar") } + I18n.with_locale(:fr) { I18n.t("animal.dog") } + + @backend.reload! + + I18n.with_locale(:en) do + refute_predicate @backend, :initialized? + end + + I18n.with_locale(:fr) do + refute_predicate @backend, :initialized? + end + end + end + + test "lazy mode: eager_load! raises UnsupportedMethod exception" do + with_lazy_mode do + exception = assert_raises(I18n::UnsupportedMethod) { @backend.eager_load! } + expected_msg = "I18n::Backend::LazyLoadable does not support the #eager_load! method. Cannot eager load translations because backend was configured with lazy_load: true." + assert_equal expected_msg, exception.message + end + end + + test "lazy mode: loads translations from files that start with current locale identifier" do + with_lazy_mode do + file_contents = { en: { alice: "bob" } }.to_yaml + + invalid_files = [ + { filename: ['translation', '.yml'] }, # No locale identifier + { filename: ['translation', '.unsupported'] }, # No locale identifier and unsupported extension + ] + + invalid_files.each do |file| + with_translation_file_in_load_path(file[:filename], file[:dir], file_contents) do + I18n.with_locale(:en) { I18n.t("foo.bar") } + assert_equal({ en: { foo: { bar: "baz" }}}, translations) + end + end + + valid_files = [ + { filename: ['en_translation', '.yml'] }, # Contains locale identifier with correct demarcation, and supported extension + { filename: ['en_', '.yml'] }, # Path component matches locale identifier exactly + ] + + valid_files.each do |file| + with_translation_file_in_load_path(file[:filename], file[:dir], file_contents) do + I18n.with_locale(:en) { I18n.t("foo.bar") } + assert_equal({ en: { foo: { bar: "baz" }, alice: "bob" }}, translations) + end + end + end + end + + test "lazy mode: files with unsupported extensions raise UnknownFileType error" do + with_lazy_mode do + file_contents = { en: { alice: "bob" } }.to_yaml + filename = ['en_translation', '.unsupported'] # Correct locale identifier, but unsupported extension + + with_translation_file_in_load_path(filename, nil, file_contents) do + assert_raises(I18n::UnknownFileType) { I18n.t("foo.bar") } + end + end + end + + test "lazy mode: #available_locales returns all locales available from load path irrespective of current locale" do + with_lazy_mode do + I18n.with_locale(:en) { assert_equal [:en, :fr], @backend.available_locales } + I18n.with_locale(:fr) { assert_equal [:en, :fr], @backend.available_locales } + end + end + + test "lazy mode: raises error if translations loaded don't correspond to locale extracted from filename" do + filename = ["en_", ".yml"] + file_contents = { fr: { dog: "chien" } }.to_yaml + + with_lazy_mode do + with_translation_file_in_load_path(filename, nil, file_contents) do |file_path| + exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") } + + expected_message = /#{Regexp.escape(file_path)} can only load translations for "en"\. Found translations for: \[\:fr\]/ + assert_match expected_message, exception.message + end + end + end + + test "lazy mode: raises error if translations for more than one locale are loaded from a single file" do + filename = ["en_", ".yml"] + file_contents = { en: { alice: "bob" }, fr: { dog: "chien" }, de: { cat: 'katze' } }.to_yaml + + with_lazy_mode do + with_translation_file_in_load_path(filename, nil, file_contents) do |file_path| + exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") } + + expected_message = /#{Regexp.escape(file_path)} can only load translations for "en"\. Found translations for: \[\:fr\, \:de\]/ + assert_match expected_message, exception.message + end + end + end + + test "lazy mode: #lookup lazy loads translations for supplied locale" do + with_lazy_mode do + @backend.reload! + assert_nil translations + + I18n.with_locale(:en) do + assert_equal "chien", @backend.lookup(:fr, "animal.dog") + end + + assert_equal({ fr: { animal: { dog: "chien" } } }, translations) + end + end + + test "eager mode: load all translations, irrespective of locale" do + with_eager_mode do + @backend.reload! + + assert_nil translations + + I18n.with_locale(:en) { I18n.t("foo.bar") } + assert_equal({ en: { foo: { bar: "baz" } }, fr: { animal: { dog: "chien" } } }, translations) + end + end + + test "eager mode: raises error if locales loaded cannot be extracted from load path names" do + with_eager_mode do + @backend.reload! + + contents = { de: { cat: 'katze' } }.to_yaml + + with_translation_file_in_load_path(['fr_translation', '.yml'], nil, contents) do |file_path| + exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") } + + expected_message = /#{Regexp.escape(file_path)} can only load translations for "fr"\. Found translations for: \[\:de\]/ + assert_match expected_message, exception.message + end + end + end + + private + + def with_lazy_mode + @backend = I18n.backend = @lazy_mode_backend + + yield + end + + def with_eager_mode + @backend = I18n.backend = @eager_mode_backend + + yield + end + + + def with_translation_file_in_load_path(name, tmpdir, file_contents) + @backend.reload! + + path_to_dir = FileUtils.mkdir_p(File.join(Dir.tmpdir, tmpdir)).first if tmpdir + locale_file = Tempfile.new(name, path_to_dir) + + locale_file.write(file_contents) + locale_file.rewind + + I18n.load_path << locale_file.path + + yield(locale_file.path) + + I18n.load_path.delete(locale_file.path) + end +end + diff --git a/test/test_data/locales/fr.yml b/test/test_data/locales/fr.yml new file mode 100644 index 00000000..0e7bb66c --- /dev/null +++ b/test/test_data/locales/fr.yml @@ -0,0 +1,3 @@ +fr: + animal: + dog: chien