From 422959a5732b5da25033e551de3aaffb1d3cfd57 Mon Sep 17 00:00:00 2001
From: Paarth Madan <paarth.madan@shopify.com>
Date: Wed, 2 Feb 2022 19:47:41 -0500
Subject: [PATCH] Introduce LazyLoadable backend

---
 lib/i18n/backend.rb                |   1 +
 lib/i18n/backend/lazy_loadable.rb  | 184 ++++++++++++++++++++++++
 lib/i18n/exceptions.rb             |  34 +++++
 test/api/lazy_loadable_test.rb     |  24 ++++
 test/backend/lazy_loadable_test.rb | 223 +++++++++++++++++++++++++++++
 test/test_data/locales/fr.yml      |   3 +
 6 files changed, 469 insertions(+)
 create mode 100644 lib/i18n/backend/lazy_loadable.rb
 create mode 100644 test/api/lazy_loadable_test.rb
 create mode 100644 test/backend/lazy_loadable_test.rb
 create mode 100644 test/test_data/locales/fr.yml

diff --git a/lib/i18n/backend.rb b/lib/i18n/backend.rb
index 52c5498f..863d6187 100644
--- a/lib/i18n/backend.rb
+++ b/lib/i18n/backend.rb
@@ -12,6 +12,7 @@ module Backend
     autoload :Gettext,               'i18n/backend/gettext'
     autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler'
     autoload :KeyValue,              'i18n/backend/key_value'
+    autoload :LazyLoadable,          'i18n/backend/lazy_loadable'
     autoload :Memoize,               'i18n/backend/memoize'
     autoload :Metadata,              'i18n/backend/metadata'
     autoload :Pluralization,         'i18n/backend/pluralization'
diff --git a/lib/i18n/backend/lazy_loadable.rb b/lib/i18n/backend/lazy_loadable.rb
new file mode 100644
index 00000000..60f21fa1
--- /dev/null
+++ b/lib/i18n/backend/lazy_loadable.rb
@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module I18n
+  module Backend
+    # Backend that lazy loads translations based on the current locale. This
+    # implementation avoids loading all translations up front. Instead, it only
+    # loads the translations that belong to the current locale. This offers a
+    # performance incentive in local development and test environments for
+    # applications with many translations for many different locales. It's
+    # particularly useful when the application only refers to a single locales'
+    # translations at a time (ex. A Rails workload).  The implementation
+    # identifies which translation files from the load path belong to the
+    # current locale by pattern matching against their path name.
+    #
+    # Specifically, a translation file is considered to belong to a locale if:
+    # a) the filename is in the I18n load path
+    # b) the filename ends in a supported extension (ie. .yml, .json, .po, .rb)
+    # c) the filename starts with the locale identifier
+    # d) the locale identifier and optional proceeding text is separated by an underscore, ie. "_".
+    #
+    # Examples:
+    # Valid files that will be selected by this backend:
+    #
+    # "files/locales/en_translation.yml" (Selected for locale "en")
+    # "files/locales/fr.po"  (Selected for locale "fr")
+    #
+    # Invalid files that won't be selected by this backend:
+    #
+    # "files/locales/translation-file"
+    # "files/locales/en-translation.unsupported"
+    # "files/locales/french/translation.yml"
+    # "files/locales/fr/translation.yml"
+    #
+    # The implementation uses this assumption to defer the loading of
+    # translation files until the current locale actually requires them.
+    #
+    # The backend has two working modes: lazy_load and eager_load.
+    #
+    # Note: This backend should only be enabled in test environments!
+    # When the mode is set to false, the backend behaves exactly like the
+    # Simple backend, with an additional check that the paths being loaded
+    # abide by the format. If paths can't be matched to the format, an error is raised.
+    #
+    # You can configure lazy loaded backends through the initializer or backends
+    # accessor:
+    #
+    #   # In test environments
+    #
+    #   I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: true)
+    #
+    #   # In other environments, such as production and CI
+    #
+    #   I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: false) # default
+    #
+    class LocaleExtractor
+      class << self
+        def locale_from_path(path)
+          name = File.basename(path, ".*")
+          locale = name.split("_").first
+          locale.to_sym unless locale.nil?
+        end
+      end
+    end
+
+    class LazyLoadable < Simple
+      def initialize(lazy_load: false)
+        @lazy_load = lazy_load
+      end
+
+      # Returns whether the current locale is initialized.
+      def initialized?
+        if lazy_load?
+          initialized_locales[I18n.locale]
+        else
+          super
+        end
+      end
+
+      # Clean up translations and uninitialize all locales.
+      def reload!
+        if lazy_load?
+          @initialized_locales = nil
+          @translations = nil
+        else
+          super
+        end
+      end
+
+      # Eager loading is not supported in the lazy context.
+      def eager_load!
+        if lazy_load?
+          raise UnsupportedMethod.new(__method__, self.class, "Cannot eager load translations because backend was configured with lazy_load: true.")
+        else
+          super
+        end
+      end
+
+      # Parse the load path and extract all locales.
+      def available_locales
+        if lazy_load?
+          I18n.load_path.map { |path| LocaleExtractor.locale_from_path(path) }
+        else
+          super
+        end
+      end
+
+      def lookup(locale, key, scope = [], options = EMPTY_HASH)
+        if lazy_load?
+          I18n.with_locale(locale) do
+            super
+          end
+        else
+          super
+        end
+      end
+
+      protected
+
+
+      # Load translations from files that belong to the current locale.
+      def init_translations
+        file_errors = if lazy_load?
+          initialized_locales[I18n.locale] = true
+          load_translations_and_collect_file_errors(filenames_for_current_locale)
+        else
+          @initialized = true
+          load_translations_and_collect_file_errors(I18n.load_path)
+        end
+
+        raise InvalidFilenames.new(file_errors) unless file_errors.empty?
+      end
+
+      def initialized_locales
+        @initialized_locales ||= Hash.new(false)
+      end
+
+      private
+
+      def lazy_load?
+        @lazy_load
+      end
+
+      class FilenameIncorrect < StandardError
+        def initialize(file, expected_locale, unexpected_locales)
+          super "#{file} can only load translations for \"#{expected_locale}\". Found translations for: #{unexpected_locales}."
+        end
+      end
+
+      # Loads each file supplied and asserts that the file only loads
+      # translations as expected by the name. The method returns a list of
+      # errors corresponding to offending files.
+      def load_translations_and_collect_file_errors(files)
+        errors = []
+
+        load_translations(files) do |file, loaded_translations|
+          assert_file_named_correctly!(file, loaded_translations)
+        rescue FilenameIncorrect => e
+          errors << e
+        end
+
+        errors
+      end
+
+      # Select all files from I18n load path that belong to current locale.
+      # These files must start with the locale identifier (ie. "en", "pt-BR"),
+      # followed by an "_" demarcation to separate proceeding text.
+      def filenames_for_current_locale
+        I18n.load_path.flatten.select do |path|
+          LocaleExtractor.locale_from_path(path) == I18n.locale
+        end
+      end
+
+      # Checks if a filename is named in correspondence to the translations it loaded.
+      # The locale extracted from the path must be the single locale loaded in the translations.
+      def assert_file_named_correctly!(file, translations)
+        loaded_locales = translations.keys.map(&:to_sym)
+        expected_locale = LocaleExtractor.locale_from_path(file)
+        unexpected_locales = loaded_locales.reject { |locale| locale == expected_locale }
+
+        raise FilenameIncorrect.new(file, expected_locale, unexpected_locales) unless unexpected_locales.empty?
+      end
+    end
+  end
+end
diff --git a/lib/i18n/exceptions.rb b/lib/i18n/exceptions.rb
index 212a9039..f66e2076 100644
--- a/lib/i18n/exceptions.rb
+++ b/lib/i18n/exceptions.rb
@@ -110,4 +110,38 @@ def initialize(type, filename)
       super "can not load translations from #{filename}, the file type #{type} is not known"
     end
   end
+
+  class UnsupportedMethod < ArgumentError
+    attr_reader :method, :backend_klass, :msg
+    def initialize(method, backend_klass, msg)
+      @method = method
+      @backend_klass = backend_klass
+      @msg = msg
+      super "#{backend_klass} does not support the ##{method} method. #{msg}"
+    end
+  end
+
+  class InvalidFilenames < ArgumentError
+    NUMBER_OF_ERRORS_SHOWN = 20
+    def initialize(file_errors)
+      super <<~MSG
+        Found #{file_errors.count} error(s).
+        The first #{[file_errors.count, NUMBER_OF_ERRORS_SHOWN].min} error(s):
+        #{file_errors.map(&:message).first(NUMBER_OF_ERRORS_SHOWN).join("\n")}
+
+        To use the LazyLoadable backend:
+        1. Filenames must start with the locale.
+        2. An underscore must separate the locale with any optional text that follows.
+        3. The file must only contain translation data for the single locale.
+
+        Example:
+        "/config/locales/fr.yml" which contains:
+        ```yml
+          fr:
+            dog:
+              chien
+        ```
+      MSG
+    end
+  end
 end
diff --git a/test/api/lazy_loadable_test.rb b/test/api/lazy_loadable_test.rb
new file mode 100644
index 00000000..a47dac47
--- /dev/null
+++ b/test/api/lazy_loadable_test.rb
@@ -0,0 +1,24 @@
+require 'test_helper'
+
+class I18nLazyLoadableBackendApiTest < I18n::TestCase
+  def setup
+    I18n.backend = I18n::Backend::LazyLoadable.new
+    super
+  end
+
+  include I18n::Tests::Basics
+  include I18n::Tests::Defaults
+  include I18n::Tests::Interpolation
+  include I18n::Tests::Link
+  include I18n::Tests::Lookup
+  include I18n::Tests::Pluralization
+  include I18n::Tests::Procs
+  include I18n::Tests::Localization::Date
+  include I18n::Tests::Localization::DateTime
+  include I18n::Tests::Localization::Time
+  include I18n::Tests::Localization::Procs
+
+  test "make sure we use the LazyLoadable backend" do
+    assert_equal I18n::Backend::LazyLoadable, I18n.backend.class
+  end
+end
diff --git a/test/backend/lazy_loadable_test.rb b/test/backend/lazy_loadable_test.rb
new file mode 100644
index 00000000..a15ab009
--- /dev/null
+++ b/test/backend/lazy_loadable_test.rb
@@ -0,0 +1,223 @@
+require 'test_helper'
+
+class I18nBackendLazyLoadableTest < I18n::TestCase
+  def setup
+    super
+
+    @lazy_mode_backend = I18n::Backend::LazyLoadable.new(lazy_load: true)
+    @eager_mode_backend = I18n::Backend::LazyLoadable.new(lazy_load: false)
+
+    I18n.load_path = [File.join(locales_dir, '/en.yml'), File.join(locales_dir,  '/fr.yml')]
+  end
+
+  test "lazy mode: only loads translations for current locale" do
+    with_lazy_mode do
+      @backend.reload!
+
+      assert_nil translations
+
+      I18n.with_locale(:en) { I18n.t("foo.bar") }
+      assert_equal({ en: { foo: { bar: "baz" }}}, translations)
+    end
+  end
+
+  test "lazy mode: merges translations for current locale with translations already existing in memory" do
+    with_lazy_mode do
+      @backend.reload!
+
+      I18n.with_locale(:en) { I18n.t("foo.bar") }
+      assert_equal({ en: { foo: { bar: "baz" }}}, translations)
+
+      I18n.with_locale(:fr) { I18n.t("animal.dog") }
+      assert_equal({ en: { foo: { bar: "baz" } }, fr: { animal: { dog: "chien" } } }, translations)
+    end
+  end
+
+  test "lazy mode: #initialized? responds based on whether current locale is initialized" do
+    with_lazy_mode do
+      @backend.reload!
+
+      I18n.with_locale(:en) do
+        refute_predicate @backend, :initialized?
+        I18n.t("foo.bar")
+        assert_predicate @backend, :initialized?
+      end
+
+      I18n.with_locale(:fr) do
+        refute_predicate @backend, :initialized?
+      end
+    end
+  end
+
+  test "lazy mode: reload! uninitializes all locales" do
+    with_lazy_mode do
+      I18n.with_locale(:en) { I18n.t("foo.bar") }
+      I18n.with_locale(:fr) { I18n.t("animal.dog") }
+
+      @backend.reload!
+
+      I18n.with_locale(:en) do
+        refute_predicate @backend, :initialized?
+      end
+
+      I18n.with_locale(:fr) do
+        refute_predicate @backend, :initialized?
+      end
+    end
+  end
+
+  test "lazy mode: eager_load! raises UnsupportedMethod exception" do
+    with_lazy_mode do
+      exception = assert_raises(I18n::UnsupportedMethod) { @backend.eager_load! }
+      expected_msg = "I18n::Backend::LazyLoadable does not support the #eager_load! method. Cannot eager load translations because backend was configured with lazy_load: true." 
+      assert_equal expected_msg, exception.message
+    end
+  end
+
+  test "lazy mode: loads translations from files that start with current locale identifier" do
+    with_lazy_mode do
+      file_contents = { en: { alice: "bob" } }.to_yaml
+
+      invalid_files = [
+        { filename: ['translation', '.yml'] },                # No locale identifier
+        { filename: ['translation', '.unsupported'] },        # No locale identifier and unsupported extension
+      ]
+
+      invalid_files.each do |file|
+        with_translation_file_in_load_path(file[:filename], file[:dir], file_contents) do
+          I18n.with_locale(:en) { I18n.t("foo.bar") }
+          assert_equal({ en: { foo: { bar: "baz" }}}, translations)
+        end
+      end
+
+      valid_files = [
+        { filename: ['en_translation', '.yml'] },         # Contains locale identifier with correct demarcation, and supported extension
+        { filename: ['en_', '.yml'] },                    # Path component matches locale identifier exactly
+      ]
+
+      valid_files.each do |file|
+        with_translation_file_in_load_path(file[:filename], file[:dir], file_contents) do
+          I18n.with_locale(:en) { I18n.t("foo.bar") }
+          assert_equal({ en: { foo: { bar: "baz" }, alice: "bob" }}, translations)
+        end
+      end
+    end
+  end
+
+  test "lazy mode: files with unsupported extensions raise UnknownFileType error" do
+    with_lazy_mode do
+      file_contents = { en: { alice: "bob" } }.to_yaml
+      filename =  ['en_translation', '.unsupported']     # Correct locale identifier, but unsupported extension
+
+      with_translation_file_in_load_path(filename, nil, file_contents) do
+        assert_raises(I18n::UnknownFileType) { I18n.t("foo.bar") }
+      end
+    end
+  end
+
+  test "lazy mode: #available_locales returns all locales available from load path irrespective of current locale" do
+    with_lazy_mode do
+      I18n.with_locale(:en) { assert_equal [:en, :fr], @backend.available_locales }
+      I18n.with_locale(:fr) { assert_equal [:en, :fr], @backend.available_locales }
+    end
+  end
+
+  test "lazy mode: raises error if translations loaded don't correspond to locale extracted from filename" do
+    filename = ["en_", ".yml"]
+    file_contents = { fr: { dog: "chien" } }.to_yaml
+
+    with_lazy_mode do
+      with_translation_file_in_load_path(filename, nil, file_contents) do |file_path|
+        exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") }
+
+        expected_message = /#{Regexp.escape(file_path)} can only load translations for "en"\. Found translations for: \[\:fr\]/
+        assert_match expected_message, exception.message
+      end
+    end
+  end
+
+  test "lazy mode: raises error if translations for more than one locale are loaded from a single file" do
+    filename = ["en_", ".yml"]
+    file_contents = { en: { alice: "bob" },  fr: { dog: "chien" }, de: { cat: 'katze' } }.to_yaml
+
+    with_lazy_mode do
+      with_translation_file_in_load_path(filename, nil, file_contents) do |file_path|
+        exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") }
+
+        expected_message = /#{Regexp.escape(file_path)} can only load translations for "en"\. Found translations for: \[\:fr\, \:de\]/
+        assert_match expected_message, exception.message
+      end
+    end
+  end
+
+  test "lazy mode: #lookup lazy loads translations for supplied locale" do
+    with_lazy_mode do
+      @backend.reload!
+      assert_nil translations
+
+      I18n.with_locale(:en) do
+        assert_equal "chien", @backend.lookup(:fr, "animal.dog")
+      end
+
+      assert_equal({ fr: { animal: { dog: "chien" } } }, translations)
+    end
+  end
+
+  test "eager mode: load all translations, irrespective of locale" do
+    with_eager_mode do
+      @backend.reload!
+
+      assert_nil translations
+
+      I18n.with_locale(:en) { I18n.t("foo.bar") }
+      assert_equal({ en: { foo: { bar: "baz" } }, fr: { animal: { dog: "chien" } } }, translations)
+    end
+  end
+
+  test "eager mode: raises error if locales loaded cannot be extracted from load path names" do
+    with_eager_mode do
+      @backend.reload!
+
+      contents = { de: { cat: 'katze' } }.to_yaml
+
+      with_translation_file_in_load_path(['fr_translation', '.yml'], nil, contents) do |file_path|
+        exception = assert_raises(I18n::InvalidFilenames) { I18n.t("foo.bar") }
+
+        expected_message = /#{Regexp.escape(file_path)} can only load translations for "fr"\. Found translations for: \[\:de\]/
+        assert_match expected_message, exception.message
+      end
+    end
+  end
+
+  private
+
+  def with_lazy_mode
+    @backend = I18n.backend = @lazy_mode_backend
+
+    yield
+  end
+
+  def with_eager_mode
+    @backend = I18n.backend = @eager_mode_backend
+
+    yield
+  end
+
+
+  def with_translation_file_in_load_path(name, tmpdir, file_contents)
+    @backend.reload!
+
+    path_to_dir = FileUtils.mkdir_p(File.join(Dir.tmpdir, tmpdir)).first if tmpdir
+    locale_file = Tempfile.new(name, path_to_dir)
+
+    locale_file.write(file_contents)
+    locale_file.rewind
+
+    I18n.load_path << locale_file.path
+
+    yield(locale_file.path)
+
+    I18n.load_path.delete(locale_file.path)
+  end
+end
+
diff --git a/test/test_data/locales/fr.yml b/test/test_data/locales/fr.yml
new file mode 100644
index 00000000..0e7bb66c
--- /dev/null
+++ b/test/test_data/locales/fr.yml
@@ -0,0 +1,3 @@
+fr:
+  animal:
+    dog: chien