albert flax

huggingface · Aug 30, 2021 · 2cfaf97 · 2cfaf97
1 parent d506495
commit 2cfaf97
Show file tree

Hide file tree

Showing 8 changed files with 1,477 additions and 2 deletions.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -321,7 +321,7 @@ Flax), PyTorch, and/or TensorFlow.
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
 |            Model            | Tokenizer slow | Tokenizer fast | PyTorch support | TensorFlow support | Flax Support |
 +=============================+================+================+=================+====================+==============+
-|           ALBERT            |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
+|           ALBERT            |       ✅       |       ✅       |       ✅        |         ✅         |      ✅      |
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
 |            BART             |       ✅       |       ✅       |       ✅        |         ✅         |      ✅      |
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+

diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst
@@ -43,7 +43,8 @@ Tips:
   similar to a BERT-like architecture with the same number of hidden layers as it has to iterate through the same
   number of (repeating) layers.
 
-This model was contributed by `lysandre <https://huggingface.co/lysandre>`__. The original code can be found `here
+This model was contributed by `lysandre <https://huggingface.co/lysandre>`__. This model jax version was contributed by
+`kamalkraj <https://huggingface.co/kamalkraj>`__. The original code can be found `here
 <https://github.com/google-research/ALBERT>`__.
 
 AlbertConfig
@@ -174,3 +175,52 @@ TFAlbertForQuestionAnswering
 
 .. autoclass:: transformers.TFAlbertForQuestionAnswering
     :members: call
+
+
+FlaxAlbertModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertModel
+    :members: __call__
+
+
+FlaxAlbertForPreTraining
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForPreTraining
+    :members: __call__
+
+
+FlaxAlbertForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForMaskedLM
+    :members: __call__
+
+
+FlaxAlbertForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForSequenceClassification
+    :members: __call__
+
+
+FlaxAlbertForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForMultipleChoice
+    :members: __call__
+
+
+FlaxAlbertForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForTokenClassification
+    :members: __call__
+
+
+FlaxAlbertForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxAlbertForQuestionAnswering
+    :members: __call__
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
@@ -1642,6 +1642,18 @@
         "FlaxTopPLogitsWarper",
     ]
     _import_structure["modeling_flax_utils"] = ["FlaxPreTrainedModel"]
+    _import_structure["models.albert"].extend(
+        [
+            "FlaxAlbertForMaskedLM",
+            "FlaxAlbertForMultipleChoice",
+            "FlaxAlbertForPreTraining",
+            "FlaxAlbertForQuestionAnswering",
+            "FlaxAlbertForSequenceClassification",
+            "FlaxAlbertForTokenClassification",
+            "FlaxAlbertModel",
+            "FlaxAlbertPreTrainedModel",
+        ]
+    )
     _import_structure["models.auto"].extend(
         [
             "FLAX_MODEL_FOR_CAUSAL_LM_MAPPING",
@@ -3152,6 +3164,16 @@
             FlaxTopPLogitsWarper,
         )
         from .modeling_flax_utils import FlaxPreTrainedModel
+        from .models.albert import (
+            FlaxAlbertForMaskedLM,
+            FlaxAlbertForMultipleChoice,
+            FlaxAlbertForPreTraining,
+            FlaxAlbertForQuestionAnswering,
+            FlaxAlbertForSequenceClassification,
+            FlaxAlbertForTokenClassification,
+            FlaxAlbertModel,
+            FlaxAlbertPreTrainedModel,
+        )
         from .models.auto import (
             FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
             FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,

diff --git a/src/transformers/models/albert/__init__.py b/src/transformers/models/albert/__init__.py
@@ -20,6 +20,7 @@
 
 from ...file_utils import (
     _LazyModule,
+    is_flax_available,
     is_sentencepiece_available,
     is_tf_available,
     is_tokenizers_available,
@@ -65,6 +66,17 @@
         "TFAlbertPreTrainedModel",
     ]
 
+if is_flax_available():
+    _import_structure["modeling_flax_albert"] = [
+        "FlaxAlbertForMaskedLM",
+        "FlaxAlbertForMultipleChoice",
+        "FlaxAlbertForPreTraining",
+        "FlaxAlbertForQuestionAnswering",
+        "FlaxAlbertForSequenceClassification",
+        "FlaxAlbertForTokenClassification",
+        "FlaxAlbertModel",
+        "FlaxAlbertPreTrainedModel",
+    ]
 
 if TYPE_CHECKING:
     from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig
@@ -103,6 +115,17 @@
             TFAlbertPreTrainedModel,
         )
 
+    if is_flax_available():
+        from .modeling_flax_albert import (
+            FlaxAlbertForMaskedLM,
+            FlaxAlbertForMultipleChoice,
+            FlaxAlbertForPreTraining,
+            FlaxAlbertForQuestionAnswering,
+            FlaxAlbertForSequenceClassification,
+            FlaxAlbertForTokenClassification,
+            FlaxAlbertModel,
+            FlaxAlbertPreTrainedModel,
+        )
 else:
     import sys